diff --git "a/all_results.json" "b/all_results.json" --- "a/all_results.json" +++ "b/all_results.json" @@ -1,43955 +1,31998 @@ { - "gpt_35_turbo_1106": { - "model_size": "Closed Source", - "model_link": "https://openai.com/blog/chatgpt", + "random": { + "model_size": "NA", + "model_link": "https://seaeval.github.io/", "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.915126050420168, + "overall_acc": 0.242436974789916, "language_acc": { - "Spanish": 0.9201680672268907, - "English": 0.9436974789915966, - "Chinese": 0.9042016806722689, - "Vietnamese": 0.892436974789916 + "Spanish": 0.22941176470588234, + "English": 0.23781512605042016, + "Chinese": 0.2605042016806723, + "Vietnamese": 0.24201680672268908 }, - "consistency_score_2": 0.900280112044818, - "consistency_score_3": 0.8569327731092438, - "consistency_score_4": 0.826890756302521, + "consistency_score_2": 0.2533613445378151, + "consistency_score_3": 0.06932773109243698, + "consistency_score_4": 0.024369747899159664, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.9285714285714286, - "Spanish,Chinese": 0.8949579831932774, - "Spanish,Vietnamese": 0.892436974789916, - "English,Chinese": 0.9050420168067227, - "English,Vietnamese": 0.9033613445378151, - "Chinese,Vietnamese": 0.8773109243697479 + "Spanish,English": 0.23781512605042016, + "Spanish,Chinese": 0.2529411764705882, + "Spanish,Vietnamese": 0.25798319327731095, + "English,Chinese": 0.26638655462184874, + "English,Vietnamese": 0.2504201680672269, + "Chinese,Vietnamese": 0.2546218487394958 }, "3_combine": { - "Spanish,English,Chinese": 0.8697478991596639, - "Spanish,English,Vietnamese": 0.8672268907563025, - "Spanish,Chinese,Vietnamese": 0.8411764705882353, - "English,Chinese,Vietnamese": 0.8495798319327731 + "Spanish,English,Chinese": 0.07226890756302522, + "Spanish,English,Vietnamese": 0.06050420168067227, + "Spanish,Chinese,Vietnamese": 0.07058823529411765, + "English,Chinese,Vietnamese": 0.07394957983193277 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.826890756302521 + "Spanish,English,Chinese,Vietnamese": 0.024369747899159664 } }, - "AC3_2": 0.9076423779936448, - "AC3_3": 0.8850738967286825, - "AC3_4": 0.8687737902187682 + "AC3_2": 0.24777880638785366, + "AC3_3": 0.10782237419812744, + "AC3_4": 0.044287699315102014 }, "prompt_2": { - "overall_acc": 0.9155462184873949, + "overall_acc": 0.2640756302521008, "language_acc": { - "Spanish": 0.9193277310924369, - "English": 0.9436974789915966, - "Chinese": 0.9058823529411765, - "Vietnamese": 0.8932773109243698 + "Spanish": 0.2571428571428571, + "English": 0.25546218487394956, + "Chinese": 0.26974789915966385, + "Vietnamese": 0.2739495798319328 }, - "consistency_score_2": 0.9009803921568628, - "consistency_score_3": 0.8571428571428572, - "consistency_score_4": 0.826890756302521, + "consistency_score_2": 0.24439775910364148, + "consistency_score_3": 0.05882352941176471, + "consistency_score_4": 0.013445378151260505, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.926890756302521, - "Spanish,Chinese": 0.8966386554621849, - "Spanish,Vietnamese": 0.892436974789916, - "English,Chinese": 0.9067226890756303, - "English,Vietnamese": 0.9033613445378151, - "Chinese,Vietnamese": 0.8798319327731092 + "Spanish,English": 0.22941176470588234, + "Spanish,Chinese": 0.246218487394958, + "Spanish,Vietnamese": 0.246218487394958, + "English,Chinese": 0.24789915966386555, + "English,Vietnamese": 0.25126050420168067, + "Chinese,Vietnamese": 0.2453781512605042 }, "3_combine": { - "Spanish,English,Chinese": 0.8697478991596639, - "Spanish,English,Vietnamese": 0.865546218487395, - "Spanish,Chinese,Vietnamese": 0.8420168067226891, - "English,Chinese,Vietnamese": 0.8512605042016806 + "Spanish,English,Chinese": 0.05966386554621849, + "Spanish,English,Vietnamese": 0.05714285714285714, + "Spanish,Chinese,Vietnamese": 0.06638655462184874, + "English,Chinese,Vietnamese": 0.052100840336134456 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.826890756302521 + "Spanish,English,Chinese,Vietnamese": 0.013445378151260505 } }, - "AC3_2": 0.9082049072020327, - "AC3_3": 0.8853824534766179, - "AC3_4": 0.8689630855460749 + "AC3_2": 0.2538559287697504, + "AC3_3": 0.09621493356888625, + "AC3_4": 0.02558794902353448 }, "prompt_3": { - "overall_acc": 0.9065126050420168, + "overall_acc": 0.24726890756302522, "language_acc": { - "Spanish": 0.9159663865546218, - "English": 0.9277310924369748, - "Chinese": 0.9008403361344538, - "Vietnamese": 0.8815126050420168 + "Spanish": 0.2495798319327731, + "English": 0.24033613445378152, + "Chinese": 0.25798319327731095, + "Vietnamese": 0.2411764705882353 }, - "consistency_score_2": 0.8900560224089636, - "consistency_score_3": 0.8407563025210084, - "consistency_score_4": 0.8050420168067227, + "consistency_score_2": 0.24663865546218489, + "consistency_score_3": 0.0592436974789916, + "consistency_score_4": 0.01764705882352941, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.9184873949579831, - "Spanish,Chinese": 0.892436974789916, - "Spanish,Vietnamese": 0.873109243697479, - "English,Chinese": 0.8974789915966387, - "English,Vietnamese": 0.8890756302521008, - "Chinese,Vietnamese": 0.8697478991596639 + "Spanish,English": 0.24369747899159663, + "Spanish,Chinese": 0.24789915966386555, + "Spanish,Vietnamese": 0.24873949579831933, + "English,Chinese": 0.24705882352941178, + "English,Vietnamese": 0.24705882352941178, + "Chinese,Vietnamese": 0.2453781512605042 }, "3_combine": { - "Spanish,English,Chinese": 0.8571428571428571, - "Spanish,English,Vietnamese": 0.846218487394958, - "Spanish,Chinese,Vietnamese": 0.8252100840336134, - "English,Chinese,Vietnamese": 0.8344537815126051 + "Spanish,English,Chinese": 0.06134453781512605, + "Spanish,English,Vietnamese": 0.0546218487394958, + "Spanish,Chinese,Vietnamese": 0.06050420168067227, + "English,Chinese,Vietnamese": 0.06050420168067227 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.8050420168067227 + "Spanish,English,Chinese,Vietnamese": 0.01764705882352941 } }, - "AC3_2": 0.8982089424628893, - "AC3_3": 0.8723971252063294, - "AC3_4": 0.8527694371714862 + "AC3_2": 0.24695337934514394, + "AC3_3": 0.09558578740236284, + "AC3_4": 0.03294304239088608 }, "prompt_4": { - "overall_acc": 0.9142857142857143, + "overall_acc": 0.25168067226890756, "language_acc": { - "Spanish": 0.9319327731092437, - "English": 0.938655462184874, - "Chinese": 0.9016806722689076, - "Vietnamese": 0.8848739495798319 + "Spanish": 0.22941176470588234, + "English": 0.26974789915966385, + "Chinese": 0.27310924369747897, + "Vietnamese": 0.23445378151260504 }, - "consistency_score_2": 0.8978991596638656, - "consistency_score_3": 0.8525210084033614, - "consistency_score_4": 0.8184873949579832, + "consistency_score_2": 0.24985994397759104, + "consistency_score_3": 0.06491596638655461, + "consistency_score_4": 0.015126050420168067, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.938655462184874, - "Spanish,Chinese": 0.8983193277310925, - "Spanish,Vietnamese": 0.8949579831932774, - "English,Chinese": 0.8983193277310925, - "English,Vietnamese": 0.8865546218487395, - "Chinese,Vietnamese": 0.8705882352941177 + "Spanish,English": 0.26554621848739496, + "Spanish,Chinese": 0.21260504201680672, + "Spanish,Vietnamese": 0.24033613445378152, + "English,Chinese": 0.24789915966386555, + "English,Vietnamese": 0.2722689075630252, + "Chinese,Vietnamese": 0.2605042016806723 }, "3_combine": { - "Spanish,English,Chinese": 0.8705882352941177, - "Spanish,English,Vietnamese": 0.8630252100840337, - "Spanish,Chinese,Vietnamese": 0.8411764705882353, - "English,Chinese,Vietnamese": 0.8352941176470589 + "Spanish,English,Chinese": 0.058823529411764705, + "Spanish,English,Vietnamese": 0.073109243697479, + "Spanish,Chinese,Vietnamese": 0.05630252100840336, + "English,Chinese,Vietnamese": 0.07142857142857142 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.8184873949579832 + "Spanish,English,Chinese,Vietnamese": 0.015126050420168067 } }, - "AC3_2": 0.9060183497893602, - "AC3_3": 0.8823237641763225, - "AC3_4": 0.863738395405328 + "AC3_2": 0.25076700320480716, + "AC3_3": 0.10321078660454978, + "AC3_4": 0.028537021096959894 }, "prompt_5": { - "overall_acc": 0.9058823529411765, + "overall_acc": 0.2548319327731092, "language_acc": { - "Spanish": 0.9168067226890756, - "English": 0.926890756302521, - "Chinese": 0.9025210084033614, - "Vietnamese": 0.8773109243697479 + "Spanish": 0.2453781512605042, + "English": 0.2546218487394958, + "Chinese": 0.27563025210084036, + "Vietnamese": 0.24369747899159663 }, - "consistency_score_2": 0.8901960784313726, - "consistency_score_3": 0.8418067226890756, - "consistency_score_4": 0.8067226890756303, + "consistency_score_2": 0.24915966386554622, + "consistency_score_3": 0.06365546218487395, + "consistency_score_4": 0.01680672268907563, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.9193277310924369, - "Spanish,Chinese": 0.8957983193277311, - "Spanish,Vietnamese": 0.8697478991596639, - "English,Chinese": 0.9016806722689076, - "English,Vietnamese": 0.8899159663865546, - "Chinese,Vietnamese": 0.8647058823529412 + "Spanish,English": 0.2630252100840336, + "Spanish,Chinese": 0.246218487394958, + "Spanish,Vietnamese": 0.23529411764705882, + "English,Chinese": 0.253781512605042, + "English,Vietnamese": 0.2495798319327731, + "Chinese,Vietnamese": 0.24705882352941178 }, "3_combine": { - "Spanish,English,Chinese": 0.8630252100840337, - "Spanish,English,Vietnamese": 0.8453781512605042, - "Spanish,Chinese,Vietnamese": 0.8235294117647058, - "English,Chinese,Vietnamese": 0.8352941176470589 + "Spanish,English,Chinese": 0.06722689075630252, + "Spanish,English,Vietnamese": 0.05798319327731093, + "Spanish,Chinese,Vietnamese": 0.06638655462184874, + "English,Chinese,Vietnamese": 0.06302521008403361 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.8067226890756303 + "Spanish,English,Chinese,Vietnamese": 0.01680672268907563 } }, - "AC3_2": 0.8979707166209516, - "AC3_3": 0.8726699334120924, - "AC3_4": 0.8534318535552013 + "AC3_2": 0.2519638784568071, + "AC3_3": 0.10186553510027453, + "AC3_4": 0.03153372716835841 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.661904761904762, + "overall_acc": 0.26285714285714284, "language_acc": { - "Vietnamese": 0.6133333333333333, - "Malay": 0.6133333333333333, - "Filipino": 0.6266666666666667, - "Indonesian": 0.66, - "Chinese": 0.6533333333333333, - "Spanish": 0.72, - "English": 0.7466666666666667 + "Filipino": 0.32666666666666666, + "Vietnamese": 0.29333333333333333, + "Chinese": 0.23333333333333334, + "Spanish": 0.26, + "Malay": 0.24, + "Indonesian": 0.25333333333333335, + "English": 0.23333333333333334 }, - "consistency_score_2": 0.6663492063492062, - "consistency_score_3": 0.5316190476190475, - "consistency_score_4": 0.449904761904762, - "consistency_score_5": 0.3923809523809524, - "consistency_score_6": 0.3485714285714286, - "consistency_score_7": 0.31333333333333335, + "consistency_score_2": 0.24730158730158736, + "consistency_score_3": 0.058666666666666666, + "consistency_score_4": 0.013142857142857138, + "consistency_score_5": 0.0031746031746031746, + "consistency_score_6": 0.0009523809523809525, + "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { - "Vietnamese,Malay": 0.5666666666666667, - "Vietnamese,Filipino": 0.6133333333333333, - "Vietnamese,Indonesian": 0.6466666666666666, - "Vietnamese,Chinese": 0.5933333333333334, - "Vietnamese,Spanish": 0.6666666666666666, - "Vietnamese,English": 0.64, - "Malay,Filipino": 0.62, - "Malay,Indonesian": 0.6866666666666666, - "Malay,Chinese": 0.64, - "Malay,Spanish": 0.6933333333333334, - "Malay,English": 0.6733333333333333, - "Filipino,Indonesian": 0.6733333333333333, - "Filipino,Chinese": 0.6133333333333333, - "Filipino,Spanish": 0.72, - "Filipino,English": 0.6666666666666666, - "Indonesian,Chinese": 0.6666666666666666, - "Indonesian,Spanish": 0.76, - "Indonesian,English": 0.7333333333333333, - "Chinese,Spanish": 0.6666666666666666, - "Chinese,English": 0.6533333333333333, - "Spanish,English": 0.8 - }, - "3_combine": { - "Vietnamese,Malay,Filipino": 0.4533333333333333, - "Vietnamese,Malay,Indonesian": 0.4866666666666667, - "Vietnamese,Malay,Chinese": 0.4533333333333333, - "Vietnamese,Malay,Spanish": 0.49333333333333335, - "Vietnamese,Malay,English": 0.4866666666666667, - "Vietnamese,Filipino,Indonesian": 0.5133333333333333, - "Vietnamese,Filipino,Chinese": 0.4533333333333333, - "Vietnamese,Filipino,Spanish": 0.54, - "Vietnamese,Filipino,English": 0.5133333333333333, - "Vietnamese,Indonesian,Chinese": 0.5, - "Vietnamese,Indonesian,Spanish": 0.5733333333333334, - "Vietnamese,Indonesian,English": 0.56, - "Vietnamese,Chinese,Spanish": 0.5, - "Vietnamese,Chinese,English": 0.4866666666666667, - "Vietnamese,Spanish,English": 0.58, - "Malay,Filipino,Indonesian": 0.5266666666666666, - "Malay,Filipino,Chinese": 0.48, - "Malay,Filipino,Spanish": 0.54, - "Malay,Filipino,English": 0.52, - "Malay,Indonesian,Chinese": 0.5266666666666666, - "Malay,Indonesian,Spanish": 0.5933333333333334, - "Malay,Indonesian,English": 0.5733333333333334, - "Malay,Chinese,Spanish": 0.5133333333333333, - "Malay,Chinese,English": 0.5133333333333333, - "Malay,Spanish,English": 0.6066666666666667, - "Filipino,Indonesian,Chinese": 0.49333333333333335, - "Filipino,Indonesian,Spanish": 0.5866666666666667, - "Filipino,Indonesian,English": 0.5533333333333333, - "Filipino,Chinese,Spanish": 0.5133333333333333, - "Filipino,Chinese,English": 0.5, - "Filipino,Spanish,English": 0.6066666666666667, - "Indonesian,Chinese,Spanish": 0.5666666666666667, - "Indonesian,Chinese,English": 0.5466666666666666, - "Indonesian,Spanish,English": 0.6733333333333333, - "Chinese,Spanish,English": 0.58 - }, - "4_combine": { - "Vietnamese,Malay,Filipino,Indonesian": 0.41333333333333333, - "Vietnamese,Malay,Filipino,Chinese": 0.38, - "Vietnamese,Malay,Filipino,Spanish": 0.4266666666666667, - "Vietnamese,Malay,Filipino,English": 0.4066666666666667, - "Vietnamese,Malay,Indonesian,Chinese": 0.4066666666666667, - "Vietnamese,Malay,Indonesian,Spanish": 0.4533333333333333, - "Vietnamese,Malay,Indonesian,English": 0.4533333333333333, - "Vietnamese,Malay,Chinese,Spanish": 0.4066666666666667, - "Vietnamese,Malay,Chinese,English": 0.4, - "Vietnamese,Malay,Spanish,English": 0.46, - "Vietnamese,Filipino,Indonesian,Chinese": 0.4, - "Vietnamese,Filipino,Indonesian,Spanish": 0.48, - "Vietnamese,Filipino,Indonesian,English": 0.46, - "Vietnamese,Filipino,Chinese,Spanish": 0.41333333333333333, - "Vietnamese,Filipino,Chinese,English": 0.4, - "Vietnamese,Filipino,Spanish,English": 0.48, - "Vietnamese,Indonesian,Chinese,Spanish": 0.46, - "Vietnamese,Indonesian,Chinese,English": 0.4533333333333333, - "Vietnamese,Indonesian,Spanish,English": 0.5266666666666666, - "Vietnamese,Chinese,Spanish,English": 0.4533333333333333, - "Malay,Filipino,Indonesian,Chinese": 0.41333333333333333, - "Malay,Filipino,Indonesian,Spanish": 0.4866666666666667, - "Malay,Filipino,Indonesian,English": 0.4666666666666667, - "Malay,Filipino,Chinese,Spanish": 0.43333333333333335, - "Malay,Filipino,Chinese,English": 0.42, - "Malay,Filipino,Spanish,English": 0.48, - "Malay,Indonesian,Chinese,Spanish": 0.46, - "Malay,Indonesian,Chinese,English": 0.46, - "Malay,Indonesian,Spanish,English": 0.54, - "Malay,Chinese,Spanish,English": 0.4666666666666667, - "Filipino,Indonesian,Chinese,Spanish": 0.44666666666666666, - "Filipino,Indonesian,Chinese,English": 0.43333333333333335, - "Filipino,Indonesian,Spanish,English": 0.5266666666666666, - "Filipino,Chinese,Spanish,English": 0.4666666666666667, - "Indonesian,Chinese,Spanish,English": 0.5133333333333333 - }, - "5_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.34, - "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.4, - "Vietnamese,Malay,Filipino,Indonesian,English": 0.38666666666666666, - "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.36, - "Vietnamese,Malay,Filipino,Chinese,English": 0.34, - "Vietnamese,Malay,Filipino,Spanish,English": 0.3933333333333333, - "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.38, - "Vietnamese,Malay,Indonesian,Chinese,English": 0.38, - "Vietnamese,Malay,Indonesian,Spanish,English": 0.4266666666666667, - "Vietnamese,Malay,Chinese,Spanish,English": 0.38, - "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.38, - "Vietnamese,Filipino,Indonesian,Chinese,English": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian,Spanish,English": 0.44, - "Vietnamese,Filipino,Chinese,Spanish,English": 0.38, - "Vietnamese,Indonesian,Chinese,Spanish,English": 0.4266666666666667, - "Malay,Filipino,Indonesian,Chinese,Spanish": 0.38666666666666666, - "Malay,Filipino,Indonesian,Chinese,English": 0.38, - "Malay,Filipino,Indonesian,Spanish,English": 0.44666666666666666, - "Malay,Filipino,Chinese,Spanish,English": 0.4, - "Malay,Indonesian,Chinese,Spanish,English": 0.43333333333333335, - "Filipino,Indonesian,Chinese,Spanish,English": 0.41333333333333333 + "Filipino,Vietnamese": 0.2, + "Filipino,Chinese": 0.25333333333333335, + "Filipino,Spanish": 0.2733333333333333, + "Filipino,Malay": 0.24, + "Filipino,Indonesian": 0.23333333333333334, + "Filipino,English": 0.2733333333333333, + "Vietnamese,Chinese": 0.29333333333333333, + "Vietnamese,Spanish": 0.2733333333333333, + "Vietnamese,Malay": 0.26, + "Vietnamese,Indonesian": 0.25333333333333335, + "Vietnamese,English": 0.21333333333333335, + "Chinese,Spanish": 0.25333333333333335, + "Chinese,Malay": 0.28, + "Chinese,Indonesian": 0.23333333333333334, + "Chinese,English": 0.26, + "Spanish,Malay": 0.19333333333333333, + "Spanish,Indonesian": 0.24666666666666667, + "Spanish,English": 0.24666666666666667, + "Malay,Indonesian": 0.2733333333333333, + "Malay,English": 0.24, + "Indonesian,English": 0.2 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.08, + "Filipino,Vietnamese,Spanish": 0.06, + "Filipino,Vietnamese,Malay": 0.06, + "Filipino,Vietnamese,Indonesian": 0.06, + "Filipino,Vietnamese,English": 0.04666666666666667, + "Filipino,Chinese,Spanish": 0.06666666666666667, + "Filipino,Chinese,Malay": 0.08, + "Filipino,Chinese,Indonesian": 0.05333333333333334, + "Filipino,Chinese,English": 0.07333333333333333, + "Filipino,Spanish,Malay": 0.04, + "Filipino,Spanish,Indonesian": 0.04, + "Filipino,Spanish,English": 0.09333333333333334, + "Filipino,Malay,Indonesian": 0.05333333333333334, + "Filipino,Malay,English": 0.05333333333333334, + "Filipino,Indonesian,English": 0.03333333333333333, + "Vietnamese,Chinese,Spanish": 0.09333333333333334, + "Vietnamese,Chinese,Malay": 0.08666666666666667, + "Vietnamese,Chinese,Indonesian": 0.05333333333333334, + "Vietnamese,Chinese,English": 0.07333333333333333, + "Vietnamese,Spanish,Malay": 0.06, + "Vietnamese,Spanish,Indonesian": 0.05333333333333334, + "Vietnamese,Spanish,English": 0.04, + "Vietnamese,Malay,Indonesian": 0.07333333333333333, + "Vietnamese,Malay,English": 0.04666666666666667, + "Vietnamese,Indonesian,English": 0.04, + "Chinese,Spanish,Malay": 0.06666666666666667, + "Chinese,Spanish,Indonesian": 0.04666666666666667, + "Chinese,Spanish,English": 0.04, + "Chinese,Malay,Indonesian": 0.07333333333333333, + "Chinese,Malay,English": 0.06666666666666667, + "Chinese,Indonesian,English": 0.04, + "Spanish,Malay,Indonesian": 0.04666666666666667, + "Spanish,Malay,English": 0.06666666666666667, + "Spanish,Indonesian,English": 0.05333333333333334, + "Malay,Indonesian,English": 0.04 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.02666666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.02, + "Filipino,Vietnamese,Chinese,Indonesian": 0.02666666666666667, + "Filipino,Vietnamese,Chinese,English": 0.02666666666666667, + "Filipino,Vietnamese,Spanish,Malay": 0.02, + "Filipino,Vietnamese,Spanish,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,English": 0.006666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.02, + "Filipino,Vietnamese,Malay,English": 0.013333333333333334, + "Filipino,Vietnamese,Indonesian,English": 0.006666666666666667, + "Filipino,Chinese,Spanish,Malay": 0.013333333333333334, + "Filipino,Chinese,Spanish,Indonesian": 0.013333333333333334, + "Filipino,Chinese,Spanish,English": 0.013333333333333334, + "Filipino,Chinese,Malay,Indonesian": 0.013333333333333334, + "Filipino,Chinese,Malay,English": 0.013333333333333334, + "Filipino,Chinese,Indonesian,English": 0.006666666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.0, + "Filipino,Spanish,Malay,English": 0.02, + "Filipino,Spanish,Indonesian,English": 0.02, + "Filipino,Malay,Indonesian,English": 0.013333333333333334, + "Vietnamese,Chinese,Spanish,Malay": 0.02666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.006666666666666667, + "Vietnamese,Chinese,Spanish,English": 0.006666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.006666666666666667, + "Vietnamese,Chinese,Malay,English": 0.02, + "Vietnamese,Chinese,Indonesian,English": 0.0, + "Vietnamese,Spanish,Malay,Indonesian": 0.006666666666666667, + "Vietnamese,Spanish,Malay,English": 0.013333333333333334, + "Vietnamese,Spanish,Indonesian,English": 0.013333333333333334, + "Vietnamese,Malay,Indonesian,English": 0.013333333333333334, + "Chinese,Spanish,Malay,Indonesian": 0.006666666666666667, + "Chinese,Spanish,Malay,English": 0.02, + "Chinese,Spanish,Indonesian,English": 0.006666666666666667, + "Chinese,Malay,Indonesian,English": 0.006666666666666667, + "Spanish,Malay,Indonesian,English": 0.006666666666666667 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.0, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Spanish,Malay,English": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.0, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.006666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Chinese,Spanish,Malay,English": 0.006666666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.006666666666666667, + "Filipino,Chinese,Malay,Indonesian,English": 0.0, + "Filipino,Spanish,Malay,Indonesian,English": 0.0, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, + "Vietnamese,Chinese,Spanish,Malay,English": 0.006666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.0, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.0, + "Chinese,Spanish,Malay,Indonesian,English": 0.0 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.0, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + } + }, + "AC3_2": 0.2548422081463306, + "AC3_3": 0.09592417058627951, + "AC3_4": 0.025034013596371876, + "AC3_5": 0.006273440161296806, + "AC3_6": 0.0018978855072743965, + "AC3_7": 0.0 + }, + "prompt_2": { + "overall_acc": 0.2580952380952381, + "language_acc": { + "Filipino": 0.2733333333333333, + "Vietnamese": 0.23333333333333334, + "Chinese": 0.26, + "Spanish": 0.22666666666666666, + "Malay": 0.25333333333333335, + "Indonesian": 0.29333333333333333, + "English": 0.26666666666666666 + }, + "consistency_score_2": 0.24000000000000007, + "consistency_score_3": 0.056761904761904756, + "consistency_score_4": 0.013142857142857138, + "consistency_score_5": 0.0022222222222222222, + "consistency_score_6": 0.0, + "consistency_score_7": 0.0, + "detailed_consistency_score": { + "2_combine": { + "Filipino,Vietnamese": 0.2866666666666667, + "Filipino,Chinese": 0.21333333333333335, + "Filipino,Spanish": 0.24666666666666667, + "Filipino,Malay": 0.23333333333333334, + "Filipino,Indonesian": 0.25333333333333335, + "Filipino,English": 0.26, + "Vietnamese,Chinese": 0.23333333333333334, + "Vietnamese,Spanish": 0.2, + "Vietnamese,Malay": 0.2, + "Vietnamese,Indonesian": 0.22, + "Vietnamese,English": 0.17333333333333334, + "Chinese,Spanish": 0.24, + "Chinese,Malay": 0.28, + "Chinese,Indonesian": 0.24, + "Chinese,English": 0.26666666666666666, + "Spanish,Malay": 0.28, + "Spanish,Indonesian": 0.24666666666666667, + "Spanish,English": 0.2733333333333333, + "Malay,Indonesian": 0.23333333333333334, + "Malay,English": 0.24, + "Indonesian,English": 0.22 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.06, + "Filipino,Vietnamese,Spanish": 0.07333333333333333, + "Filipino,Vietnamese,Malay": 0.06, + "Filipino,Vietnamese,Indonesian": 0.07333333333333333, + "Filipino,Vietnamese,English": 0.04666666666666667, + "Filipino,Chinese,Spanish": 0.04666666666666667, + "Filipino,Chinese,Malay": 0.06666666666666667, + "Filipino,Chinese,Indonesian": 0.04666666666666667, + "Filipino,Chinese,English": 0.06666666666666667, + "Filipino,Spanish,Malay": 0.06666666666666667, + "Filipino,Spanish,Indonesian": 0.06, + "Filipino,Spanish,English": 0.08, + "Filipino,Malay,Indonesian": 0.05333333333333334, + "Filipino,Malay,English": 0.06666666666666667, + "Filipino,Indonesian,English": 0.03333333333333333, + "Vietnamese,Chinese,Spanish": 0.04, + "Vietnamese,Chinese,Malay": 0.05333333333333334, + "Vietnamese,Chinese,Indonesian": 0.04666666666666667, + "Vietnamese,Chinese,English": 0.05333333333333334, + "Vietnamese,Spanish,Malay": 0.05333333333333334, + "Vietnamese,Spanish,Indonesian": 0.07333333333333333, + "Vietnamese,Spanish,English": 0.02666666666666667, + "Vietnamese,Malay,Indonesian": 0.03333333333333333, + "Vietnamese,Malay,English": 0.02666666666666667, + "Vietnamese,Indonesian,English": 0.013333333333333334, + "Chinese,Spanish,Malay": 0.07333333333333333, + "Chinese,Spanish,Indonesian": 0.06, + "Chinese,Spanish,English": 0.08666666666666667, + "Chinese,Malay,Indonesian": 0.04666666666666667, + "Chinese,Malay,English": 0.07333333333333333, + "Chinese,Indonesian,English": 0.06666666666666667, + "Spanish,Malay,Indonesian": 0.06666666666666667, + "Spanish,Malay,English": 0.06, + "Spanish,Indonesian,English": 0.06666666666666667, + "Malay,Indonesian,English": 0.06666666666666667 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.013333333333333334, + "Filipino,Vietnamese,Chinese,Malay": 0.02, + "Filipino,Vietnamese,Chinese,Indonesian": 0.02, + "Filipino,Vietnamese,Chinese,English": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,Malay": 0.02, + "Filipino,Vietnamese,Spanish,Indonesian": 0.03333333333333333, + "Filipino,Vietnamese,Spanish,English": 0.006666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.013333333333333334, + "Filipino,Vietnamese,Malay,English": 0.0, + "Filipino,Vietnamese,Indonesian,English": 0.0, + "Filipino,Chinese,Spanish,Malay": 0.006666666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.013333333333333334, + "Filipino,Chinese,Spanish,English": 0.013333333333333334, + "Filipino,Chinese,Malay,Indonesian": 0.006666666666666667, + "Filipino,Chinese,Malay,English": 0.006666666666666667, + "Filipino,Chinese,Indonesian,English": 0.013333333333333334, + "Filipino,Spanish,Malay,Indonesian": 0.02, + "Filipino,Spanish,Malay,English": 0.03333333333333333, + "Filipino,Spanish,Indonesian,English": 0.006666666666666667, + "Filipino,Malay,Indonesian,English": 0.013333333333333334, + "Vietnamese,Chinese,Spanish,Malay": 0.013333333333333334, + "Vietnamese,Chinese,Spanish,Indonesian": 0.02, + "Vietnamese,Chinese,Spanish,English": 0.006666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.006666666666666667, + "Vietnamese,Chinese,Malay,English": 0.013333333333333334, + "Vietnamese,Chinese,Indonesian,English": 0.0, + "Vietnamese,Spanish,Malay,Indonesian": 0.013333333333333334, + "Vietnamese,Spanish,Malay,English": 0.0, + "Vietnamese,Spanish,Indonesian,English": 0.006666666666666667, + "Vietnamese,Malay,Indonesian,English": 0.006666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.02, + "Chinese,Spanish,Malay,English": 0.02666666666666667, + "Chinese,Spanish,Indonesian,English": 0.02666666666666667, + "Chinese,Malay,Indonesian,English": 0.02, + "Spanish,Malay,Indonesian,English": 0.013333333333333334 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.013333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.0, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Chinese,Malay,English": 0.0, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.0, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.0, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.0, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.0, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Chinese,Spanish,Malay,English": 0.006666666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.0, + "Filipino,Chinese,Malay,Indonesian,English": 0.0, + "Filipino,Spanish,Malay,Indonesian,English": 0.006666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.006666666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.0, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.0, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.0, + "Chinese,Spanish,Malay,Indonesian,English": 0.006666666666666667 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.0, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + } + }, + "AC3_2": 0.24871892920436814, + "AC3_3": 0.09305793215723321, + "AC3_4": 0.02501203851405304, + "AC3_5": 0.004406504063347907, + "AC3_6": 0.0, + "AC3_7": 0.0 + }, + "prompt_3": { + "overall_acc": 0.24095238095238095, + "language_acc": { + "Filipino": 0.22666666666666666, + "Vietnamese": 0.18, + "Chinese": 0.28, + "Spanish": 0.24666666666666667, + "Malay": 0.21333333333333335, + "Indonesian": 0.2733333333333333, + "English": 0.26666666666666666 + }, + "consistency_score_2": 0.25968253968253974, + "consistency_score_3": 0.06914285714285713, + "consistency_score_4": 0.01923809523809524, + "consistency_score_5": 0.005079365079365079, + "consistency_score_6": 0.0009523809523809525, + "consistency_score_7": 0.0, + "detailed_consistency_score": { + "2_combine": { + "Filipino,Vietnamese": 0.24666666666666667, + "Filipino,Chinese": 0.26666666666666666, + "Filipino,Spanish": 0.30666666666666664, + "Filipino,Malay": 0.26, + "Filipino,Indonesian": 0.20666666666666667, + "Filipino,English": 0.26, + "Vietnamese,Chinese": 0.25333333333333335, + "Vietnamese,Spanish": 0.22666666666666666, + "Vietnamese,Malay": 0.22666666666666666, + "Vietnamese,Indonesian": 0.2866666666666667, + "Vietnamese,English": 0.2733333333333333, + "Chinese,Spanish": 0.22666666666666666, + "Chinese,Malay": 0.22, + "Chinese,Indonesian": 0.23333333333333334, + "Chinese,English": 0.26, + "Spanish,Malay": 0.2733333333333333, + "Spanish,Indonesian": 0.24, + "Spanish,English": 0.3, + "Malay,Indonesian": 0.32666666666666666, + "Malay,English": 0.24, + "Indonesian,English": 0.32 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.08666666666666667, + "Filipino,Vietnamese,Spanish": 0.07333333333333333, + "Filipino,Vietnamese,Malay": 0.06666666666666667, + "Filipino,Vietnamese,Indonesian": 0.04666666666666667, + "Filipino,Vietnamese,English": 0.08, + "Filipino,Chinese,Spanish": 0.06, + "Filipino,Chinese,Malay": 0.07333333333333333, + "Filipino,Chinese,Indonesian": 0.04666666666666667, + "Filipino,Chinese,English": 0.07333333333333333, + "Filipino,Spanish,Malay": 0.06666666666666667, + "Filipino,Spanish,Indonesian": 0.05333333333333334, + "Filipino,Spanish,English": 0.1, + "Filipino,Malay,Indonesian": 0.07333333333333333, + "Filipino,Malay,English": 0.06666666666666667, + "Filipino,Indonesian,English": 0.08, + "Vietnamese,Chinese,Spanish": 0.05333333333333334, + "Vietnamese,Chinese,Malay": 0.05333333333333334, + "Vietnamese,Chinese,Indonesian": 0.05333333333333334, + "Vietnamese,Chinese,English": 0.08666666666666667, + "Vietnamese,Spanish,Malay": 0.05333333333333334, + "Vietnamese,Spanish,Indonesian": 0.06666666666666667, + "Vietnamese,Spanish,English": 0.06, + "Vietnamese,Malay,Indonesian": 0.06666666666666667, + "Vietnamese,Malay,English": 0.04666666666666667, + "Vietnamese,Indonesian,English": 0.09333333333333334, + "Chinese,Spanish,Malay": 0.04666666666666667, + "Chinese,Spanish,Indonesian": 0.07333333333333333, + "Chinese,Spanish,English": 0.08, + "Chinese,Malay,Indonesian": 0.05333333333333334, + "Chinese,Malay,English": 0.06, + "Chinese,Indonesian,English": 0.08, + "Spanish,Malay,Indonesian": 0.09333333333333334, + "Spanish,Malay,English": 0.06, + "Spanish,Indonesian,English": 0.09333333333333334, + "Malay,Indonesian,English": 0.1 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.02666666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.02, + "Filipino,Vietnamese,Chinese,Indonesian": 0.013333333333333334, + "Filipino,Vietnamese,Chinese,English": 0.04, + "Filipino,Vietnamese,Spanish,Malay": 0.013333333333333334, + "Filipino,Vietnamese,Spanish,Indonesian": 0.02, + "Filipino,Vietnamese,Spanish,English": 0.02666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.013333333333333334, + "Filipino,Vietnamese,Malay,English": 0.013333333333333334, + "Filipino,Vietnamese,Indonesian,English": 0.02, + "Filipino,Chinese,Spanish,Malay": 0.013333333333333334, + "Filipino,Chinese,Spanish,Indonesian": 0.02, + "Filipino,Chinese,Spanish,English": 0.02, + "Filipino,Chinese,Malay,Indonesian": 0.013333333333333334, + "Filipino,Chinese,Malay,English": 0.02, + "Filipino,Chinese,Indonesian,English": 0.02666666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.02, + "Filipino,Spanish,Malay,English": 0.013333333333333334, + "Filipino,Spanish,Indonesian,English": 0.02666666666666667, + "Filipino,Malay,Indonesian,English": 0.02666666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.006666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.013333333333333334, + "Vietnamese,Chinese,Spanish,English": 0.02666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.013333333333333334, + "Vietnamese,Chinese,Malay,English": 0.02, + "Vietnamese,Chinese,Indonesian,English": 0.03333333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.02, + "Vietnamese,Spanish,Malay,English": 0.006666666666666667, + "Vietnamese,Spanish,Indonesian,English": 0.02666666666666667, + "Vietnamese,Malay,Indonesian,English": 0.006666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.006666666666666667, + "Chinese,Spanish,Malay,English": 0.006666666666666667, + "Chinese,Spanish,Indonesian,English": 0.04666666666666667, + "Chinese,Malay,Indonesian,English": 0.013333333333333334, + "Spanish,Malay,Indonesian,English": 0.02 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.013333333333333334, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Chinese,Malay,English": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.013333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.0, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.006666666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.0, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Chinese,Spanish,Malay,English": 0.0, + "Filipino,Chinese,Spanish,Indonesian,English": 0.013333333333333334, + "Filipino,Chinese,Malay,Indonesian,English": 0.006666666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.013333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, + "Vietnamese,Chinese,Spanish,Malay,English": 0.0, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.013333333333333334, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.006666666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.0, + "Chinese,Spanish,Malay,Indonesian,English": 0.0 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.0, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + } + }, + "AC3_2": 0.24996708634074552, + "AC3_3": 0.10745173741708622, + "AC3_4": 0.0356313183994689, + "AC3_5": 0.009949001532054521, + "AC3_6": 0.0018972628413604306, + "AC3_7": 0.0 + }, + "prompt_4": { + "overall_acc": 0.2638095238095238, + "language_acc": { + "Filipino": 0.21333333333333335, + "Vietnamese": 0.30666666666666664, + "Chinese": 0.20666666666666667, + "Spanish": 0.25333333333333335, + "Malay": 0.31333333333333335, + "Indonesian": 0.2733333333333333, + "English": 0.28 + }, + "consistency_score_2": 0.26031746031746034, + "consistency_score_3": 0.07180952380952384, + "consistency_score_4": 0.021904761904761906, + "consistency_score_5": 0.006349206349206349, + "consistency_score_6": 0.0009523809523809525, + "consistency_score_7": 0.0, + "detailed_consistency_score": { + "2_combine": { + "Filipino,Vietnamese": 0.26, + "Filipino,Chinese": 0.2866666666666667, + "Filipino,Spanish": 0.26, + "Filipino,Malay": 0.2733333333333333, + "Filipino,Indonesian": 0.29333333333333333, + "Filipino,English": 0.21333333333333335, + "Vietnamese,Chinese": 0.21333333333333335, + "Vietnamese,Spanish": 0.2733333333333333, + "Vietnamese,Malay": 0.35333333333333333, + "Vietnamese,Indonesian": 0.25333333333333335, + "Vietnamese,English": 0.2733333333333333, + "Chinese,Spanish": 0.29333333333333333, + "Chinese,Malay": 0.20666666666666667, + "Chinese,Indonesian": 0.24666666666666667, + "Chinese,English": 0.2, + "Spanish,Malay": 0.2733333333333333, + "Spanish,Indonesian": 0.25333333333333335, + "Spanish,English": 0.32666666666666666, + "Malay,Indonesian": 0.26, + "Malay,English": 0.21333333333333335, + "Indonesian,English": 0.24 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.05333333333333334, + "Filipino,Vietnamese,Spanish": 0.1, + "Filipino,Vietnamese,Malay": 0.10666666666666667, + "Filipino,Vietnamese,Indonesian": 0.06, + "Filipino,Vietnamese,English": 0.06666666666666667, + "Filipino,Chinese,Spanish": 0.10666666666666667, + "Filipino,Chinese,Malay": 0.03333333333333333, + "Filipino,Chinese,Indonesian": 0.09333333333333334, + "Filipino,Chinese,English": 0.06666666666666667, + "Filipino,Spanish,Malay": 0.06, + "Filipino,Spanish,Indonesian": 0.07333333333333333, + "Filipino,Spanish,English": 0.08666666666666667, + "Filipino,Malay,Indonesian": 0.1, + "Filipino,Malay,English": 0.05333333333333334, + "Filipino,Indonesian,English": 0.08, + "Vietnamese,Chinese,Spanish": 0.07333333333333333, + "Vietnamese,Chinese,Malay": 0.05333333333333334, + "Vietnamese,Chinese,Indonesian": 0.05333333333333334, + "Vietnamese,Chinese,English": 0.04, + "Vietnamese,Spanish,Malay": 0.10666666666666667, + "Vietnamese,Spanish,Indonesian": 0.04666666666666667, + "Vietnamese,Spanish,English": 0.11333333333333333, + "Vietnamese,Malay,Indonesian": 0.12, + "Vietnamese,Malay,English": 0.08666666666666667, + "Vietnamese,Indonesian,English": 0.06, + "Chinese,Spanish,Malay": 0.06666666666666667, + "Chinese,Spanish,Indonesian": 0.08666666666666667, + "Chinese,Spanish,English": 0.07333333333333333, + "Chinese,Malay,Indonesian": 0.06, + "Chinese,Malay,English": 0.006666666666666667, + "Chinese,Indonesian,English": 0.06, + "Spanish,Malay,Indonesian": 0.08666666666666667, + "Spanish,Malay,English": 0.06, + "Spanish,Indonesian,English": 0.07333333333333333, + "Malay,Indonesian,English": 0.04666666666666667 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.03333333333333333, + "Filipino,Vietnamese,Chinese,Malay": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.013333333333333334, + "Filipino,Vietnamese,Chinese,English": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,Malay": 0.03333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,English": 0.03333333333333333, + "Filipino,Vietnamese,Malay,Indonesian": 0.04666666666666667, + "Filipino,Vietnamese,Malay,English": 0.03333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.006666666666666667, + "Filipino,Chinese,Spanish,Malay": 0.013333333333333334, + "Filipino,Chinese,Spanish,Indonesian": 0.04, + "Filipino,Chinese,Spanish,English": 0.04, + "Filipino,Chinese,Malay,Indonesian": 0.02, + "Filipino,Chinese,Malay,English": 0.0, + "Filipino,Chinese,Indonesian,English": 0.03333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.02, + "Filipino,Spanish,Malay,English": 0.02666666666666667, + "Filipino,Spanish,Indonesian,English": 0.03333333333333333, + "Filipino,Malay,Indonesian,English": 0.02, + "Vietnamese,Chinese,Spanish,Malay": 0.02666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.02, + "Vietnamese,Chinese,Spanish,English": 0.02, + "Vietnamese,Chinese,Malay,Indonesian": 0.02, + "Vietnamese,Chinese,Malay,English": 0.0, + "Vietnamese,Chinese,Indonesian,English": 0.006666666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.04, + "Vietnamese,Spanish,Malay,English": 0.04, + "Vietnamese,Spanish,Indonesian,English": 0.013333333333333334, + "Vietnamese,Malay,Indonesian,English": 0.02666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.04, + "Chinese,Spanish,Malay,English": 0.0, + "Chinese,Spanish,Indonesian,English": 0.02666666666666667, + "Chinese,Malay,Indonesian,English": 0.0, + "Spanish,Malay,Indonesian,English": 0.02 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.0, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.0, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.02, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.0, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.006666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.013333333333333334, + "Filipino,Chinese,Spanish,Malay,English": 0.0, + "Filipino,Chinese,Spanish,Indonesian,English": 0.02, + "Filipino,Chinese,Malay,Indonesian,English": 0.0, + "Filipino,Spanish,Malay,Indonesian,English": 0.006666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.02, + "Vietnamese,Chinese,Spanish,Malay,English": 0.0, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.0, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.013333333333333334, + "Chinese,Spanish,Malay,Indonesian,English": 0.0 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.0, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + } + }, + "AC3_2": 0.2620518588522548, + "AC3_3": 0.11289011401425296, + "AC3_4": 0.040450793636635875, + "AC3_5": 0.012399977612779037, + "AC3_6": 0.001897910242517148, + "AC3_7": 0.0 + }, + "prompt_5": { + "overall_acc": 0.24095238095238095, + "language_acc": { + "Filipino": 0.22666666666666666, + "Vietnamese": 0.16666666666666666, + "Chinese": 0.2866666666666667, + "Spanish": 0.26666666666666666, + "Malay": 0.22, + "Indonesian": 0.25333333333333335, + "English": 0.26666666666666666 + }, + "consistency_score_2": 0.25555555555555554, + "consistency_score_3": 0.06019047619047618, + "consistency_score_4": 0.011428571428571425, + "consistency_score_5": 0.0012698412698412698, + "consistency_score_6": 0.0, + "consistency_score_7": 0.0, + "detailed_consistency_score": { + "2_combine": { + "Filipino,Vietnamese": 0.25333333333333335, + "Filipino,Chinese": 0.22, + "Filipino,Spanish": 0.3333333333333333, + "Filipino,Malay": 0.2733333333333333, + "Filipino,Indonesian": 0.24, + "Filipino,English": 0.18666666666666668, + "Vietnamese,Chinese": 0.3, + "Vietnamese,Spanish": 0.24, + "Vietnamese,Malay": 0.22666666666666666, + "Vietnamese,Indonesian": 0.20666666666666667, + "Vietnamese,English": 0.28, + "Chinese,Spanish": 0.23333333333333334, + "Chinese,Malay": 0.26, + "Chinese,Indonesian": 0.26666666666666666, + "Chinese,English": 0.20666666666666667, + "Spanish,Malay": 0.2733333333333333, + "Spanish,Indonesian": 0.26, + "Spanish,English": 0.21333333333333335, + "Malay,Indonesian": 0.31333333333333335, + "Malay,English": 0.24666666666666667, + "Indonesian,English": 0.3333333333333333 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.06666666666666667, + "Filipino,Vietnamese,Spanish": 0.06666666666666667, + "Filipino,Vietnamese,Malay": 0.04, + "Filipino,Vietnamese,Indonesian": 0.02, + "Filipino,Vietnamese,English": 0.06666666666666667, + "Filipino,Chinese,Spanish": 0.07333333333333333, + "Filipino,Chinese,Malay": 0.05333333333333334, + "Filipino,Chinese,Indonesian": 0.06, + "Filipino,Chinese,English": 0.02, + "Filipino,Spanish,Malay": 0.07333333333333333, + "Filipino,Spanish,Indonesian": 0.06666666666666667, + "Filipino,Spanish,English": 0.03333333333333333, + "Filipino,Malay,Indonesian": 0.08, + "Filipino,Malay,English": 0.05333333333333334, + "Filipino,Indonesian,English": 0.06, + "Vietnamese,Chinese,Spanish": 0.07333333333333333, + "Vietnamese,Chinese,Malay": 0.06666666666666667, + "Vietnamese,Chinese,Indonesian": 0.09333333333333334, + "Vietnamese,Chinese,English": 0.07333333333333333, + "Vietnamese,Spanish,Malay": 0.06, + "Vietnamese,Spanish,Indonesian": 0.04, + "Vietnamese,Spanish,English": 0.06, + "Vietnamese,Malay,Indonesian": 0.04666666666666667, + "Vietnamese,Malay,English": 0.06666666666666667, + "Vietnamese,Indonesian,English": 0.06666666666666667, + "Chinese,Spanish,Malay": 0.06, + "Chinese,Spanish,Indonesian": 0.06666666666666667, + "Chinese,Spanish,English": 0.03333333333333333, + "Chinese,Malay,Indonesian": 0.09333333333333334, + "Chinese,Malay,English": 0.04, + "Chinese,Indonesian,English": 0.08, + "Spanish,Malay,Indonesian": 0.05333333333333334, + "Spanish,Malay,English": 0.05333333333333334, + "Spanish,Indonesian,English": 0.05333333333333334, + "Malay,Indonesian,English": 0.09333333333333334 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.02666666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.013333333333333334, + "Filipino,Vietnamese,Chinese,English": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,Malay": 0.0, + "Filipino,Vietnamese,Spanish,Indonesian": 0.0, + "Filipino,Vietnamese,Spanish,English": 0.013333333333333334, + "Filipino,Vietnamese,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Malay,English": 0.02, + "Filipino,Vietnamese,Indonesian,English": 0.006666666666666667, + "Filipino,Chinese,Spanish,Malay": 0.013333333333333334, + "Filipino,Chinese,Spanish,Indonesian": 0.02, + "Filipino,Chinese,Spanish,English": 0.006666666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.02666666666666667, + "Filipino,Chinese,Malay,English": 0.0, + "Filipino,Chinese,Indonesian,English": 0.006666666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.013333333333333334, + "Filipino,Spanish,Malay,English": 0.0, + "Filipino,Spanish,Indonesian,English": 0.0, + "Filipino,Malay,Indonesian,English": 0.02, + "Vietnamese,Chinese,Spanish,Malay": 0.02, + "Vietnamese,Chinese,Spanish,Indonesian": 0.013333333333333334, + "Vietnamese,Chinese,Spanish,English": 0.02, + "Vietnamese,Chinese,Malay,Indonesian": 0.013333333333333334, + "Vietnamese,Chinese,Malay,English": 0.013333333333333334, + "Vietnamese,Chinese,Indonesian,English": 0.03333333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.006666666666666667, + "Vietnamese,Spanish,Malay,English": 0.02, + "Vietnamese,Spanish,Indonesian,English": 0.0, + "Vietnamese,Malay,Indonesian,English": 0.006666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.02, + "Chinese,Spanish,Malay,English": 0.013333333333333334, + "Chinese,Spanish,Indonesian,English": 0.006666666666666667, + "Chinese,Malay,Indonesian,English": 0.013333333333333334, + "Spanish,Malay,Indonesian,English": 0.0 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Chinese,Malay,English": 0.0, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.0, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Spanish,Malay,English": 0.0, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.0, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.0, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.013333333333333334, + "Filipino,Chinese,Spanish,Malay,English": 0.0, + "Filipino,Chinese,Spanish,Indonesian,English": 0.0, + "Filipino,Chinese,Malay,Indonesian,English": 0.0, + "Filipino,Spanish,Malay,Indonesian,English": 0.0, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, + "Vietnamese,Chinese,Spanish,Malay,English": 0.006666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.0, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.0, + "Chinese,Spanish,Malay,Indonesian,English": 0.0 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.0, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + } + }, + "AC3_2": 0.24803921563631776, + "AC3_3": 0.09631999032975126, + "AC3_4": 0.02182210241722951, + "AC3_5": 0.0025263683444603424, + "AC3_6": 0.0, + "AC3_7": 0.0 + } + }, + "cross_logiqa": { + "prompt_1": { + "overall_acc": 0.26623376623376627, + "language_acc": { + "Indonesian": 0.2556818181818182, + "English": 0.25, + "Filipino": 0.30113636363636365, + "Spanish": 0.2897727272727273, + "Chinese": 0.2897727272727273, + "Malay": 0.21022727272727273, + "Vietnamese": 0.26704545454545453 + }, + "consistency_score_2": 0.23457792207792205, + "consistency_score_3": 0.05113636363636364, + "consistency_score_4": 0.008928571428571425, + "consistency_score_5": 0.0005411255411255411, + "consistency_score_6": 0.0, + "consistency_score_7": 0.0, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.22727272727272727, + "Indonesian,Filipino": 0.26136363636363635, + "Indonesian,Spanish": 0.22727272727272727, + "Indonesian,Chinese": 0.23295454545454544, + "Indonesian,Malay": 0.2784090909090909, + "Indonesian,Vietnamese": 0.19318181818181818, + "English,Filipino": 0.24431818181818182, + "English,Spanish": 0.29545454545454547, + "English,Chinese": 0.23295454545454544, + "English,Malay": 0.2840909090909091, + "English,Vietnamese": 0.19886363636363635, + "Filipino,Spanish": 0.17613636363636365, + "Filipino,Chinese": 0.2727272727272727, + "Filipino,Malay": 0.2840909090909091, + "Filipino,Vietnamese": 0.2159090909090909, + "Spanish,Chinese": 0.1875, + "Spanish,Malay": 0.21022727272727273, + "Spanish,Vietnamese": 0.22727272727272727, + "Chinese,Malay": 0.1875, + "Chinese,Vietnamese": 0.3068181818181818, + "Malay,Vietnamese": 0.18181818181818182 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.045454545454545456, + "Indonesian,English,Spanish": 0.045454545454545456, + "Indonesian,English,Chinese": 0.056818181818181816, + "Indonesian,English,Malay": 0.056818181818181816, + "Indonesian,English,Vietnamese": 0.017045454545454544, + "Indonesian,Filipino,Spanish": 0.03977272727272727, + "Indonesian,Filipino,Chinese": 0.07954545454545454, + "Indonesian,Filipino,Malay": 0.09659090909090909, + "Indonesian,Filipino,Vietnamese": 0.022727272727272728, + "Indonesian,Spanish,Chinese": 0.03409090909090909, + "Indonesian,Spanish,Malay": 0.03977272727272727, + "Indonesian,Spanish,Vietnamese": 0.045454545454545456, + "Indonesian,Chinese,Malay": 0.045454545454545456, + "Indonesian,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,Malay,Vietnamese": 0.05113636363636364, + "English,Filipino,Spanish": 0.06818181818181818, + "English,Filipino,Chinese": 0.056818181818181816, + "English,Filipino,Malay": 0.10795454545454546, + "English,Filipino,Vietnamese": 0.045454545454545456, + "English,Spanish,Chinese": 0.06818181818181818, + "English,Spanish,Malay": 0.07954545454545454, + "English,Spanish,Vietnamese": 0.0625, + "English,Chinese,Malay": 0.045454545454545456, + "English,Chinese,Vietnamese": 0.0625, + "English,Malay,Vietnamese": 0.05113636363636364, + "Filipino,Spanish,Chinese": 0.03409090909090909, + "Filipino,Spanish,Malay": 0.03409090909090909, + "Filipino,Spanish,Vietnamese": 0.03977272727272727, + "Filipino,Chinese,Malay": 0.06818181818181818, + "Filipino,Chinese,Vietnamese": 0.0625, + "Filipino,Malay,Vietnamese": 0.03409090909090909, + "Spanish,Chinese,Malay": 0.011363636363636364, + "Spanish,Chinese,Vietnamese": 0.0625, + "Spanish,Malay,Vietnamese": 0.03977272727272727, + "Chinese,Malay,Vietnamese": 0.056818181818181816 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.0, + "Indonesian,English,Filipino,Chinese": 0.005681818181818182, + "Indonesian,English,Filipino,Malay": 0.028409090909090908, + "Indonesian,English,Filipino,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese": 0.011363636363636364, + "Indonesian,English,Spanish,Malay": 0.0, + "Indonesian,English,Spanish,Vietnamese": 0.005681818181818182, + "Indonesian,English,Chinese,Malay": 0.0, + "Indonesian,English,Chinese,Vietnamese": 0.0, + "Indonesian,English,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Chinese": 0.0, + "Indonesian,Filipino,Spanish,Malay": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Vietnamese": 0.0, + "Indonesian,Filipino,Chinese,Malay": 0.028409090909090908, + "Indonesian,Filipino,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Spanish,Chinese,Malay": 0.0, + "Indonesian,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,Spanish,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Chinese,Malay,Vietnamese": 0.017045454545454544, + "English,Filipino,Spanish,Chinese": 0.022727272727272728, + "English,Filipino,Spanish,Malay": 0.022727272727272728, + "English,Filipino,Spanish,Vietnamese": 0.011363636363636364, + "English,Filipino,Chinese,Malay": 0.011363636363636364, + "English,Filipino,Chinese,Vietnamese": 0.011363636363636364, + "English,Filipino,Malay,Vietnamese": 0.011363636363636364, + "English,Spanish,Chinese,Malay": 0.005681818181818182, + "English,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "English,Spanish,Malay,Vietnamese": 0.011363636363636364, + "English,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Filipino,Spanish,Chinese,Malay": 0.0, + "Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, + "Filipino,Spanish,Malay,Vietnamese": 0.0, + "Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.0, + "Indonesian,English,Filipino,Spanish,Malay": 0.0, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0, + "Indonesian,English,Filipino,Chinese,Malay": 0.0, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.0, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese,Malay": 0.0, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0, + "English,Filipino,Spanish,Chinese,Malay": 0.0, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "English,Filipino,Spanish,Malay,Vietnamese": 0.0, + "English,Filipino,Chinese,Malay,Vietnamese": 0.0, + "English,Spanish,Chinese,Malay,Vietnamese": 0.0, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "6_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.3333333333333333, - "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.32, - "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.37333333333333335, - "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.3333333333333333, - "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.36, - "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.35333333333333333, - "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.36666666666666664 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "7_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.31333333333333335 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 } }, - "AC3_2": 0.6641195483429715, - "AC3_3": 0.5896508746637111, - "AC3_4": 0.5356926666021684, - "AC3_5": 0.49269153004317184, - "AC3_6": 0.45665813917656617, - "AC3_7": 0.4253255207897209 + "AC3_2": 0.24940537577372796, + "AC3_3": 0.08579400136799162, + "AC3_4": 0.01727770753691517, + "AC3_5": 0.00108005584638527, + "AC3_6": 0.0, + "AC3_7": 0.0 }, "prompt_2": { - "overall_acc": 0.6885714285714286, + "overall_acc": 0.25243506493506496, "language_acc": { - "Vietnamese": 0.6333333333333333, - "Malay": 0.6533333333333333, - "Filipino": 0.6666666666666666, - "Indonesian": 0.6933333333333334, - "Chinese": 0.6533333333333333, - "Spanish": 0.7333333333333333, - "English": 0.7866666666666666 + "Indonesian": 0.23295454545454544, + "English": 0.2897727272727273, + "Filipino": 0.2556818181818182, + "Spanish": 0.2556818181818182, + "Chinese": 0.26136363636363635, + "Malay": 0.23295454545454544, + "Vietnamese": 0.23863636363636365 }, - "consistency_score_2": 0.6926984126984127, - "consistency_score_3": 0.5647619047619048, - "consistency_score_4": 0.4864761904761906, - "consistency_score_5": 0.43142857142857144, - "consistency_score_6": 0.3904761904761905, - "consistency_score_7": 0.36, + "consistency_score_2": 0.24729437229437234, + "consistency_score_3": 0.05746753246753245, + "consistency_score_4": 0.012012987012987007, + "consistency_score_5": 0.0018939393939393942, + "consistency_score_6": 0.0, + "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { - "Vietnamese,Malay": 0.62, - "Vietnamese,Filipino": 0.66, - "Vietnamese,Indonesian": 0.7066666666666667, - "Vietnamese,Chinese": 0.6533333333333333, - "Vietnamese,Spanish": 0.6666666666666666, - "Vietnamese,English": 0.68, - "Malay,Filipino": 0.6466666666666666, - "Malay,Indonesian": 0.6866666666666666, - "Malay,Chinese": 0.66, - "Malay,Spanish": 0.68, - "Malay,English": 0.6733333333333333, - "Filipino,Indonesian": 0.72, - "Filipino,Chinese": 0.6066666666666667, - "Filipino,Spanish": 0.7066666666666667, - "Filipino,English": 0.7333333333333333, - "Indonesian,Chinese": 0.6933333333333334, - "Indonesian,Spanish": 0.7666666666666667, - "Indonesian,English": 0.7933333333333333, - "Chinese,Spanish": 0.6533333333333333, - "Chinese,English": 0.7, - "Spanish,English": 0.84 - }, - "3_combine": { - "Vietnamese,Malay,Filipino": 0.49333333333333335, - "Vietnamese,Malay,Indonesian": 0.5333333333333333, - "Vietnamese,Malay,Chinese": 0.5, - "Vietnamese,Malay,Spanish": 0.5066666666666667, - "Vietnamese,Malay,English": 0.52, - "Vietnamese,Filipino,Indonesian": 0.5666666666666667, - "Vietnamese,Filipino,Chinese": 0.4866666666666667, - "Vietnamese,Filipino,Spanish": 0.54, - "Vietnamese,Filipino,English": 0.5666666666666667, - "Vietnamese,Indonesian,Chinese": 0.5466666666666666, - "Vietnamese,Indonesian,Spanish": 0.58, - "Vietnamese,Indonesian,English": 0.6133333333333333, - "Vietnamese,Chinese,Spanish": 0.5066666666666667, - "Vietnamese,Chinese,English": 0.54, - "Vietnamese,Spanish,English": 0.6066666666666667, - "Malay,Filipino,Indonesian": 0.56, - "Malay,Filipino,Chinese": 0.49333333333333335, - "Malay,Filipino,Spanish": 0.5533333333333333, - "Malay,Filipino,English": 0.56, - "Malay,Indonesian,Chinese": 0.54, - "Malay,Indonesian,Spanish": 0.5933333333333334, - "Malay,Indonesian,English": 0.6066666666666667, - "Malay,Chinese,Spanish": 0.5333333333333333, - "Malay,Chinese,English": 0.5533333333333333, - "Malay,Spanish,English": 0.6133333333333333, - "Filipino,Indonesian,Chinese": 0.5333333333333333, - "Filipino,Indonesian,Spanish": 0.6333333333333333, - "Filipino,Indonesian,English": 0.6533333333333333, - "Filipino,Chinese,Spanish": 0.52, - "Filipino,Chinese,English": 0.5533333333333333, - "Filipino,Spanish,English": 0.66, - "Indonesian,Chinese,Spanish": 0.5733333333333334, - "Indonesian,Chinese,English": 0.6133333333333333, - "Indonesian,Spanish,English": 0.7066666666666667, - "Chinese,Spanish,English": 0.6066666666666667 - }, - "4_combine": { - "Vietnamese,Malay,Filipino,Indonesian": 0.46, - "Vietnamese,Malay,Filipino,Chinese": 0.4066666666666667, - "Vietnamese,Malay,Filipino,Spanish": 0.44, - "Vietnamese,Malay,Filipino,English": 0.46, - "Vietnamese,Malay,Indonesian,Chinese": 0.44, - "Vietnamese,Malay,Indonesian,Spanish": 0.48, - "Vietnamese,Malay,Indonesian,English": 0.4866666666666667, - "Vietnamese,Malay,Chinese,Spanish": 0.4266666666666667, - "Vietnamese,Malay,Chinese,English": 0.44666666666666666, - "Vietnamese,Malay,Spanish,English": 0.47333333333333333, - "Vietnamese,Filipino,Indonesian,Chinese": 0.44, - "Vietnamese,Filipino,Indonesian,Spanish": 0.5066666666666667, - "Vietnamese,Filipino,Indonesian,English": 0.52, - "Vietnamese,Filipino,Chinese,Spanish": 0.4266666666666667, - "Vietnamese,Filipino,Chinese,English": 0.46, - "Vietnamese,Filipino,Spanish,English": 0.52, - "Vietnamese,Indonesian,Chinese,Spanish": 0.4666666666666667, - "Vietnamese,Indonesian,Chinese,English": 0.5, - "Vietnamese,Indonesian,Spanish,English": 0.5533333333333333, - "Vietnamese,Chinese,Spanish,English": 0.4866666666666667, - "Malay,Filipino,Indonesian,Chinese": 0.44666666666666666, - "Malay,Filipino,Indonesian,Spanish": 0.5133333333333333, - "Malay,Filipino,Indonesian,English": 0.5266666666666666, - "Malay,Filipino,Chinese,Spanish": 0.44666666666666666, - "Malay,Filipino,Chinese,English": 0.47333333333333333, - "Malay,Filipino,Spanish,English": 0.5133333333333333, - "Malay,Indonesian,Chinese,Spanish": 0.4866666666666667, - "Malay,Indonesian,Chinese,English": 0.5066666666666667, - "Malay,Indonesian,Spanish,English": 0.56, - "Malay,Chinese,Spanish,English": 0.5133333333333333, - "Filipino,Indonesian,Chinese,Spanish": 0.47333333333333333, - "Filipino,Indonesian,Chinese,English": 0.5, - "Filipino,Indonesian,Spanish,English": 0.6066666666666667, - "Filipino,Chinese,Spanish,English": 0.5066666666666667, - "Indonesian,Chinese,Spanish,English": 0.5533333333333333 - }, - "5_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.38, - "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.4266666666666667, - "Vietnamese,Malay,Filipino,Indonesian,English": 0.43333333333333335, - "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.37333333333333335, - "Vietnamese,Malay,Filipino,Chinese,English": 0.4, - "Vietnamese,Malay,Filipino,Spanish,English": 0.42, - "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.4066666666666667, - "Vietnamese,Malay,Indonesian,Chinese,English": 0.42, - "Vietnamese,Malay,Indonesian,Spanish,English": 0.4533333333333333, - "Vietnamese,Malay,Chinese,Spanish,English": 0.41333333333333333, - "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.4, - "Vietnamese,Filipino,Indonesian,Chinese,English": 0.42, - "Vietnamese,Filipino,Indonesian,Spanish,English": 0.4866666666666667, - "Vietnamese,Filipino,Chinese,Spanish,English": 0.4266666666666667, - "Vietnamese,Indonesian,Chinese,Spanish,English": 0.46, - "Malay,Filipino,Indonesian,Chinese,Spanish": 0.42, - "Malay,Filipino,Indonesian,Chinese,English": 0.44, - "Malay,Filipino,Indonesian,Spanish,English": 0.49333333333333335, - "Malay,Filipino,Chinese,Spanish,English": 0.44, - "Malay,Indonesian,Chinese,Spanish,English": 0.48, - "Filipino,Indonesian,Chinese,Spanish,English": 0.4666666666666667 + "Indonesian,English": 0.26136363636363635, + "Indonesian,Filipino": 0.3465909090909091, + "Indonesian,Spanish": 0.20454545454545456, + "Indonesian,Chinese": 0.2159090909090909, + "Indonesian,Malay": 0.2159090909090909, + "Indonesian,Vietnamese": 0.2556818181818182, + "English,Filipino": 0.2556818181818182, + "English,Spanish": 0.26136363636363635, + "English,Chinese": 0.23863636363636365, + "English,Malay": 0.2556818181818182, + "English,Vietnamese": 0.19886363636363635, + "Filipino,Spanish": 0.2159090909090909, + "Filipino,Chinese": 0.23295454545454544, + "Filipino,Malay": 0.26136363636363635, + "Filipino,Vietnamese": 0.2840909090909091, + "Spanish,Chinese": 0.23863636363636365, + "Spanish,Malay": 0.2727272727272727, + "Spanish,Vietnamese": 0.25, + "Chinese,Malay": 0.23295454545454544, + "Chinese,Vietnamese": 0.23295454545454544, + "Malay,Vietnamese": 0.26136363636363635 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.11931818181818182, + "Indonesian,English,Spanish": 0.0625, + "Indonesian,English,Chinese": 0.056818181818181816, + "Indonesian,English,Malay": 0.0625, + "Indonesian,English,Vietnamese": 0.028409090909090908, + "Indonesian,Filipino,Spanish": 0.045454545454545456, + "Indonesian,Filipino,Chinese": 0.06818181818181818, + "Indonesian,Filipino,Malay": 0.06818181818181818, + "Indonesian,Filipino,Vietnamese": 0.09090909090909091, + "Indonesian,Spanish,Chinese": 0.03977272727272727, + "Indonesian,Spanish,Malay": 0.045454545454545456, + "Indonesian,Spanish,Vietnamese": 0.03409090909090909, + "Indonesian,Chinese,Malay": 0.028409090909090908, + "Indonesian,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,Malay,Vietnamese": 0.05113636363636364, + "English,Filipino,Spanish": 0.06818181818181818, + "English,Filipino,Chinese": 0.056818181818181816, + "English,Filipino,Malay": 0.056818181818181816, + "English,Filipino,Vietnamese": 0.0625, + "English,Spanish,Chinese": 0.056818181818181816, + "English,Spanish,Malay": 0.06818181818181818, + "English,Spanish,Vietnamese": 0.0625, + "English,Chinese,Malay": 0.045454545454545456, + "English,Chinese,Vietnamese": 0.045454545454545456, + "English,Malay,Vietnamese": 0.0625, + "Filipino,Spanish,Chinese": 0.056818181818181816, + "Filipino,Spanish,Malay": 0.05113636363636364, + "Filipino,Spanish,Vietnamese": 0.05113636363636364, + "Filipino,Chinese,Malay": 0.0625, + "Filipino,Chinese,Vietnamese": 0.05113636363636364, + "Filipino,Malay,Vietnamese": 0.06818181818181818, + "Spanish,Chinese,Malay": 0.06818181818181818, + "Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Spanish,Malay,Vietnamese": 0.07954545454545454, + "Chinese,Malay,Vietnamese": 0.03409090909090909 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.028409090909090908, + "Indonesian,English,Filipino,Chinese": 0.028409090909090908, + "Indonesian,English,Filipino,Malay": 0.017045454545454544, + "Indonesian,English,Filipino,Vietnamese": 0.017045454545454544, + "Indonesian,English,Spanish,Chinese": 0.011363636363636364, + "Indonesian,English,Spanish,Malay": 0.017045454545454544, + "Indonesian,English,Spanish,Vietnamese": 0.0, + "Indonesian,English,Chinese,Malay": 0.005681818181818182, + "Indonesian,English,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,English,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Chinese": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Malay": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Chinese,Malay": 0.0, + "Indonesian,Filipino,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Malay,Vietnamese": 0.022727272727272728, + "Indonesian,Spanish,Chinese,Malay": 0.005681818181818182, + "Indonesian,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,Spanish,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Chinese,Malay,Vietnamese": 0.0, + "English,Filipino,Spanish,Chinese": 0.022727272727272728, + "English,Filipino,Spanish,Malay": 0.011363636363636364, + "English,Filipino,Spanish,Vietnamese": 0.0, + "English,Filipino,Chinese,Malay": 0.011363636363636364, + "English,Filipino,Chinese,Vietnamese": 0.011363636363636364, + "English,Filipino,Malay,Vietnamese": 0.022727272727272728, + "English,Spanish,Chinese,Malay": 0.005681818181818182, + "English,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "English,Spanish,Malay,Vietnamese": 0.022727272727272728, + "English,Chinese,Malay,Vietnamese": 0.0, + "Filipino,Spanish,Chinese,Malay": 0.017045454545454544, + "Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "Filipino,Spanish,Malay,Vietnamese": 0.017045454545454544, + "Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.011363636363636364, + "Indonesian,English,Filipino,Spanish,Malay": 0.005681818181818182, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0, + "Indonesian,English,Filipino,Chinese,Malay": 0.0, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,English,Spanish,Chinese,Malay": 0.0, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0, + "English,Filipino,Spanish,Chinese,Malay": 0.0, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.0, + "English,Filipino,Spanish,Malay,Vietnamese": 0.0, + "English,Filipino,Chinese,Malay,Vietnamese": 0.0, + "English,Spanish,Chinese,Malay,Vietnamese": 0.0, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "6_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.36, - "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.37333333333333335, - "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.4066666666666667, - "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.37333333333333335, - "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.4, - "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.4, - "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.42 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "7_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.36 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 } }, - "AC3_2": 0.6906287552443499, - "AC3_3": 0.6205514545748909, - "AC3_4": 0.570144732832474, - "AC3_5": 0.5304810495153178, - "AC3_6": 0.49834825364187546, - "AC3_7": 0.4728065394644463 + "AC3_2": 0.24983827753598187, + "AC3_3": 0.0936218050897616, + "AC3_4": 0.022934554693837875, + "AC3_5": 0.0037596711784056757, + "AC3_6": 0.0, + "AC3_7": 0.0 }, "prompt_3": { - "overall_acc": 0.7009523809523809, + "overall_acc": 0.2508116883116883, "language_acc": { - "Vietnamese": 0.6266666666666667, - "Malay": 0.68, - "Filipino": 0.6933333333333334, - "Indonesian": 0.6933333333333334, - "Chinese": 0.6533333333333333, - "Spanish": 0.7333333333333333, - "English": 0.8266666666666667 + "Indonesian": 0.24431818181818182, + "English": 0.26136363636363635, + "Filipino": 0.24431818181818182, + "Spanish": 0.23295454545454544, + "Chinese": 0.23863636363636365, + "Malay": 0.26704545454545453, + "Vietnamese": 0.26704545454545453 }, - "consistency_score_2": 0.6974603174603174, - "consistency_score_3": 0.5767619047619048, - "consistency_score_4": 0.5036190476190475, - "consistency_score_5": 0.45269841269841266, - "consistency_score_6": 0.41523809523809524, - "consistency_score_7": 0.38666666666666666, + "consistency_score_2": 0.2440476190476191, + "consistency_score_3": 0.05503246753246754, + "consistency_score_4": 0.010714285714285711, + "consistency_score_5": 0.001352813852813853, + "consistency_score_6": 0.0, + "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { - "Vietnamese,Malay": 0.6466666666666666, - "Vietnamese,Filipino": 0.68, - "Vietnamese,Indonesian": 0.64, - "Vietnamese,Chinese": 0.62, - "Vietnamese,Spanish": 0.6533333333333333, - "Vietnamese,English": 0.64, - "Malay,Filipino": 0.7, - "Malay,Indonesian": 0.7466666666666667, - "Malay,Chinese": 0.6466666666666666, - "Malay,Spanish": 0.7666666666666667, - "Malay,English": 0.74, - "Filipino,Indonesian": 0.7, - "Filipino,Chinese": 0.6466666666666666, - "Filipino,Spanish": 0.7266666666666667, - "Filipino,English": 0.7466666666666667, - "Indonesian,Chinese": 0.6466666666666666, - "Indonesian,Spanish": 0.74, - "Indonesian,English": 0.76, - "Chinese,Spanish": 0.68, - "Chinese,English": 0.6866666666666666, - "Spanish,English": 0.8333333333333334 - }, - "3_combine": { - "Vietnamese,Malay,Filipino": 0.5466666666666666, - "Vietnamese,Malay,Indonesian": 0.5533333333333333, - "Vietnamese,Malay,Chinese": 0.5133333333333333, - "Vietnamese,Malay,Spanish": 0.56, - "Vietnamese,Malay,English": 0.5466666666666666, - "Vietnamese,Filipino,Indonesian": 0.5533333333333333, - "Vietnamese,Filipino,Chinese": 0.5133333333333333, - "Vietnamese,Filipino,Spanish": 0.5666666666666667, - "Vietnamese,Filipino,English": 0.5666666666666667, - "Vietnamese,Indonesian,Chinese": 0.49333333333333335, - "Vietnamese,Indonesian,Spanish": 0.5466666666666666, - "Vietnamese,Indonesian,English": 0.56, - "Vietnamese,Chinese,Spanish": 0.52, - "Vietnamese,Chinese,English": 0.52, - "Vietnamese,Spanish,English": 0.5933333333333334, - "Malay,Filipino,Indonesian": 0.6, - "Malay,Filipino,Chinese": 0.5266666666666666, - "Malay,Filipino,Spanish": 0.62, - "Malay,Filipino,English": 0.62, - "Malay,Indonesian,Chinese": 0.5466666666666666, - "Malay,Indonesian,Spanish": 0.64, - "Malay,Indonesian,English": 0.6466666666666666, - "Malay,Chinese,Spanish": 0.5733333333333334, - "Malay,Chinese,English": 0.5666666666666667, - "Malay,Spanish,English": 0.6866666666666666, - "Filipino,Indonesian,Chinese": 0.5266666666666666, - "Filipino,Indonesian,Spanish": 0.62, - "Filipino,Indonesian,English": 0.6333333333333333, - "Filipino,Chinese,Spanish": 0.5666666666666667, - "Filipino,Chinese,English": 0.56, - "Filipino,Spanish,English": 0.6666666666666666, - "Indonesian,Chinese,Spanish": 0.56, - "Indonesian,Chinese,English": 0.5666666666666667, - "Indonesian,Spanish,English": 0.6866666666666666, - "Chinese,Spanish,English": 0.62 - }, - "4_combine": { - "Vietnamese,Malay,Filipino,Indonesian": 0.49333333333333335, - "Vietnamese,Malay,Filipino,Chinese": 0.44666666666666666, - "Vietnamese,Malay,Filipino,Spanish": 0.5066666666666667, - "Vietnamese,Malay,Filipino,English": 0.5, - "Vietnamese,Malay,Indonesian,Chinese": 0.44, - "Vietnamese,Malay,Indonesian,Spanish": 0.5, - "Vietnamese,Malay,Indonesian,English": 0.49333333333333335, - "Vietnamese,Malay,Chinese,Spanish": 0.47333333333333333, - "Vietnamese,Malay,Chinese,English": 0.4666666666666667, - "Vietnamese,Malay,Spanish,English": 0.5266666666666666, - "Vietnamese,Filipino,Indonesian,Chinese": 0.44666666666666666, - "Vietnamese,Filipino,Indonesian,Spanish": 0.5, - "Vietnamese,Filipino,Indonesian,English": 0.5133333333333333, - "Vietnamese,Filipino,Chinese,Spanish": 0.47333333333333333, - "Vietnamese,Filipino,Chinese,English": 0.47333333333333333, - "Vietnamese,Filipino,Spanish,English": 0.54, - "Vietnamese,Indonesian,Chinese,Spanish": 0.4533333333333333, - "Vietnamese,Indonesian,Chinese,English": 0.46, - "Vietnamese,Indonesian,Spanish,English": 0.5266666666666666, - "Vietnamese,Chinese,Spanish,English": 0.49333333333333335, - "Malay,Filipino,Indonesian,Chinese": 0.4666666666666667, - "Malay,Filipino,Indonesian,Spanish": 0.54, - "Malay,Filipino,Indonesian,English": 0.5533333333333333, - "Malay,Filipino,Chinese,Spanish": 0.5, - "Malay,Filipino,Chinese,English": 0.4866666666666667, - "Malay,Filipino,Spanish,English": 0.58, - "Malay,Indonesian,Chinese,Spanish": 0.5066666666666667, - "Malay,Indonesian,Chinese,English": 0.5066666666666667, - "Malay,Indonesian,Spanish,English": 0.6066666666666667, - "Malay,Chinese,Spanish,English": 0.54, - "Filipino,Indonesian,Chinese,Spanish": 0.4866666666666667, - "Filipino,Indonesian,Chinese,English": 0.4866666666666667, - "Filipino,Indonesian,Spanish,English": 0.5866666666666667, - "Filipino,Chinese,Spanish,English": 0.5266666666666666, - "Indonesian,Chinese,Spanish,English": 0.5266666666666666 - }, - "5_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.4066666666666667, - "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.46, - "Vietnamese,Malay,Filipino,Indonesian,English": 0.46, - "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.43333333333333335, - "Vietnamese,Malay,Filipino,Chinese,English": 0.4266666666666667, - "Vietnamese,Malay,Filipino,Spanish,English": 0.4866666666666667, - "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.4266666666666667, - "Vietnamese,Malay,Indonesian,Chinese,English": 0.42, - "Vietnamese,Malay,Indonesian,Spanish,English": 0.48, - "Vietnamese,Malay,Chinese,Spanish,English": 0.4533333333333333, - "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.42, - "Vietnamese,Filipino,Indonesian,Chinese,English": 0.4266666666666667, - "Vietnamese,Filipino,Indonesian,Spanish,English": 0.4866666666666667, - "Vietnamese,Filipino,Chinese,Spanish,English": 0.4533333333333333, - "Vietnamese,Indonesian,Chinese,Spanish,English": 0.44, - "Malay,Filipino,Indonesian,Chinese,Spanish": 0.44666666666666666, - "Malay,Filipino,Indonesian,Chinese,English": 0.44, - "Malay,Filipino,Indonesian,Spanish,English": 0.52, - "Malay,Filipino,Chinese,Spanish,English": 0.47333333333333333, - "Malay,Indonesian,Chinese,Spanish,English": 0.4866666666666667, - "Filipino,Indonesian,Chinese,Spanish,English": 0.46 + "Indonesian,English": 0.2159090909090909, + "Indonesian,Filipino": 0.19318181818181818, + "Indonesian,Spanish": 0.2556818181818182, + "Indonesian,Chinese": 0.21022727272727273, + "Indonesian,Malay": 0.2727272727272727, + "Indonesian,Vietnamese": 0.29545454545454547, + "English,Filipino": 0.24431818181818182, + "English,Spanish": 0.2840909090909091, + "English,Chinese": 0.17045454545454544, + "English,Malay": 0.2556818181818182, + "English,Vietnamese": 0.23295454545454544, + "Filipino,Spanish": 0.23863636363636365, + "Filipino,Chinese": 0.24431818181818182, + "Filipino,Malay": 0.22727272727272727, + "Filipino,Vietnamese": 0.3181818181818182, + "Spanish,Chinese": 0.3068181818181818, + "Spanish,Malay": 0.26136363636363635, + "Spanish,Vietnamese": 0.2215909090909091, + "Chinese,Malay": 0.2840909090909091, + "Chinese,Vietnamese": 0.19886363636363635, + "Malay,Vietnamese": 0.19318181818181818 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.017045454545454544, + "Indonesian,English,Spanish": 0.05113636363636364, + "Indonesian,English,Chinese": 0.03977272727272727, + "Indonesian,English,Malay": 0.08522727272727272, + "Indonesian,English,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish": 0.03977272727272727, + "Indonesian,Filipino,Chinese": 0.022727272727272728, + "Indonesian,Filipino,Malay": 0.03977272727272727, + "Indonesian,Filipino,Vietnamese": 0.07386363636363637, + "Indonesian,Spanish,Chinese": 0.07386363636363637, + "Indonesian,Spanish,Malay": 0.07386363636363637, + "Indonesian,Spanish,Vietnamese": 0.056818181818181816, + "Indonesian,Chinese,Malay": 0.07386363636363637, + "Indonesian,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,Malay,Vietnamese": 0.05113636363636364, + "English,Filipino,Spanish": 0.08522727272727272, + "English,Filipino,Chinese": 0.045454545454545456, + "English,Filipino,Malay": 0.06818181818181818, + "English,Filipino,Vietnamese": 0.045454545454545456, + "English,Spanish,Chinese": 0.056818181818181816, + "English,Spanish,Malay": 0.07954545454545454, + "English,Spanish,Vietnamese": 0.07386363636363637, + "English,Chinese,Malay": 0.045454545454545456, + "English,Chinese,Vietnamese": 0.028409090909090908, + "English,Malay,Vietnamese": 0.07386363636363637, + "Filipino,Spanish,Chinese": 0.0625, + "Filipino,Spanish,Malay": 0.03977272727272727, + "Filipino,Spanish,Vietnamese": 0.0625, + "Filipino,Chinese,Malay": 0.056818181818181816, + "Filipino,Chinese,Vietnamese": 0.056818181818181816, + "Filipino,Malay,Vietnamese": 0.028409090909090908, + "Spanish,Chinese,Malay": 0.07386363636363637, + "Spanish,Chinese,Vietnamese": 0.0625, + "Spanish,Malay,Vietnamese": 0.05113636363636364, + "Chinese,Malay,Vietnamese": 0.03977272727272727 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.0, + "Indonesian,English,Filipino,Chinese": 0.005681818181818182, + "Indonesian,English,Filipino,Malay": 0.011363636363636364, + "Indonesian,English,Filipino,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese": 0.011363636363636364, + "Indonesian,English,Spanish,Malay": 0.022727272727272728, + "Indonesian,English,Spanish,Vietnamese": 0.005681818181818182, + "Indonesian,English,Chinese,Malay": 0.017045454545454544, + "Indonesian,English,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,English,Malay,Vietnamese": 0.017045454545454544, + "Indonesian,Filipino,Spanish,Chinese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Malay": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Chinese,Malay": 0.017045454545454544, + "Indonesian,Filipino,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Spanish,Chinese,Malay": 0.011363636363636364, + "Indonesian,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,Spanish,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Chinese,Malay,Vietnamese": 0.005681818181818182, + "English,Filipino,Spanish,Chinese": 0.022727272727272728, + "English,Filipino,Spanish,Malay": 0.017045454545454544, + "English,Filipino,Spanish,Vietnamese": 0.017045454545454544, + "English,Filipino,Chinese,Malay": 0.005681818181818182, + "English,Filipino,Chinese,Vietnamese": 0.005681818181818182, + "English,Filipino,Malay,Vietnamese": 0.011363636363636364, + "English,Spanish,Chinese,Malay": 0.011363636363636364, + "English,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "English,Spanish,Malay,Vietnamese": 0.03409090909090909, + "English,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Filipino,Spanish,Chinese,Malay": 0.005681818181818182, + "Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Filipino,Spanish,Malay,Vietnamese": 0.0, + "Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.0, + "Indonesian,English,Filipino,Spanish,Malay": 0.0, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0, + "Indonesian,English,Filipino,Chinese,Malay": 0.005681818181818182, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.0, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese,Malay": 0.005681818181818182, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0, + "English,Filipino,Spanish,Chinese,Malay": 0.0, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "English,Filipino,Spanish,Malay,Vietnamese": 0.0, + "English,Filipino,Chinese,Malay,Vietnamese": 0.0, + "English,Spanish,Chinese,Malay,Vietnamese": 0.0, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "6_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.4, - "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.3933333333333333, - "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.44666666666666666, - "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.42, - "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.41333333333333333, - "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.4066666666666667, - "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.4266666666666667 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "7_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.38666666666666666 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 } }, - "AC3_2": 0.6992019890316715, - "AC3_3": 0.6328216485860745, - "AC3_4": 0.5861221047072226, - "AC3_5": 0.5501145267757028, - "AC3_6": 0.5215277100134094, - "AC3_7": 0.4984004669711045 + "AC3_2": 0.24738342570959695, + "AC3_3": 0.09026025722915307, + "AC3_4": 0.020550678364049425, + "AC3_5": 0.0026911125344659314, + "AC3_6": 0.0, + "AC3_7": 0.0 }, "prompt_4": { - "overall_acc": 0.6647619047619048, + "overall_acc": 0.2394480519480519, "language_acc": { - "Vietnamese": 0.5733333333333334, - "Malay": 0.6333333333333333, - "Filipino": 0.6266666666666667, - "Indonesian": 0.6933333333333334, - "Chinese": 0.6266666666666667, - "Spanish": 0.72, - "English": 0.78 + "Indonesian": 0.2556818181818182, + "English": 0.23295454545454544, + "Filipino": 0.26136363636363635, + "Spanish": 0.21022727272727273, + "Chinese": 0.2556818181818182, + "Malay": 0.26136363636363635, + "Vietnamese": 0.19886363636363635 }, - "consistency_score_2": 0.6571428571428571, - "consistency_score_3": 0.520952380952381, - "consistency_score_4": 0.44552380952380943, - "consistency_score_5": 0.39587301587301593, - "consistency_score_6": 0.35999999999999993, - "consistency_score_7": 0.3333333333333333, + "consistency_score_2": 0.25676406926406925, + "consistency_score_3": 0.0681818181818182, + "consistency_score_4": 0.021753246753246754, + "consistency_score_5": 0.009740259740259737, + "consistency_score_6": 0.006493506493506494, + "consistency_score_7": 0.005681818181818182, "detailed_consistency_score": { "2_combine": { - "Vietnamese,Malay": 0.58, - "Vietnamese,Filipino": 0.6333333333333333, - "Vietnamese,Indonesian": 0.6266666666666667, - "Vietnamese,Chinese": 0.6533333333333333, - "Vietnamese,Spanish": 0.6133333333333333, - "Vietnamese,English": 0.6, - "Malay,Filipino": 0.62, - "Malay,Indonesian": 0.6666666666666666, - "Malay,Chinese": 0.5666666666666667, - "Malay,Spanish": 0.68, - "Malay,English": 0.6266666666666667, - "Filipino,Indonesian": 0.7, - "Filipino,Chinese": 0.6, - "Filipino,Spanish": 0.68, - "Filipino,English": 0.68, - "Indonesian,Chinese": 0.64, - "Indonesian,Spanish": 0.72, - "Indonesian,English": 0.7466666666666667, - "Chinese,Spanish": 0.6333333333333333, - "Chinese,English": 0.7, - "Spanish,English": 0.8333333333333334 - }, - "3_combine": { - "Vietnamese,Malay,Filipino": 0.46, - "Vietnamese,Malay,Indonesian": 0.47333333333333333, - "Vietnamese,Malay,Chinese": 0.4533333333333333, - "Vietnamese,Malay,Spanish": 0.49333333333333335, - "Vietnamese,Malay,English": 0.4533333333333333, - "Vietnamese,Filipino,Indonesian": 0.5133333333333333, - "Vietnamese,Filipino,Chinese": 0.48, - "Vietnamese,Filipino,Spanish": 0.5, - "Vietnamese,Filipino,English": 0.5, - "Vietnamese,Indonesian,Chinese": 0.5133333333333333, - "Vietnamese,Indonesian,Spanish": 0.5066666666666667, - "Vietnamese,Indonesian,English": 0.5333333333333333, - "Vietnamese,Chinese,Spanish": 0.4866666666666667, - "Vietnamese,Chinese,English": 0.5133333333333333, - "Vietnamese,Spanish,English": 0.54, - "Malay,Filipino,Indonesian": 0.5333333333333333, - "Malay,Filipino,Chinese": 0.44, - "Malay,Filipino,Spanish": 0.5133333333333333, - "Malay,Filipino,English": 0.5066666666666667, - "Malay,Indonesian,Chinese": 0.4666666666666667, - "Malay,Indonesian,Spanish": 0.5733333333333334, - "Malay,Indonesian,English": 0.56, - "Malay,Chinese,Spanish": 0.4866666666666667, - "Malay,Chinese,English": 0.4866666666666667, - "Malay,Spanish,English": 0.5933333333333334, - "Filipino,Indonesian,Chinese": 0.5066666666666667, - "Filipino,Indonesian,Spanish": 0.5733333333333334, - "Filipino,Indonesian,English": 0.5866666666666667, - "Filipino,Chinese,Spanish": 0.5, - "Filipino,Chinese,English": 0.52, - "Filipino,Spanish,English": 0.6133333333333333, - "Indonesian,Chinese,Spanish": 0.5266666666666666, - "Indonesian,Chinese,English": 0.56, - "Indonesian,Spanish,English": 0.6666666666666666, - "Chinese,Spanish,English": 0.6 - }, - "4_combine": { - "Vietnamese,Malay,Filipino,Indonesian": 0.41333333333333333, - "Vietnamese,Malay,Filipino,Chinese": 0.38666666666666666, - "Vietnamese,Malay,Filipino,Spanish": 0.4266666666666667, - "Vietnamese,Malay,Filipino,English": 0.4, - "Vietnamese,Malay,Indonesian,Chinese": 0.3933333333333333, - "Vietnamese,Malay,Indonesian,Spanish": 0.44, - "Vietnamese,Malay,Indonesian,English": 0.4266666666666667, - "Vietnamese,Malay,Chinese,Spanish": 0.4066666666666667, - "Vietnamese,Malay,Chinese,English": 0.4, - "Vietnamese,Malay,Spanish,English": 0.44666666666666666, - "Vietnamese,Filipino,Indonesian,Chinese": 0.42, - "Vietnamese,Filipino,Indonesian,Spanish": 0.4533333333333333, - "Vietnamese,Filipino,Indonesian,English": 0.46, - "Vietnamese,Filipino,Chinese,Spanish": 0.41333333333333333, - "Vietnamese,Filipino,Chinese,English": 0.43333333333333335, - "Vietnamese,Filipino,Spanish,English": 0.4666666666666667, - "Vietnamese,Indonesian,Chinese,Spanish": 0.42, - "Vietnamese,Indonesian,Chinese,English": 0.46, - "Vietnamese,Indonesian,Spanish,English": 0.48, - "Vietnamese,Chinese,Spanish,English": 0.4666666666666667, - "Malay,Filipino,Indonesian,Chinese": 0.3933333333333333, - "Malay,Filipino,Indonesian,Spanish": 0.48, - "Malay,Filipino,Indonesian,English": 0.47333333333333333, - "Malay,Filipino,Chinese,Spanish": 0.41333333333333333, - "Malay,Filipino,Chinese,English": 0.4, - "Malay,Filipino,Spanish,English": 0.4866666666666667, - "Malay,Indonesian,Chinese,Spanish": 0.44, - "Malay,Indonesian,Chinese,English": 0.44, - "Malay,Indonesian,Spanish,English": 0.54, - "Malay,Chinese,Spanish,English": 0.47333333333333333, - "Filipino,Indonesian,Chinese,Spanish": 0.4533333333333333, - "Filipino,Indonesian,Chinese,English": 0.47333333333333333, - "Filipino,Indonesian,Spanish,English": 0.5333333333333333, - "Filipino,Chinese,Spanish,English": 0.47333333333333333, - "Indonesian,Chinese,Spanish,English": 0.5066666666666667 - }, - "5_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.3466666666666667, - "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.4, - "Vietnamese,Malay,Filipino,Indonesian,English": 0.38, - "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.36666666666666664, - "Vietnamese,Malay,Filipino,Chinese,English": 0.35333333333333333, - "Vietnamese,Malay,Filipino,Spanish,English": 0.4, - "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.37333333333333335, - "Vietnamese,Malay,Indonesian,Chinese,English": 0.37333333333333335, - "Vietnamese,Malay,Indonesian,Spanish,English": 0.42, - "Vietnamese,Malay,Chinese,Spanish,English": 0.3933333333333333, - "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.37333333333333335, - "Vietnamese,Filipino,Indonesian,Chinese,English": 0.4, - "Vietnamese,Filipino,Indonesian,Spanish,English": 0.4266666666666667, - "Vietnamese,Filipino,Chinese,Spanish,English": 0.4, - "Vietnamese,Indonesian,Chinese,Spanish,English": 0.41333333333333333, - "Malay,Filipino,Indonesian,Chinese,Spanish": 0.38666666666666666, - "Malay,Filipino,Indonesian,Chinese,English": 0.38, - "Malay,Filipino,Indonesian,Spanish,English": 0.46, - "Malay,Filipino,Chinese,Spanish,English": 0.4, - "Malay,Indonesian,Chinese,Spanish,English": 0.43333333333333335, - "Filipino,Indonesian,Chinese,Spanish,English": 0.43333333333333335 + "Indonesian,English": 0.2727272727272727, + "Indonesian,Filipino": 0.2159090909090909, + "Indonesian,Spanish": 0.2215909090909091, + "Indonesian,Chinese": 0.23295454545454544, + "Indonesian,Malay": 0.25, + "Indonesian,Vietnamese": 0.29545454545454547, + "English,Filipino": 0.2159090909090909, + "English,Spanish": 0.2556818181818182, + "English,Chinese": 0.26136363636363635, + "English,Malay": 0.23863636363636365, + "English,Vietnamese": 0.3181818181818182, + "Filipino,Spanish": 0.26704545454545453, + "Filipino,Chinese": 0.32386363636363635, + "Filipino,Malay": 0.2556818181818182, + "Filipino,Vietnamese": 0.26704545454545453, + "Spanish,Chinese": 0.26136363636363635, + "Spanish,Malay": 0.25, + "Spanish,Vietnamese": 0.24431818181818182, + "Chinese,Malay": 0.25, + "Chinese,Vietnamese": 0.1875, + "Malay,Vietnamese": 0.3068181818181818 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.045454545454545456, + "Indonesian,English,Spanish": 0.056818181818181816, + "Indonesian,English,Chinese": 0.0625, + "Indonesian,English,Malay": 0.056818181818181816, + "Indonesian,English,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Spanish": 0.045454545454545456, + "Indonesian,Filipino,Chinese": 0.06818181818181818, + "Indonesian,Filipino,Malay": 0.056818181818181816, + "Indonesian,Filipino,Vietnamese": 0.08522727272727272, + "Indonesian,Spanish,Chinese": 0.0625, + "Indonesian,Spanish,Malay": 0.05113636363636364, + "Indonesian,Spanish,Vietnamese": 0.056818181818181816, + "Indonesian,Chinese,Malay": 0.07954545454545454, + "Indonesian,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,Malay,Vietnamese": 0.07386363636363637, + "English,Filipino,Spanish": 0.056818181818181816, + "English,Filipino,Chinese": 0.08522727272727272, + "English,Filipino,Malay": 0.056818181818181816, + "English,Filipino,Vietnamese": 0.09659090909090909, + "English,Spanish,Chinese": 0.07954545454545454, + "English,Spanish,Malay": 0.07954545454545454, + "English,Spanish,Vietnamese": 0.07386363636363637, + "English,Chinese,Malay": 0.0625, + "English,Chinese,Vietnamese": 0.045454545454545456, + "English,Malay,Vietnamese": 0.07954545454545454, + "Filipino,Spanish,Chinese": 0.09090909090909091, + "Filipino,Spanish,Malay": 0.07386363636363637, + "Filipino,Spanish,Vietnamese": 0.06818181818181818, + "Filipino,Chinese,Malay": 0.07954545454545454, + "Filipino,Chinese,Vietnamese": 0.07386363636363637, + "Filipino,Malay,Vietnamese": 0.09659090909090909, + "Spanish,Chinese,Malay": 0.06818181818181818, + "Spanish,Chinese,Vietnamese": 0.045454545454545456, + "Spanish,Malay,Vietnamese": 0.05113636363636364, + "Chinese,Malay,Vietnamese": 0.06818181818181818 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.011363636363636364, + "Indonesian,English,Filipino,Chinese": 0.011363636363636364, + "Indonesian,English,Filipino,Malay": 0.011363636363636364, + "Indonesian,English,Filipino,Vietnamese": 0.03977272727272727, + "Indonesian,English,Spanish,Chinese": 0.017045454545454544, + "Indonesian,English,Spanish,Malay": 0.022727272727272728, + "Indonesian,English,Spanish,Vietnamese": 0.022727272727272728, + "Indonesian,English,Chinese,Malay": 0.022727272727272728, + "Indonesian,English,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Chinese": 0.022727272727272728, + "Indonesian,Filipino,Spanish,Malay": 0.017045454545454544, + "Indonesian,Filipino,Spanish,Vietnamese": 0.017045454545454544, + "Indonesian,Filipino,Chinese,Malay": 0.022727272727272728, + "Indonesian,Filipino,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,Filipino,Malay,Vietnamese": 0.028409090909090908, + "Indonesian,Spanish,Chinese,Malay": 0.022727272727272728, + "Indonesian,Spanish,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,Spanish,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Chinese,Malay,Vietnamese": 0.017045454545454544, + "English,Filipino,Spanish,Chinese": 0.022727272727272728, + "English,Filipino,Spanish,Malay": 0.03409090909090909, + "English,Filipino,Spanish,Vietnamese": 0.022727272727272728, + "English,Filipino,Chinese,Malay": 0.022727272727272728, + "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, + "English,Filipino,Malay,Vietnamese": 0.028409090909090908, + "English,Spanish,Chinese,Malay": 0.03409090909090909, + "English,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "English,Spanish,Malay,Vietnamese": 0.022727272727272728, + "English,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Filipino,Spanish,Chinese,Malay": 0.028409090909090908, + "Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Filipino,Spanish,Malay,Vietnamese": 0.022727272727272728, + "Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.005681818181818182, + "Indonesian,English,Filipino,Spanish,Malay": 0.011363636363636364, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.005681818181818182, + "Indonesian,English,Filipino,Chinese,Malay": 0.005681818181818182, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,English,Spanish,Chinese,Malay": 0.011363636363636364, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, + "English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, + "English,Filipino,Spanish,Malay,Vietnamese": 0.017045454545454544, + "English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, + "English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 }, "6_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.34, - "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.3333333333333333, - "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.38, - "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.35333333333333333, - "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.36666666666666664, - "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.38 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.005681818181818182, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 }, "7_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.3333333333333333 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 } }, - "AC3_2": 0.6609304239928177, - "AC3_3": 0.5841361636531642, - "AC3_4": 0.5334973735110832, - "AC3_5": 0.4962335199999633, - "AC3_6": 0.4670631969804445, - "AC3_7": 0.4440203561896099 + "AC3_2": 0.24780392721145814, + "AC3_3": 0.10614056125117736, + "AC3_4": 0.03988320565413914, + "AC3_5": 0.018719065942824287, + "AC3_6": 0.012644121549871535, + "AC3_7": 0.011100240814255554 }, "prompt_5": { - "overall_acc": 0.699047619047619, + "overall_acc": 0.2605519480519481, "language_acc": { - "Vietnamese": 0.62, - "Malay": 0.68, - "Filipino": 0.6733333333333333, - "Indonesian": 0.7, - "Chinese": 0.6933333333333334, - "Spanish": 0.7266666666666667, - "English": 0.8 + "Indonesian": 0.2215909090909091, + "English": 0.2215909090909091, + "Filipino": 0.30113636363636365, + "Spanish": 0.23863636363636365, + "Chinese": 0.2727272727272727, + "Malay": 0.2727272727272727, + "Vietnamese": 0.29545454545454547 }, - "consistency_score_2": 0.7066666666666666, - "consistency_score_3": 0.5843809523809523, - "consistency_score_4": 0.5060952380952382, - "consistency_score_5": 0.448888888888889, - "consistency_score_6": 0.4038095238095238, - "consistency_score_7": 0.36666666666666664, + "consistency_score_2": 0.25162337662337664, + "consistency_score_3": 0.06185064935064933, + "consistency_score_4": 0.01233766233766233, + "consistency_score_5": 0.0010822510822510823, + "consistency_score_6": 0.0, + "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { - "Vietnamese,Malay": 0.6533333333333333, - "Vietnamese,Filipino": 0.6533333333333333, - "Vietnamese,Indonesian": 0.66, - "Vietnamese,Chinese": 0.64, - "Vietnamese,Spanish": 0.6333333333333333, - "Vietnamese,English": 0.6533333333333333, - "Malay,Filipino": 0.6866666666666666, - "Malay,Indonesian": 0.7466666666666667, - "Malay,Chinese": 0.6733333333333333, - "Malay,Spanish": 0.72, - "Malay,English": 0.74, - "Filipino,Indonesian": 0.72, - "Filipino,Chinese": 0.68, - "Filipino,Spanish": 0.7466666666666667, - "Filipino,English": 0.7333333333333333, - "Indonesian,Chinese": 0.6933333333333334, - "Indonesian,Spanish": 0.7466666666666667, - "Indonesian,English": 0.76, - "Chinese,Spanish": 0.7066666666666667, - "Chinese,English": 0.7666666666666667, - "Spanish,English": 0.8266666666666667 - }, - "3_combine": { - "Vietnamese,Malay,Filipino": 0.52, - "Vietnamese,Malay,Indonesian": 0.5466666666666666, - "Vietnamese,Malay,Chinese": 0.5066666666666667, - "Vietnamese,Malay,Spanish": 0.5466666666666666, - "Vietnamese,Malay,English": 0.56, - "Vietnamese,Filipino,Indonesian": 0.54, - "Vietnamese,Filipino,Chinese": 0.5133333333333333, - "Vietnamese,Filipino,Spanish": 0.5466666666666666, - "Vietnamese,Filipino,English": 0.5533333333333333, - "Vietnamese,Indonesian,Chinese": 0.5133333333333333, - "Vietnamese,Indonesian,Spanish": 0.5466666666666666, - "Vietnamese,Indonesian,English": 0.5666666666666667, - "Vietnamese,Chinese,Spanish": 0.5133333333333333, - "Vietnamese,Chinese,English": 0.5533333333333333, - "Vietnamese,Spanish,English": 0.5866666666666667, - "Malay,Filipino,Indonesian": 0.6066666666666667, - "Malay,Filipino,Chinese": 0.54, - "Malay,Filipino,Spanish": 0.6066666666666667, - "Malay,Filipino,English": 0.6066666666666667, - "Malay,Indonesian,Chinese": 0.58, - "Malay,Indonesian,Spanish": 0.6333333333333333, - "Malay,Indonesian,English": 0.6533333333333333, - "Malay,Chinese,Spanish": 0.5666666666666667, - "Malay,Chinese,English": 0.6066666666666667, - "Malay,Spanish,English": 0.66, - "Filipino,Indonesian,Chinese": 0.5666666666666667, - "Filipino,Indonesian,Spanish": 0.6333333333333333, - "Filipino,Indonesian,English": 0.64, - "Filipino,Chinese,Spanish": 0.5933333333333334, - "Filipino,Chinese,English": 0.6133333333333333, - "Filipino,Spanish,English": 0.68, - "Indonesian,Chinese,Spanish": 0.5866666666666667, - "Indonesian,Chinese,English": 0.6266666666666667, - "Indonesian,Spanish,English": 0.68, - "Chinese,Spanish,English": 0.66 - }, - "4_combine": { - "Vietnamese,Malay,Filipino,Indonesian": 0.4666666666666667, - "Vietnamese,Malay,Filipino,Chinese": 0.43333333333333335, - "Vietnamese,Malay,Filipino,Spanish": 0.47333333333333333, - "Vietnamese,Malay,Filipino,English": 0.48, - "Vietnamese,Malay,Indonesian,Chinese": 0.44666666666666666, - "Vietnamese,Malay,Indonesian,Spanish": 0.4866666666666667, - "Vietnamese,Malay,Indonesian,English": 0.5, - "Vietnamese,Malay,Chinese,Spanish": 0.4533333333333333, - "Vietnamese,Malay,Chinese,English": 0.48, - "Vietnamese,Malay,Spanish,English": 0.5133333333333333, - "Vietnamese,Filipino,Indonesian,Chinese": 0.44, - "Vietnamese,Filipino,Indonesian,Spanish": 0.4866666666666667, - "Vietnamese,Filipino,Indonesian,English": 0.5, - "Vietnamese,Filipino,Chinese,Spanish": 0.46, - "Vietnamese,Filipino,Chinese,English": 0.4866666666666667, - "Vietnamese,Filipino,Spanish,English": 0.52, - "Vietnamese,Indonesian,Chinese,Spanish": 0.44666666666666666, - "Vietnamese,Indonesian,Chinese,English": 0.4866666666666667, - "Vietnamese,Indonesian,Spanish,English": 0.52, - "Vietnamese,Chinese,Spanish,English": 0.5, - "Malay,Filipino,Indonesian,Chinese": 0.4866666666666667, - "Malay,Filipino,Indonesian,Spanish": 0.5533333333333333, - "Malay,Filipino,Indonesian,English": 0.5533333333333333, - "Malay,Filipino,Chinese,Spanish": 0.5, - "Malay,Filipino,Chinese,English": 0.52, - "Malay,Filipino,Spanish,English": 0.5666666666666667, - "Malay,Indonesian,Chinese,Spanish": 0.5133333333333333, - "Malay,Indonesian,Chinese,English": 0.5466666666666666, - "Malay,Indonesian,Spanish,English": 0.5933333333333334, - "Malay,Chinese,Spanish,English": 0.54, - "Filipino,Indonesian,Chinese,Spanish": 0.5133333333333333, - "Filipino,Indonesian,Chinese,English": 0.5333333333333333, - "Filipino,Indonesian,Spanish,English": 0.6, - "Filipino,Chinese,Spanish,English": 0.56, - "Indonesian,Chinese,Spanish,English": 0.5533333333333333 - }, - "5_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.38666666666666666, - "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.43333333333333335, - "Vietnamese,Malay,Filipino,Indonesian,English": 0.44, - "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.4066666666666667, - "Vietnamese,Malay,Filipino,Chinese,English": 0.4266666666666667, - "Vietnamese,Malay,Filipino,Spanish,English": 0.4533333333333333, - "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.4066666666666667, - "Vietnamese,Malay,Indonesian,Chinese,English": 0.43333333333333335, - "Vietnamese,Malay,Indonesian,Spanish,English": 0.4666666666666667, - "Vietnamese,Malay,Chinese,Spanish,English": 0.44, - "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.4066666666666667, - "Vietnamese,Filipino,Indonesian,Chinese,English": 0.43333333333333335, - "Vietnamese,Filipino,Indonesian,Spanish,English": 0.47333333333333333, - "Vietnamese,Filipino,Chinese,Spanish,English": 0.4533333333333333, - "Vietnamese,Indonesian,Chinese,Spanish,English": 0.44, - "Malay,Filipino,Indonesian,Chinese,Spanish": 0.46, - "Malay,Filipino,Indonesian,Chinese,English": 0.47333333333333333, - "Malay,Filipino,Indonesian,Spanish,English": 0.5266666666666666, - "Malay,Filipino,Chinese,Spanish,English": 0.48, - "Malay,Indonesian,Chinese,Spanish,English": 0.49333333333333335, - "Filipino,Indonesian,Chinese,Spanish,English": 0.49333333333333335 + "Indonesian,English": 0.23863636363636365, + "Indonesian,Filipino": 0.2784090909090909, + "Indonesian,Spanish": 0.3068181818181818, + "Indonesian,Chinese": 0.26136363636363635, + "Indonesian,Malay": 0.2215909090909091, + "Indonesian,Vietnamese": 0.2159090909090909, + "English,Filipino": 0.23863636363636365, + "English,Spanish": 0.2897727272727273, + "English,Chinese": 0.2556818181818182, + "English,Malay": 0.2784090909090909, + "English,Vietnamese": 0.23863636363636365, + "Filipino,Spanish": 0.2897727272727273, + "Filipino,Chinese": 0.20454545454545456, + "Filipino,Malay": 0.2215909090909091, + "Filipino,Vietnamese": 0.22727272727272727, + "Spanish,Chinese": 0.22727272727272727, + "Spanish,Malay": 0.23295454545454544, + "Spanish,Vietnamese": 0.2897727272727273, + "Chinese,Malay": 0.23863636363636365, + "Chinese,Vietnamese": 0.26136363636363635, + "Malay,Vietnamese": 0.26704545454545453 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.0625, + "Indonesian,English,Spanish": 0.07386363636363637, + "Indonesian,English,Chinese": 0.06818181818181818, + "Indonesian,English,Malay": 0.07386363636363637, + "Indonesian,English,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Spanish": 0.10227272727272728, + "Indonesian,Filipino,Chinese": 0.0625, + "Indonesian,Filipino,Malay": 0.05113636363636364, + "Indonesian,Filipino,Vietnamese": 0.045454545454545456, + "Indonesian,Spanish,Chinese": 0.05113636363636364, + "Indonesian,Spanish,Malay": 0.056818181818181816, + "Indonesian,Spanish,Vietnamese": 0.056818181818181816, + "Indonesian,Chinese,Malay": 0.07386363636363637, + "Indonesian,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,Malay,Vietnamese": 0.05113636363636364, + "English,Filipino,Spanish": 0.07954545454545454, + "English,Filipino,Chinese": 0.03977272727272727, + "English,Filipino,Malay": 0.0625, + "English,Filipino,Vietnamese": 0.0625, + "English,Spanish,Chinese": 0.0625, + "English,Spanish,Malay": 0.06818181818181818, + "English,Spanish,Vietnamese": 0.06818181818181818, + "English,Chinese,Malay": 0.045454545454545456, + "English,Chinese,Vietnamese": 0.06818181818181818, + "English,Malay,Vietnamese": 0.08522727272727272, + "Filipino,Spanish,Chinese": 0.07386363636363637, + "Filipino,Spanish,Malay": 0.056818181818181816, + "Filipino,Spanish,Vietnamese": 0.06818181818181818, + "Filipino,Chinese,Malay": 0.045454545454545456, + "Filipino,Chinese,Vietnamese": 0.03977272727272727, + "Filipino,Malay,Vietnamese": 0.05113636363636364, + "Spanish,Chinese,Malay": 0.05113636363636364, + "Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Spanish,Malay,Vietnamese": 0.07386363636363637, + "Chinese,Malay,Vietnamese": 0.0625 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.017045454545454544, + "Indonesian,English,Filipino,Chinese": 0.011363636363636364, + "Indonesian,English,Filipino,Malay": 0.011363636363636364, + "Indonesian,English,Filipino,Vietnamese": 0.005681818181818182, + "Indonesian,English,Spanish,Chinese": 0.017045454545454544, + "Indonesian,English,Spanish,Malay": 0.011363636363636364, + "Indonesian,English,Spanish,Vietnamese": 0.005681818181818182, + "Indonesian,English,Chinese,Malay": 0.017045454545454544, + "Indonesian,English,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,English,Malay,Vietnamese": 0.022727272727272728, + "Indonesian,Filipino,Spanish,Chinese": 0.017045454545454544, + "Indonesian,Filipino,Spanish,Malay": 0.017045454545454544, + "Indonesian,Filipino,Spanish,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Chinese,Malay": 0.011363636363636364, + "Indonesian,Filipino,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Malay,Vietnamese": 0.0, + "Indonesian,Spanish,Chinese,Malay": 0.011363636363636364, + "Indonesian,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,Spanish,Malay,Vietnamese": 0.017045454545454544, + "Indonesian,Chinese,Malay,Vietnamese": 0.011363636363636364, + "English,Filipino,Spanish,Chinese": 0.011363636363636364, + "English,Filipino,Spanish,Malay": 0.017045454545454544, + "English,Filipino,Spanish,Vietnamese": 0.011363636363636364, + "English,Filipino,Chinese,Malay": 0.0, + "English,Filipino,Chinese,Vietnamese": 0.005681818181818182, + "English,Filipino,Malay,Vietnamese": 0.022727272727272728, + "English,Spanish,Chinese,Malay": 0.005681818181818182, + "English,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "English,Spanish,Malay,Vietnamese": 0.011363636363636364, + "English,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Filipino,Spanish,Chinese,Malay": 0.011363636363636364, + "Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, + "Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.0, + "Indonesian,English,Filipino,Spanish,Malay": 0.0, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0, + "Indonesian,English,Filipino,Chinese,Malay": 0.0, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.0, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese,Malay": 0.0, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, + "English,Filipino,Spanish,Chinese,Malay": 0.0, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.0, + "English,Filipino,Spanish,Malay,Vietnamese": 0.0, + "English,Filipino,Chinese,Malay,Vietnamese": 0.0, + "English,Spanish,Chinese,Malay,Vietnamese": 0.0, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "6_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.36666666666666664, - "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.38666666666666666, - "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.42, - "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.4, - "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.4, - "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.4066666666666667, - "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.44666666666666666 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "7_combine": { - "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.36666666666666664 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 } }, - "AC3_2": 0.7028364949816184, - "AC3_3": 0.6365918952422053, - "AC3_4": 0.5871248691814143, - "AC3_5": 0.5467109144066519, - "AC3_6": 0.5119105189107344, - "AC3_7": 0.48102472441122346 + "AC3_2": 0.2560098379725315, + "AC3_3": 0.09997008275629547, + "AC3_4": 0.023559724026997794, + "AC3_5": 0.0021555486904021653, + "AC3_6": 0.0, + "AC3_7": 0.0 } }, - "cross_logiqa": { + "sg_eval": { "prompt_1": { - "overall_acc": 0.47970779220779225, - "language_acc": { - "English": 0.5625, - "Filipino": 0.3806818181818182, - "Vietnamese": 0.4375, - "Chinese": 0.5568181818181818, - "Indonesian": 0.4375, - "Malay": 0.4431818181818182, - "Spanish": 0.5397727272727273 - }, - "consistency_score_2": 0.549512987012987, - "consistency_score_3": 0.37905844155844165, - "consistency_score_4": 0.29107142857142865, - "consistency_score_5": 0.2372835497835498, - "consistency_score_6": 0.2012987012987013, - "consistency_score_7": 0.17613636363636365, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4772727272727273, - "English,Vietnamese": 0.5738636363636364, - "English,Chinese": 0.5795454545454546, - "English,Indonesian": 0.6079545454545454, - "English,Malay": 0.6193181818181818, - "English,Spanish": 0.7443181818181818, - "Filipino,Vietnamese": 0.4602272727272727, - "Filipino,Chinese": 0.44886363636363635, - "Filipino,Indonesian": 0.44886363636363635, - "Filipino,Malay": 0.48863636363636365, - "Filipino,Spanish": 0.4602272727272727, - "Vietnamese,Chinese": 0.5170454545454546, - "Vietnamese,Indonesian": 0.5397727272727273, - "Vietnamese,Malay": 0.5170454545454546, - "Vietnamese,Spanish": 0.5965909090909091, - "Chinese,Indonesian": 0.5511363636363636, - "Chinese,Malay": 0.5170454545454546, - "Chinese,Spanish": 0.6022727272727273, - "Indonesian,Malay": 0.5909090909090909, - "Indonesian,Spanish": 0.625, - "Malay,Spanish": 0.5738636363636364 - }, - "3_combine": { - "English,Filipino,Vietnamese": 0.3068181818181818, - "English,Filipino,Chinese": 0.3125, - "English,Filipino,Indonesian": 0.3409090909090909, - "English,Filipino,Malay": 0.35795454545454547, - "English,Filipino,Spanish": 0.3693181818181818, - "English,Vietnamese,Chinese": 0.375, - "English,Vietnamese,Indonesian": 0.42045454545454547, - "English,Vietnamese,Malay": 0.4090909090909091, - "English,Vietnamese,Spanish": 0.48295454545454547, - "English,Chinese,Indonesian": 0.42613636363636365, - "English,Chinese,Malay": 0.4034090909090909, - "English,Chinese,Spanish": 0.48863636363636365, - "English,Indonesian,Malay": 0.4659090909090909, - "English,Indonesian,Spanish": 0.5170454545454546, - "English,Malay,Spanish": 0.5, - "Filipino,Vietnamese,Chinese": 0.2840909090909091, - "Filipino,Vietnamese,Indonesian": 0.29545454545454547, - "Filipino,Vietnamese,Malay": 0.30113636363636365, - "Filipino,Vietnamese,Spanish": 0.3125, - "Filipino,Chinese,Indonesian": 0.30113636363636365, - "Filipino,Chinese,Malay": 0.29545454545454547, - "Filipino,Chinese,Spanish": 0.3181818181818182, - "Filipino,Indonesian,Malay": 0.32954545454545453, - "Filipino,Indonesian,Spanish": 0.3181818181818182, - "Filipino,Malay,Spanish": 0.32386363636363635, - "Vietnamese,Chinese,Indonesian": 0.375, - "Vietnamese,Chinese,Malay": 0.32954545454545453, - "Vietnamese,Chinese,Spanish": 0.4090909090909091, - "Vietnamese,Indonesian,Malay": 0.3806818181818182, - "Vietnamese,Indonesian,Spanish": 0.4431818181818182, - "Vietnamese,Malay,Spanish": 0.38636363636363635, - "Chinese,Indonesian,Malay": 0.3977272727272727, - "Chinese,Indonesian,Spanish": 0.4431818181818182, - "Chinese,Malay,Spanish": 0.38636363636363635, - "Indonesian,Malay,Spanish": 0.4602272727272727 - }, - "4_combine": { - "English,Filipino,Vietnamese,Chinese": 0.23295454545454544, - "English,Filipino,Vietnamese,Indonesian": 0.2556818181818182, - "English,Filipino,Vietnamese,Malay": 0.2556818181818182, - "English,Filipino,Vietnamese,Spanish": 0.2727272727272727, - "English,Filipino,Chinese,Indonesian": 0.25, - "English,Filipino,Chinese,Malay": 0.25, - "English,Filipino,Chinese,Spanish": 0.2727272727272727, - "English,Filipino,Indonesian,Malay": 0.29545454545454547, - "English,Filipino,Indonesian,Spanish": 0.2840909090909091, - "English,Filipino,Malay,Spanish": 0.29545454545454547, - "English,Vietnamese,Chinese,Indonesian": 0.3125, - "English,Vietnamese,Chinese,Malay": 0.2840909090909091, - "English,Vietnamese,Chinese,Spanish": 0.3409090909090909, - "English,Vietnamese,Indonesian,Malay": 0.32954545454545453, - "English,Vietnamese,Indonesian,Spanish": 0.375, - "English,Vietnamese,Malay,Spanish": 0.3465909090909091, - "English,Chinese,Indonesian,Malay": 0.3465909090909091, - "English,Chinese,Indonesian,Spanish": 0.38636363636363635, - "English,Chinese,Malay,Spanish": 0.35795454545454547, - "English,Indonesian,Malay,Spanish": 0.4090909090909091, - "Filipino,Vietnamese,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Vietnamese,Chinese,Malay": 0.2215909090909091, - "Filipino,Vietnamese,Chinese,Spanish": 0.25, - "Filipino,Vietnamese,Indonesian,Malay": 0.25, - "Filipino,Vietnamese,Indonesian,Spanish": 0.24431818181818182, - "Filipino,Vietnamese,Malay,Spanish": 0.23863636363636365, - "Filipino,Chinese,Indonesian,Malay": 0.2556818181818182, - "Filipino,Chinese,Indonesian,Spanish": 0.24431818181818182, - "Filipino,Chinese,Malay,Spanish": 0.24431818181818182, - "Filipino,Indonesian,Malay,Spanish": 0.2727272727272727, - "Vietnamese,Chinese,Indonesian,Malay": 0.2840909090909091, - "Vietnamese,Chinese,Indonesian,Spanish": 0.3409090909090909, - "Vietnamese,Chinese,Malay,Spanish": 0.2897727272727273, - "Vietnamese,Indonesian,Malay,Spanish": 0.32954545454545453, - "Chinese,Indonesian,Malay,Spanish": 0.3409090909090909 - }, - "5_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian": 0.20454545454545456, - "English,Filipino,Vietnamese,Chinese,Malay": 0.19886363636363635, - "English,Filipino,Vietnamese,Chinese,Spanish": 0.2215909090909091, - "English,Filipino,Vietnamese,Indonesian,Malay": 0.23295454545454544, - "English,Filipino,Vietnamese,Indonesian,Spanish": 0.22727272727272727, - "English,Filipino,Vietnamese,Malay,Spanish": 0.2215909090909091, - "English,Filipino,Chinese,Indonesian,Malay": 0.22727272727272727, - "English,Filipino,Chinese,Indonesian,Spanish": 0.22727272727272727, - "English,Filipino,Chinese,Malay,Spanish": 0.22727272727272727, - "English,Filipino,Indonesian,Malay,Spanish": 0.2556818181818182, - "English,Vietnamese,Chinese,Indonesian,Malay": 0.2556818181818182, - "English,Vietnamese,Chinese,Indonesian,Spanish": 0.2897727272727273, - "English,Vietnamese,Chinese,Malay,Spanish": 0.26136363636363635, - "English,Vietnamese,Indonesian,Malay,Spanish": 0.29545454545454547, - "English,Chinese,Indonesian,Malay,Spanish": 0.3181818181818182, - "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.20454545454545456, - "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.21022727272727273, - "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.20454545454545456, - "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.2159090909090909, - "Filipino,Chinese,Indonesian,Malay,Spanish": 0.2215909090909091, - "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.26136363636363635 - }, - "6_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.1875, - "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.19318181818181818, - "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.1875, - "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.20454545454545456, - "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.21022727272727273, - "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.23863636363636365, - "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.1875 - }, - "7_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.17613636363636365 - } - }, - "AC3_2": 0.5122431786957747, - "AC3_3": 0.42348495072769815, - "AC3_4": 0.3623066854747182, - "AC3_5": 0.3175122518547604, - "AC3_6": 0.2835936411205318, - "AC3_7": 0.2576648289436108 + "accuracy": 0.27184466019417475 }, "prompt_2": { - "overall_acc": 0.4878246753246754, - "language_acc": { - "English": 0.5909090909090909, - "Filipino": 0.4147727272727273, - "Vietnamese": 0.4318181818181818, - "Chinese": 0.5625, - "Indonesian": 0.4772727272727273, - "Malay": 0.4318181818181818, - "Spanish": 0.5056818181818182 - }, - "consistency_score_2": 0.5744047619047619, - "consistency_score_3": 0.40649350649350646, - "consistency_score_4": 0.3180194805194805, - "consistency_score_5": 0.26379870129870125, - "consistency_score_6": 0.22646103896103895, - "consistency_score_7": 0.19886363636363635, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.5284090909090909, - "English,Vietnamese": 0.5454545454545454, - "English,Chinese": 0.5795454545454546, - "English,Indonesian": 0.6818181818181818, - "English,Malay": 0.5795454545454546, - "English,Spanish": 0.6875, - "Filipino,Vietnamese": 0.5170454545454546, - "Filipino,Chinese": 0.5227272727272727, - "Filipino,Indonesian": 0.5568181818181818, - "Filipino,Malay": 0.5965909090909091, - "Filipino,Spanish": 0.5511363636363636, - "Vietnamese,Chinese": 0.5170454545454546, - "Vietnamese,Indonesian": 0.5625, - "Vietnamese,Malay": 0.5284090909090909, - "Vietnamese,Spanish": 0.5681818181818182, - "Chinese,Indonesian": 0.5795454545454546, - "Chinese,Malay": 0.5681818181818182, - "Chinese,Spanish": 0.5852272727272727, - "Indonesian,Malay": 0.6420454545454546, - "Indonesian,Spanish": 0.6136363636363636, - "Malay,Spanish": 0.5511363636363636 - }, - "3_combine": { - "English,Filipino,Vietnamese": 0.3522727272727273, - "English,Filipino,Chinese": 0.36363636363636365, - "English,Filipino,Indonesian": 0.42613636363636365, - "English,Filipino,Malay": 0.4034090909090909, - "English,Filipino,Spanish": 0.42045454545454547, - "English,Vietnamese,Chinese": 0.375, - "English,Vietnamese,Indonesian": 0.44886363636363635, - "English,Vietnamese,Malay": 0.39204545454545453, - "English,Vietnamese,Spanish": 0.42613636363636365, - "English,Chinese,Indonesian": 0.44886363636363635, - "English,Chinese,Malay": 0.3977272727272727, - "English,Chinese,Spanish": 0.4602272727272727, - "English,Indonesian,Malay": 0.4772727272727273, - "English,Indonesian,Spanish": 0.5227272727272727, - "English,Malay,Spanish": 0.4318181818181818, - "Filipino,Vietnamese,Chinese": 0.32954545454545453, - "Filipino,Vietnamese,Indonesian": 0.3806818181818182, - "Filipino,Vietnamese,Malay": 0.375, - "Filipino,Vietnamese,Spanish": 0.375, - "Filipino,Chinese,Indonesian": 0.3806818181818182, - "Filipino,Chinese,Malay": 0.39204545454545453, - "Filipino,Chinese,Spanish": 0.3806818181818182, - "Filipino,Indonesian,Malay": 0.4431818181818182, - "Filipino,Indonesian,Spanish": 0.3977272727272727, - "Filipino,Malay,Spanish": 0.4034090909090909, - "Vietnamese,Chinese,Indonesian": 0.38636363636363635, - "Vietnamese,Chinese,Malay": 0.35795454545454547, - "Vietnamese,Chinese,Spanish": 0.38636363636363635, - "Vietnamese,Indonesian,Malay": 0.4090909090909091, - "Vietnamese,Indonesian,Spanish": 0.42045454545454547, - "Vietnamese,Malay,Spanish": 0.39204545454545453, - "Chinese,Indonesian,Malay": 0.4147727272727273, - "Chinese,Indonesian,Spanish": 0.42613636363636365, - "Chinese,Malay,Spanish": 0.39204545454545453, - "Indonesian,Malay,Spanish": 0.4375 - }, - "4_combine": { - "English,Filipino,Vietnamese,Chinese": 0.2556818181818182, - "English,Filipino,Vietnamese,Indonesian": 0.3181818181818182, - "English,Filipino,Vietnamese,Malay": 0.29545454545454547, - "English,Filipino,Vietnamese,Spanish": 0.30113636363636365, - "English,Filipino,Chinese,Indonesian": 0.3068181818181818, - "English,Filipino,Chinese,Malay": 0.29545454545454547, - "English,Filipino,Chinese,Spanish": 0.30113636363636365, - "English,Filipino,Indonesian,Malay": 0.3465909090909091, - "English,Filipino,Indonesian,Spanish": 0.3522727272727273, - "English,Filipino,Malay,Spanish": 0.3409090909090909, - "English,Vietnamese,Chinese,Indonesian": 0.32386363636363635, - "English,Vietnamese,Chinese,Malay": 0.2897727272727273, - "English,Vietnamese,Chinese,Spanish": 0.32954545454545453, - "English,Vietnamese,Indonesian,Malay": 0.3352272727272727, - "English,Vietnamese,Indonesian,Spanish": 0.36363636363636365, - "English,Vietnamese,Malay,Spanish": 0.3181818181818182, - "English,Chinese,Indonesian,Malay": 0.32954545454545453, - "English,Chinese,Indonesian,Spanish": 0.3806818181818182, - "English,Chinese,Malay,Spanish": 0.32386363636363635, - "English,Indonesian,Malay,Spanish": 0.38636363636363635, - "Filipino,Vietnamese,Chinese,Indonesian": 0.26704545454545453, - "Filipino,Vietnamese,Chinese,Malay": 0.2784090909090909, - "Filipino,Vietnamese,Chinese,Spanish": 0.2840909090909091, - "Filipino,Vietnamese,Indonesian,Malay": 0.3125, - "Filipino,Vietnamese,Indonesian,Spanish": 0.3181818181818182, - "Filipino,Vietnamese,Malay,Spanish": 0.3125, - "Filipino,Chinese,Indonesian,Malay": 0.3125, - "Filipino,Chinese,Indonesian,Spanish": 0.30113636363636365, - "Filipino,Chinese,Malay,Spanish": 0.3125, - "Filipino,Indonesian,Malay,Spanish": 0.3465909090909091, - "Vietnamese,Chinese,Indonesian,Malay": 0.30113636363636365, - "Vietnamese,Chinese,Indonesian,Spanish": 0.32954545454545453, - "Vietnamese,Chinese,Malay,Spanish": 0.30113636363636365, - "Vietnamese,Indonesian,Malay,Spanish": 0.32954545454545453, - "Chinese,Indonesian,Malay,Spanish": 0.32954545454545453 - }, - "5_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian": 0.22727272727272727, - "English,Filipino,Vietnamese,Chinese,Malay": 0.22727272727272727, - "English,Filipino,Vietnamese,Chinese,Spanish": 0.23863636363636365, - "English,Filipino,Vietnamese,Indonesian,Malay": 0.26136363636363635, - "English,Filipino,Vietnamese,Indonesian,Spanish": 0.2784090909090909, - "English,Filipino,Vietnamese,Malay,Spanish": 0.26704545454545453, - "English,Filipino,Chinese,Indonesian,Malay": 0.2556818181818182, - "English,Filipino,Chinese,Indonesian,Spanish": 0.26704545454545453, - "English,Filipino,Chinese,Malay,Spanish": 0.2556818181818182, - "English,Filipino,Indonesian,Malay,Spanish": 0.3068181818181818, - "English,Vietnamese,Chinese,Indonesian,Malay": 0.2556818181818182, - "English,Vietnamese,Chinese,Indonesian,Spanish": 0.29545454545454547, - "English,Vietnamese,Chinese,Malay,Spanish": 0.26136363636363635, - "English,Vietnamese,Indonesian,Malay,Spanish": 0.2897727272727273, - "English,Chinese,Indonesian,Malay,Spanish": 0.29545454545454547, - "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.23295454545454544, - "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.25, - "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2556818181818182, - "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.2784090909090909, - "Filipino,Chinese,Indonesian,Malay,Spanish": 0.26704545454545453, - "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.2727272727272727 - }, - "6_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.19886363636363635, - "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.2215909090909091, - "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2159090909090909, - "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.24431818181818182, - "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.23295454545454544, - "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.24431818181818182, - "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.22727272727272727 - }, - "7_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.19886363636363635 - } - }, - "AC3_2": 0.5275862382153784, - "AC3_3": 0.44346087742835405, - "AC3_4": 0.38503164336923823, - "AC3_5": 0.3424255279901211, - "AC3_6": 0.30932519181029544, - "AC3_7": 0.28254620670718816 + "accuracy": 0.2912621359223301 }, "prompt_3": { - "overall_acc": 0.4805194805194805, - "language_acc": { - "English": 0.5454545454545454, - "Filipino": 0.4375, - "Vietnamese": 0.4715909090909091, - "Chinese": 0.5397727272727273, - "Indonesian": 0.4431818181818182, - "Malay": 0.4375, - "Spanish": 0.48863636363636365 - }, - "consistency_score_2": 0.5708874458874458, - "consistency_score_3": 0.40568181818181825, - "consistency_score_4": 0.3199675324675324, - "consistency_score_5": 0.2673160173160173, - "consistency_score_6": 0.23133116883116883, - "consistency_score_7": 0.20454545454545456, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.5284090909090909, - "English,Vietnamese": 0.5738636363636364, - "English,Chinese": 0.5738636363636364, - "English,Indonesian": 0.5795454545454546, - "English,Malay": 0.5852272727272727, - "English,Spanish": 0.6931818181818182, - "Filipino,Vietnamese": 0.5170454545454546, - "Filipino,Chinese": 0.5340909090909091, - "Filipino,Indonesian": 0.5340909090909091, - "Filipino,Malay": 0.5909090909090909, - "Filipino,Spanish": 0.5909090909090909, - "Vietnamese,Chinese": 0.5284090909090909, - "Vietnamese,Indonesian": 0.5625, - "Vietnamese,Malay": 0.5397727272727273, - "Vietnamese,Spanish": 0.6079545454545454, - "Chinese,Indonesian": 0.5397727272727273, - "Chinese,Malay": 0.5397727272727273, - "Chinese,Spanish": 0.5625, - "Indonesian,Malay": 0.6136363636363636, - "Indonesian,Spanish": 0.5909090909090909, - "Malay,Spanish": 0.6022727272727273 - }, - "3_combine": { - "English,Filipino,Vietnamese": 0.36363636363636365, - "English,Filipino,Chinese": 0.3693181818181818, - "English,Filipino,Indonesian": 0.3693181818181818, - "English,Filipino,Malay": 0.4090909090909091, - "English,Filipino,Spanish": 0.4375, - "English,Vietnamese,Chinese": 0.3806818181818182, - "English,Vietnamese,Indonesian": 0.4034090909090909, - "English,Vietnamese,Malay": 0.4034090909090909, - "English,Vietnamese,Spanish": 0.4943181818181818, - "English,Chinese,Indonesian": 0.3977272727272727, - "English,Chinese,Malay": 0.39204545454545453, - "English,Chinese,Spanish": 0.4772727272727273, - "English,Indonesian,Malay": 0.4431818181818182, - "English,Indonesian,Spanish": 0.4715909090909091, - "English,Malay,Spanish": 0.48863636363636365, - "Filipino,Vietnamese,Chinese": 0.3465909090909091, - "Filipino,Vietnamese,Indonesian": 0.3693181818181818, - "Filipino,Vietnamese,Malay": 0.375, - "Filipino,Vietnamese,Spanish": 0.42613636363636365, - "Filipino,Chinese,Indonesian": 0.36363636363636365, - "Filipino,Chinese,Malay": 0.38636363636363635, - "Filipino,Chinese,Spanish": 0.3977272727272727, - "Filipino,Indonesian,Malay": 0.4147727272727273, - "Filipino,Indonesian,Spanish": 0.4034090909090909, - "Filipino,Malay,Spanish": 0.4431818181818182, - "Vietnamese,Chinese,Indonesian": 0.3522727272727273, - "Vietnamese,Chinese,Malay": 0.3465909090909091, - "Vietnamese,Chinese,Spanish": 0.39204545454545453, - "Vietnamese,Indonesian,Malay": 0.4034090909090909, - "Vietnamese,Indonesian,Spanish": 0.42613636363636365, - "Vietnamese,Malay,Spanish": 0.4375, - "Chinese,Indonesian,Malay": 0.38636363636363635, - "Chinese,Indonesian,Spanish": 0.39204545454545453, - "Chinese,Malay,Spanish": 0.3977272727272727, - "Indonesian,Malay,Spanish": 0.4375 - }, - "4_combine": { - "English,Filipino,Vietnamese,Chinese": 0.2727272727272727, - "English,Filipino,Vietnamese,Indonesian": 0.2897727272727273, - "English,Filipino,Vietnamese,Malay": 0.29545454545454547, - "English,Filipino,Vietnamese,Spanish": 0.3522727272727273, - "English,Filipino,Chinese,Indonesian": 0.29545454545454547, - "English,Filipino,Chinese,Malay": 0.3068181818181818, - "English,Filipino,Chinese,Spanish": 0.3409090909090909, - "English,Filipino,Indonesian,Malay": 0.32954545454545453, - "English,Filipino,Indonesian,Spanish": 0.32954545454545453, - "English,Filipino,Malay,Spanish": 0.3693181818181818, - "English,Vietnamese,Chinese,Indonesian": 0.30113636363636365, - "English,Vietnamese,Chinese,Malay": 0.2897727272727273, - "English,Vietnamese,Chinese,Spanish": 0.3522727272727273, - "English,Vietnamese,Indonesian,Malay": 0.32954545454545453, - "English,Vietnamese,Indonesian,Spanish": 0.36363636363636365, - "English,Vietnamese,Malay,Spanish": 0.375, - "English,Chinese,Indonesian,Malay": 0.32386363636363635, - "English,Chinese,Indonesian,Spanish": 0.35795454545454547, - "English,Chinese,Malay,Spanish": 0.36363636363636365, - "English,Indonesian,Malay,Spanish": 0.375, - "Filipino,Vietnamese,Chinese,Indonesian": 0.26136363636363635, - "Filipino,Vietnamese,Chinese,Malay": 0.2556818181818182, - "Filipino,Vietnamese,Chinese,Spanish": 0.30113636363636365, - "Filipino,Vietnamese,Indonesian,Malay": 0.30113636363636365, - "Filipino,Vietnamese,Indonesian,Spanish": 0.3181818181818182, - "Filipino,Vietnamese,Malay,Spanish": 0.3352272727272727, - "Filipino,Chinese,Indonesian,Malay": 0.29545454545454547, - "Filipino,Chinese,Indonesian,Spanish": 0.3125, - "Filipino,Chinese,Malay,Spanish": 0.3125, - "Filipino,Indonesian,Malay,Spanish": 0.35795454545454547, - "Vietnamese,Chinese,Indonesian,Malay": 0.2784090909090909, - "Vietnamese,Chinese,Indonesian,Spanish": 0.30113636363636365, - "Vietnamese,Chinese,Malay,Spanish": 0.29545454545454547, - "Vietnamese,Indonesian,Malay,Spanish": 0.3465909090909091, - "Chinese,Indonesian,Malay,Spanish": 0.3125 - }, - "5_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian": 0.23295454545454544, - "English,Filipino,Vietnamese,Chinese,Malay": 0.2215909090909091, - "English,Filipino,Vietnamese,Chinese,Spanish": 0.2727272727272727, - "English,Filipino,Vietnamese,Indonesian,Malay": 0.2556818181818182, - "English,Filipino,Vietnamese,Indonesian,Spanish": 0.2784090909090909, - "English,Filipino,Vietnamese,Malay,Spanish": 0.2897727272727273, - "English,Filipino,Chinese,Indonesian,Malay": 0.26704545454545453, - "English,Filipino,Chinese,Indonesian,Spanish": 0.2840909090909091, - "English,Filipino,Chinese,Malay,Spanish": 0.2897727272727273, - "English,Filipino,Indonesian,Malay,Spanish": 0.30113636363636365, - "English,Vietnamese,Chinese,Indonesian,Malay": 0.2556818181818182, - "English,Vietnamese,Chinese,Indonesian,Spanish": 0.2840909090909091, - "English,Vietnamese,Chinese,Malay,Spanish": 0.2784090909090909, - "English,Vietnamese,Indonesian,Malay,Spanish": 0.3068181818181818, - "English,Chinese,Indonesian,Malay,Spanish": 0.30113636363636365, - "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.2159090909090909, - "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.24431818181818182, - "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.23295454545454544, - "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.2840909090909091, - "Filipino,Chinese,Indonesian,Malay,Spanish": 0.26704545454545453, - "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.25 - }, - "6_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.20454545454545456, - "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.23295454545454544, - "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2215909090909091, - "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.25, - "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.2556818181818182, - "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.24431818181818182, - "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.21022727272727273 - }, - "7_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.20454545454545456 - } - }, - "AC3_2": 0.5218199196086831, - "AC3_3": 0.43994071503303905, - "AC3_4": 0.38414272804120053, - "AC3_5": 0.3435262277060514, - "AC3_6": 0.31231026665635947, - "AC3_7": 0.2869452821640599 + "accuracy": 0.32038834951456313 }, "prompt_4": { - "overall_acc": 0.5137987012987013, - "language_acc": { - "English": 0.5681818181818182, - "Filipino": 0.4772727272727273, - "Vietnamese": 0.4318181818181818, - "Chinese": 0.5397727272727273, - "Indonesian": 0.5284090909090909, - "Malay": 0.5056818181818182, - "Spanish": 0.5454545454545454 - }, - "consistency_score_2": 0.5868506493506492, - "consistency_score_3": 0.4237012987012987, - "consistency_score_4": 0.34139610389610386, - "consistency_score_5": 0.2924783549783549, - "consistency_score_6": 0.25892857142857145, - "consistency_score_7": 0.23295454545454544, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.5625, - "English,Vietnamese": 0.5852272727272727, - "English,Chinese": 0.5681818181818182, - "English,Indonesian": 0.6136363636363636, - "English,Malay": 0.6022727272727273, - "English,Spanish": 0.7443181818181818, - "Filipino,Vietnamese": 0.5625, - "Filipino,Chinese": 0.5454545454545454, - "Filipino,Indonesian": 0.5454545454545454, - "Filipino,Malay": 0.5852272727272727, - "Filipino,Spanish": 0.5625, - "Vietnamese,Chinese": 0.5113636363636364, - "Vietnamese,Indonesian": 0.5056818181818182, - "Vietnamese,Malay": 0.5454545454545454, - "Vietnamese,Spanish": 0.6079545454545454, - "Chinese,Indonesian": 0.5454545454545454, - "Chinese,Malay": 0.5795454545454546, - "Chinese,Spanish": 0.5511363636363636, - "Indonesian,Malay": 0.6931818181818182, - "Indonesian,Spanish": 0.6306818181818182, - "Malay,Spanish": 0.6761363636363636 - }, - "3_combine": { - "English,Filipino,Vietnamese": 0.3977272727272727, - "English,Filipino,Chinese": 0.38636363636363635, - "English,Filipino,Indonesian": 0.4090909090909091, - "English,Filipino,Malay": 0.42613636363636365, - "English,Filipino,Spanish": 0.4659090909090909, - "English,Vietnamese,Chinese": 0.3693181818181818, - "English,Vietnamese,Indonesian": 0.4090909090909091, - "English,Vietnamese,Malay": 0.4147727272727273, - "English,Vietnamese,Spanish": 0.4943181818181818, - "English,Chinese,Indonesian": 0.3977272727272727, - "English,Chinese,Malay": 0.42045454545454547, - "English,Chinese,Spanish": 0.4772727272727273, - "English,Indonesian,Malay": 0.4943181818181818, - "English,Indonesian,Spanish": 0.5170454545454546, - "English,Malay,Spanish": 0.5397727272727273, - "Filipino,Vietnamese,Chinese": 0.3465909090909091, - "Filipino,Vietnamese,Indonesian": 0.3806818181818182, - "Filipino,Vietnamese,Malay": 0.3977272727272727, - "Filipino,Vietnamese,Spanish": 0.4147727272727273, - "Filipino,Chinese,Indonesian": 0.375, - "Filipino,Chinese,Malay": 0.4090909090909091, - "Filipino,Chinese,Spanish": 0.3806818181818182, - "Filipino,Indonesian,Malay": 0.4602272727272727, - "Filipino,Indonesian,Spanish": 0.4318181818181818, - "Filipino,Malay,Spanish": 0.4602272727272727, - "Vietnamese,Chinese,Indonesian": 0.32386363636363635, - "Vietnamese,Chinese,Malay": 0.36363636363636365, - "Vietnamese,Chinese,Spanish": 0.3806818181818182, - "Vietnamese,Indonesian,Malay": 0.4090909090909091, - "Vietnamese,Indonesian,Spanish": 0.4090909090909091, - "Vietnamese,Malay,Spanish": 0.45454545454545453, - "Chinese,Indonesian,Malay": 0.4375, - "Chinese,Indonesian,Spanish": 0.4034090909090909, - "Chinese,Malay,Spanish": 0.4375, - "Indonesian,Malay,Spanish": 0.5340909090909091 - }, - "4_combine": { - "English,Filipino,Vietnamese,Chinese": 0.2784090909090909, - "English,Filipino,Vietnamese,Indonesian": 0.32954545454545453, - "English,Filipino,Vietnamese,Malay": 0.3409090909090909, - "English,Filipino,Vietnamese,Spanish": 0.35795454545454547, - "English,Filipino,Chinese,Indonesian": 0.3125, - "English,Filipino,Chinese,Malay": 0.32954545454545453, - "English,Filipino,Chinese,Spanish": 0.3465909090909091, - "English,Filipino,Indonesian,Malay": 0.3693181818181818, - "English,Filipino,Indonesian,Spanish": 0.3806818181818182, - "English,Filipino,Malay,Spanish": 0.3977272727272727, - "English,Vietnamese,Chinese,Indonesian": 0.2784090909090909, - "English,Vietnamese,Chinese,Malay": 0.29545454545454547, - "English,Vietnamese,Chinese,Spanish": 0.32954545454545453, - "English,Vietnamese,Indonesian,Malay": 0.3465909090909091, - "English,Vietnamese,Indonesian,Spanish": 0.36363636363636365, - "English,Vietnamese,Malay,Spanish": 0.39204545454545453, - "English,Chinese,Indonesian,Malay": 0.36363636363636365, - "English,Chinese,Indonesian,Spanish": 0.36363636363636365, - "English,Chinese,Malay,Spanish": 0.39204545454545453, - "English,Indonesian,Malay,Spanish": 0.44886363636363635, - "Filipino,Vietnamese,Chinese,Indonesian": 0.2784090909090909, - "Filipino,Vietnamese,Chinese,Malay": 0.3068181818181818, - "Filipino,Vietnamese,Chinese,Spanish": 0.2840909090909091, - "Filipino,Vietnamese,Indonesian,Malay": 0.3409090909090909, - "Filipino,Vietnamese,Indonesian,Spanish": 0.32954545454545453, - "Filipino,Vietnamese,Malay,Spanish": 0.36363636363636365, - "Filipino,Chinese,Indonesian,Malay": 0.32954545454545453, - "Filipino,Chinese,Indonesian,Spanish": 0.3125, - "Filipino,Chinese,Malay,Spanish": 0.3352272727272727, - "Filipino,Indonesian,Malay,Spanish": 0.4090909090909091, - "Vietnamese,Chinese,Indonesian,Malay": 0.29545454545454547, - "Vietnamese,Chinese,Indonesian,Spanish": 0.2897727272727273, - "Vietnamese,Chinese,Malay,Spanish": 0.3181818181818182, - "Vietnamese,Indonesian,Malay,Spanish": 0.36363636363636365, - "Chinese,Indonesian,Malay,Spanish": 0.375 - }, - "5_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian": 0.25, - "English,Filipino,Vietnamese,Chinese,Malay": 0.26136363636363635, - "English,Filipino,Vietnamese,Chinese,Spanish": 0.26136363636363635, - "English,Filipino,Vietnamese,Indonesian,Malay": 0.30113636363636365, - "English,Filipino,Vietnamese,Indonesian,Spanish": 0.3068181818181818, - "English,Filipino,Vietnamese,Malay,Spanish": 0.32954545454545453, - "English,Filipino,Chinese,Indonesian,Malay": 0.29545454545454547, - "English,Filipino,Chinese,Indonesian,Spanish": 0.30113636363636365, - "English,Filipino,Chinese,Malay,Spanish": 0.3125, - "English,Filipino,Indonesian,Malay,Spanish": 0.35795454545454547, - "English,Vietnamese,Chinese,Indonesian,Malay": 0.26136363636363635, - "English,Vietnamese,Chinese,Indonesian,Spanish": 0.26136363636363635, - "English,Vietnamese,Chinese,Malay,Spanish": 0.2840909090909091, - "English,Vietnamese,Indonesian,Malay,Spanish": 0.32954545454545453, - "English,Chinese,Indonesian,Malay,Spanish": 0.3465909090909091, - "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.26704545454545453, - "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.25, - "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2727272727272727, - "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.3181818181818182, - "Filipino,Chinese,Indonesian,Malay,Spanish": 0.30113636363636365, - "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.2727272727272727 - }, - "6_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.23863636363636365, - "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.23863636363636365, - "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.25, - "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.29545454545454547, - "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.2897727272727273, - "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.2556818181818182, - "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.24431818181818182 - }, - "7_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.23295454545454544 - } - }, - "AC3_2": 0.5479003849634371, - "AC3_3": 0.46442064424123597, - "AC3_4": 0.4102196919912001, - "AC3_5": 0.3727626819026909, - "AC3_6": 0.3443314825484767, - "AC3_7": 0.32056571141952267 - }, - "prompt_5": { - "overall_acc": 0.49025974025974023, - "language_acc": { - "English": 0.5454545454545454, - "Filipino": 0.4090909090909091, - "Vietnamese": 0.48295454545454547, - "Chinese": 0.5397727272727273, - "Indonesian": 0.42613636363636365, - "Malay": 0.4715909090909091, - "Spanish": 0.5568181818181818 - }, - "consistency_score_2": 0.5622294372294371, - "consistency_score_3": 0.39285714285714296, - "consistency_score_4": 0.30909090909090897, - "consistency_score_5": 0.26136363636363635, - "consistency_score_6": 0.23133116883116883, - "consistency_score_7": 0.21022727272727273, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.48863636363636365, - "English,Vietnamese": 0.5852272727272727, - "English,Chinese": 0.5625, - "English,Indonesian": 0.5738636363636364, - "English,Malay": 0.6193181818181818, - "English,Spanish": 0.7045454545454546, - "Filipino,Vietnamese": 0.5340909090909091, - "Filipino,Chinese": 0.5113636363636364, - "Filipino,Indonesian": 0.5340909090909091, - "Filipino,Malay": 0.5568181818181818, - "Filipino,Spanish": 0.5056818181818182, - "Vietnamese,Chinese": 0.5113636363636364, - "Vietnamese,Indonesian": 0.48863636363636365, - "Vietnamese,Malay": 0.5909090909090909, - "Vietnamese,Spanish": 0.5738636363636364, - "Chinese,Indonesian": 0.5397727272727273, - "Chinese,Malay": 0.5511363636363636, - "Chinese,Spanish": 0.5511363636363636, - "Indonesian,Malay": 0.625, - "Indonesian,Spanish": 0.6022727272727273, - "Malay,Spanish": 0.5965909090909091 - }, - "3_combine": { - "English,Filipino,Vietnamese": 0.35795454545454547, - "English,Filipino,Chinese": 0.3409090909090909, - "English,Filipino,Indonesian": 0.3465909090909091, - "English,Filipino,Malay": 0.3693181818181818, - "English,Filipino,Spanish": 0.38636363636363635, - "English,Vietnamese,Chinese": 0.3693181818181818, - "English,Vietnamese,Indonesian": 0.3806818181818182, - "English,Vietnamese,Malay": 0.4375, - "English,Vietnamese,Spanish": 0.4659090909090909, - "English,Chinese,Indonesian": 0.39204545454545453, - "English,Chinese,Malay": 0.3977272727272727, - "English,Chinese,Spanish": 0.44886363636363635, - "English,Indonesian,Malay": 0.44886363636363635, - "English,Indonesian,Spanish": 0.4715909090909091, - "English,Malay,Spanish": 0.48863636363636365, - "Filipino,Vietnamese,Chinese": 0.3465909090909091, - "Filipino,Vietnamese,Indonesian": 0.3465909090909091, - "Filipino,Vietnamese,Malay": 0.39204545454545453, - "Filipino,Vietnamese,Spanish": 0.36363636363636365, - "Filipino,Chinese,Indonesian": 0.3465909090909091, - "Filipino,Chinese,Malay": 0.375, - "Filipino,Chinese,Spanish": 0.3352272727272727, - "Filipino,Indonesian,Malay": 0.4034090909090909, - "Filipino,Indonesian,Spanish": 0.375, - "Filipino,Malay,Spanish": 0.3693181818181818, - "Vietnamese,Chinese,Indonesian": 0.3352272727272727, - "Vietnamese,Chinese,Malay": 0.36363636363636365, - "Vietnamese,Chinese,Spanish": 0.38636363636363635, - "Vietnamese,Indonesian,Malay": 0.3977272727272727, - "Vietnamese,Indonesian,Spanish": 0.3977272727272727, - "Vietnamese,Malay,Spanish": 0.42045454545454547, - "Chinese,Indonesian,Malay": 0.4147727272727273, - "Chinese,Indonesian,Spanish": 0.42045454545454547, - "Chinese,Malay,Spanish": 0.39204545454545453, - "Indonesian,Malay,Spanish": 0.4659090909090909 - }, - "4_combine": { - "English,Filipino,Vietnamese,Chinese": 0.26704545454545453, - "English,Filipino,Vietnamese,Indonesian": 0.2840909090909091, - "English,Filipino,Vietnamese,Malay": 0.30113636363636365, - "English,Filipino,Vietnamese,Spanish": 0.3125, - "English,Filipino,Chinese,Indonesian": 0.2727272727272727, - "English,Filipino,Chinese,Malay": 0.2784090909090909, - "English,Filipino,Chinese,Spanish": 0.2840909090909091, - "English,Filipino,Indonesian,Malay": 0.30113636363636365, - "English,Filipino,Indonesian,Spanish": 0.3068181818181818, - "English,Filipino,Malay,Spanish": 0.3068181818181818, - "English,Vietnamese,Chinese,Indonesian": 0.2840909090909091, - "English,Vietnamese,Chinese,Malay": 0.2897727272727273, - "English,Vietnamese,Chinese,Spanish": 0.3352272727272727, - "English,Vietnamese,Indonesian,Malay": 0.32954545454545453, - "English,Vietnamese,Indonesian,Spanish": 0.3409090909090909, - "English,Vietnamese,Malay,Spanish": 0.35795454545454547, - "English,Chinese,Indonesian,Malay": 0.3352272727272727, - "English,Chinese,Indonesian,Spanish": 0.36363636363636365, - "English,Chinese,Malay,Spanish": 0.3465909090909091, - "English,Indonesian,Malay,Spanish": 0.39204545454545453, - "Filipino,Vietnamese,Chinese,Indonesian": 0.2727272727272727, - "Filipino,Vietnamese,Chinese,Malay": 0.2840909090909091, - "Filipino,Vietnamese,Chinese,Spanish": 0.2727272727272727, - "Filipino,Vietnamese,Indonesian,Malay": 0.30113636363636365, - "Filipino,Vietnamese,Indonesian,Spanish": 0.30113636363636365, - "Filipino,Vietnamese,Malay,Spanish": 0.3068181818181818, - "Filipino,Chinese,Indonesian,Malay": 0.30113636363636365, - "Filipino,Chinese,Indonesian,Spanish": 0.29545454545454547, - "Filipino,Chinese,Malay,Spanish": 0.2727272727272727, - "Filipino,Indonesian,Malay,Spanish": 0.3125, - "Vietnamese,Chinese,Indonesian,Malay": 0.29545454545454547, - "Vietnamese,Chinese,Indonesian,Spanish": 0.3125, - "Vietnamese,Chinese,Malay,Spanish": 0.3068181818181818, - "Vietnamese,Indonesian,Malay,Spanish": 0.3409090909090909, - "Chinese,Indonesian,Malay,Spanish": 0.3522727272727273 - }, - "5_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian": 0.23863636363636365, - "English,Filipino,Vietnamese,Chinese,Malay": 0.23295454545454544, - "English,Filipino,Vietnamese,Chinese,Spanish": 0.24431818181818182, - "English,Filipino,Vietnamese,Indonesian,Malay": 0.2556818181818182, - "English,Filipino,Vietnamese,Indonesian,Spanish": 0.26704545454545453, - "English,Filipino,Vietnamese,Malay,Spanish": 0.26704545454545453, - "English,Filipino,Chinese,Indonesian,Malay": 0.24431818181818182, - "English,Filipino,Chinese,Indonesian,Spanish": 0.26136363636363635, - "English,Filipino,Chinese,Malay,Spanish": 0.24431818181818182, - "English,Filipino,Indonesian,Malay,Spanish": 0.2727272727272727, - "English,Vietnamese,Chinese,Indonesian,Malay": 0.2556818181818182, - "English,Vietnamese,Chinese,Indonesian,Spanish": 0.2784090909090909, - "English,Vietnamese,Chinese,Malay,Spanish": 0.26704545454545453, - "English,Vietnamese,Indonesian,Malay,Spanish": 0.29545454545454547, - "English,Chinese,Indonesian,Malay,Spanish": 0.32386363636363635, - "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.24431818181818182, - "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.2556818181818182, - "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.23863636363636365, - "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.26704545454545453, - "Filipino,Chinese,Indonesian,Malay,Spanish": 0.2556818181818182, - "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.2784090909090909 - }, - "6_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.2159090909090909, - "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.23295454545454544, - "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2159090909090909, - "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.23863636363636365, - "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.23863636363636365, - "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.25, - "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.22727272727272727 - }, - "7_combine": { - "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.21022727272727273 - } - }, - "AC3_2": 0.5237839281043418, - "AC3_3": 0.43618697474052426, - "AC3_4": 0.37914481938983285, - "AC3_5": 0.3409581778458603, - "AC3_6": 0.31433976592725005, - "AC3_7": 0.29426946166661816 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.6504854368932039 - }, - "prompt_2": { - "accuracy": 0.6213592233009708 - }, - "prompt_3": { - "accuracy": 0.6407766990291263 - }, - "prompt_4": { - "accuracy": 0.6699029126213593 + "accuracy": 0.24271844660194175 }, "prompt_5": { - "accuracy": 0.6407766990291263 + "accuracy": 0.21359223300970873 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.4666666666666667 + "accuracy": 0.29523809523809524 }, "prompt_2": { - "accuracy": 0.4666666666666667 + "accuracy": 0.2 }, "prompt_3": { - "accuracy": 0.47619047619047616 + "accuracy": 0.2571428571428571 }, "prompt_4": { - "accuracy": 0.49523809523809526 + "accuracy": 0.2761904761904762 }, "prompt_5": { - "accuracy": 0.5142857142857142 + "accuracy": 0.21904761904761905 } }, "us_eval": { "prompt_1": { - "accuracy": 0.7570093457943925 + "accuracy": 0.2897196261682243 }, "prompt_2": { - "accuracy": 0.7476635514018691 + "accuracy": 0.2523364485981308 }, "prompt_3": { - "accuracy": 0.7570093457943925 + "accuracy": 0.2523364485981308 }, "prompt_4": { - "accuracy": 0.7476635514018691 + "accuracy": 0.19626168224299065 }, "prompt_5": { - "accuracy": 0.7383177570093458 + "accuracy": 0.19626168224299065 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.61, + "accuracy": 0.22, "category_acc": { - "brand": 0.5, + "brand": 0.2, "demographics": 0.2, - "biology": 0.7, - "history": 0.4666666666666667, - "literature": 0.6, - "politics": 0.9, - "culture": 0.8, - "film": 0.5, - "law": 0.6, - "geography": 0.7 + "biology": 0.3, + "history": 0.06666666666666667, + "literature": 0.3, + "politics": 0.1, + "culture": 0.1, + "film": 0.3, + "law": 0.3, + "geography": 0.4 } }, "prompt_2": { - "accuracy": 0.58, + "accuracy": 0.25, "category_acc": { - "brand": 0.4, - "demographics": 0.0, - "biology": 0.8, - "history": 0.4666666666666667, - "literature": 0.5, - "politics": 0.9, - "culture": 0.8, + "brand": 0.5, + "demographics": 0.4, + "biology": 0.3, + "history": 0.2, + "literature": 0.1, + "politics": 0.2, + "culture": 0.1, "film": 0.4, - "law": 0.6, - "geography": 0.7 + "law": 0.1, + "geography": 0.3 } }, "prompt_3": { - "accuracy": 0.55, + "accuracy": 0.19, "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.7, - "history": 0.4, - "literature": 0.5, + "brand": 0.0, + "demographics": 0.4, + "biology": 0.2, + "history": 0.0, + "literature": 0.0, "politics": 0.8, - "culture": 0.7, - "film": 0.6, - "law": 0.6, - "geography": 0.6 + "culture": 0.1, + "film": 0.2, + "law": 0.2, + "geography": 0.2 } }, "prompt_4": { - "accuracy": 0.6, + "accuracy": 0.26, "category_acc": { - "brand": 0.4, + "brand": 0.1, "demographics": 0.4, - "biology": 0.7, - "history": 0.5333333333333333, - "literature": 0.4, - "politics": 0.8, - "culture": 0.8, - "film": 0.6, - "law": 0.7, - "geography": 0.6 + "biology": 0.4, + "history": 0.3333333333333333, + "literature": 0.1, + "politics": 0.1, + "culture": 0.5, + "film": 0.3, + "law": 0.2, + "geography": 0.2 } }, "prompt_5": { - "accuracy": 0.58, + "accuracy": 0.25, "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.8, - "history": 0.5333333333333333, - "literature": 0.5, - "politics": 0.8, - "culture": 0.6, - "film": 0.4, - "law": 0.6, - "geography": 0.8 + "brand": 0.1, + "demographics": 0.4, + "biology": 0.3, + "history": 0.2, + "literature": 0.3, + "politics": 0.1, + "culture": 0.4, + "film": 0.1, + "law": 0.4, + "geography": 0.3 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.2794452574404052 + "bleu_score": 0.00845529439051568 }, "prompt_2": { - "bleu_score": 0.2674775026836445 + "bleu_score": 0.00845589251198986 }, "prompt_3": { - "bleu_score": 0.2722339627774794 + "bleu_score": 0.00888315083476906 }, "prompt_4": { - "bleu_score": 0.26992687523049974 + "bleu_score": 0.0086399802884862 }, "prompt_5": { - "bleu_score": 0.2339828457497402 + "bleu_score": 0.008537367494631853 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "prompt_1": { + "accuracy": 0.23880098804993657, + "category_acc": { + "History": 0.1746987951807229, + "Geography": 0.22040816326530613, + "Lampungic": 0.2789115646258503, + "Social science": 0.24540901502504173, + "Balinese": 0.23142250530785563, + "Makassarese": 0.25268817204301075, + "Banjarese": 0.2569444444444444, + "Chemistry": 0.22481751824817517, + "Biology": 0.22603550295857988, + "Science": 0.2528379772961816, + "Christian religion": 0.24378109452736318, + "Art": 0.2579034941763727, + "Islam religion": 0.2119487908961593, + "Hindu religion": 0.24, + "Madurese": 0.2983050847457627, + "Sport": 0.2972972972972973, + "Indonesian language": 0.25062266500622665, + "Physics": 0.21818181818181817, + "Minangkabau culture": 0.23618090452261306, + "Dayak language": 0.26605504587155965, + "Sociology": 0.18548387096774194, + "Economy": 0.2192622950819672, + "Sundanese": 0.27052722558340536, + "Javanese": 0.24294354838709678, + "Civic education": 0.2117310443490701 + } + }, + "prompt_2": { + "accuracy": 0.2410708324988317, + "category_acc": { + "History": 0.1827309236947791, + "Geography": 0.18571428571428572, + "Lampungic": 0.2789115646258503, + "Social science": 0.27378964941569284, + "Balinese": 0.27176220806794055, + "Makassarese": 0.26344086021505375, + "Banjarese": 0.2777777777777778, + "Chemistry": 0.2116788321167883, + "Biology": 0.2, + "Science": 0.25077399380804954, + "Christian religion": 0.24875621890547264, + "Art": 0.24459234608985025, + "Islam religion": 0.2546230440967283, + "Hindu religion": 0.28, + "Madurese": 0.25084745762711863, + "Sport": 0.25675675675675674, + "Indonesian language": 0.24283935242839352, + "Physics": 0.1797979797979798, + "Minangkabau culture": 0.22613065326633167, + "Dayak language": 0.1834862385321101, + "Sociology": 0.20766129032258066, + "Economy": 0.2069672131147541, + "Sundanese": 0.28262748487467587, + "Javanese": 0.2852822580645161, + "Civic education": 0.24606580829756797 + } + }, + "prompt_3": { + "accuracy": 0.2364643834701916, + "category_acc": { + "History": 0.19678714859437751, + "Geography": 0.20612244897959184, + "Lampungic": 0.24489795918367346, + "Social science": 0.2838063439065108, + "Balinese": 0.2760084925690021, + "Makassarese": 0.23118279569892472, + "Banjarese": 0.2986111111111111, + "Chemistry": 0.1927007299270073, + "Biology": 0.22248520710059172, + "Science": 0.26831785345717235, + "Christian religion": 0.20398009950248755, + "Art": 0.2512479201331115, + "Islam religion": 0.20910384068278806, + "Hindu religion": 0.25333333333333335, + "Madurese": 0.2135593220338983, + "Sport": 0.22297297297297297, + "Indonesian language": 0.24470734744707348, + "Physics": 0.2202020202020202, + "Minangkabau culture": 0.23618090452261306, + "Dayak language": 0.1743119266055046, + "Sociology": 0.21975806451612903, + "Economy": 0.2192622950819672, + "Sundanese": 0.23681936041486604, + "Javanese": 0.26411290322580644, + "Civic education": 0.2217453505007153 + } + }, + "prompt_4": { + "accuracy": 0.23559650176914346, + "category_acc": { + "History": 0.19678714859437751, + "Geography": 0.22653061224489796, + "Lampungic": 0.24489795918367346, + "Social science": 0.2671118530884808, + "Balinese": 0.25477707006369427, + "Makassarese": 0.22580645161290322, + "Banjarese": 0.2013888888888889, + "Chemistry": 0.1897810218978102, + "Biology": 0.1881656804733728, + "Science": 0.25386996904024767, + "Christian religion": 0.23880597014925373, + "Art": 0.24126455906821964, + "Islam religion": 0.2802275960170697, + "Hindu religion": 0.2866666666666667, + "Madurese": 0.19661016949152543, + "Sport": 0.18243243243243243, + "Indonesian language": 0.24533001245330013, + "Physics": 0.18585858585858586, + "Minangkabau culture": 0.24623115577889448, + "Dayak language": 0.25688073394495414, + "Sociology": 0.21975806451612903, + "Economy": 0.19467213114754098, + "Sundanese": 0.26534140017286084, + "Javanese": 0.24798387096774194, + "Civic education": 0.2374821173104435 + } + }, + "prompt_5": { + "accuracy": 0.23145737365645236, + "category_acc": { + "History": 0.21686746987951808, + "Geography": 0.21836734693877552, + "Lampungic": 0.2653061224489796, + "Social science": 0.2888146911519199, + "Balinese": 0.24203821656050956, + "Makassarese": 0.24731182795698925, + "Banjarese": 0.2708333333333333, + "Chemistry": 0.21313868613138687, + "Biology": 0.2, + "Science": 0.2218782249742002, + "Christian religion": 0.21393034825870647, + "Art": 0.2113144758735441, + "Islam religion": 0.22617354196301565, + "Hindu religion": 0.2733333333333333, + "Madurese": 0.2542372881355932, + "Sport": 0.21621621621621623, + "Indonesian language": 0.24221668742216687, + "Physics": 0.20404040404040405, + "Minangkabau culture": 0.22110552763819097, + "Dayak language": 0.23853211009174313, + "Sociology": 0.22379032258064516, + "Economy": 0.20081967213114754, + "Sundanese": 0.2644770959377701, + "Javanese": 0.22681451612903225, + "Civic education": 0.20743919885550788 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.38084676314905563 + "bleu_score": 0.0094989992928008 }, "prompt_2": { - "bleu_score": 0.3911982896928193 + "bleu_score": 0.00948411159412573 }, "prompt_3": { - "bleu_score": 0.39332694536667584 + "bleu_score": 0.009230011413355381 }, "prompt_4": { - "bleu_score": 0.3948636422462026 + "bleu_score": 0.009319569974615713 }, "prompt_5": { - "bleu_score": 0.36809815124796963 + "bleu_score": 0.009396177509831877 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.3111744174894432 + "bleu_score": 0.009454892363036137 }, "prompt_2": { - "bleu_score": 0.3175458861102973 + "bleu_score": 0.00936148871734506 }, "prompt_3": { - "bleu_score": 0.31221700320284107 + "bleu_score": 0.009360333481711986 }, "prompt_4": { - "bleu_score": 0.31870733530187106 + "bleu_score": 0.009209909575947904 }, "prompt_5": { - "bleu_score": 0.2972648972980948 + "bleu_score": 0.009456497491476989 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.24954533306484694 + "bleu_score": 0.009551814087188693 }, "prompt_2": { - "bleu_score": 0.24872110088444996 + "bleu_score": 0.009370587804251214 }, "prompt_3": { - "bleu_score": 0.24951988951653503 + "bleu_score": 0.009307087651375135 }, "prompt_4": { - "bleu_score": 0.2497913431912669 + "bleu_score": 0.009352766278754754 }, "prompt_5": { - "bleu_score": 0.23957916096630588 + "bleu_score": 0.00934145442627295 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.3901065940971776 + "bleu_score": 0.009286655895261653 }, "prompt_2": { - "bleu_score": 0.4022235711140328 + "bleu_score": 0.009381494437991243 }, "prompt_3": { - "bleu_score": 0.4029939240859687 + "bleu_score": 0.009176759653251399 }, "prompt_4": { - "bleu_score": 0.4027192292749885 + "bleu_score": 0.009414395334233324 }, "prompt_5": { - "bleu_score": 0.3765211117066765 + "bleu_score": 0.009602584340295471 } }, "mmlu": { "prompt_1": { - "accuracy": 0.6721120186697783 + "accuracy": 0.2718786464410735 }, "prompt_2": { - "accuracy": 0.6779463243873979 + "accuracy": 0.24504084014002334 }, "prompt_3": { - "accuracy": 0.661610268378063 + "accuracy": 0.22753792298716452 }, "prompt_4": { - "accuracy": 0.6779463243873979 + "accuracy": 0.24154025670945156 }, "prompt_5": { - "accuracy": 0.6837806301050176 - } - }, - "mmlu_full": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "c_eval": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "c_eval_full": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "cmmlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "cmmlu_full": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "zbench": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "ind_emotion": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "ocnli": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "c3": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "dream": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "samsum": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "dialogsum": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "sst2": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "cola": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "qqp": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "mnli": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "qnli": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "wnli": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "rte": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "mrpc": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.8081932773109244, - "language_acc": { - "Spanish": 0.8487394957983193, - "English": 0.9084033613445378, - "Chinese": 0.7663865546218488, - "Vietnamese": 0.7092436974789916 - }, - "consistency_score_2": 0.7292717086834735, - "consistency_score_3": 0.6239495798319328, - "consistency_score_4": 0.5521008403361345, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8546218487394958, - "Spanish,Chinese": 0.7285714285714285, - "Spanish,Vietnamese": 0.6873949579831933, - "English,Chinese": 0.753781512605042, - "English,Vietnamese": 0.7109243697478992, - "Chinese,Vietnamese": 0.6403361344537815 - }, - "3_combine": { - "Spanish,English,Chinese": 0.6882352941176471, - "Spanish,English,Vietnamese": 0.646218487394958, - "Spanish,Chinese,Vietnamese": 0.5689075630252101, - "English,Chinese,Vietnamese": 0.592436974789916 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.5521008403361345 - } - }, - "AC3_2": 0.7667068812906891, - "AC3_3": 0.7042200479524251, - "AC3_4": 0.6560407513874816 + "accuracy": 0.2322053675612602 } }, - "cross_mmlu": { - "prompt_1": -1 - }, - "cross_logiqa": { - "prompt_1": -1 - }, - "sg_eval": { - "prompt_1": -1 - }, - "cn_eval": { - "prompt_1": -1 - }, - "us_eval": { - "prompt_1": -1 - }, - "ph_eval": { - "prompt_1": -1 - }, - "sing2eng": { - "prompt_1": -1 - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": -1 - }, - "flores_vie2eng": { - "prompt_1": -1 - }, - "flores_zho2eng": { - "prompt_1": -1 - }, - "flores_zsm2eng": { - "prompt_1": -1 - }, - "mmlu": { - "prompt_1": -1 - }, "mmlu_full": { - "prompt_1": -1 - }, - "c_eval": { - "prompt_1": -1 - }, - "c_eval_full": { - "prompt_1": -1 - }, - "cmmlu": { - "prompt_1": -1 - }, - "cmmlu_full": { - "prompt_1": -1 - }, - "zbench": { - "prompt_1": -1 - }, - "ind_emotion": { - "prompt_1": -1 - }, - "ocnli": { - "prompt_1": -1 - }, - "c3": { - "prompt_1": -1 - }, - "dream": { - "prompt_1": -1 - }, - "samsum": { - "prompt_1": -1 - }, - "dialogsum": { - "prompt_1": -1 - }, - "sst2": { - "prompt_1": -1 - }, - "cola": { - "prompt_1": -1 - }, - "qqp": { - "prompt_1": -1 - }, - "mnli": { - "prompt_1": -1 - }, - "qnli": { - "prompt_1": -1 - }, - "wnli": { - "prompt_1": -1 - }, - "rte": { - "prompt_1": -1 - }, - "mrpc": { - "prompt_1": -1 - } - } - }, - "mistral_7b_v0_1": { - "model_size": "7B", - "model_link": "https://huggingface.co/mistralai/Mistral-7B-v0.1", - "zero_shot": { - "cross_xquad": { "prompt_1": { - "overall_acc": 0.8514705882352942, - "language_acc": { - "English": 0.8941176470588236, - "Chinese": 0.83109243697479, - "Spanish": 0.8705882352941177, - "Vietnamese": 0.8100840336134454 - }, - "consistency_score_2": 0.8267507002801119, - "consistency_score_3": 0.7512605042016807, - "consistency_score_4": 0.7008403361344537, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8302521008403362, - "English,Spanish": 0.880672268907563, - "English,Vietnamese": 0.8100840336134454, - "Chinese,Spanish": 0.8260504201680672, - "Chinese,Vietnamese": 0.7932773109243697, - "Spanish,Vietnamese": 0.8201680672268907 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7789915966386555, - "English,Chinese,Vietnamese": 0.7294117647058823, - "English,Spanish,Vietnamese": 0.7638655462184873, - "Chinese,Spanish,Vietnamese": 0.7327731092436974 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7008403361344537 - } - }, - "AC3_2": 0.8389285845280107, - "AC3_3": 0.7982327495974394, - "AC3_4": 0.7688471734118163 + "accuracy": 0.2474794422595638, + "category_acc": { + "high_school_european_history": 0.2926829268292683, + "business_ethics": 0.25252525252525254, + "clinical_knowledge": 0.23484848484848486, + "medical_genetics": 0.24242424242424243, + "high_school_us_history": 0.21674876847290642, + "high_school_physics": 0.26666666666666666, + "high_school_world_history": 0.24152542372881355, + "virology": 0.18787878787878787, + "high_school_microeconomics": 0.22784810126582278, + "econometrics": 0.20353982300884957, + "college_computer_science": 0.23232323232323232, + "high_school_biology": 0.2168284789644013, + "abstract_algebra": 0.26262626262626265, + "professional_accounting": 0.2313167259786477, + "philosophy": 0.2709677419354839, + "professional_medicine": 0.1992619926199262, + "nutrition": 0.24918032786885247, + "global_facts": 0.23232323232323232, + "machine_learning": 0.2702702702702703, + "security_studies": 0.3319672131147541, + "public_relations": 0.1834862385321101, + "professional_psychology": 0.24549918166939444, + "prehistory": 0.25386996904024767, + "anatomy": 0.26119402985074625, + "human_sexuality": 0.3, + "college_medicine": 0.27906976744186046, + "high_school_government_and_politics": 0.2552083333333333, + "college_chemistry": 0.26262626262626265, + "logical_fallacies": 0.2345679012345679, + "high_school_geography": 0.2182741116751269, + "elementary_mathematics": 0.2413793103448276, + "human_aging": 0.25225225225225223, + "college_mathematics": 0.25252525252525254, + "high_school_psychology": 0.25183823529411764, + "formal_logic": 0.328, + "high_school_statistics": 0.2558139534883721, + "international_law": 0.25833333333333336, + "high_school_mathematics": 0.20446096654275092, + "high_school_computer_science": 0.20202020202020202, + "conceptual_physics": 0.2264957264957265, + "miscellaneous": 0.24808184143222506, + "high_school_chemistry": 0.2524752475247525, + "marketing": 0.2446351931330472, + "professional_law": 0.24787997390737115, + "management": 0.3137254901960784, + "college_physics": 0.19801980198019803, + "jurisprudence": 0.2616822429906542, + "world_religions": 0.27058823529411763, + "sociology": 0.255, + "us_foreign_policy": 0.2727272727272727, + "high_school_macroeconomics": 0.2442159383033419, + "computer_security": 0.24242424242424243, + "moral_scenarios": 0.2539149888143177, + "moral_disputes": 0.2608695652173913, + "electrical_engineering": 0.2638888888888889, + "astronomy": 0.2119205298013245, + "college_biology": 0.26573426573426573 + } }, "prompt_2": { - "overall_acc": 0.8581932773109244, - "language_acc": { - "English": 0.892436974789916, - "Chinese": 0.8445378151260504, - "Spanish": 0.8798319327731092, - "Vietnamese": 0.8159663865546218 - }, - "consistency_score_2": 0.8403361344537815, - "consistency_score_3": 0.7697478991596639, - "consistency_score_4": 0.7226890756302521, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8495798319327731, - "English,Spanish": 0.9008403361344538, - "English,Vietnamese": 0.8159663865546218, - "Chinese,Spanish": 0.8529411764705882, - "Chinese,Vietnamese": 0.8, - "Spanish,Vietnamese": 0.8226890756302521 - }, - "3_combine": { - "English,Chinese,Spanish": 0.807563025210084, - "English,Chinese,Vietnamese": 0.7453781512605042, - "English,Spanish,Vietnamese": 0.7764705882352941, - "Chinese,Spanish,Vietnamese": 0.7495798319327731 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7226890756302521 - } - }, - "AC3_2": 0.8491708371160408, - "AC3_3": 0.8115679875788036, - "AC3_4": 0.7846338534917838 + "accuracy": 0.2531283518055059, + "category_acc": { + "high_school_european_history": 0.2865853658536585, + "business_ethics": 0.30303030303030304, + "clinical_knowledge": 0.26515151515151514, + "medical_genetics": 0.29292929292929293, + "high_school_us_history": 0.21674876847290642, + "high_school_physics": 0.21333333333333335, + "high_school_world_history": 0.2966101694915254, + "virology": 0.23030303030303031, + "high_school_microeconomics": 0.3206751054852321, + "econometrics": 0.2743362831858407, + "college_computer_science": 0.32323232323232326, + "high_school_biology": 0.23948220064724918, + "abstract_algebra": 0.30303030303030304, + "professional_accounting": 0.2846975088967972, + "philosophy": 0.23548387096774193, + "professional_medicine": 0.2730627306273063, + "nutrition": 0.23934426229508196, + "global_facts": 0.23232323232323232, + "machine_learning": 0.24324324324324326, + "security_studies": 0.26229508196721313, + "public_relations": 0.3211009174311927, + "professional_psychology": 0.23731587561374795, + "prehistory": 0.26006191950464397, + "anatomy": 0.20149253731343283, + "human_sexuality": 0.27692307692307694, + "college_medicine": 0.23837209302325582, + "high_school_government_and_politics": 0.22916666666666666, + "college_chemistry": 0.2222222222222222, + "logical_fallacies": 0.21604938271604937, + "high_school_geography": 0.233502538071066, + "elementary_mathematics": 0.22281167108753316, + "human_aging": 0.25675675675675674, + "college_mathematics": 0.2727272727272727, + "high_school_psychology": 0.24080882352941177, + "formal_logic": 0.24, + "high_school_statistics": 0.24186046511627907, + "international_law": 0.25833333333333336, + "high_school_mathematics": 0.24535315985130113, + "high_school_computer_science": 0.32323232323232326, + "conceptual_physics": 0.23504273504273504, + "miscellaneous": 0.27877237851662406, + "high_school_chemistry": 0.25742574257425743, + "marketing": 0.21030042918454936, + "professional_law": 0.2563600782778865, + "management": 0.22549019607843138, + "college_physics": 0.26732673267326734, + "jurisprudence": 0.308411214953271, + "world_religions": 0.2529411764705882, + "sociology": 0.255, + "us_foreign_policy": 0.1919191919191919, + "high_school_macroeconomics": 0.2442159383033419, + "computer_security": 0.2222222222222222, + "moral_scenarios": 0.2360178970917226, + "moral_disputes": 0.26666666666666666, + "electrical_engineering": 0.2222222222222222, + "astronomy": 0.2781456953642384, + "college_biology": 0.2867132867132867 + } }, "prompt_3": { - "overall_acc": 0.8491596638655463, - "language_acc": { - "English": 0.8831932773109243, - "Chinese": 0.8378151260504202, - "Spanish": 0.8764705882352941, - "Vietnamese": 0.7991596638655463 - }, - "consistency_score_2": 0.8261904761904763, - "consistency_score_3": 0.75, - "consistency_score_4": 0.6991596638655462, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8352941176470589, - "English,Spanish": 0.8907563025210085, - "English,Vietnamese": 0.7991596638655463, - "Chinese,Spanish": 0.8411764705882353, - "Chinese,Vietnamese": 0.7873949579831933, - "Spanish,Vietnamese": 0.8033613445378152 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7915966386554621, - "English,Chinese,Vietnamese": 0.7235294117647059, - "English,Spanish,Vietnamese": 0.7546218487394958, - "Chinese,Spanish,Vietnamese": 0.7302521008403361 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.6991596638655462 - } - }, - "AC3_2": 0.83751761525562, - "AC3_3": 0.7965055175539758, - "AC3_4": 0.7668937208038038 + "accuracy": 0.24669288523417948, + "category_acc": { + "high_school_european_history": 0.24390243902439024, + "business_ethics": 0.2222222222222222, + "clinical_knowledge": 0.26136363636363635, + "medical_genetics": 0.25252525252525254, + "high_school_us_history": 0.26108374384236455, + "high_school_physics": 0.22, + "high_school_world_history": 0.3135593220338983, + "virology": 0.2787878787878788, + "high_school_microeconomics": 0.2320675105485232, + "econometrics": 0.22123893805309736, + "college_computer_science": 0.21212121212121213, + "high_school_biology": 0.20388349514563106, + "abstract_algebra": 0.18181818181818182, + "professional_accounting": 0.20640569395017794, + "philosophy": 0.2129032258064516, + "professional_medicine": 0.2656826568265683, + "nutrition": 0.26229508196721313, + "global_facts": 0.26262626262626265, + "machine_learning": 0.24324324324324326, + "security_studies": 0.21721311475409835, + "public_relations": 0.29357798165137616, + "professional_psychology": 0.2225859247135843, + "prehistory": 0.3126934984520124, + "anatomy": 0.27611940298507465, + "human_sexuality": 0.2692307692307692, + "college_medicine": 0.23255813953488372, + "high_school_government_and_politics": 0.2604166666666667, + "college_chemistry": 0.25252525252525254, + "logical_fallacies": 0.20987654320987653, + "high_school_geography": 0.2131979695431472, + "elementary_mathematics": 0.23076923076923078, + "human_aging": 0.2927927927927928, + "college_mathematics": 0.26262626262626265, + "high_school_psychology": 0.25, + "formal_logic": 0.2, + "high_school_statistics": 0.2186046511627907, + "international_law": 0.2833333333333333, + "high_school_mathematics": 0.26394052044609667, + "high_school_computer_science": 0.2828282828282828, + "conceptual_physics": 0.23076923076923078, + "miscellaneous": 0.26854219948849106, + "high_school_chemistry": 0.26732673267326734, + "marketing": 0.26180257510729615, + "professional_law": 0.24266144814090018, + "management": 0.19607843137254902, + "college_physics": 0.25742574257425743, + "jurisprudence": 0.27102803738317754, + "world_religions": 0.22941176470588234, + "sociology": 0.25, + "us_foreign_policy": 0.23232323232323232, + "high_school_macroeconomics": 0.2442159383033419, + "computer_security": 0.29292929292929293, + "moral_scenarios": 0.24049217002237136, + "moral_disputes": 0.2463768115942029, + "electrical_engineering": 0.19444444444444445, + "astronomy": 0.24503311258278146, + "college_biology": 0.32167832167832167 + } }, "prompt_4": { - "overall_acc": 0.8531512605042018, - "language_acc": { - "English": 0.9025210084033614, - "Chinese": 0.8302521008403362, - "Spanish": 0.8689075630252101, - "Vietnamese": 0.8109243697478992 - }, - "consistency_score_2": 0.8316526610644258, - "consistency_score_3": 0.7579831932773109, - "consistency_score_4": 0.7092436974789916, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8378151260504202, - "English,Spanish": 0.8865546218487395, - "English,Vietnamese": 0.8176470588235294, - "Chinese,Spanish": 0.8302521008403362, - "Chinese,Vietnamese": 0.7991596638655463, - "Spanish,Vietnamese": 0.8184873949579832 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7840336134453781, - "English,Chinese,Vietnamese": 0.7403361344537815, - "English,Spanish,Vietnamese": 0.7697478991596639, - "Chinese,Spanish,Vietnamese": 0.7378151260504202 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7092436974789916 - } - }, - "AC3_2": 0.8422647964708859, - "AC3_3": 0.8027564865583788, - "AC3_4": 0.7745700296565382 + "accuracy": 0.2474794422595638, + "category_acc": { + "high_school_european_history": 0.23170731707317074, + "business_ethics": 0.2222222222222222, + "clinical_knowledge": 0.2765151515151515, + "medical_genetics": 0.1919191919191919, + "high_school_us_history": 0.18226600985221675, + "high_school_physics": 0.22, + "high_school_world_history": 0.24152542372881355, + "virology": 0.3090909090909091, + "high_school_microeconomics": 0.22362869198312235, + "econometrics": 0.24778761061946902, + "college_computer_science": 0.21212121212121213, + "high_school_biology": 0.24919093851132687, + "abstract_algebra": 0.32323232323232326, + "professional_accounting": 0.2669039145907473, + "philosophy": 0.2645161290322581, + "professional_medicine": 0.2693726937269373, + "nutrition": 0.24262295081967214, + "global_facts": 0.1919191919191919, + "machine_learning": 0.18018018018018017, + "security_studies": 0.25, + "public_relations": 0.1834862385321101, + "professional_psychology": 0.25859247135842883, + "prehistory": 0.23219814241486067, + "anatomy": 0.2537313432835821, + "human_sexuality": 0.25384615384615383, + "college_medicine": 0.20930232558139536, + "high_school_government_and_politics": 0.234375, + "college_chemistry": 0.32323232323232326, + "logical_fallacies": 0.2777777777777778, + "high_school_geography": 0.2639593908629442, + "elementary_mathematics": 0.2440318302387268, + "human_aging": 0.25675675675675674, + "college_mathematics": 0.2727272727272727, + "high_school_psychology": 0.2775735294117647, + "formal_logic": 0.264, + "high_school_statistics": 0.19069767441860466, + "international_law": 0.21666666666666667, + "high_school_mathematics": 0.26765799256505574, + "high_school_computer_science": 0.30303030303030304, + "conceptual_physics": 0.21794871794871795, + "miscellaneous": 0.24808184143222506, + "high_school_chemistry": 0.2722772277227723, + "marketing": 0.2532188841201717, + "professional_law": 0.23352902804957598, + "management": 0.3137254901960784, + "college_physics": 0.37623762376237624, + "jurisprudence": 0.17757009345794392, + "world_religions": 0.27058823529411763, + "sociology": 0.26, + "us_foreign_policy": 0.2222222222222222, + "high_school_macroeconomics": 0.2519280205655527, + "computer_security": 0.2828282828282828, + "moral_scenarios": 0.24161073825503357, + "moral_disputes": 0.25507246376811593, + "electrical_engineering": 0.20833333333333334, + "astronomy": 0.2119205298013245, + "college_biology": 0.2727272727272727 + } }, "prompt_5": { - "overall_acc": 0.8430672268907563, - "language_acc": { - "English": 0.8966386554621849, - "Chinese": 0.8201680672268907, - "Spanish": 0.8613445378151261, - "Vietnamese": 0.7941176470588235 - }, - "consistency_score_2": 0.8144257703081231, - "consistency_score_3": 0.7338235294117648, - "consistency_score_4": 0.6798319327731093, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8142857142857143, - "English,Spanish": 0.8798319327731092, - "English,Vietnamese": 0.8008403361344538, - "Chinese,Spanish": 0.8092436974789916, - "Chinese,Vietnamese": 0.7815126050420168, - "Spanish,Vietnamese": 0.8008403361344538 - }, - "3_combine": { - "English,Chinese,Spanish": 0.761344537815126, - "English,Chinese,Vietnamese": 0.7126050420168067, - "English,Spanish,Vietnamese": 0.7504201680672269, - "Chinese,Spanish,Vietnamese": 0.7109243697478992 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.6798319327731093 - } - }, - "AC3_2": 0.828499036558194, - "AC3_3": 0.7846612905256227, - "AC3_4": 0.752701212867801 + "accuracy": 0.24461923489452986, + "category_acc": { + "high_school_european_history": 0.23170731707317074, + "business_ethics": 0.23232323232323232, + "clinical_knowledge": 0.21212121212121213, + "medical_genetics": 0.3333333333333333, + "high_school_us_history": 0.22660098522167488, + "high_school_physics": 0.19333333333333333, + "high_school_world_history": 0.2457627118644068, + "virology": 0.21818181818181817, + "high_school_microeconomics": 0.24472573839662448, + "econometrics": 0.26548672566371684, + "college_computer_science": 0.25252525252525254, + "high_school_biology": 0.2588996763754045, + "abstract_algebra": 0.1919191919191919, + "professional_accounting": 0.20284697508896798, + "philosophy": 0.2709677419354839, + "professional_medicine": 0.23985239852398524, + "nutrition": 0.25245901639344265, + "global_facts": 0.2828282828282828, + "machine_learning": 0.26126126126126126, + "security_studies": 0.2459016393442623, + "public_relations": 0.25688073394495414, + "professional_psychology": 0.2569558101472995, + "prehistory": 0.22291021671826625, + "anatomy": 0.21641791044776118, + "human_sexuality": 0.26153846153846155, + "college_medicine": 0.22674418604651161, + "high_school_government_and_politics": 0.2552083333333333, + "college_chemistry": 0.21212121212121213, + "logical_fallacies": 0.2654320987654321, + "high_school_geography": 0.28426395939086296, + "elementary_mathematics": 0.26525198938992045, + "human_aging": 0.23873873873873874, + "college_mathematics": 0.24242424242424243, + "high_school_psychology": 0.25183823529411764, + "formal_logic": 0.264, + "high_school_statistics": 0.24186046511627907, + "international_law": 0.20833333333333334, + "high_school_mathematics": 0.21189591078066913, + "high_school_computer_science": 0.2222222222222222, + "conceptual_physics": 0.2564102564102564, + "miscellaneous": 0.22506393861892582, + "high_school_chemistry": 0.20297029702970298, + "marketing": 0.21888412017167383, + "professional_law": 0.2517938682322244, + "management": 0.23529411764705882, + "college_physics": 0.31683168316831684, + "jurisprudence": 0.29906542056074764, + "world_religions": 0.2823529411764706, + "sociology": 0.24, + "us_foreign_policy": 0.2222222222222222, + "high_school_macroeconomics": 0.2442159383033419, + "computer_security": 0.23232323232323232, + "moral_scenarios": 0.24720357941834453, + "moral_disputes": 0.2579710144927536, + "electrical_engineering": 0.2152777777777778, + "astronomy": 0.31788079470198677, + "college_biology": 0.22377622377622378 + } } }, - "cross_mmlu": { + "c_eval": { "prompt_1": { - "overall_acc": 0.5095238095238096, - "language_acc": { - "Spanish": 0.6, - "English": 0.6333333333333333, - "Chinese": 0.5, - "Vietnamese": 0.4266666666666667, - "Filipino": 0.4866666666666667, - "Indonesian": 0.49333333333333335, - "Malay": 0.4266666666666667 - }, - "consistency_score_2": 0.6073015873015872, - "consistency_score_3": 0.4497142857142858, - "consistency_score_4": 0.35714285714285715, - "consistency_score_5": 0.29365079365079355, - "consistency_score_6": 0.24571428571428575, - "consistency_score_7": 0.20666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.72, - "Spanish,Chinese": 0.6466666666666666, - "Spanish,Vietnamese": 0.5866666666666667, - "Spanish,Filipino": 0.5866666666666667, - "Spanish,Indonesian": 0.6666666666666666, - "Spanish,Malay": 0.5933333333333334, - "English,Chinese": 0.6, - "English,Vietnamese": 0.5666666666666667, - "English,Filipino": 0.6266666666666667, - "English,Indonesian": 0.6266666666666667, - "English,Malay": 0.5666666666666667, - "Chinese,Vietnamese": 0.6266666666666667, - "Chinese,Filipino": 0.5533333333333333, - "Chinese,Indonesian": 0.5866666666666667, - "Chinese,Malay": 0.56, - "Vietnamese,Filipino": 0.54, - "Vietnamese,Indonesian": 0.6066666666666667, - "Vietnamese,Malay": 0.5866666666666667, - "Filipino,Indonesian": 0.6, - "Filipino,Malay": 0.6066666666666667, - "Indonesian,Malay": 0.7 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5066666666666667, - "Spanish,English,Vietnamese": 0.47333333333333333, - "Spanish,English,Filipino": 0.4866666666666667, - "Spanish,English,Indonesian": 0.52, - "Spanish,English,Malay": 0.46, - "Spanish,Chinese,Vietnamese": 0.47333333333333333, - "Spanish,Chinese,Filipino": 0.44666666666666666, - "Spanish,Chinese,Indonesian": 0.4866666666666667, - "Spanish,Chinese,Malay": 0.43333333333333335, - "Spanish,Vietnamese,Filipino": 0.4, - "Spanish,Vietnamese,Indonesian": 0.4666666666666667, - "Spanish,Vietnamese,Malay": 0.43333333333333335, - "Spanish,Filipino,Indonesian": 0.4533333333333333, - "Spanish,Filipino,Malay": 0.44, - "Spanish,Indonesian,Malay": 0.5066666666666667, - "English,Chinese,Vietnamese": 0.46, - "English,Chinese,Filipino": 0.42, - "English,Chinese,Indonesian": 0.44, - "English,Chinese,Malay": 0.4, - "English,Vietnamese,Filipino": 0.41333333333333333, - "English,Vietnamese,Indonesian": 0.44, - "English,Vietnamese,Malay": 0.4066666666666667, - "English,Filipino,Indonesian": 0.46, - "English,Filipino,Malay": 0.44666666666666666, - "English,Indonesian,Malay": 0.47333333333333333, - "Chinese,Vietnamese,Filipino": 0.41333333333333333, - "Chinese,Vietnamese,Indonesian": 0.4666666666666667, - "Chinese,Vietnamese,Malay": 0.43333333333333335, - "Chinese,Filipino,Indonesian": 0.4266666666666667, - "Chinese,Filipino,Malay": 0.41333333333333333, - "Chinese,Indonesian,Malay": 0.44, - "Vietnamese,Filipino,Indonesian": 0.4266666666666667, - "Vietnamese,Filipino,Malay": 0.4, - "Vietnamese,Indonesian,Malay": 0.4866666666666667, - "Filipino,Indonesian,Malay": 0.4866666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4, - "Spanish,English,Chinese,Filipino": 0.35333333333333333, - "Spanish,English,Chinese,Indonesian": 0.3933333333333333, - "Spanish,English,Chinese,Malay": 0.3466666666666667, - "Spanish,English,Vietnamese,Filipino": 0.34, - "Spanish,English,Vietnamese,Indonesian": 0.38, - "Spanish,English,Vietnamese,Malay": 0.35333333333333333, - "Spanish,English,Filipino,Indonesian": 0.37333333333333335, - "Spanish,English,Filipino,Malay": 0.36, - "Spanish,English,Indonesian,Malay": 0.38666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.34, - "Spanish,Chinese,Vietnamese,Indonesian": 0.38666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.35333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.36, - "Spanish,Chinese,Filipino,Malay": 0.34, - "Spanish,Chinese,Indonesian,Malay": 0.37333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.32, - "Spanish,Vietnamese,Indonesian,Malay": 0.4, - "Spanish,Filipino,Indonesian,Malay": 0.38666666666666666, - "English,Chinese,Vietnamese,Filipino": 0.34, - "English,Chinese,Vietnamese,Indonesian": 0.36666666666666664, - "English,Chinese,Vietnamese,Malay": 0.34, - "English,Chinese,Filipino,Indonesian": 0.3333333333333333, - "English,Chinese,Filipino,Malay": 0.32, - "English,Chinese,Indonesian,Malay": 0.3333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "English,Vietnamese,Filipino,Malay": 0.31333333333333335, - "English,Vietnamese,Indonesian,Malay": 0.36666666666666664, - "English,Filipino,Indonesian,Malay": 0.37333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.36, - "Vietnamese,Filipino,Indonesian,Malay": 0.36 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.32666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.3, - "Spanish,English,Chinese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.29333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,English,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.32, - "Spanish,English,Filipino,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.32, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.30666666666666664, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.26, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.3, - "English,Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.22, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 - } - }, - "AC3_2": 0.5541324886479657, - "AC3_3": 0.4777544535982144, - "AC3_4": 0.41993720560303716, - "AC3_5": 0.3725766986172671, - "AC3_6": 0.3315438659261962, - "AC3_7": 0.2940602836468843 + "accuracy": 0.2585438335809807 }, "prompt_2": { - "overall_acc": 0.5, - "language_acc": { - "Spanish": 0.5733333333333334, - "English": 0.56, - "Chinese": 0.49333333333333335, - "Vietnamese": 0.4533333333333333, - "Filipino": 0.5133333333333333, - "Indonesian": 0.4666666666666667, - "Malay": 0.44 - }, - "consistency_score_2": 0.6022222222222222, - "consistency_score_3": 0.453904761904762, - "consistency_score_4": 0.3714285714285714, - "consistency_score_5": 0.31587301587301586, - "consistency_score_6": 0.2742857142857143, - "consistency_score_7": 0.24, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.7, - "Spanish,Chinese": 0.6466666666666666, - "Spanish,Vietnamese": 0.5666666666666667, - "Spanish,Filipino": 0.6066666666666667, - "Spanish,Indonesian": 0.64, - "Spanish,Malay": 0.6533333333333333, - "English,Chinese": 0.6133333333333333, - "English,Vietnamese": 0.5533333333333333, - "English,Filipino": 0.5666666666666667, - "English,Indonesian": 0.6133333333333333, - "English,Malay": 0.5533333333333333, - "Chinese,Vietnamese": 0.5133333333333333, - "Chinese,Filipino": 0.5666666666666667, - "Chinese,Indonesian": 0.56, - "Chinese,Malay": 0.58, - "Vietnamese,Filipino": 0.6133333333333333, - "Vietnamese,Indonesian": 0.6133333333333333, - "Vietnamese,Malay": 0.62, - "Filipino,Indonesian": 0.5866666666666667, - "Filipino,Malay": 0.5933333333333334, - "Indonesian,Malay": 0.6866666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5066666666666667, - "Spanish,English,Vietnamese": 0.4533333333333333, - "Spanish,English,Filipino": 0.47333333333333333, - "Spanish,English,Indonesian": 0.5066666666666667, - "Spanish,English,Malay": 0.5, - "Spanish,Chinese,Vietnamese": 0.4266666666666667, - "Spanish,Chinese,Filipino": 0.4666666666666667, - "Spanish,Chinese,Indonesian": 0.47333333333333333, - "Spanish,Chinese,Malay": 0.4866666666666667, - "Spanish,Vietnamese,Filipino": 0.44, - "Spanish,Vietnamese,Indonesian": 0.47333333333333333, - "Spanish,Vietnamese,Malay": 0.47333333333333333, - "Spanish,Filipino,Indonesian": 0.4666666666666667, - "Spanish,Filipino,Malay": 0.48, - "Spanish,Indonesian,Malay": 0.5266666666666666, - "English,Chinese,Vietnamese": 0.38666666666666666, - "English,Chinese,Filipino": 0.42, - "English,Chinese,Indonesian": 0.44, - "English,Chinese,Malay": 0.42, - "English,Vietnamese,Filipino": 0.4, - "English,Vietnamese,Indonesian": 0.44, - "English,Vietnamese,Malay": 0.4266666666666667, - "English,Filipino,Indonesian": 0.4266666666666667, - "English,Filipino,Malay": 0.42, - "English,Indonesian,Malay": 0.4666666666666667, - "Chinese,Vietnamese,Filipino": 0.42, - "Chinese,Vietnamese,Indonesian": 0.41333333333333333, - "Chinese,Vietnamese,Malay": 0.41333333333333333, - "Chinese,Filipino,Indonesian": 0.4266666666666667, - "Chinese,Filipino,Malay": 0.43333333333333335, - "Chinese,Indonesian,Malay": 0.46, - "Vietnamese,Filipino,Indonesian": 0.46, - "Vietnamese,Filipino,Malay": 0.46, - "Vietnamese,Indonesian,Malay": 0.5066666666666667, - "Filipino,Indonesian,Malay": 0.49333333333333335 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.35333333333333333, - "Spanish,English,Chinese,Filipino": 0.38, - "Spanish,English,Chinese,Indonesian": 0.3933333333333333, - "Spanish,English,Chinese,Malay": 0.4066666666666667, - "Spanish,English,Vietnamese,Filipino": 0.35333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.38666666666666666, - "Spanish,English,Vietnamese,Malay": 0.3933333333333333, - "Spanish,English,Filipino,Indonesian": 0.38666666666666666, - "Spanish,English,Filipino,Malay": 0.4, - "Spanish,English,Indonesian,Malay": 0.43333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.36, - "Spanish,Chinese,Vietnamese,Indonesian": 0.37333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.36666666666666664, - "Spanish,Chinese,Filipino,Indonesian": 0.37333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.38666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.41333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.38, - "Spanish,Vietnamese,Filipino,Malay": 0.38666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.42, - "Spanish,Filipino,Indonesian,Malay": 0.42, - "English,Chinese,Vietnamese,Filipino": 0.31333333333333335, - "English,Chinese,Vietnamese,Indonesian": 0.32666666666666666, - "English,Chinese,Vietnamese,Malay": 0.32, - "English,Chinese,Filipino,Indonesian": 0.34, - "English,Chinese,Filipino,Malay": 0.34, - "English,Chinese,Indonesian,Malay": 0.35333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "English,Vietnamese,Filipino,Malay": 0.34, - "English,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "English,Filipino,Indonesian,Malay": 0.37333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.3466666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.38, - "Vietnamese,Filipino,Indonesian,Malay": 0.4 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.29333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.30666666666666664, - "Spanish,English,Chinese,Vietnamese,Malay": 0.31333333333333335, - "Spanish,English,Chinese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,English,Chinese,Filipino,Malay": 0.3333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.3466666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Spanish,English,Vietnamese,Filipino,Malay": 0.32666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.36, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.3466666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.28, - "English,Chinese,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.3, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.28, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.3, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24 - } - }, - "AC3_2": 0.5463709676923655, - "AC3_3": 0.47583865809708165, - "AC3_4": 0.4262295081478097, - "AC3_5": 0.38715953302647654, - "AC3_6": 0.35424354238967337, - "AC3_7": 0.3243243242804967 + "accuracy": 0.25037147102526003 }, "prompt_3": { - "overall_acc": 0.4961904761904762, - "language_acc": { - "Spanish": 0.5666666666666667, - "English": 0.62, - "Chinese": 0.4533333333333333, - "Vietnamese": 0.46, - "Filipino": 0.5133333333333333, - "Indonesian": 0.49333333333333335, - "Malay": 0.36666666666666664 - }, - "consistency_score_2": 0.6076190476190477, - "consistency_score_3": 0.45466666666666666, - "consistency_score_4": 0.36971428571428583, - "consistency_score_5": 0.3126984126984127, - "consistency_score_6": 0.26952380952380955, - "consistency_score_7": 0.23333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.76, - "Spanish,Chinese": 0.6533333333333333, - "Spanish,Vietnamese": 0.6133333333333333, - "Spanish,Filipino": 0.62, - "Spanish,Indonesian": 0.62, - "Spanish,Malay": 0.6, - "English,Chinese": 0.6333333333333333, - "English,Vietnamese": 0.6333333333333333, - "English,Filipino": 0.6533333333333333, - "English,Indonesian": 0.6466666666666666, - "English,Malay": 0.5733333333333334, - "Chinese,Vietnamese": 0.5333333333333333, - "Chinese,Filipino": 0.5266666666666666, - "Chinese,Indonesian": 0.56, - "Chinese,Malay": 0.5266666666666666, - "Vietnamese,Filipino": 0.64, - "Vietnamese,Indonesian": 0.6066666666666667, - "Vietnamese,Malay": 0.56, - "Filipino,Indonesian": 0.5733333333333334, - "Filipino,Malay": 0.52, - "Indonesian,Malay": 0.7066666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.54, - "Spanish,English,Vietnamese": 0.54, - "Spanish,English,Filipino": 0.5466666666666666, - "Spanish,English,Indonesian": 0.5533333333333333, - "Spanish,English,Malay": 0.5066666666666667, - "Spanish,Chinese,Vietnamese": 0.44666666666666666, - "Spanish,Chinese,Filipino": 0.43333333333333335, - "Spanish,Chinese,Indonesian": 0.46, - "Spanish,Chinese,Malay": 0.42, - "Spanish,Vietnamese,Filipino": 0.48, - "Spanish,Vietnamese,Indonesian": 0.47333333333333333, - "Spanish,Vietnamese,Malay": 0.44, - "Spanish,Filipino,Indonesian": 0.44666666666666666, - "Spanish,Filipino,Malay": 0.4266666666666667, - "Spanish,Indonesian,Malay": 0.5066666666666667, - "English,Chinese,Vietnamese": 0.44, - "English,Chinese,Filipino": 0.44666666666666666, - "English,Chinese,Indonesian": 0.48, - "English,Chinese,Malay": 0.4066666666666667, - "English,Vietnamese,Filipino": 0.5, - "English,Vietnamese,Indonesian": 0.48, - "English,Vietnamese,Malay": 0.43333333333333335, - "English,Filipino,Indonesian": 0.4666666666666667, - "English,Filipino,Malay": 0.42, - "English,Indonesian,Malay": 0.5, - "Chinese,Vietnamese,Filipino": 0.4066666666666667, - "Chinese,Vietnamese,Indonesian": 0.41333333333333333, - "Chinese,Vietnamese,Malay": 0.37333333333333335, - "Chinese,Filipino,Indonesian": 0.38666666666666666, - "Chinese,Filipino,Malay": 0.34, - "Chinese,Indonesian,Malay": 0.4266666666666667, - "Vietnamese,Filipino,Indonesian": 0.44, - "Vietnamese,Filipino,Malay": 0.4066666666666667, - "Vietnamese,Indonesian,Malay": 0.4866666666666667, - "Filipino,Indonesian,Malay": 0.44 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4, - "Spanish,English,Chinese,Filipino": 0.4, - "Spanish,English,Chinese,Indonesian": 0.4266666666666667, - "Spanish,English,Chinese,Malay": 0.37333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.44, - "Spanish,English,Vietnamese,Indonesian": 0.44666666666666666, - "Spanish,English,Vietnamese,Malay": 0.41333333333333333, - "Spanish,English,Filipino,Indonesian": 0.42, - "Spanish,English,Filipino,Malay": 0.3933333333333333, - "Spanish,English,Indonesian,Malay": 0.44666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.36, - "Spanish,Chinese,Vietnamese,Indonesian": 0.37333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.34, - "Spanish,Chinese,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.3, - "Spanish,Chinese,Indonesian,Malay": 0.36, - "Spanish,Vietnamese,Filipino,Indonesian": 0.37333333333333335, - "Spanish,Vietnamese,Filipino,Malay": 0.35333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.4, - "Spanish,Filipino,Indonesian,Malay": 0.37333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.36, - "English,Chinese,Vietnamese,Indonesian": 0.38, - "English,Chinese,Vietnamese,Malay": 0.3333333333333333, - "English,Chinese,Filipino,Indonesian": 0.36666666666666664, - "English,Chinese,Filipino,Malay": 0.30666666666666664, - "English,Chinese,Indonesian,Malay": 0.36, - "English,Vietnamese,Filipino,Indonesian": 0.38, - "English,Vietnamese,Filipino,Malay": 0.35333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.4, - "English,Filipino,Indonesian,Malay": 0.38, - "Chinese,Vietnamese,Filipino,Indonesian": 0.32, - "Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.2866666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.36 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.35333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.32, - "Spanish,English,Chinese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.2866666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.3333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.36, - "Spanish,English,Vietnamese,Filipino,Malay": 0.34, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.38, - "Spanish,English,Filipino,Indonesian,Malay": 0.35333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.26, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Filipino,Indonesian,Malay": 0.28, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.32, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.26, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.26, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - } - }, - "AC3_2": 0.5462804551869613, - "AC3_3": 0.47452190165949704, - "AC3_4": 0.42371566835657243, - "AC3_5": 0.38363235399309026, - "AC3_6": 0.34930822075513945, - "AC3_7": 0.317406440339433 + "accuracy": 0.2652303120356612 }, "prompt_4": { - "overall_acc": 0.5152380952380952, - "language_acc": { - "Spanish": 0.6, - "English": 0.6466666666666666, - "Chinese": 0.4533333333333333, - "Vietnamese": 0.46, - "Filipino": 0.5333333333333333, - "Indonesian": 0.5, - "Malay": 0.41333333333333333 - }, - "consistency_score_2": 0.6165079365079364, - "consistency_score_3": 0.46323809523809517, - "consistency_score_4": 0.3716190476190476, - "consistency_score_5": 0.30698412698412697, - "consistency_score_6": 0.2580952380952381, - "consistency_score_7": 0.22, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.74, - "Spanish,Chinese": 0.6266666666666667, - "Spanish,Vietnamese": 0.6133333333333333, - "Spanish,Filipino": 0.6133333333333333, - "Spanish,Indonesian": 0.6533333333333333, - "Spanish,Malay": 0.6266666666666667, - "English,Chinese": 0.5866666666666667, - "English,Vietnamese": 0.5866666666666667, - "English,Filipino": 0.6333333333333333, - "English,Indonesian": 0.6533333333333333, - "English,Malay": 0.6, - "Chinese,Vietnamese": 0.6066666666666667, - "Chinese,Filipino": 0.5666666666666667, - "Chinese,Indonesian": 0.5533333333333333, - "Chinese,Malay": 0.5333333333333333, - "Vietnamese,Filipino": 0.5733333333333334, - "Vietnamese,Indonesian": 0.6466666666666666, - "Vietnamese,Malay": 0.6466666666666666, - "Filipino,Indonesian": 0.62, - "Filipino,Malay": 0.58, - "Indonesian,Malay": 0.6866666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5, - "Spanish,English,Vietnamese": 0.5, - "Spanish,English,Filipino": 0.5133333333333333, - "Spanish,English,Indonesian": 0.54, - "Spanish,English,Malay": 0.5066666666666667, - "Spanish,Chinese,Vietnamese": 0.47333333333333333, - "Spanish,Chinese,Filipino": 0.44666666666666666, - "Spanish,Chinese,Indonesian": 0.4533333333333333, - "Spanish,Chinese,Malay": 0.44, - "Spanish,Vietnamese,Filipino": 0.44, - "Spanish,Vietnamese,Indonesian": 0.49333333333333335, - "Spanish,Vietnamese,Malay": 0.49333333333333335, - "Spanish,Filipino,Indonesian": 0.48, - "Spanish,Filipino,Malay": 0.44, - "Spanish,Indonesian,Malay": 0.5066666666666667, - "English,Chinese,Vietnamese": 0.44666666666666666, - "English,Chinese,Filipino": 0.43333333333333335, - "English,Chinese,Indonesian": 0.4266666666666667, - "English,Chinese,Malay": 0.3933333333333333, - "English,Vietnamese,Filipino": 0.44, - "English,Vietnamese,Indonesian": 0.48, - "English,Vietnamese,Malay": 0.4533333333333333, - "English,Filipino,Indonesian": 0.49333333333333335, - "English,Filipino,Malay": 0.44, - "English,Indonesian,Malay": 0.5, - "Chinese,Vietnamese,Filipino": 0.4266666666666667, - "Chinese,Vietnamese,Indonesian": 0.4533333333333333, - "Chinese,Vietnamese,Malay": 0.4666666666666667, - "Chinese,Filipino,Indonesian": 0.42, - "Chinese,Filipino,Malay": 0.38666666666666666, - "Chinese,Indonesian,Malay": 0.4266666666666667, - "Vietnamese,Filipino,Indonesian": 0.4666666666666667, - "Vietnamese,Filipino,Malay": 0.43333333333333335, - "Vietnamese,Indonesian,Malay": 0.5266666666666666, - "Filipino,Indonesian,Malay": 0.47333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4066666666666667, - "Spanish,English,Chinese,Filipino": 0.38, - "Spanish,English,Chinese,Indonesian": 0.37333333333333335, - "Spanish,English,Chinese,Malay": 0.36, - "Spanish,English,Vietnamese,Filipino": 0.38, - "Spanish,English,Vietnamese,Indonesian": 0.41333333333333333, - "Spanish,English,Vietnamese,Malay": 0.42, - "Spanish,English,Filipino,Indonesian": 0.41333333333333333, - "Spanish,English,Filipino,Malay": 0.37333333333333335, - "Spanish,English,Indonesian,Malay": 0.4266666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.36, - "Spanish,Chinese,Vietnamese,Indonesian": 0.37333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.4, - "Spanish,Chinese,Filipino,Indonesian": 0.36, - "Spanish,Chinese,Filipino,Malay": 0.34, - "Spanish,Chinese,Indonesian,Malay": 0.36666666666666664, - "Spanish,Vietnamese,Filipino,Indonesian": 0.37333333333333335, - "Spanish,Vietnamese,Filipino,Malay": 0.36666666666666664, - "Spanish,Vietnamese,Indonesian,Malay": 0.43333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.38666666666666666, - "English,Chinese,Vietnamese,Filipino": 0.34, - "English,Chinese,Vietnamese,Indonesian": 0.36, - "English,Chinese,Vietnamese,Malay": 0.35333333333333333, - "English,Chinese,Filipino,Indonesian": 0.34, - "English,Chinese,Filipino,Malay": 0.3, - "English,Chinese,Indonesian,Malay": 0.32666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.38, - "English,Vietnamese,Filipino,Malay": 0.34, - "English,Vietnamese,Indonesian,Malay": 0.4, - "English,Filipino,Indonesian,Malay": 0.37333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.34, - "Chinese,Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.32666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.38 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.32, - "Spanish,English,Chinese,Vietnamese,Malay": 0.34, - "Spanish,English,Chinese,Filipino,Indonesian": 0.3, - "Spanish,English,Chinese,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.3, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.31333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Spanish,English,Filipino,Indonesian,Malay": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.34, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.3, - "English,Chinese,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22 - } - }, - "AC3_2": 0.561342149152297, - "AC3_3": 0.4878563548728772, - "AC3_4": 0.43179962358083174, - "AC3_5": 0.384737635547986, - "AC3_6": 0.3439150832302174, - "AC3_7": 0.3083419688699794 + "accuracy": 0.2540861812778603 }, "prompt_5": { - "overall_acc": 0.5180952380952382, - "language_acc": { - "Spanish": 0.5933333333333334, - "English": 0.6533333333333333, - "Chinese": 0.49333333333333335, - "Vietnamese": 0.4533333333333333, - "Filipino": 0.5, - "Indonesian": 0.5066666666666667, - "Malay": 0.4266666666666667 - }, - "consistency_score_2": 0.634920634920635, - "consistency_score_3": 0.48666666666666664, - "consistency_score_4": 0.4020952380952381, - "consistency_score_5": 0.3460317460317461, - "consistency_score_6": 0.30666666666666664, - "consistency_score_7": 0.28, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.7666666666666667, - "Spanish,Chinese": 0.7, - "Spanish,Vietnamese": 0.6533333333333333, - "Spanish,Filipino": 0.68, - "Spanish,Indonesian": 0.6733333333333333, - "Spanish,Malay": 0.6666666666666666, - "English,Chinese": 0.5866666666666667, - "English,Vietnamese": 0.5866666666666667, - "English,Filipino": 0.66, - "English,Indonesian": 0.6466666666666666, - "English,Malay": 0.6266666666666667, - "Chinese,Vietnamese": 0.5933333333333334, - "Chinese,Filipino": 0.5933333333333334, - "Chinese,Indonesian": 0.6, - "Chinese,Malay": 0.56, - "Vietnamese,Filipino": 0.6133333333333333, - "Vietnamese,Indonesian": 0.62, - "Vietnamese,Malay": 0.58, - "Filipino,Indonesian": 0.6266666666666667, - "Filipino,Malay": 0.6066666666666667, - "Indonesian,Malay": 0.6933333333333334 - }, - "3_combine": { - "Spanish,English,Chinese": 0.54, - "Spanish,English,Vietnamese": 0.52, - "Spanish,English,Filipino": 0.5733333333333334, - "Spanish,English,Indonesian": 0.56, - "Spanish,English,Malay": 0.5466666666666666, - "Spanish,Chinese,Vietnamese": 0.5066666666666667, - "Spanish,Chinese,Filipino": 0.5266666666666666, - "Spanish,Chinese,Indonesian": 0.5266666666666666, - "Spanish,Chinese,Malay": 0.4866666666666667, - "Spanish,Vietnamese,Filipino": 0.49333333333333335, - "Spanish,Vietnamese,Indonesian": 0.49333333333333335, - "Spanish,Vietnamese,Malay": 0.48, - "Spanish,Filipino,Indonesian": 0.52, - "Spanish,Filipino,Malay": 0.5133333333333333, - "Spanish,Indonesian,Malay": 0.5333333333333333, - "English,Chinese,Vietnamese": 0.44, - "English,Chinese,Filipino": 0.4533333333333333, - "English,Chinese,Indonesian": 0.4533333333333333, - "English,Chinese,Malay": 0.41333333333333333, - "English,Vietnamese,Filipino": 0.46, - "English,Vietnamese,Indonesian": 0.46, - "English,Vietnamese,Malay": 0.44, - "English,Filipino,Indonesian": 0.5, - "English,Filipino,Malay": 0.48, - "English,Indonesian,Malay": 0.52, - "Chinese,Vietnamese,Filipino": 0.4533333333333333, - "Chinese,Vietnamese,Indonesian": 0.46, - "Chinese,Vietnamese,Malay": 0.4266666666666667, - "Chinese,Filipino,Indonesian": 0.4666666666666667, - "Chinese,Filipino,Malay": 0.4266666666666667, - "Chinese,Indonesian,Malay": 0.4666666666666667, - "Vietnamese,Filipino,Indonesian": 0.47333333333333333, - "Vietnamese,Filipino,Malay": 0.44, - "Vietnamese,Indonesian,Malay": 0.4866666666666667, - "Filipino,Indonesian,Malay": 0.49333333333333335 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4266666666666667, - "Spanish,English,Chinese,Filipino": 0.43333333333333335, - "Spanish,English,Chinese,Indonesian": 0.43333333333333335, - "Spanish,English,Chinese,Malay": 0.4, - "Spanish,English,Vietnamese,Filipino": 0.42, - "Spanish,English,Vietnamese,Indonesian": 0.42, - "Spanish,English,Vietnamese,Malay": 0.41333333333333333, - "Spanish,English,Filipino,Indonesian": 0.4533333333333333, - "Spanish,English,Filipino,Malay": 0.44666666666666666, - "Spanish,English,Indonesian,Malay": 0.46, - "Spanish,Chinese,Vietnamese,Filipino": 0.41333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.41333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.3933333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.44, - "Spanish,Chinese,Filipino,Malay": 0.41333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.41333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.4, - "Spanish,Vietnamese,Filipino,Malay": 0.38666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.42, - "Spanish,Filipino,Indonesian,Malay": 0.44666666666666666, - "English,Chinese,Vietnamese,Filipino": 0.36666666666666664, - "English,Chinese,Vietnamese,Indonesian": 0.36666666666666664, - "English,Chinese,Vietnamese,Malay": 0.34, - "English,Chinese,Filipino,Indonesian": 0.38666666666666666, - "English,Chinese,Filipino,Malay": 0.3466666666666667, - "English,Chinese,Indonesian,Malay": 0.36, - "English,Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "English,Vietnamese,Filipino,Malay": 0.36666666666666664, - "English,Vietnamese,Indonesian,Malay": 0.4, - "English,Filipino,Indonesian,Malay": 0.42, - "Chinese,Vietnamese,Filipino,Indonesian": 0.38, - "Chinese,Vietnamese,Filipino,Malay": 0.3466666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.38666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.38666666666666666 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.35333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.35333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.34, - "Spanish,English,Chinese,Filipino,Indonesian": 0.37333333333333335, - "Spanish,English,Chinese,Filipino,Malay": 0.3466666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.35333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.3466666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.3933333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.3333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.37333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.35333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.32, - "English,Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Filipino,Indonesian,Malay": 0.32, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.34, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.32666666666666666 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.32, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.32, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.28 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.28 - } - }, - "AC3_2": 0.5705894692183967, - "AC3_3": 0.5018894154318815, - "AC3_4": 0.45278370834203774, - "AC3_5": 0.41493299739783807, - "AC3_6": 0.3852809853266499, - "AC3_7": 0.3635322195248558 + "accuracy": 0.24888558692421991 } }, - "cross_logiqa": { + "c_eval_full": { "prompt_1": { - "overall_acc": 0.3969155844155844, - "language_acc": { - "English": 0.4772727272727273, - "Filipino": 0.3181818181818182, - "Spanish": 0.42613636363636365, - "Malay": 0.36363636363636365, - "Chinese": 0.4318181818181818, - "Indonesian": 0.39204545454545453, - "Vietnamese": 0.3693181818181818 - }, - "consistency_score_2": 0.4872835497835498, - "consistency_score_3": 0.29610389610389604, - "consistency_score_4": 0.20162337662337662, - "consistency_score_5": 0.14826839826839827, - "consistency_score_6": 0.11607142857142858, - "consistency_score_7": 0.09659090909090909, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4659090909090909, - "English,Spanish": 0.5852272727272727, - "English,Malay": 0.4943181818181818, - "English,Chinese": 0.5056818181818182, - "English,Indonesian": 0.4715909090909091, - "English,Vietnamese": 0.44886363636363635, - "Filipino,Spanish": 0.4375, - "Filipino,Malay": 0.5056818181818182, - "Filipino,Chinese": 0.3522727272727273, - "Filipino,Indonesian": 0.45454545454545453, - "Filipino,Vietnamese": 0.39204545454545453, - "Spanish,Malay": 0.5909090909090909, - "Spanish,Chinese": 0.5056818181818182, - "Spanish,Indonesian": 0.5397727272727273, - "Spanish,Vietnamese": 0.4431818181818182, - "Malay,Chinese": 0.4772727272727273, - "Malay,Indonesian": 0.6647727272727273, - "Malay,Vietnamese": 0.4659090909090909, - "Chinese,Indonesian": 0.48295454545454547, - "Chinese,Vietnamese": 0.4602272727272727, - "Indonesian,Vietnamese": 0.48863636363636365 - }, - "3_combine": { - "English,Filipino,Spanish": 0.30113636363636365, - "English,Filipino,Malay": 0.29545454545454547, - "English,Filipino,Chinese": 0.22727272727272727, - "English,Filipino,Indonesian": 0.26136363636363635, - "English,Filipino,Vietnamese": 0.23863636363636365, - "English,Spanish,Malay": 0.38636363636363635, - "English,Spanish,Chinese": 0.3522727272727273, - "English,Spanish,Indonesian": 0.3352272727272727, - "English,Spanish,Vietnamese": 0.30113636363636365, - "English,Malay,Chinese": 0.3068181818181818, - "English,Malay,Indonesian": 0.3522727272727273, - "English,Malay,Vietnamese": 0.2840909090909091, - "English,Chinese,Indonesian": 0.30113636363636365, - "English,Chinese,Vietnamese": 0.2727272727272727, - "English,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Malay": 0.3181818181818182, - "Filipino,Spanish,Chinese": 0.2215909090909091, - "Filipino,Spanish,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese": 0.2556818181818182, - "Filipino,Malay,Indonesian": 0.36363636363636365, - "Filipino,Malay,Vietnamese": 0.2556818181818182, - "Filipino,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Indonesian,Vietnamese": 0.23863636363636365, - "Spanish,Malay,Chinese": 0.3522727272727273, - "Spanish,Malay,Indonesian": 0.4318181818181818, - "Spanish,Malay,Vietnamese": 0.3181818181818182, - "Spanish,Chinese,Indonesian": 0.32386363636363635, - "Spanish,Chinese,Vietnamese": 0.2727272727272727, - "Spanish,Indonesian,Vietnamese": 0.30113636363636365, - "Malay,Chinese,Indonesian": 0.3693181818181818, - "Malay,Chinese,Vietnamese": 0.2840909090909091, - "Malay,Indonesian,Vietnamese": 0.36363636363636365, - "Chinese,Indonesian,Vietnamese": 0.2897727272727273 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.23863636363636365, - "English,Filipino,Spanish,Chinese": 0.17613636363636365, - "English,Filipino,Spanish,Indonesian": 0.1875, - "English,Filipino,Spanish,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Chinese": 0.18181818181818182, - "English,Filipino,Malay,Indonesian": 0.22727272727272727, - "English,Filipino,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese": 0.25, - "English,Spanish,Malay,Indonesian": 0.2840909090909091, - "English,Spanish,Malay,Vietnamese": 0.23863636363636365, - "English,Spanish,Chinese,Indonesian": 0.23863636363636365, - "English,Spanish,Chinese,Vietnamese": 0.20454545454545456, - "English,Spanish,Indonesian,Vietnamese": 0.19886363636363635, - "English,Malay,Chinese,Indonesian": 0.25, - "English,Malay,Chinese,Vietnamese": 0.19318181818181818, - "English,Malay,Indonesian,Vietnamese": 0.2159090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Malay,Chinese": 0.1875, - "Filipino,Spanish,Malay,Indonesian": 0.23863636363636365, - "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.20454545454545456, - "Filipino,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.26704545454545453, - "Spanish,Malay,Chinese,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.25, - "Spanish,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.25 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Indonesian": 0.18181818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian": 0.20454545454545456, - "English,Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1875 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - } - }, - "AC3_2": 0.43748162022743015, - "AC3_3": 0.33917733702017294, - "AC3_4": 0.26740936032644086, - "AC3_5": 0.21589056101225182, - "AC3_6": 0.179616862532798, - "AC3_7": 0.15537156097330151 + "accuracy": 0.24097135740971357, + "category_acc": { + "computer_network": 0.16666666666666666, + "operating_system": 0.041666666666666664, + "computer_architecture": 0.3076923076923077, + "college_programming": 0.23809523809523808, + "college_physics": 0.375, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.21739130434782608, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.27586206896551724, + "high_school_mathematics": 0.13043478260869565, + "high_school_physics": 0.3333333333333333, + "high_school_chemistry": 0.125, + "high_school_biology": 0.25, + "middle_school_mathematics": 0.3333333333333333, + "middle_school_biology": 0.11538461538461539, + "middle_school_physics": 0.20833333333333334, + "middle_school_chemistry": 0.36, + "veterinary_medicine": 0.39285714285714285, + "college_economics": 0.31666666666666665, + "business_administration": 0.15789473684210525, + "marxism": 0.25, + "mao_zedong_thought": 0.27586206896551724, + "education_science": 0.17647058823529413, + "teacher_qualification": 0.1836734693877551, + "high_school_politics": 0.4166666666666667, + "high_school_geography": 0.20833333333333334, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.23529411764705882, + "modern_chinese_history": 0.21428571428571427, + "ideological_and_moral_cultivation": 0.375, + "logic": 0.25925925925925924, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.34210526315789475, + "professional_tour_guide": 0.29411764705882354, + "legal_professional": 0.14285714285714285, + "high_school_chinese": 0.16666666666666666, + "high_school_history": 0.24, + "middle_school_history": 0.1111111111111111, + "civil_servant": 0.17307692307692307, + "sports_science": 0.20833333333333334, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.16666666666666666, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.23529411764705882, + "accountant": 0.25925925925925924, + "fire_engineer": 0.2222222222222222, + "environmental_impact_assessment_engineer": 0.3055555555555556, + "tax_accountant": 0.24074074074074073, + "physician": 0.25925925925925924 + } }, "prompt_2": { - "overall_acc": 0.4342532467532468, - "language_acc": { - "English": 0.5113636363636364, - "Filipino": 0.3125, - "Spanish": 0.48295454545454547, - "Malay": 0.4090909090909091, - "Chinese": 0.4602272727272727, - "Indonesian": 0.4659090909090909, - "Vietnamese": 0.3977272727272727 - }, - "consistency_score_2": 0.5338203463203464, - "consistency_score_3": 0.3521103896103896, - "consistency_score_4": 0.25503246753246755, - "consistency_score_5": 0.19480519480519481, - "consistency_score_6": 0.15422077922077923, - "consistency_score_7": 0.125, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4147727272727273, - "English,Spanish": 0.6931818181818182, - "English,Malay": 0.5340909090909091, - "English,Chinese": 0.5454545454545454, - "English,Indonesian": 0.5511363636363636, - "English,Vietnamese": 0.5284090909090909, - "Filipino,Spanish": 0.4772727272727273, - "Filipino,Malay": 0.5397727272727273, - "Filipino,Chinese": 0.3977272727272727, - "Filipino,Indonesian": 0.5454545454545454, - "Filipino,Vietnamese": 0.48863636363636365, - "Spanish,Malay": 0.5568181818181818, - "Spanish,Chinese": 0.5397727272727273, - "Spanish,Indonesian": 0.6193181818181818, - "Spanish,Vietnamese": 0.5738636363636364, - "Malay,Chinese": 0.5284090909090909, - "Malay,Indonesian": 0.6477272727272727, - "Malay,Vietnamese": 0.4943181818181818, - "Chinese,Indonesian": 0.5170454545454546, - "Chinese,Vietnamese": 0.4943181818181818, - "Indonesian,Vietnamese": 0.5227272727272727 - }, - "3_combine": { - "English,Filipino,Spanish": 0.32954545454545453, - "English,Filipino,Malay": 0.29545454545454547, - "English,Filipino,Chinese": 0.26136363636363635, - "English,Filipino,Indonesian": 0.3125, - "English,Filipino,Vietnamese": 0.2840909090909091, - "English,Spanish,Malay": 0.4318181818181818, - "English,Spanish,Chinese": 0.4375, - "English,Spanish,Indonesian": 0.4602272727272727, - "English,Spanish,Vietnamese": 0.42613636363636365, - "English,Malay,Chinese": 0.3693181818181818, - "English,Malay,Indonesian": 0.4090909090909091, - "English,Malay,Vietnamese": 0.3409090909090909, - "English,Chinese,Indonesian": 0.3522727272727273, - "English,Chinese,Vietnamese": 0.3465909090909091, - "English,Indonesian,Vietnamese": 0.3465909090909091, - "Filipino,Spanish,Malay": 0.3409090909090909, - "Filipino,Spanish,Chinese": 0.2784090909090909, - "Filipino,Spanish,Indonesian": 0.36363636363636365, - "Filipino,Spanish,Vietnamese": 0.3181818181818182, - "Filipino,Malay,Chinese": 0.3068181818181818, - "Filipino,Malay,Indonesian": 0.3977272727272727, - "Filipino,Malay,Vietnamese": 0.3125, - "Filipino,Chinese,Indonesian": 0.3068181818181818, - "Filipino,Chinese,Vietnamese": 0.26136363636363635, - "Filipino,Indonesian,Vietnamese": 0.32386363636363635, - "Spanish,Malay,Chinese": 0.3693181818181818, - "Spanish,Malay,Indonesian": 0.4431818181818182, - "Spanish,Malay,Vietnamese": 0.36363636363636365, - "Spanish,Chinese,Indonesian": 0.36363636363636365, - "Spanish,Chinese,Vietnamese": 0.35795454545454547, - "Spanish,Indonesian,Vietnamese": 0.39204545454545453, - "Malay,Chinese,Indonesian": 0.4034090909090909, - "Malay,Chinese,Vietnamese": 0.3181818181818182, - "Malay,Indonesian,Vietnamese": 0.375, - "Chinese,Indonesian,Vietnamese": 0.32386363636363635 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.25, - "English,Filipino,Spanish,Chinese": 0.2215909090909091, - "English,Filipino,Spanish,Indonesian": 0.2556818181818182, - "English,Filipino,Spanish,Vietnamese": 0.23295454545454544, - "English,Filipino,Malay,Chinese": 0.20454545454545456, - "English,Filipino,Malay,Indonesian": 0.2556818181818182, - "English,Filipino,Malay,Vietnamese": 0.20454545454545456, - "English,Filipino,Chinese,Indonesian": 0.19886363636363635, - "English,Filipino,Chinese,Vietnamese": 0.1875, - "English,Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay,Chinese": 0.32386363636363635, - "English,Spanish,Malay,Indonesian": 0.3522727272727273, - "English,Spanish,Malay,Vietnamese": 0.29545454545454547, - "English,Spanish,Chinese,Indonesian": 0.3068181818181818, - "English,Spanish,Chinese,Vietnamese": 0.3068181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.3068181818181818, - "English,Malay,Chinese,Indonesian": 0.2897727272727273, - "English,Malay,Chinese,Vietnamese": 0.25, - "English,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Malay,Chinese": 0.23295454545454544, - "Filipino,Spanish,Malay,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.2556818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.3068181818181818, - "Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.1875, - "English,Filipino,Spanish,Malay,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Chinese,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese,Indonesian": 0.17045454545454544, - "English,Filipino,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese,Indonesian": 0.26136363636363635, - "English,Spanish,Malay,Chinese,Vietnamese": 0.23295454545454544, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.25, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1875, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125 - } - }, - "AC3_2": 0.4789165208248475, - "AC3_3": 0.38889153265401055, - "AC3_4": 0.3213433116111055, - "AC3_5": 0.2689568495592555, - "AC3_6": 0.22760859825957572, - "AC3_7": 0.19412191578531812 + "accuracy": 0.2696139476961395, + "category_acc": { + "computer_network": 0.3333333333333333, + "operating_system": 0.3333333333333333, + "computer_architecture": 0.34615384615384615, + "college_programming": 0.21428571428571427, + "college_physics": 0.16666666666666666, + "college_chemistry": 0.3793103448275862, + "advanced_mathematics": 0.16666666666666666, + "probability_and_statistics": 0.21739130434782608, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.23809523809523808, + "metrology_engineer": 0.1724137931034483, + "high_school_mathematics": 0.21739130434782608, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.20833333333333334, + "high_school_biology": 0.2916666666666667, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.3076923076923077, + "middle_school_physics": 0.16666666666666666, + "middle_school_chemistry": 0.24, + "veterinary_medicine": 0.14285714285714285, + "college_economics": 0.25, + "business_administration": 0.23684210526315788, + "marxism": 0.125, + "mao_zedong_thought": 0.3103448275862069, + "education_science": 0.35294117647058826, + "teacher_qualification": 0.24489795918367346, + "high_school_politics": 0.375, + "high_school_geography": 0.3333333333333333, + "middle_school_politics": 0.23076923076923078, + "middle_school_geography": 0.47058823529411764, + "modern_chinese_history": 0.25, + "ideological_and_moral_cultivation": 0.20833333333333334, + "logic": 0.3333333333333333, + "law": 0.20689655172413793, + "chinese_language_and_literature": 0.14285714285714285, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.4117647058823529, + "legal_professional": 0.25, + "high_school_chinese": 0.3333333333333333, + "high_school_history": 0.28, + "middle_school_history": 0.1111111111111111, + "civil_servant": 0.36538461538461536, + "sports_science": 0.20833333333333334, + "plant_protection": 0.3333333333333333, + "basic_medicine": 0.3333333333333333, + "clinical_medicine": 0.18518518518518517, + "urban_and_rural_planner": 0.1568627450980392, + "accountant": 0.37037037037037035, + "fire_engineer": 0.3888888888888889, + "environmental_impact_assessment_engineer": 0.3333333333333333, + "tax_accountant": 0.2962962962962963, + "physician": 0.2777777777777778 + } }, "prompt_3": { - "overall_acc": 0.4350649350649351, - "language_acc": { - "English": 0.5056818181818182, - "Filipino": 0.3409090909090909, - "Spanish": 0.48295454545454547, - "Malay": 0.3977272727272727, - "Chinese": 0.4772727272727273, - "Indonesian": 0.4431818181818182, - "Vietnamese": 0.3977272727272727 - }, - "consistency_score_2": 0.5178571428571429, - "consistency_score_3": 0.3336038961038961, - "consistency_score_4": 0.2313311688311688, - "consistency_score_5": 0.16612554112554112, - "consistency_score_6": 0.12175324675324675, - "consistency_score_7": 0.09090909090909091, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4090909090909091, - "English,Spanish": 0.5909090909090909, - "English,Malay": 0.5170454545454546, - "English,Chinese": 0.48863636363636365, - "English,Indonesian": 0.5227272727272727, - "English,Vietnamese": 0.4602272727272727, - "Filipino,Spanish": 0.4602272727272727, - "Filipino,Malay": 0.5227272727272727, - "Filipino,Chinese": 0.4090909090909091, - "Filipino,Indonesian": 0.5227272727272727, - "Filipino,Vietnamese": 0.4318181818181818, - "Spanish,Malay": 0.5681818181818182, - "Spanish,Chinese": 0.5568181818181818, - "Spanish,Indonesian": 0.6193181818181818, - "Spanish,Vietnamese": 0.5681818181818182, - "Malay,Chinese": 0.5113636363636364, - "Malay,Indonesian": 0.6136363636363636, - "Malay,Vietnamese": 0.5170454545454546, - "Chinese,Indonesian": 0.48295454545454547, - "Chinese,Vietnamese": 0.5284090909090909, - "Indonesian,Vietnamese": 0.5738636363636364 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Malay": 0.2897727272727273, - "English,Filipino,Chinese": 0.23295454545454544, - "English,Filipino,Indonesian": 0.30113636363636365, - "English,Filipino,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay": 0.4034090909090909, - "English,Spanish,Chinese": 0.3693181818181818, - "English,Spanish,Indonesian": 0.4034090909090909, - "English,Spanish,Vietnamese": 0.35795454545454547, - "English,Malay,Chinese": 0.32386363636363635, - "English,Malay,Indonesian": 0.3693181818181818, - "English,Malay,Vietnamese": 0.32386363636363635, - "English,Chinese,Indonesian": 0.29545454545454547, - "English,Chinese,Vietnamese": 0.29545454545454547, - "English,Indonesian,Vietnamese": 0.32386363636363635, - "Filipino,Spanish,Malay": 0.3409090909090909, - "Filipino,Spanish,Chinese": 0.2784090909090909, - "Filipino,Spanish,Indonesian": 0.35795454545454547, - "Filipino,Spanish,Vietnamese": 0.2897727272727273, - "Filipino,Malay,Chinese": 0.2897727272727273, - "Filipino,Malay,Indonesian": 0.375, - "Filipino,Malay,Vietnamese": 0.29545454545454547, - "Filipino,Chinese,Indonesian": 0.2840909090909091, - "Filipino,Chinese,Vietnamese": 0.26136363636363635, - "Filipino,Indonesian,Vietnamese": 0.3125, - "Spanish,Malay,Chinese": 0.36363636363636365, - "Spanish,Malay,Indonesian": 0.4318181818181818, - "Spanish,Malay,Vietnamese": 0.3806818181818182, - "Spanish,Chinese,Indonesian": 0.3693181818181818, - "Spanish,Chinese,Vietnamese": 0.3693181818181818, - "Spanish,Indonesian,Vietnamese": 0.42045454545454547, - "Malay,Chinese,Indonesian": 0.35795454545454547, - "Malay,Chinese,Vietnamese": 0.32954545454545453, - "Malay,Indonesian,Vietnamese": 0.3977272727272727, - "Chinese,Indonesian,Vietnamese": 0.3465909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.22727272727272727, - "English,Filipino,Spanish,Chinese": 0.19886363636363635, - "English,Filipino,Spanish,Indonesian": 0.22727272727272727, - "English,Filipino,Spanish,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Chinese": 0.18181818181818182, - "English,Filipino,Malay,Indonesian": 0.22727272727272727, - "English,Filipino,Malay,Vietnamese": 0.19318181818181818, - "English,Filipino,Chinese,Indonesian": 0.17613636363636365, - "English,Filipino,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay,Chinese": 0.2784090909090909, - "English,Spanish,Malay,Indonesian": 0.3068181818181818, - "English,Spanish,Malay,Vietnamese": 0.26704545454545453, - "English,Spanish,Chinese,Indonesian": 0.26136363636363635, - "English,Spanish,Chinese,Vietnamese": 0.24431818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.26704545454545453, - "English,Malay,Chinese,Indonesian": 0.23863636363636365, - "English,Malay,Chinese,Vietnamese": 0.20454545454545456, - "English,Malay,Indonesian,Vietnamese": 0.25, - "English,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay,Chinese": 0.21022727272727273, - "Filipino,Spanish,Malay,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.1875, - "Filipino,Spanish,Indonesian,Vietnamese": 0.25, - "Filipino,Malay,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese,Indonesian": 0.2840909090909091, - "Spanish,Malay,Chinese,Vietnamese": 0.2556818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.3068181818181818, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.18181818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese,Indonesian": 0.2159090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - } - }, - "AC3_2": 0.47286444385400367, - "AC3_3": 0.3776382012427156, - "AC3_4": 0.3020548269790497, - "AC3_5": 0.2404409271696099, - "AC3_6": 0.190261633376152, - "AC3_7": 0.15039281703089052 - }, - "prompt_4": { - "overall_acc": 0.43100649350649356, - "language_acc": { - "English": 0.48295454545454547, - "Filipino": 0.3465909090909091, - "Spanish": 0.4715909090909091, - "Malay": 0.4090909090909091, - "Chinese": 0.45454545454545453, - "Indonesian": 0.42613636363636365, - "Vietnamese": 0.42613636363636365 - }, - "consistency_score_2": 0.5186688311688312, - "consistency_score_3": 0.33603896103896097, - "consistency_score_4": 0.24155844155844158, - "consistency_score_5": 0.1858766233766234, - "consistency_score_6": 0.15178571428571427, - "consistency_score_7": 0.13068181818181818, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.5, - "English,Spanish": 0.6420454545454546, - "English,Malay": 0.5284090909090909, - "English,Chinese": 0.5113636363636364, - "English,Indonesian": 0.48863636363636365, - "English,Vietnamese": 0.4602272727272727, - "Filipino,Spanish": 0.44886363636363635, - "Filipino,Malay": 0.5284090909090909, - "Filipino,Chinese": 0.4375, - "Filipino,Indonesian": 0.5227272727272727, - "Filipino,Vietnamese": 0.5056818181818182, - "Spanish,Malay": 0.5852272727272727, - "Spanish,Chinese": 0.5284090909090909, - "Spanish,Indonesian": 0.5738636363636364, - "Spanish,Vietnamese": 0.5170454545454546, - "Malay,Chinese": 0.4943181818181818, - "Malay,Indonesian": 0.6363636363636364, - "Malay,Vietnamese": 0.5113636363636364, - "Chinese,Indonesian": 0.5113636363636364, - "Chinese,Vietnamese": 0.4318181818181818, - "Indonesian,Vietnamese": 0.5284090909090909 - }, - "3_combine": { - "English,Filipino,Spanish": 0.36363636363636365, - "English,Filipino,Malay": 0.3522727272727273, - "English,Filipino,Chinese": 0.3068181818181818, - "English,Filipino,Indonesian": 0.3465909090909091, - "English,Filipino,Vietnamese": 0.29545454545454547, - "English,Spanish,Malay": 0.42045454545454547, - "English,Spanish,Chinese": 0.38636363636363635, - "English,Spanish,Indonesian": 0.3806818181818182, - "English,Spanish,Vietnamese": 0.3522727272727273, - "English,Malay,Chinese": 0.3352272727272727, - "English,Malay,Indonesian": 0.3693181818181818, - "English,Malay,Vietnamese": 0.3068181818181818, - "English,Chinese,Indonesian": 0.32386363636363635, - "English,Chinese,Vietnamese": 0.2727272727272727, - "English,Indonesian,Vietnamese": 0.2840909090909091, - "Filipino,Spanish,Malay": 0.32954545454545453, - "Filipino,Spanish,Chinese": 0.2727272727272727, - "Filipino,Spanish,Indonesian": 0.32954545454545453, - "Filipino,Spanish,Vietnamese": 0.29545454545454547, - "Filipino,Malay,Chinese": 0.3068181818181818, - "Filipino,Malay,Indonesian": 0.39204545454545453, - "Filipino,Malay,Vietnamese": 0.32386363636363635, - "Filipino,Chinese,Indonesian": 0.29545454545454547, - "Filipino,Chinese,Vietnamese": 0.25, - "Filipino,Indonesian,Vietnamese": 0.3352272727272727, - "Spanish,Malay,Chinese": 0.3693181818181818, - "Spanish,Malay,Indonesian": 0.42613636363636365, - "Spanish,Malay,Vietnamese": 0.35795454545454547, - "Spanish,Chinese,Indonesian": 0.36363636363636365, - "Spanish,Chinese,Vietnamese": 0.30113636363636365, - "Spanish,Indonesian,Vietnamese": 0.35795454545454547, - "Malay,Chinese,Indonesian": 0.3693181818181818, - "Malay,Chinese,Vietnamese": 0.2840909090909091, - "Malay,Indonesian,Vietnamese": 0.3977272727272727, - "Chinese,Indonesian,Vietnamese": 0.3068181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.2784090909090909, - "English,Filipino,Spanish,Chinese": 0.23863636363636365, - "English,Filipino,Spanish,Indonesian": 0.2556818181818182, - "English,Filipino,Spanish,Vietnamese": 0.23295454545454544, - "English,Filipino,Malay,Chinese": 0.25, - "English,Filipino,Malay,Indonesian": 0.2840909090909091, - "English,Filipino,Malay,Vietnamese": 0.2215909090909091, - "English,Filipino,Chinese,Indonesian": 0.23295454545454544, - "English,Filipino,Chinese,Vietnamese": 0.19886363636363635, - "English,Filipino,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Chinese": 0.2840909090909091, - "English,Spanish,Malay,Indonesian": 0.30113636363636365, - "English,Spanish,Malay,Vietnamese": 0.2556818181818182, - "English,Spanish,Chinese,Indonesian": 0.2727272727272727, - "English,Spanish,Chinese,Vietnamese": 0.2215909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese,Indonesian": 0.26704545454545453, - "English,Malay,Chinese,Vietnamese": 0.19318181818181818, - "English,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "English,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Malay,Chinese": 0.2159090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Chinese,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.2556818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese,Indonesian": 0.29545454545454547, - "Spanish,Malay,Chinese,Vietnamese": 0.23863636363636365, - "Spanish,Malay,Indonesian,Vietnamese": 0.2897727272727273, - "Spanish,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.19886363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.2215909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, - "English,Filipino,Spanish,Chinese,Indonesian": 0.1875, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Indonesian": 0.2159090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.22727272727272727, - "English,Spanish,Malay,Chinese,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - } - }, - "AC3_2": 0.4707917082421344, - "AC3_3": 0.3776437847374083, - "AC3_4": 0.3096006093373427, - "AC3_5": 0.25973812367628196, - "AC3_6": 0.22450756064591815, - "AC3_7": 0.20055504463061058 - }, - "prompt_5": { - "overall_acc": 0.42857142857142866, - "language_acc": { - "English": 0.5056818181818182, - "Filipino": 0.3522727272727273, - "Spanish": 0.45454545454545453, - "Malay": 0.4090909090909091, - "Chinese": 0.4431818181818182, - "Indonesian": 0.4090909090909091, - "Vietnamese": 0.42613636363636365 - }, - "consistency_score_2": 0.5000000000000001, - "consistency_score_3": 0.3048701298701299, - "consistency_score_4": 0.20681818181818185, - "consistency_score_5": 0.15205627705627706, - "consistency_score_6": 0.12012987012987013, - "consistency_score_7": 0.10227272727272728, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4772727272727273, - "English,Spanish": 0.6022727272727273, - "English,Malay": 0.4772727272727273, - "English,Chinese": 0.5, - "English,Indonesian": 0.5056818181818182, - "English,Vietnamese": 0.4772727272727273, - "Filipino,Spanish": 0.4034090909090909, - "Filipino,Malay": 0.5454545454545454, - "Filipino,Chinese": 0.39204545454545453, - "Filipino,Indonesian": 0.5, - "Filipino,Vietnamese": 0.4943181818181818, - "Spanish,Malay": 0.5227272727272727, - "Spanish,Chinese": 0.5397727272727273, - "Spanish,Indonesian": 0.5511363636363636, - "Spanish,Vietnamese": 0.48295454545454547, - "Malay,Chinese": 0.44886363636363635, - "Malay,Indonesian": 0.6079545454545454, - "Malay,Vietnamese": 0.5170454545454546, - "Chinese,Indonesian": 0.48863636363636365, - "Chinese,Vietnamese": 0.44886363636363635, - "Indonesian,Vietnamese": 0.5170454545454546 - }, - "3_combine": { - "English,Filipino,Spanish": 0.30113636363636365, - "English,Filipino,Malay": 0.30113636363636365, - "English,Filipino,Chinese": 0.2556818181818182, - "English,Filipino,Indonesian": 0.3125, - "English,Filipino,Vietnamese": 0.2840909090909091, - "English,Spanish,Malay": 0.3465909090909091, - "English,Spanish,Chinese": 0.375, - "English,Spanish,Indonesian": 0.3693181818181818, - "English,Spanish,Vietnamese": 0.3352272727272727, - "English,Malay,Chinese": 0.2727272727272727, - "English,Malay,Indonesian": 0.3409090909090909, - "English,Malay,Vietnamese": 0.2840909090909091, - "English,Chinese,Indonesian": 0.3068181818181818, - "English,Chinese,Vietnamese": 0.2840909090909091, - "English,Indonesian,Vietnamese": 0.3181818181818182, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.24431818181818182, - "Filipino,Spanish,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Vietnamese": 0.24431818181818182, - "Filipino,Malay,Chinese": 0.26704545454545453, - "Filipino,Malay,Indonesian": 0.375, - "Filipino,Malay,Vietnamese": 0.3181818181818182, - "Filipino,Chinese,Indonesian": 0.2556818181818182, - "Filipino,Chinese,Vietnamese": 0.23295454545454544, - "Filipino,Indonesian,Vietnamese": 0.3125, - "Spanish,Malay,Chinese": 0.3181818181818182, - "Spanish,Malay,Indonesian": 0.3693181818181818, - "Spanish,Malay,Vietnamese": 0.3068181818181818, - "Spanish,Chinese,Indonesian": 0.3352272727272727, - "Spanish,Chinese,Vietnamese": 0.30113636363636365, - "Spanish,Indonesian,Vietnamese": 0.3181818181818182, - "Malay,Chinese,Indonesian": 0.32386363636363635, - "Malay,Chinese,Vietnamese": 0.26136363636363635, - "Malay,Indonesian,Vietnamese": 0.36363636363636365, - "Chinese,Indonesian,Vietnamese": 0.2784090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.2159090909090909, - "English,Filipino,Spanish,Chinese": 0.19318181818181818, - "English,Filipino,Spanish,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Chinese": 0.18181818181818182, - "English,Filipino,Malay,Indonesian": 0.23295454545454544, - "English,Filipino,Malay,Vietnamese": 0.19886363636363635, - "English,Filipino,Chinese,Indonesian": 0.1875, - "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Chinese": 0.22727272727272727, - "English,Spanish,Malay,Indonesian": 0.26704545454545453, - "English,Spanish,Malay,Vietnamese": 0.21022727272727273, - "English,Spanish,Chinese,Indonesian": 0.25, - "English,Spanish,Chinese,Vietnamese": 0.23295454545454544, - "English,Spanish,Indonesian,Vietnamese": 0.23863636363636365, - "English,Malay,Chinese,Indonesian": 0.2159090909090909, - "English,Malay,Chinese,Vietnamese": 0.17613636363636365, - "English,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese": 0.18181818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.20454545454545456, - "Filipino,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese,Indonesian": 0.1875, - "English,Spanish,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728 - } - }, - "AC3_2": 0.4615384614887575, - "AC3_3": 0.3562891291489181, - "AC3_4": 0.2789984669974801, - "AC3_5": 0.2244707761562541, - "AC3_6": 0.18765849531660256, - "AC3_7": 0.16513761464779061 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.5436893203883495 - }, - "prompt_2": { - "accuracy": 0.42718446601941745 - }, - "prompt_3": { - "accuracy": 0.47572815533980584 - }, - "prompt_4": { - "accuracy": 0.3883495145631068 - }, - "prompt_5": { - "accuracy": 0.5728155339805825 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.37142857142857144 - }, - "prompt_2": { - "accuracy": 0.37142857142857144 - }, - "prompt_3": { - "accuracy": 0.34285714285714286 + "accuracy": 0.2465753424657534, + "category_acc": { + "computer_network": 0.125, + "operating_system": 0.25, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.30952380952380953, + "college_physics": 0.25, + "college_chemistry": 0.3103448275862069, + "advanced_mathematics": 0.3333333333333333, + "probability_and_statistics": 0.17391304347826086, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.19047619047619047, + "metrology_engineer": 0.2413793103448276, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.25, + "high_school_chemistry": 0.08333333333333333, + "high_school_biology": 0.4166666666666667, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.15384615384615385, + "middle_school_physics": 0.375, + "middle_school_chemistry": 0.36, + "veterinary_medicine": 0.32142857142857145, + "college_economics": 0.18333333333333332, + "business_administration": 0.3157894736842105, + "marxism": 0.125, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.14285714285714285, + "high_school_politics": 0.2916666666666667, + "high_school_geography": 0.25, + "middle_school_politics": 0.23076923076923078, + "middle_school_geography": 0.17647058823529413, + "modern_chinese_history": 0.25, + "ideological_and_moral_cultivation": 0.2916666666666667, + "logic": 0.18518518518518517, + "law": 0.2413793103448276, + "chinese_language_and_literature": 0.42857142857142855, + "art_studies": 0.23684210526315788, + "professional_tour_guide": 0.20588235294117646, + "legal_professional": 0.21428571428571427, + "high_school_chinese": 0.375, + "high_school_history": 0.2, + "middle_school_history": 0.2222222222222222, + "civil_servant": 0.25, + "sports_science": 0.375, + "plant_protection": 0.2962962962962963, + "basic_medicine": 0.25, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.23529411764705882, + "accountant": 0.2037037037037037, + "fire_engineer": 0.19444444444444445, + "environmental_impact_assessment_engineer": 0.1388888888888889, + "tax_accountant": 0.37037037037037035, + "physician": 0.2037037037037037 + } }, "prompt_4": { - "accuracy": 0.3619047619047619 + "accuracy": 0.2546699875466999, + "category_acc": { + "computer_network": 0.375, + "operating_system": 0.16666666666666666, + "computer_architecture": 0.3076923076923077, + "college_programming": 0.30952380952380953, + "college_physics": 0.2916666666666667, + "college_chemistry": 0.3103448275862069, + "advanced_mathematics": 0.375, + "probability_and_statistics": 0.21739130434782608, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.09523809523809523, + "metrology_engineer": 0.3103448275862069, + "high_school_mathematics": 0.17391304347826086, + "high_school_physics": 0.375, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.20833333333333334, + "middle_school_mathematics": 0.20833333333333334, + "middle_school_biology": 0.07692307692307693, + "middle_school_physics": 0.125, + "middle_school_chemistry": 0.4, + "veterinary_medicine": 0.25, + "college_economics": 0.21666666666666667, + "business_administration": 0.2894736842105263, + "marxism": 0.20833333333333334, + "mao_zedong_thought": 0.2413793103448276, + "education_science": 0.17647058823529413, + "teacher_qualification": 0.30612244897959184, + "high_school_politics": 0.4166666666666667, + "high_school_geography": 0.041666666666666664, + "middle_school_politics": 0.23076923076923078, + "middle_school_geography": 0.23529411764705882, + "modern_chinese_history": 0.25, + "ideological_and_moral_cultivation": 0.25, + "logic": 0.14814814814814814, + "law": 0.20689655172413793, + "chinese_language_and_literature": 0.17857142857142858, + "art_studies": 0.2631578947368421, + "professional_tour_guide": 0.38235294117647056, + "legal_professional": 0.21428571428571427, + "high_school_chinese": 0.25, + "high_school_history": 0.32, + "middle_school_history": 0.4074074074074074, + "civil_servant": 0.23076923076923078, + "sports_science": 0.2916666666666667, + "plant_protection": 0.3333333333333333, + "basic_medicine": 0.20833333333333334, + "clinical_medicine": 0.3333333333333333, + "urban_and_rural_planner": 0.2549019607843137, + "accountant": 0.3333333333333333, + "fire_engineer": 0.1111111111111111, + "environmental_impact_assessment_engineer": 0.2222222222222222, + "tax_accountant": 0.2777777777777778, + "physician": 0.25925925925925924 + } }, "prompt_5": { - "accuracy": 0.3523809523809524 + "accuracy": 0.24408468244084683, + "category_acc": { + "computer_network": 0.25, + "operating_system": 0.2916666666666667, + "computer_architecture": 0.19230769230769232, + "college_programming": 0.30952380952380953, + "college_physics": 0.25, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.4166666666666667, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.2857142857142857, + "metrology_engineer": 0.27586206896551724, + "high_school_mathematics": 0.08695652173913043, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.2916666666666667, + "high_school_biology": 0.20833333333333334, + "middle_school_mathematics": 0.125, + "middle_school_biology": 0.19230769230769232, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.36, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.2833333333333333, + "business_administration": 0.18421052631578946, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.3103448275862069, + "education_science": 0.17647058823529413, + "teacher_qualification": 0.20408163265306123, + "high_school_politics": 0.375, + "high_school_geography": 0.125, + "middle_school_politics": 0.07692307692307693, + "middle_school_geography": 0.23529411764705882, + "modern_chinese_history": 0.21428571428571427, + "ideological_and_moral_cultivation": 0.125, + "logic": 0.14814814814814814, + "law": 0.27586206896551724, + "chinese_language_and_literature": 0.14285714285714285, + "art_studies": 0.34210526315789475, + "professional_tour_guide": 0.20588235294117646, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.25, + "high_school_history": 0.2, + "middle_school_history": 0.2222222222222222, + "civil_servant": 0.25, + "sports_science": 0.375, + "plant_protection": 0.2962962962962963, + "basic_medicine": 0.16666666666666666, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.19607843137254902, + "accountant": 0.24074074074074073, + "fire_engineer": 0.2777777777777778, + "environmental_impact_assessment_engineer": 0.25, + "tax_accountant": 0.24074074074074073, + "physician": 0.25925925925925924 + } } }, - "us_eval": { + "cmmlu": { "prompt_1": { - "accuracy": 0.6448598130841121 + "accuracy": 0.26881720430107525 }, "prompt_2": { - "accuracy": 0.411214953271028 + "accuracy": 0.22939068100358423 }, "prompt_3": { - "accuracy": 0.37383177570093457 + "accuracy": 0.17562724014336917 }, "prompt_4": { - "accuracy": 0.38317757009345793 + "accuracy": 0.24014336917562723 }, "prompt_5": { - "accuracy": 0.6448598130841121 + "accuracy": 0.26881720430107525 } }, - "ph_eval": { + "cmmlu_full": { "prompt_1": { - "accuracy": 0.47, + "accuracy": 0.25375582800897944, "category_acc": { - "brand": 0.5, - "demographics": 0.0, - "biology": 0.3, - "history": 0.4, - "literature": 0.4, - "politics": 0.7, - "culture": 0.7, - "film": 0.5, - "law": 0.3, - "geography": 0.7 + "agronomy": 0.24260355029585798, + "anatomy": 0.22297297297297297, + "ancient_chinese": 0.2865853658536585, + "arts": 0.275, + "astronomy": 0.23030303030303031, + "business_ethics": 0.23923444976076555, + "chinese_civil_service_exam": 0.25, + "chinese_driving_rule": 0.2595419847328244, + "chinese_food_culture": 0.25, + "chinese_foreign_policy": 0.27102803738317754, + "chinese_history": 0.2724458204334365, + "chinese_literature": 0.2549019607843137, + "chinese_teacher_qualification": 0.24581005586592178, + "clinical_knowledge": 0.2616033755274262, + "college_actuarial_science": 0.2641509433962264, + "college_education": 0.22429906542056074, + "college_engineering_hydrology": 0.2169811320754717, + "college_law": 0.24074074074074073, + "college_mathematics": 0.22857142857142856, + "college_medical_statistics": 0.2358490566037736, + "college_medicine": 0.2967032967032967, + "computer_science": 0.2647058823529412, + "computer_security": 0.2573099415204678, + "conceptual_physics": 0.2585034013605442, + "construction_project_management": 0.2302158273381295, + "economics": 0.22012578616352202, + "education": 0.2085889570552147, + "electrical_engineering": 0.2616279069767442, + "elementary_chinese": 0.27380952380952384, + "elementary_commonsense": 0.2676767676767677, + "elementary_information_and_technology": 0.2689075630252101, + "elementary_mathematics": 0.26956521739130435, + "ethnology": 0.3037037037037037, + "food_science": 0.23076923076923078, + "genetics": 0.25, + "global_facts": 0.24161073825503357, + "high_school_biology": 0.22485207100591717, + "high_school_chemistry": 0.2196969696969697, + "high_school_geography": 0.2457627118644068, + "high_school_mathematics": 0.17073170731707318, + "high_school_physics": 0.21818181818181817, + "high_school_politics": 0.2097902097902098, + "human_sexuality": 0.3333333333333333, + "international_law": 0.22162162162162163, + "journalism": 0.27906976744186046, + "jurisprudence": 0.2749391727493917, + "legal_and_moral_basis": 0.2850467289719626, + "logical": 0.21951219512195122, + "machine_learning": 0.1721311475409836, + "management": 0.24761904761904763, + "marketing": 0.2611111111111111, + "marxist_theory": 0.2698412698412698, + "modern_chinese": 0.19827586206896552, + "nutrition": 0.25517241379310346, + "philosophy": 0.22857142857142856, + "professional_accounting": 0.21714285714285714, + "professional_law": 0.2985781990521327, + "professional_medicine": 0.2898936170212766, + "professional_psychology": 0.3146551724137931, + "public_relations": 0.28160919540229884, + "security_study": 0.2518518518518518, + "sociology": 0.23008849557522124, + "sports_science": 0.23030303030303031, + "traditional_chinese_medicine": 0.1837837837837838, + "virology": 0.23076923076923078, + "world_history": 0.32298136645962733, + "world_religions": 0.2625 } }, "prompt_2": { - "accuracy": 0.42, + "accuracy": 0.2502158521844241, "category_acc": { - "brand": 0.6, - "demographics": 0.0, - "biology": 0.4, - "history": 0.3333333333333333, - "literature": 0.3, - "politics": 0.5, - "culture": 0.5, - "film": 0.4, - "law": 0.4, - "geography": 0.6 + "agronomy": 0.21893491124260356, + "anatomy": 0.28378378378378377, + "ancient_chinese": 0.27439024390243905, + "arts": 0.275, + "astronomy": 0.26666666666666666, + "business_ethics": 0.2631578947368421, + "chinese_civil_service_exam": 0.25, + "chinese_driving_rule": 0.22137404580152673, + "chinese_food_culture": 0.23529411764705882, + "chinese_foreign_policy": 0.22429906542056074, + "chinese_history": 0.25077399380804954, + "chinese_literature": 0.25980392156862747, + "chinese_teacher_qualification": 0.2681564245810056, + "clinical_knowledge": 0.24050632911392406, + "college_actuarial_science": 0.18867924528301888, + "college_education": 0.2336448598130841, + "college_engineering_hydrology": 0.25471698113207547, + "college_law": 0.23148148148148148, + "college_mathematics": 0.22857142857142856, + "college_medical_statistics": 0.25471698113207547, + "college_medicine": 0.27106227106227104, + "computer_science": 0.23529411764705882, + "computer_security": 0.24561403508771928, + "conceptual_physics": 0.19727891156462585, + "construction_project_management": 0.2949640287769784, + "economics": 0.25157232704402516, + "education": 0.26380368098159507, + "electrical_engineering": 0.20348837209302326, + "elementary_chinese": 0.27380952380952384, + "elementary_commonsense": 0.23232323232323232, + "elementary_information_and_technology": 0.25630252100840334, + "elementary_mathematics": 0.23478260869565218, + "ethnology": 0.24444444444444444, + "food_science": 0.24475524475524477, + "genetics": 0.24431818181818182, + "global_facts": 0.24161073825503357, + "high_school_biology": 0.24260355029585798, + "high_school_chemistry": 0.32575757575757575, + "high_school_geography": 0.22033898305084745, + "high_school_mathematics": 0.27439024390243905, + "high_school_physics": 0.21818181818181817, + "high_school_politics": 0.24475524475524477, + "human_sexuality": 0.23809523809523808, + "international_law": 0.22162162162162163, + "journalism": 0.26744186046511625, + "jurisprudence": 0.25304136253041365, + "legal_and_moral_basis": 0.21962616822429906, + "logical": 0.2682926829268293, + "machine_learning": 0.2459016393442623, + "management": 0.22380952380952382, + "marketing": 0.26666666666666666, + "marxist_theory": 0.2698412698412698, + "modern_chinese": 0.20689655172413793, + "nutrition": 0.18620689655172415, + "philosophy": 0.3142857142857143, + "professional_accounting": 0.29714285714285715, + "professional_law": 0.2796208530805687, + "professional_medicine": 0.26861702127659576, + "professional_psychology": 0.21551724137931033, + "public_relations": 0.2471264367816092, + "security_study": 0.31851851851851853, + "sociology": 0.23893805309734514, + "sports_science": 0.26666666666666666, + "traditional_chinese_medicine": 0.23243243243243245, + "virology": 0.26627218934911245, + "world_history": 0.2484472049689441, + "world_religions": 0.25625 } }, "prompt_3": { - "accuracy": 0.46, + "accuracy": 0.24650319461232947, "category_acc": { - "brand": 0.8, - "demographics": 0.0, - "biology": 0.3, - "history": 0.3333333333333333, - "literature": 0.4, - "politics": 0.7, - "culture": 0.4, - "film": 0.6, - "law": 0.4, - "geography": 0.5 + "agronomy": 0.20710059171597633, + "anatomy": 0.2702702702702703, + "ancient_chinese": 0.2682926829268293, + "arts": 0.2375, + "astronomy": 0.24242424242424243, + "business_ethics": 0.18660287081339713, + "chinese_civil_service_exam": 0.2625, + "chinese_driving_rule": 0.22900763358778625, + "chinese_food_culture": 0.22794117647058823, + "chinese_foreign_policy": 0.22429906542056074, + "chinese_history": 0.25077399380804954, + "chinese_literature": 0.2549019607843137, + "chinese_teacher_qualification": 0.24581005586592178, + "clinical_knowledge": 0.2320675105485232, + "college_actuarial_science": 0.29245283018867924, + "college_education": 0.21495327102803738, + "college_engineering_hydrology": 0.1509433962264151, + "college_law": 0.16666666666666666, + "college_mathematics": 0.19047619047619047, + "college_medical_statistics": 0.25471698113207547, + "college_medicine": 0.27106227106227104, + "computer_science": 0.20588235294117646, + "computer_security": 0.24561403508771928, + "conceptual_physics": 0.23809523809523808, + "construction_project_management": 0.2446043165467626, + "economics": 0.29559748427672955, + "education": 0.2085889570552147, + "electrical_engineering": 0.2616279069767442, + "elementary_chinese": 0.29365079365079366, + "elementary_commonsense": 0.21212121212121213, + "elementary_information_and_technology": 0.25630252100840334, + "elementary_mathematics": 0.24347826086956523, + "ethnology": 0.26666666666666666, + "food_science": 0.23776223776223776, + "genetics": 0.2727272727272727, + "global_facts": 0.26174496644295303, + "high_school_biology": 0.2485207100591716, + "high_school_chemistry": 0.2196969696969697, + "high_school_geography": 0.23728813559322035, + "high_school_mathematics": 0.22560975609756098, + "high_school_physics": 0.2818181818181818, + "high_school_politics": 0.22377622377622378, + "human_sexuality": 0.24603174603174602, + "international_law": 0.23783783783783785, + "journalism": 0.3081395348837209, + "jurisprudence": 0.27007299270072993, + "legal_and_moral_basis": 0.2803738317757009, + "logical": 0.21951219512195122, + "machine_learning": 0.1885245901639344, + "management": 0.23333333333333334, + "marketing": 0.23333333333333334, + "marxist_theory": 0.26455026455026454, + "modern_chinese": 0.1810344827586207, + "nutrition": 0.22758620689655173, + "philosophy": 0.22857142857142856, + "professional_accounting": 0.2571428571428571, + "professional_law": 0.25118483412322273, + "professional_medicine": 0.2765957446808511, + "professional_psychology": 0.27155172413793105, + "public_relations": 0.2988505747126437, + "security_study": 0.2222222222222222, + "sociology": 0.26991150442477874, + "sports_science": 0.2727272727272727, + "traditional_chinese_medicine": 0.2594594594594595, + "virology": 0.26627218934911245, + "world_history": 0.2111801242236025, + "world_religions": 0.21875 } }, "prompt_4": { - "accuracy": 0.39, - "category_acc": { - "brand": 0.5, - "demographics": 0.0, - "biology": 0.2, - "history": 0.3333333333333333, - "literature": 0.3, - "politics": 0.6, - "culture": 0.5, - "film": 0.5, - "law": 0.3, - "geography": 0.5 - } - }, - "prompt_5": { - "accuracy": 0.47, + "accuracy": 0.24624417199102056, "category_acc": { - "brand": 0.5, - "demographics": 0.0, - "biology": 0.3, - "history": 0.4, - "literature": 0.4, - "politics": 0.7, - "culture": 0.7, - "film": 0.5, - "law": 0.3, - "geography": 0.7 + "agronomy": 0.3136094674556213, + "anatomy": 0.22972972972972974, + "ancient_chinese": 0.18292682926829268, + "arts": 0.275, + "astronomy": 0.17575757575757575, + "business_ethics": 0.18181818181818182, + "chinese_civil_service_exam": 0.275, + "chinese_driving_rule": 0.33587786259541985, + "chinese_food_culture": 0.25735294117647056, + "chinese_foreign_policy": 0.16822429906542055, + "chinese_history": 0.23529411764705882, + "chinese_literature": 0.27450980392156865, + "chinese_teacher_qualification": 0.2849162011173184, + "clinical_knowledge": 0.2869198312236287, + "college_actuarial_science": 0.18867924528301888, + "college_education": 0.2336448598130841, + "college_engineering_hydrology": 0.24528301886792453, + "college_law": 0.25925925925925924, + "college_mathematics": 0.2857142857142857, + "college_medical_statistics": 0.24528301886792453, + "college_medicine": 0.23809523809523808, + "computer_science": 0.25980392156862747, + "computer_security": 0.23976608187134502, + "conceptual_physics": 0.22448979591836735, + "construction_project_management": 0.2949640287769784, + "economics": 0.27044025157232704, + "education": 0.2331288343558282, + "electrical_engineering": 0.26744186046511625, + "elementary_chinese": 0.2222222222222222, + "elementary_commonsense": 0.30808080808080807, + "elementary_information_and_technology": 0.24789915966386555, + "elementary_mathematics": 0.24782608695652175, + "ethnology": 0.3037037037037037, + "food_science": 0.2867132867132867, + "genetics": 0.23295454545454544, + "global_facts": 0.20134228187919462, + "high_school_biology": 0.26627218934911245, + "high_school_chemistry": 0.22727272727272727, + "high_school_geography": 0.22033898305084745, + "high_school_mathematics": 0.22560975609756098, + "high_school_physics": 0.2636363636363636, + "high_school_politics": 0.1888111888111888, + "human_sexuality": 0.24603174603174602, + "international_law": 0.22702702702702704, + "journalism": 0.28488372093023256, + "jurisprudence": 0.22384428223844283, + "legal_and_moral_basis": 0.22897196261682243, + "logical": 0.3170731707317073, + "machine_learning": 0.29508196721311475, + "management": 0.2619047619047619, + "marketing": 0.2722222222222222, + "marxist_theory": 0.2222222222222222, + "modern_chinese": 0.21551724137931033, + "nutrition": 0.1793103448275862, + "philosophy": 0.18095238095238095, + "professional_accounting": 0.24, + "professional_law": 0.23222748815165878, + "professional_medicine": 0.24202127659574468, + "professional_psychology": 0.27586206896551724, + "public_relations": 0.28735632183908044, + "security_study": 0.2222222222222222, + "sociology": 0.19469026548672566, + "sports_science": 0.24242424242424243, + "traditional_chinese_medicine": 0.2702702702702703, + "virology": 0.26627218934911245, + "world_history": 0.2608695652173913, + "world_religions": 0.225 + } + }, + "prompt_5": { + "accuracy": 0.24641685373855984, + "category_acc": { + "agronomy": 0.21893491124260356, + "anatomy": 0.25675675675675674, + "ancient_chinese": 0.2804878048780488, + "arts": 0.3, + "astronomy": 0.2545454545454545, + "business_ethics": 0.22009569377990432, + "chinese_civil_service_exam": 0.225, + "chinese_driving_rule": 0.16793893129770993, + "chinese_food_culture": 0.25, + "chinese_foreign_policy": 0.21495327102803738, + "chinese_history": 0.23839009287925697, + "chinese_literature": 0.22549019607843138, + "chinese_teacher_qualification": 0.29608938547486036, + "clinical_knowledge": 0.2489451476793249, + "college_actuarial_science": 0.2169811320754717, + "college_education": 0.2523364485981308, + "college_engineering_hydrology": 0.2358490566037736, + "college_law": 0.23148148148148148, + "college_mathematics": 0.2761904761904762, + "college_medical_statistics": 0.20754716981132076, + "college_medicine": 0.23076923076923078, + "computer_science": 0.27941176470588236, + "computer_security": 0.24561403508771928, + "conceptual_physics": 0.2789115646258503, + "construction_project_management": 0.18705035971223022, + "economics": 0.25157232704402516, + "education": 0.1901840490797546, + "electrical_engineering": 0.20930232558139536, + "elementary_chinese": 0.23015873015873015, + "elementary_commonsense": 0.23737373737373738, + "elementary_information_and_technology": 0.29411764705882354, + "elementary_mathematics": 0.24347826086956523, + "ethnology": 0.2740740740740741, + "food_science": 0.25874125874125875, + "genetics": 0.22727272727272727, + "global_facts": 0.3087248322147651, + "high_school_biology": 0.21893491124260356, + "high_school_chemistry": 0.18181818181818182, + "high_school_geography": 0.2542372881355932, + "high_school_mathematics": 0.3048780487804878, + "high_school_physics": 0.2, + "high_school_politics": 0.2727272727272727, + "human_sexuality": 0.2222222222222222, + "international_law": 0.24324324324324326, + "journalism": 0.22674418604651161, + "jurisprudence": 0.19951338199513383, + "legal_and_moral_basis": 0.308411214953271, + "logical": 0.2682926829268293, + "machine_learning": 0.3114754098360656, + "management": 0.23333333333333334, + "marketing": 0.28888888888888886, + "marxist_theory": 0.2698412698412698, + "modern_chinese": 0.21551724137931033, + "nutrition": 0.2827586206896552, + "philosophy": 0.26666666666666666, + "professional_accounting": 0.21142857142857144, + "professional_law": 0.2796208530805687, + "professional_medicine": 0.26861702127659576, + "professional_psychology": 0.22844827586206898, + "public_relations": 0.21839080459770116, + "security_study": 0.26666666666666666, + "sociology": 0.22123893805309736, + "sports_science": 0.2606060606060606, + "traditional_chinese_medicine": 0.23243243243243245, + "virology": 0.2603550295857988, + "world_history": 0.2732919254658385, + "world_religions": 0.2625 } } }, - "sing2eng": { + "zbench": { "prompt_1": { - "bleu_score": 0.24923081505591138 + "accuracy": 0.21212121212121213 }, "prompt_2": { - "bleu_score": 0.1921659885601938 + "accuracy": 0.21212121212121213 }, "prompt_3": { - "bleu_score": 0.20243229013807867 + "accuracy": 0.21212121212121213 }, "prompt_4": { - "bleu_score": 0.19881948930424093 + "accuracy": 0.12121212121212122 }, "prompt_5": { - "bleu_score": 0.07295198332847215 + "accuracy": 0.15151515151515152 } }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "ind_emotion": { + "prompt_1": { + "accuracy": 0.1318181818181818 + }, + "prompt_2": { + "accuracy": 0.16136363636363638 + }, + "prompt_3": { + "accuracy": 0.16136363636363638 + }, + "prompt_4": { + "accuracy": 0.17727272727272728 + }, + "prompt_5": { + "accuracy": 0.17272727272727273 + } }, - "flores_ind2eng": { + "ocnli": { "prompt_1": { - "bleu_score": 0.09360424130140148 + "accuracy": 0.34 }, "prompt_2": { - "bleu_score": 0.08739365107698188 + "accuracy": 0.3254237288135593 }, "prompt_3": { - "bleu_score": 0.10367065982168196 + "accuracy": 0.34067796610169493 }, "prompt_4": { - "bleu_score": 0.13229681868281634 + "accuracy": 0.34271186440677964 }, "prompt_5": { - "bleu_score": 0.21831994340734986 + "accuracy": 0.33559322033898303 } }, - "flores_vie2eng": { + "c3": { "prompt_1": { - "bleu_score": 0.07215646878577925 + "accuracy": 0.2774869109947644 }, "prompt_2": { - "bleu_score": 0.07706532852312316 + "accuracy": 0.26925953627524307 }, "prompt_3": { - "bleu_score": 0.08240165425155263 + "accuracy": 0.27299925205684367 }, "prompt_4": { - "bleu_score": 0.10098221803609395 + "accuracy": 0.27486910994764396 }, "prompt_5": { - "bleu_score": 0.11848612020248014 + "accuracy": 0.28421839940164545 } }, - "flores_zho2eng": { + "dream": { "prompt_1": { - "bleu_score": 0.06351506062918416 + "accuracy": 0.34296913277804997 }, "prompt_2": { - "bleu_score": 0.06908564130176305 + "accuracy": 0.3522782949534542 }, "prompt_3": { - "bleu_score": 0.0725844948495155 + "accuracy": 0.3297403233708966 }, "prompt_4": { - "bleu_score": 0.08864645402394244 + "accuracy": 0.3400293973542381 }, "prompt_5": { - "bleu_score": 0.12960749353704404 + "accuracy": 0.3395394414502695 } }, - "flores_zsm2eng": { + "samsum": { "prompt_1": { - "bleu_score": 0.09496439946533222 + "rouge1": 0.08732788348986167, + "rouge2": 0.0014923956624199408, + "rougeL": 0.07139937267026018, + "avg_rouge": 0.05340655060751393 }, "prompt_2": { - "bleu_score": 0.08903986801672446 + "rouge1": 0.09429276780188824, + "rouge2": 0.0015571897217856114, + "rougeL": 0.07590220186880957, + "avg_rouge": 0.05725071979749447 }, "prompt_3": { - "bleu_score": 0.10612942361911483 + "rouge1": 0.09149543908197538, + "rouge2": 0.0013711964459392765, + "rougeL": 0.07469201145443315, + "avg_rouge": 0.05585288232744926 }, "prompt_4": { - "bleu_score": 0.12237196934699915 + "rouge1": 0.08950232953091344, + "rouge2": 0.00146326662774582, + "rougeL": 0.07261221266441964, + "avg_rouge": 0.054525936274359636 }, "prompt_5": { - "bleu_score": 0.2112000588487331 + "rouge1": 0.09373426610282494, + "rouge2": 0.0016362694068140476, + "rougeL": 0.07685534271487146, + "avg_rouge": 0.05740862607483682 } }, - "mmlu": { + "dialogsum": { "prompt_1": { - "accuracy": 0.5799299883313885 + "rouge1": 0.10636719666265786, + "rouge2": 0.001910313699982673, + "rougeL": 0.08618104814226087, + "avg_rouge": 0.0648195195016338 }, "prompt_2": { - "accuracy": 0.5215869311551925 + "rouge1": 0.1075911884138222, + "rouge2": 0.002064888511841431, + "rougeL": 0.08842720122501456, + "avg_rouge": 0.0660277593835594 }, "prompt_3": { - "accuracy": 0.5659276546091015 + "rouge1": 0.10375648507527026, + "rouge2": 0.0016752661752168182, + "rougeL": 0.08609410346879956, + "avg_rouge": 0.06384195157309554 }, "prompt_4": { - "accuracy": 0.6126021003500584 + "rouge1": 0.1044636520106956, + "rouge2": 0.0016187605912823585, + "rougeL": 0.08534402243673829, + "avg_rouge": 0.06380881167957209 }, "prompt_5": { - "accuracy": 0.6149358226371062 + "rouge1": 0.10376988698752433, + "rouge2": 0.0012622705102603058, + "rougeL": 0.08537471802223869, + "avg_rouge": 0.06346895850667444 } }, - "mmlu_full": { + "sst2": { "prompt_1": { - "accuracy": 0.5660350375402217, - "category_acc": { - "high_school_european_history": 0.7317073170731707, - "business_ethics": 0.5555555555555556, - "clinical_knowledge": 0.5757575757575758, - "medical_genetics": 0.5656565656565656, - "high_school_us_history": 0.7586206896551724, - "high_school_physics": 0.34, - "high_school_world_history": 0.7245762711864406, - "virology": 0.4484848484848485, - "high_school_microeconomics": 0.5063291139240507, - "econometrics": 0.4247787610619469, - "college_computer_science": 0.48484848484848486, - "high_school_biology": 0.7152103559870551, - "abstract_algebra": 0.3434343434343434, - "professional_accounting": 0.4234875444839858, - "philosophy": 0.635483870967742, - "professional_medicine": 0.6974169741697417, - "nutrition": 0.7278688524590164, - "global_facts": 0.37373737373737376, - "machine_learning": 0.40540540540540543, - "security_studies": 0.5819672131147541, - "public_relations": 0.5321100917431193, - "professional_psychology": 0.574468085106383, - "prehistory": 0.6501547987616099, - "anatomy": 0.5522388059701493, - "human_sexuality": 0.7307692307692307, - "college_medicine": 0.5930232558139535, - "high_school_government_and_politics": 0.7135416666666666, - "college_chemistry": 0.42424242424242425, - "logical_fallacies": 0.7160493827160493, - "high_school_geography": 0.7360406091370558, - "elementary_mathematics": 0.4907161803713528, - "human_aging": 0.6396396396396397, - "college_mathematics": 0.37373737373737376, - "high_school_psychology": 0.7426470588235294, - "formal_logic": 0.392, - "high_school_statistics": 0.4, - "international_law": 0.7, - "high_school_mathematics": 0.2825278810408922, - "high_school_computer_science": 0.5858585858585859, - "conceptual_physics": 0.5213675213675214, - "miscellaneous": 0.7787723785166241, - "high_school_chemistry": 0.46534653465346537, - "marketing": 0.759656652360515, - "professional_law": 0.4292237442922374, - "management": 0.7156862745098039, - "college_physics": 0.4158415841584158, - "jurisprudence": 0.6542056074766355, - "world_religions": 0.8176470588235294, - "sociology": 0.77, - "us_foreign_policy": 0.7575757575757576, - "high_school_macroeconomics": 0.5141388174807198, - "computer_security": 0.7272727272727273, - "moral_scenarios": 0.24272930648769575, - "moral_disputes": 0.672463768115942, - "electrical_engineering": 0.5555555555555556, - "astronomy": 0.609271523178808, - "college_biology": 0.7272727272727273 - } + "accuracy": 0.518348623853211 }, "prompt_2": { - "accuracy": 0.49045405791919916, - "category_acc": { - "high_school_european_history": 0.7439024390243902, - "business_ethics": 0.48484848484848486, - "clinical_knowledge": 0.553030303030303, - "medical_genetics": 0.5656565656565656, - "high_school_us_history": 0.7684729064039408, - "high_school_physics": 0.32666666666666666, - "high_school_world_history": 0.7288135593220338, - "virology": 0.3575757575757576, - "high_school_microeconomics": 0.5654008438818565, - "econometrics": 0.415929203539823, - "college_computer_science": 0.48484848484848486, - "high_school_biology": 0.5760517799352751, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.4483985765124555, - "philosophy": 0.5387096774193548, - "professional_medicine": 0.6125461254612546, - "nutrition": 0.5934426229508196, - "global_facts": 0.26262626262626265, - "machine_learning": 0.43243243243243246, - "security_studies": 0.6024590163934426, - "public_relations": 0.4036697247706422, - "professional_psychology": 0.49754500818330605, - "prehistory": 0.5325077399380805, - "anatomy": 0.41044776119402987, - "human_sexuality": 0.5538461538461539, - "college_medicine": 0.5581395348837209, - "high_school_government_and_politics": 0.609375, - "college_chemistry": 0.3838383838383838, - "logical_fallacies": 0.6296296296296297, - "high_school_geography": 0.6192893401015228, - "elementary_mathematics": 0.3925729442970822, - "human_aging": 0.47297297297297297, - "college_mathematics": 0.3939393939393939, - "high_school_psychology": 0.5477941176470589, - "formal_logic": 0.376, - "high_school_statistics": 0.4558139534883721, - "international_law": 0.6333333333333333, - "high_school_mathematics": 0.3159851301115242, - "high_school_computer_science": 0.6161616161616161, - "conceptual_physics": 0.4230769230769231, - "miscellaneous": 0.4053708439897698, - "high_school_chemistry": 0.38613861386138615, - "marketing": 0.8025751072961373, - "professional_law": 0.4272667971298108, - "management": 0.5098039215686274, - "college_physics": 0.3564356435643564, - "jurisprudence": 0.6261682242990654, - "world_religions": 0.47058823529411764, - "sociology": 0.715, - "us_foreign_policy": 0.5555555555555556, - "high_school_macroeconomics": 0.5167095115681234, - "computer_security": 0.6262626262626263, - "moral_scenarios": 0.24161073825503357, - "moral_disputes": 0.6086956521739131, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.5231788079470199, - "college_biology": 0.5384615384615384 - } + "accuracy": 0.4690366972477064 }, "prompt_3": { - "accuracy": 0.5541651769753307, - "category_acc": { - "high_school_european_history": 0.7439024390243902, - "business_ethics": 0.5858585858585859, - "clinical_knowledge": 0.625, - "medical_genetics": 0.6565656565656566, - "high_school_us_history": 0.7783251231527094, - "high_school_physics": 0.32666666666666666, - "high_school_world_history": 0.7330508474576272, - "virology": 0.45454545454545453, - "high_school_microeconomics": 0.6118143459915611, - "econometrics": 0.40707964601769914, - "college_computer_science": 0.5151515151515151, - "high_school_biology": 0.6472491909385113, - "abstract_algebra": 0.25252525252525254, - "professional_accounting": 0.4555160142348754, - "philosophy": 0.6290322580645161, - "professional_medicine": 0.6900369003690037, - "nutrition": 0.6688524590163935, - "global_facts": 0.4444444444444444, - "machine_learning": 0.46846846846846846, - "security_studies": 0.6598360655737705, - "public_relations": 0.5412844036697247, - "professional_psychology": 0.5482815057283142, - "prehistory": 0.6222910216718266, - "anatomy": 0.5895522388059702, - "human_sexuality": 0.7076923076923077, - "college_medicine": 0.6162790697674418, - "high_school_government_and_politics": 0.7239583333333334, - "college_chemistry": 0.45454545454545453, - "logical_fallacies": 0.654320987654321, - "high_school_geography": 0.7106598984771574, - "elementary_mathematics": 0.38726790450928383, - "human_aging": 0.536036036036036, - "college_mathematics": 0.37373737373737376, - "high_school_psychology": 0.6617647058823529, - "formal_logic": 0.384, - "high_school_statistics": 0.4511627906976744, - "international_law": 0.7, - "high_school_mathematics": 0.30855018587360594, - "high_school_computer_science": 0.5656565656565656, - "conceptual_physics": 0.49145299145299143, - "miscellaneous": 0.731457800511509, - "high_school_chemistry": 0.4504950495049505, - "marketing": 0.8025751072961373, - "professional_law": 0.44031311154598823, - "management": 0.7058823529411765, - "college_physics": 0.36633663366336633, - "jurisprudence": 0.6915887850467289, - "world_religions": 0.7705882352941177, - "sociology": 0.645, - "us_foreign_policy": 0.7070707070707071, - "high_school_macroeconomics": 0.5398457583547558, - "computer_security": 0.7171717171717171, - "moral_scenarios": 0.2348993288590604, - "moral_disputes": 0.6115942028985507, - "electrical_engineering": 0.5208333333333334, - "astronomy": 0.6026490066225165, - "college_biology": 0.6573426573426573 - } + "accuracy": 0.481651376146789 }, "prompt_4": { - "accuracy": 0.589560243117626, - "category_acc": { - "high_school_european_history": 0.7317073170731707, - "business_ethics": 0.5555555555555556, - "clinical_knowledge": 0.6401515151515151, - "medical_genetics": 0.6767676767676768, - "high_school_us_history": 0.7586206896551724, - "high_school_physics": 0.36, - "high_school_world_history": 0.7415254237288136, - "virology": 0.4727272727272727, - "high_school_microeconomics": 0.6244725738396625, - "econometrics": 0.4336283185840708, - "college_computer_science": 0.5151515151515151, - "high_school_biology": 0.7378640776699029, - "abstract_algebra": 0.36363636363636365, - "professional_accounting": 0.4697508896797153, - "philosophy": 0.6903225806451613, - "professional_medicine": 0.7158671586715867, - "nutrition": 0.7213114754098361, - "global_facts": 0.40404040404040403, - "machine_learning": 0.4594594594594595, - "security_studies": 0.6762295081967213, - "public_relations": 0.6330275229357798, - "professional_psychology": 0.6350245499181669, - "prehistory": 0.6904024767801857, - "anatomy": 0.6119402985074627, - "human_sexuality": 0.7384615384615385, - "college_medicine": 0.6337209302325582, - "high_school_government_and_politics": 0.828125, - "college_chemistry": 0.45454545454545453, - "logical_fallacies": 0.7283950617283951, - "high_school_geography": 0.766497461928934, - "elementary_mathematics": 0.363395225464191, - "human_aging": 0.6711711711711712, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.7628676470588235, - "formal_logic": 0.448, - "high_school_statistics": 0.4511627906976744, - "international_law": 0.7583333333333333, - "high_school_mathematics": 0.3382899628252788, - "high_school_computer_science": 0.6363636363636364, - "conceptual_physics": 0.49572649572649574, - "miscellaneous": 0.7851662404092071, - "high_school_chemistry": 0.47029702970297027, - "marketing": 0.8326180257510729, - "professional_law": 0.4337899543378995, - "management": 0.7647058823529411, - "college_physics": 0.38613861386138615, - "jurisprudence": 0.7383177570093458, - "world_religions": 0.8176470588235294, - "sociology": 0.775, - "us_foreign_policy": 0.8383838383838383, - "high_school_macroeconomics": 0.5784061696658098, - "computer_security": 0.7171717171717171, - "moral_scenarios": 0.25279642058165547, - "moral_disputes": 0.6347826086956522, - "electrical_engineering": 0.5625, - "astronomy": 0.609271523178808, - "college_biology": 0.7342657342657343 - } + "accuracy": 0.49311926605504586 }, "prompt_5": { - "accuracy": 0.590561315695388, - "category_acc": { - "high_school_european_history": 0.7378048780487805, - "business_ethics": 0.5555555555555556, - "clinical_knowledge": 0.6628787878787878, - "medical_genetics": 0.6767676767676768, - "high_school_us_history": 0.7684729064039408, - "high_school_physics": 0.3333333333333333, - "high_school_world_history": 0.7161016949152542, - "virology": 0.47878787878787876, - "high_school_microeconomics": 0.6329113924050633, - "econometrics": 0.4336283185840708, - "college_computer_science": 0.5151515151515151, - "high_school_biology": 0.7216828478964401, - "abstract_algebra": 0.35353535353535354, - "professional_accounting": 0.47330960854092524, - "philosophy": 0.6741935483870968, - "professional_medicine": 0.6826568265682657, - "nutrition": 0.7081967213114754, - "global_facts": 0.3838383838383838, - "machine_learning": 0.46846846846846846, - "security_studies": 0.6885245901639344, - "public_relations": 0.6422018348623854, - "professional_psychology": 0.6170212765957447, - "prehistory": 0.7089783281733746, - "anatomy": 0.5447761194029851, - "human_sexuality": 0.7769230769230769, - "college_medicine": 0.6046511627906976, - "high_school_government_and_politics": 0.8333333333333334, - "college_chemistry": 0.45454545454545453, - "logical_fallacies": 0.6975308641975309, - "high_school_geography": 0.766497461928934, - "elementary_mathematics": 0.38726790450928383, - "human_aging": 0.6576576576576577, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.7794117647058824, - "formal_logic": 0.424, - "high_school_statistics": 0.44651162790697674, - "international_law": 0.7583333333333333, - "high_school_mathematics": 0.31970260223048325, - "high_school_computer_science": 0.6060606060606061, - "conceptual_physics": 0.5128205128205128, - "miscellaneous": 0.7928388746803069, - "high_school_chemistry": 0.4900990099009901, - "marketing": 0.8755364806866953, - "professional_law": 0.436399217221135, - "management": 0.7745098039215687, - "college_physics": 0.40594059405940597, - "jurisprudence": 0.7570093457943925, - "world_religions": 0.8176470588235294, - "sociology": 0.81, - "us_foreign_policy": 0.8383838383838383, - "high_school_macroeconomics": 0.5861182519280206, - "computer_security": 0.7272727272727273, - "moral_scenarios": 0.24496644295302014, - "moral_disputes": 0.6782608695652174, - "electrical_engineering": 0.5694444444444444, - "astronomy": 0.609271523178808, - "college_biology": 0.6923076923076923 - } + "accuracy": 0.5206422018348624 } }, - "c_eval": { + "cola": { "prompt_1": { - "accuracy": 0.40713224368499257 + "accuracy": 0.49089165867689355 }, "prompt_2": { - "accuracy": 0.3848439821693908 + "accuracy": 0.4937679769894535 }, "prompt_3": { - "accuracy": 0.39598811292719166 + "accuracy": 0.513902205177373 }, "prompt_4": { - "accuracy": 0.413075780089153 + "accuracy": 0.5244487056567594 }, "prompt_5": { - "accuracy": 0.3588410104011887 + "accuracy": 0.49952061361457334 } }, - "c_eval_full": { + "qqp": { "prompt_1": { - "accuracy": 0.4190535491905355, - "category_acc": { - "computer_network": 0.5, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.6923076923076923, - "college_programming": 0.47619047619047616, - "college_physics": 0.25, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.42857142857142855, - "electrical_engineer": 0.5, - "metrology_engineer": 0.4482758620689655, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.5769230769230769, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.44, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.5333333333333333, - "business_administration": 0.4473684210526316, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.5102040816326531, - "high_school_politics": 0.4166666666666667, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.5416666666666666, - "logic": 0.2962962962962963, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.23684210526315788, - "professional_tour_guide": 0.5, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.64, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.4423076923076923, - "sports_science": 0.25, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.5, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.37254901960784315, - "accountant": 0.37037037037037035, - "fire_engineer": 0.5, - "environmental_impact_assessment_engineer": 0.3888888888888889, - "tax_accountant": 0.2777777777777778, - "physician": 0.4074074074074074 - } + "accuracy": 0.498 }, "prompt_2": { - "accuracy": 0.3816936488169365, - "category_acc": { - "computer_network": 0.5416666666666666, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.5769230769230769, - "college_programming": 0.42857142857142855, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.40476190476190477, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.5, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.4, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.45, - "business_administration": 0.5263157894736842, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.4482758620689655, - "education_science": 0.5, - "teacher_qualification": 0.46938775510204084, - "high_school_politics": 0.4166666666666667, - "high_school_geography": 0.4583333333333333, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.32142857142857145, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.4444444444444444, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.42857142857142855, - "art_studies": 0.2631578947368421, - "professional_tour_guide": 0.4117647058823529, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.56, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.4807692307692308, - "sports_science": 0.16666666666666666, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.375, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.4444444444444444, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.4444444444444444, - "tax_accountant": 0.25925925925925924, - "physician": 0.2777777777777778 - } + "accuracy": 0.5065 }, "prompt_3": { - "accuracy": 0.37733499377334995, - "category_acc": { - "computer_network": 0.5, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.6153846153846154, - "college_programming": 0.4523809523809524, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.40476190476190477, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.25, - "high_school_chemistry": 0.16666666666666666, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.45, - "business_administration": 0.47368421052631576, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.5172413793103449, - "education_science": 0.4117647058823529, - "teacher_qualification": 0.5306122448979592, - "high_school_politics": 0.4583333333333333, - "high_school_geography": 0.5416666666666666, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.35714285714285715, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.48148148148148145, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.23684210526315788, - "professional_tour_guide": 0.4117647058823529, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.64, - "middle_school_history": 0.4074074074074074, - "civil_servant": 0.38461538461538464, - "sports_science": 0.125, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.5, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.2037037037037037, - "physician": 0.2962962962962963 - } + "accuracy": 0.4915 }, "prompt_4": { - "accuracy": 0.4190535491905355, - "category_acc": { - "computer_network": 0.5, - "operating_system": 0.625, - "computer_architecture": 0.5, - "college_programming": 0.5238095238095238, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.42857142857142855, - "metrology_engineer": 0.4827586206896552, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.5769230769230769, - "middle_school_physics": 0.4166666666666667, - "middle_school_chemistry": 0.44, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.5333333333333333, - "business_administration": 0.4473684210526316, - "marxism": 0.375, - "mao_zedong_thought": 0.5172413793103449, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.4897959183673469, - "high_school_politics": 0.5, - "high_school_geography": 0.5, - "middle_school_politics": 0.5, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.35714285714285715, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.37037037037037035, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.4642857142857143, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.5, - "legal_professional": 0.4642857142857143, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.68, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.38461538461538464, - "sports_science": 0.2916666666666667, - "plant_protection": 0.5185185185185185, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.48148148148148145, - "fire_engineer": 0.5, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.2962962962962963, - "physician": 0.37037037037037035 - } + "accuracy": 0.508 }, "prompt_5": { - "accuracy": 0.36924034869240346, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.5, - "college_programming": 0.42857142857142855, - "college_physics": 0.25, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.125, - "high_school_biology": 0.4166666666666667, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.46153846153846156, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.4, - "business_administration": 0.2894736842105263, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.41379310344827586, - "education_science": 0.5, - "teacher_qualification": 0.4897959183673469, - "high_school_politics": 0.5, - "high_school_geography": 0.4583333333333333, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.32142857142857145, - "ideological_and_moral_cultivation": 0.5, - "logic": 0.4074074074074074, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.4411764705882353, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.6, - "middle_school_history": 0.4444444444444444, - "civil_servant": 0.38461538461538464, - "sports_science": 0.20833333333333334, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.3333333333333333, - "fire_engineer": 0.4444444444444444, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.2777777777777778, - "physician": 0.2777777777777778 - } + "accuracy": 0.4905 } }, - "cmmlu": { + "mnli": { "prompt_1": { - "accuracy": 0.3870967741935484 + "accuracy": 0.3 }, "prompt_2": { - "accuracy": 0.3835125448028674 + "accuracy": 0.4 }, "prompt_3": { - "accuracy": 0.3835125448028674 + "accuracy": 0.3 }, "prompt_4": { - "accuracy": 0.3978494623655914 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.4050179211469534 + "accuracy": 0.5 } }, - "cmmlu_full": { + "qnli": { "prompt_1": { - "accuracy": 0.3952685201174236, - "category_acc": { - "agronomy": 0.3136094674556213, - "anatomy": 0.28378378378378377, - "ancient_chinese": 0.21341463414634146, - "arts": 0.41875, - "astronomy": 0.2909090909090909, - "business_ethics": 0.44019138755980863, - "chinese_civil_service_exam": 0.35625, - "chinese_driving_rule": 0.5038167938931297, - "chinese_food_culture": 0.3014705882352941, - "chinese_foreign_policy": 0.4766355140186916, - "chinese_history": 0.43034055727554177, - "chinese_literature": 0.29411764705882354, - "chinese_teacher_qualification": 0.4581005586592179, - "clinical_knowledge": 0.3080168776371308, - "college_actuarial_science": 0.3113207547169811, - "college_education": 0.4766355140186916, - "college_engineering_hydrology": 0.3867924528301887, - "college_law": 0.3333333333333333, - "college_mathematics": 0.3333333333333333, - "college_medical_statistics": 0.3867924528301887, - "college_medicine": 0.37362637362637363, - "computer_science": 0.47549019607843135, - "computer_security": 0.5555555555555556, - "conceptual_physics": 0.4013605442176871, - "construction_project_management": 0.4028776978417266, - "economics": 0.4779874213836478, - "education": 0.38650306748466257, - "electrical_engineering": 0.43023255813953487, - "elementary_chinese": 0.3134920634920635, - "elementary_commonsense": 0.3686868686868687, - "elementary_information_and_technology": 0.6218487394957983, - "elementary_mathematics": 0.29130434782608694, - "ethnology": 0.2962962962962963, - "food_science": 0.4125874125874126, - "genetics": 0.375, - "global_facts": 0.3959731543624161, - "high_school_biology": 0.3431952662721893, - "high_school_chemistry": 0.25, - "high_school_geography": 0.3474576271186441, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.3986013986013986, - "human_sexuality": 0.4126984126984127, - "international_law": 0.33513513513513515, - "journalism": 0.4476744186046512, - "jurisprudence": 0.45012165450121655, - "legal_and_moral_basis": 0.6355140186915887, - "logical": 0.4878048780487805, - "machine_learning": 0.4262295081967213, - "management": 0.45714285714285713, - "marketing": 0.5277777777777778, - "marxist_theory": 0.5185185185185185, - "modern_chinese": 0.3103448275862069, - "nutrition": 0.3586206896551724, - "philosophy": 0.4, - "professional_accounting": 0.37714285714285717, - "professional_law": 0.3175355450236967, - "professional_medicine": 0.3191489361702128, - "professional_psychology": 0.41810344827586204, - "public_relations": 0.5229885057471264, - "security_study": 0.4, - "sociology": 0.4247787610619469, - "sports_science": 0.42424242424242425, - "traditional_chinese_medicine": 0.2756756756756757, - "virology": 0.42011834319526625, - "world_history": 0.43478260869565216, - "world_religions": 0.3875 - } + "accuracy": 0.4 }, "prompt_2": { - "accuracy": 0.38508029701260577, - "category_acc": { - "agronomy": 0.3254437869822485, - "anatomy": 0.30405405405405406, - "ancient_chinese": 0.1951219512195122, - "arts": 0.4, - "astronomy": 0.26666666666666666, - "business_ethics": 0.42105263157894735, - "chinese_civil_service_exam": 0.4, - "chinese_driving_rule": 0.4732824427480916, - "chinese_food_culture": 0.36764705882352944, - "chinese_foreign_policy": 0.45794392523364486, - "chinese_history": 0.4086687306501548, - "chinese_literature": 0.3333333333333333, - "chinese_teacher_qualification": 0.4245810055865922, - "clinical_knowledge": 0.33755274261603374, - "college_actuarial_science": 0.32075471698113206, - "college_education": 0.48598130841121495, - "college_engineering_hydrology": 0.3584905660377358, - "college_law": 0.3055555555555556, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.4056603773584906, - "college_medicine": 0.2967032967032967, - "computer_science": 0.46078431372549017, - "computer_security": 0.5614035087719298, - "conceptual_physics": 0.36054421768707484, - "construction_project_management": 0.2949640287769784, - "economics": 0.44654088050314467, - "education": 0.44785276073619634, - "electrical_engineering": 0.43023255813953487, - "elementary_chinese": 0.2777777777777778, - "elementary_commonsense": 0.3333333333333333, - "elementary_information_and_technology": 0.5252100840336135, - "elementary_mathematics": 0.30869565217391304, - "ethnology": 0.34814814814814815, - "food_science": 0.40559440559440557, - "genetics": 0.3409090909090909, - "global_facts": 0.4563758389261745, - "high_school_biology": 0.3431952662721893, - "high_school_chemistry": 0.2803030303030303, - "high_school_geography": 0.4067796610169492, - "high_school_mathematics": 0.2926829268292683, - "high_school_physics": 0.3181818181818182, - "high_school_politics": 0.3776223776223776, - "human_sexuality": 0.3412698412698413, - "international_law": 0.32432432432432434, - "journalism": 0.46511627906976744, - "jurisprudence": 0.46228710462287104, - "legal_and_moral_basis": 0.6448598130841121, - "logical": 0.36585365853658536, - "machine_learning": 0.4344262295081967, - "management": 0.4238095238095238, - "marketing": 0.5055555555555555, - "marxist_theory": 0.49206349206349204, - "modern_chinese": 0.33620689655172414, - "nutrition": 0.4, - "philosophy": 0.34285714285714286, - "professional_accounting": 0.38857142857142857, - "professional_law": 0.3412322274881517, - "professional_medicine": 0.3191489361702128, - "professional_psychology": 0.3448275862068966, - "public_relations": 0.4367816091954023, - "security_study": 0.3925925925925926, - "sociology": 0.42035398230088494, - "sports_science": 0.3515151515151515, - "traditional_chinese_medicine": 0.2702702702702703, - "virology": 0.40236686390532544, - "world_history": 0.453416149068323, - "world_religions": 0.39375 - } + "accuracy": 0.5 }, "prompt_3": { - "accuracy": 0.3827490934208254, - "category_acc": { - "agronomy": 0.30177514792899407, - "anatomy": 0.2905405405405405, - "ancient_chinese": 0.21951219512195122, - "arts": 0.39375, - "astronomy": 0.24848484848484848, - "business_ethics": 0.41626794258373206, - "chinese_civil_service_exam": 0.375, - "chinese_driving_rule": 0.5267175572519084, - "chinese_food_culture": 0.33088235294117646, - "chinese_foreign_policy": 0.40186915887850466, - "chinese_history": 0.4117647058823529, - "chinese_literature": 0.31862745098039214, - "chinese_teacher_qualification": 0.44692737430167595, - "clinical_knowledge": 0.3670886075949367, - "college_actuarial_science": 0.32075471698113206, - "college_education": 0.4672897196261682, - "college_engineering_hydrology": 0.3490566037735849, - "college_law": 0.3333333333333333, - "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.37735849056603776, - "college_medicine": 0.3443223443223443, - "computer_science": 0.46568627450980393, - "computer_security": 0.5555555555555556, - "conceptual_physics": 0.35374149659863946, - "construction_project_management": 0.2805755395683453, - "economics": 0.4088050314465409, - "education": 0.4233128834355828, - "electrical_engineering": 0.4069767441860465, - "elementary_chinese": 0.30952380952380953, - "elementary_commonsense": 0.3686868686868687, - "elementary_information_and_technology": 0.5, - "elementary_mathematics": 0.28695652173913044, - "ethnology": 0.31851851851851853, - "food_science": 0.43356643356643354, - "genetics": 0.32954545454545453, - "global_facts": 0.4161073825503356, - "high_school_biology": 0.33727810650887574, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.3474576271186441, - "high_school_mathematics": 0.29878048780487804, - "high_school_physics": 0.32727272727272727, - "high_school_politics": 0.34965034965034963, - "human_sexuality": 0.38095238095238093, - "international_law": 0.2918918918918919, - "journalism": 0.48255813953488375, - "jurisprudence": 0.44038929440389296, - "legal_and_moral_basis": 0.6495327102803738, - "logical": 0.34959349593495936, - "machine_learning": 0.45081967213114754, - "management": 0.40476190476190477, - "marketing": 0.48333333333333334, - "marxist_theory": 0.47619047619047616, - "modern_chinese": 0.3103448275862069, - "nutrition": 0.3793103448275862, - "philosophy": 0.41904761904761906, - "professional_accounting": 0.4228571428571429, - "professional_law": 0.3175355450236967, - "professional_medicine": 0.300531914893617, - "professional_psychology": 0.3922413793103448, - "public_relations": 0.40804597701149425, - "security_study": 0.4222222222222222, - "sociology": 0.4690265486725664, - "sports_science": 0.3939393939393939, - "traditional_chinese_medicine": 0.2702702702702703, - "virology": 0.41420118343195267, - "world_history": 0.42857142857142855, - "world_religions": 0.39375 - } + "accuracy": 0.5 }, "prompt_4": { - "accuracy": 0.40226213089276464, - "category_acc": { - "agronomy": 0.34911242603550297, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.24390243902439024, - "arts": 0.425, - "astronomy": 0.296969696969697, - "business_ethics": 0.4258373205741627, - "chinese_civil_service_exam": 0.3375, - "chinese_driving_rule": 0.4732824427480916, - "chinese_food_culture": 0.33088235294117646, - "chinese_foreign_policy": 0.42990654205607476, - "chinese_history": 0.4179566563467492, - "chinese_literature": 0.31862745098039214, - "chinese_teacher_qualification": 0.4301675977653631, - "clinical_knowledge": 0.3206751054852321, - "college_actuarial_science": 0.32075471698113206, - "college_education": 0.4953271028037383, - "college_engineering_hydrology": 0.36792452830188677, - "college_law": 0.3148148148148148, - "college_mathematics": 0.3523809523809524, - "college_medical_statistics": 0.39622641509433965, - "college_medicine": 0.358974358974359, - "computer_science": 0.5343137254901961, - "computer_security": 0.543859649122807, - "conceptual_physics": 0.36054421768707484, - "construction_project_management": 0.39568345323741005, - "economics": 0.5031446540880503, - "education": 0.3987730061349693, - "electrical_engineering": 0.43023255813953487, - "elementary_chinese": 0.2976190476190476, - "elementary_commonsense": 0.4444444444444444, - "elementary_information_and_technology": 0.6386554621848739, - "elementary_mathematics": 0.3173913043478261, - "ethnology": 0.3925925925925926, - "food_science": 0.3916083916083916, - "genetics": 0.35795454545454547, - "global_facts": 0.436241610738255, - "high_school_biology": 0.33136094674556216, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.3813559322033898, - "high_school_mathematics": 0.27439024390243905, - "high_school_physics": 0.3181818181818182, - "high_school_politics": 0.3776223776223776, - "human_sexuality": 0.4126984126984127, - "international_law": 0.3783783783783784, - "journalism": 0.4476744186046512, - "jurisprudence": 0.4282238442822384, - "legal_and_moral_basis": 0.677570093457944, - "logical": 0.4878048780487805, - "machine_learning": 0.4180327868852459, - "management": 0.48095238095238096, - "marketing": 0.5, - "marxist_theory": 0.5291005291005291, - "modern_chinese": 0.3275862068965517, - "nutrition": 0.43448275862068964, - "philosophy": 0.3904761904761905, - "professional_accounting": 0.44, - "professional_law": 0.3791469194312796, - "professional_medicine": 0.29521276595744683, - "professional_psychology": 0.3922413793103448, - "public_relations": 0.47701149425287354, - "security_study": 0.43703703703703706, - "sociology": 0.4469026548672566, - "sports_science": 0.4121212121212121, - "traditional_chinese_medicine": 0.2702702702702703, - "virology": 0.4378698224852071, - "world_history": 0.40993788819875776, - "world_religions": 0.425 - } + "accuracy": 0.3 }, "prompt_5": { - "accuracy": 0.36150923847349337, - "category_acc": { - "agronomy": 0.31952662721893493, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.27439024390243905, - "arts": 0.325, - "astronomy": 0.30303030303030304, - "business_ethics": 0.3875598086124402, - "chinese_civil_service_exam": 0.29375, - "chinese_driving_rule": 0.3816793893129771, - "chinese_food_culture": 0.2867647058823529, - "chinese_foreign_policy": 0.4672897196261682, - "chinese_history": 0.4024767801857585, - "chinese_literature": 0.3088235294117647, - "chinese_teacher_qualification": 0.4022346368715084, - "clinical_knowledge": 0.3037974683544304, - "college_actuarial_science": 0.32075471698113206, - "college_education": 0.4766355140186916, - "college_engineering_hydrology": 0.37735849056603776, - "college_law": 0.3148148148148148, - "college_mathematics": 0.29523809523809524, - "college_medical_statistics": 0.37735849056603776, - "college_medicine": 0.336996336996337, - "computer_science": 0.43137254901960786, - "computer_security": 0.4678362573099415, - "conceptual_physics": 0.3401360544217687, - "construction_project_management": 0.2805755395683453, - "economics": 0.44025157232704404, - "education": 0.4110429447852761, - "electrical_engineering": 0.4011627906976744, - "elementary_chinese": 0.2857142857142857, - "elementary_commonsense": 0.31313131313131315, - "elementary_information_and_technology": 0.47478991596638653, - "elementary_mathematics": 0.3130434782608696, - "ethnology": 0.2814814814814815, - "food_science": 0.4125874125874126, - "genetics": 0.3181818181818182, - "global_facts": 0.4563758389261745, - "high_school_biology": 0.3076923076923077, - "high_school_chemistry": 0.25, - "high_school_geography": 0.3389830508474576, - "high_school_mathematics": 0.2865853658536585, - "high_school_physics": 0.2818181818181818, - "high_school_politics": 0.35664335664335667, - "human_sexuality": 0.3412698412698413, - "international_law": 0.3081081081081081, - "journalism": 0.37209302325581395, - "jurisprudence": 0.3746958637469586, - "legal_and_moral_basis": 0.5747663551401869, - "logical": 0.4065040650406504, - "machine_learning": 0.39344262295081966, - "management": 0.35714285714285715, - "marketing": 0.42777777777777776, - "marxist_theory": 0.4444444444444444, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.3724137931034483, - "philosophy": 0.3523809523809524, - "professional_accounting": 0.3657142857142857, - "professional_law": 0.2938388625592417, - "professional_medicine": 0.30319148936170215, - "professional_psychology": 0.39655172413793105, - "public_relations": 0.40804597701149425, - "security_study": 0.42962962962962964, - "sociology": 0.4026548672566372, - "sports_science": 0.37575757575757573, - "traditional_chinese_medicine": 0.3081081081081081, - "virology": 0.41420118343195267, - "world_history": 0.36645962732919257, - "world_religions": 0.38125 - } + "accuracy": 0.5 } }, - "zbench": { + "wnli": { "prompt_1": { - "accuracy": 0.36363636363636365 + "accuracy": 0.6 }, "prompt_2": { - "accuracy": 0.3939393939393939 + "accuracy": 0.5 }, "prompt_3": { - "accuracy": 0.36363636363636365 + "accuracy": 0.3 }, "prompt_4": { - "accuracy": 0.3939393939393939 + "accuracy": 0.4 }, "prompt_5": { - "accuracy": 0.30303030303030304 + "accuracy": 0.6 } }, - "ind_emotion": { + "rte": { "prompt_1": { - "accuracy": 0.3704545454545455 + "accuracy": 0.6 }, "prompt_2": { - "accuracy": 0.39545454545454545 + "accuracy": 0.4 }, "prompt_3": { - "accuracy": 0.3181818181818182 + "accuracy": 0.5 }, "prompt_4": { - "accuracy": 0.575 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.5613636363636364 + "accuracy": 0.5 } }, - "ocnli": { + "mrpc": { "prompt_1": { - "accuracy": 0.31152542372881353 + "accuracy": 0.4 }, "prompt_2": { - "accuracy": 0.34440677966101696 + "accuracy": 0.3 }, "prompt_3": { - "accuracy": 0.3064406779661017 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.33322033898305087 + "accuracy": 0.3 }, "prompt_5": { - "accuracy": 0.3152542372881356 + "accuracy": 0.3 + } + } + }, + "five_shot": { + "cross_xquad": { + "prompt_1": { + "overall_acc": 0.2573529411764706, + "language_acc": { + "Spanish": 0.2596638655462185, + "English": 0.2596638655462185, + "Chinese": 0.2453781512605042, + "Vietnamese": 0.2647058823529412 + }, + "consistency_score_2": 0.24971988795518207, + "consistency_score_3": 0.06302521008403361, + "consistency_score_4": 0.015966386554621848, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.25882352941176473, + "Spanish,Chinese": 0.2605042016806723, + "Spanish,Vietnamese": 0.25546218487394956, + "English,Chinese": 0.27310924369747897, + "English,Vietnamese": 0.20588235294117646, + "Chinese,Vietnamese": 0.2445378151260504 + }, + "3_combine": { + "Spanish,English,Chinese": 0.06890756302521009, + "Spanish,English,Vietnamese": 0.05546218487394958, + "Spanish,Chinese,Vietnamese": 0.073109243697479, + "English,Chinese,Vietnamese": 0.0546218487394958 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.015966386554621848 + } + }, + "AC3_2": 0.25347896369405065, + "AC3_3": 0.10125361616897419, + "AC3_4": 0.03006736898117714 } }, - "c3": { + "cross_mmlu": { "prompt_1": { - "accuracy": 0.637247569184742 - }, - "prompt_2": { - "accuracy": 0.6544502617801047 - }, - "prompt_3": { - "accuracy": 0.6335078534031413 - }, - "prompt_4": { - "accuracy": 0.6615557217651459 - }, - "prompt_5": { - "accuracy": 0.6596858638743456 + "overall_acc": 0.2552380952380952, + "language_acc": { + "Filipino": 0.23333333333333334, + "Vietnamese": 0.26666666666666666, + "Chinese": 0.24666666666666667, + "Spanish": 0.26666666666666666, + "Malay": 0.32666666666666666, + "Indonesian": 0.2, + "English": 0.24666666666666667 + }, + "consistency_score_2": 0.24603174603174605, + "consistency_score_3": 0.05809523809523809, + "consistency_score_4": 0.011999999999999995, + "consistency_score_5": 0.0019047619047619048, + "consistency_score_6": 0.0, + "consistency_score_7": 0.0, + "detailed_consistency_score": { + "2_combine": { + "Filipino,Vietnamese": 0.25333333333333335, + "Filipino,Chinese": 0.17333333333333334, + "Filipino,Spanish": 0.24, + "Filipino,Malay": 0.26, + "Filipino,Indonesian": 0.28, + "Filipino,English": 0.2733333333333333, + "Vietnamese,Chinese": 0.2733333333333333, + "Vietnamese,Spanish": 0.2733333333333333, + "Vietnamese,Malay": 0.28, + "Vietnamese,Indonesian": 0.23333333333333334, + "Vietnamese,English": 0.24666666666666667, + "Chinese,Spanish": 0.29333333333333333, + "Chinese,Malay": 0.28, + "Chinese,Indonesian": 0.24666666666666667, + "Chinese,English": 0.26666666666666666, + "Spanish,Malay": 0.20666666666666667, + "Spanish,Indonesian": 0.19333333333333333, + "Spanish,English": 0.22666666666666666, + "Malay,Indonesian": 0.20666666666666667, + "Malay,English": 0.21333333333333335, + "Indonesian,English": 0.24666666666666667 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.04666666666666667, + "Filipino,Vietnamese,Spanish": 0.08, + "Filipino,Vietnamese,Malay": 0.08, + "Filipino,Vietnamese,Indonesian": 0.06666666666666667, + "Filipino,Vietnamese,English": 0.06, + "Filipino,Chinese,Spanish": 0.04666666666666667, + "Filipino,Chinese,Malay": 0.04, + "Filipino,Chinese,Indonesian": 0.06666666666666667, + "Filipino,Chinese,English": 0.04666666666666667, + "Filipino,Spanish,Malay": 0.04, + "Filipino,Spanish,Indonesian": 0.05333333333333334, + "Filipino,Spanish,English": 0.05333333333333334, + "Filipino,Malay,Indonesian": 0.05333333333333334, + "Filipino,Malay,English": 0.06, + "Filipino,Indonesian,English": 0.07333333333333333, + "Vietnamese,Chinese,Spanish": 0.08, + "Vietnamese,Chinese,Malay": 0.10666666666666667, + "Vietnamese,Chinese,Indonesian": 0.09333333333333334, + "Vietnamese,Chinese,English": 0.06, + "Vietnamese,Spanish,Malay": 0.04666666666666667, + "Vietnamese,Spanish,Indonesian": 0.06666666666666667, + "Vietnamese,Spanish,English": 0.04666666666666667, + "Vietnamese,Malay,Indonesian": 0.05333333333333334, + "Vietnamese,Malay,English": 0.04666666666666667, + "Vietnamese,Indonesian,English": 0.06, + "Chinese,Spanish,Malay": 0.06, + "Chinese,Spanish,Indonesian": 0.06666666666666667, + "Chinese,Spanish,English": 0.06666666666666667, + "Chinese,Malay,Indonesian": 0.03333333333333333, + "Chinese,Malay,English": 0.04666666666666667, + "Chinese,Indonesian,English": 0.06, + "Spanish,Malay,Indonesian": 0.04666666666666667, + "Spanish,Malay,English": 0.02, + "Spanish,Indonesian,English": 0.04666666666666667, + "Malay,Indonesian,English": 0.06 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.02, + "Filipino,Vietnamese,Chinese,Malay": 0.02, + "Filipino,Vietnamese,Chinese,Indonesian": 0.02666666666666667, + "Filipino,Vietnamese,Chinese,English": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,Malay": 0.02, + "Filipino,Vietnamese,Spanish,Indonesian": 0.03333333333333333, + "Filipino,Vietnamese,Spanish,English": 0.006666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.02, + "Filipino,Vietnamese,Malay,English": 0.006666666666666667, + "Filipino,Vietnamese,Indonesian,English": 0.006666666666666667, + "Filipino,Chinese,Spanish,Malay": 0.0, + "Filipino,Chinese,Spanish,Indonesian": 0.02666666666666667, + "Filipino,Chinese,Spanish,English": 0.0, + "Filipino,Chinese,Malay,Indonesian": 0.006666666666666667, + "Filipino,Chinese,Malay,English": 0.0, + "Filipino,Chinese,Indonesian,English": 0.006666666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.0, + "Filipino,Spanish,Malay,English": 0.006666666666666667, + "Filipino,Spanish,Indonesian,English": 0.006666666666666667, + "Filipino,Malay,Indonesian,English": 0.013333333333333334, + "Vietnamese,Chinese,Spanish,Malay": 0.013333333333333334, + "Vietnamese,Chinese,Spanish,Indonesian": 0.03333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.006666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.02, + "Vietnamese,Chinese,Malay,English": 0.006666666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.02666666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.013333333333333334, + "Vietnamese,Spanish,Malay,English": 0.006666666666666667, + "Vietnamese,Spanish,Indonesian,English": 0.02, + "Vietnamese,Malay,Indonesian,English": 0.013333333333333334, + "Chinese,Spanish,Malay,Indonesian": 0.0, + "Chinese,Spanish,Malay,English": 0.0, + "Chinese,Spanish,Indonesian,English": 0.006666666666666667, + "Chinese,Malay,Indonesian,English": 0.006666666666666667, + "Spanish,Malay,Indonesian,English": 0.013333333333333334 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.02, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.0, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.0, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.0, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Spanish,Malay,English": 0.0, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.0, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.0, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Chinese,Spanish,Malay,English": 0.0, + "Filipino,Chinese,Spanish,Indonesian,English": 0.0, + "Filipino,Chinese,Malay,Indonesian,English": 0.0, + "Filipino,Spanish,Malay,Indonesian,English": 0.0, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, + "Vietnamese,Chinese,Spanish,Malay,English": 0.0, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.006666666666666667, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.0, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.006666666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.0 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.0, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + } + }, + "AC3_2": 0.25055037842946437, + "AC3_3": 0.09464756112191373, + "AC3_4": 0.022922309328557215, + "AC3_5": 0.0037813051131679403, + "AC3_6": 0.0, + "AC3_7": 0.0 } }, - "dream": { + "cross_logiqa": { "prompt_1": { - "accuracy": 0.8481136697697207 - }, - "prompt_2": { - "accuracy": 0.8138167564919158 - }, - "prompt_3": { - "accuracy": 0.8476237138657521 - }, - "prompt_4": { - "accuracy": 0.8432141107300343 - }, - "prompt_5": { - "accuracy": 0.8446839784419402 + "overall_acc": 0.24756493506493502, + "language_acc": { + "Indonesian": 0.2897727272727273, + "English": 0.2784090909090909, + "Filipino": 0.25, + "Spanish": 0.2784090909090909, + "Chinese": 0.2556818181818182, + "Malay": 0.19886363636363635, + "Vietnamese": 0.18181818181818182 + }, + "consistency_score_2": 0.25216450216450215, + "consistency_score_3": 0.06428571428571428, + "consistency_score_4": 0.01655844155844155, + "consistency_score_5": 0.004058441558441559, + "consistency_score_6": 0.0008116883116883117, + "consistency_score_7": 0.0, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.25, + "Indonesian,Filipino": 0.23863636363636365, + "Indonesian,Spanish": 0.26136363636363635, + "Indonesian,Chinese": 0.23863636363636365, + "Indonesian,Malay": 0.26136363636363635, + "Indonesian,Vietnamese": 0.2727272727272727, + "English,Filipino": 0.2215909090909091, + "English,Spanish": 0.2556818181818182, + "English,Chinese": 0.23295454545454544, + "English,Malay": 0.23295454545454544, + "English,Vietnamese": 0.2215909090909091, + "Filipino,Spanish": 0.2727272727272727, + "Filipino,Chinese": 0.26136363636363635, + "Filipino,Malay": 0.26136363636363635, + "Filipino,Vietnamese": 0.2556818181818182, + "Spanish,Chinese": 0.2556818181818182, + "Spanish,Malay": 0.30113636363636365, + "Spanish,Vietnamese": 0.19886363636363635, + "Chinese,Malay": 0.2727272727272727, + "Chinese,Vietnamese": 0.26136363636363635, + "Malay,Vietnamese": 0.26704545454545453 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.05113636363636364, + "Indonesian,English,Spanish": 0.056818181818181816, + "Indonesian,English,Chinese": 0.0625, + "Indonesian,English,Malay": 0.056818181818181816, + "Indonesian,English,Vietnamese": 0.05113636363636364, + "Indonesian,Filipino,Spanish": 0.0625, + "Indonesian,Filipino,Chinese": 0.07386363636363637, + "Indonesian,Filipino,Malay": 0.0625, + "Indonesian,Filipino,Vietnamese": 0.0625, + "Indonesian,Spanish,Chinese": 0.0625, + "Indonesian,Spanish,Malay": 0.08522727272727272, + "Indonesian,Spanish,Vietnamese": 0.056818181818181816, + "Indonesian,Chinese,Malay": 0.07386363636363637, + "Indonesian,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,Malay,Vietnamese": 0.07954545454545454, + "English,Filipino,Spanish": 0.06818181818181818, + "English,Filipino,Chinese": 0.05113636363636364, + "English,Filipino,Malay": 0.05113636363636364, + "English,Filipino,Vietnamese": 0.03977272727272727, + "English,Spanish,Chinese": 0.0625, + "English,Spanish,Malay": 0.07386363636363637, + "English,Spanish,Vietnamese": 0.045454545454545456, + "English,Chinese,Malay": 0.05113636363636364, + "English,Chinese,Vietnamese": 0.028409090909090908, + "English,Malay,Vietnamese": 0.03977272727272727, + "Filipino,Spanish,Chinese": 0.06818181818181818, + "Filipino,Spanish,Malay": 0.06818181818181818, + "Filipino,Spanish,Vietnamese": 0.0625, + "Filipino,Chinese,Malay": 0.07954545454545454, + "Filipino,Chinese,Vietnamese": 0.08522727272727272, + "Filipino,Malay,Vietnamese": 0.07954545454545454, + "Spanish,Chinese,Malay": 0.10795454545454546, + "Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Spanish,Malay,Vietnamese": 0.07954545454545454, + "Chinese,Malay,Vietnamese": 0.07954545454545454 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.022727272727272728, + "Indonesian,English,Filipino,Chinese": 0.017045454545454544, + "Indonesian,English,Filipino,Malay": 0.005681818181818182, + "Indonesian,English,Filipino,Vietnamese": 0.011363636363636364, + "Indonesian,English,Spanish,Chinese": 0.011363636363636364, + "Indonesian,English,Spanish,Malay": 0.017045454545454544, + "Indonesian,English,Spanish,Vietnamese": 0.005681818181818182, + "Indonesian,English,Chinese,Malay": 0.011363636363636364, + "Indonesian,English,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,English,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Chinese": 0.017045454545454544, + "Indonesian,Filipino,Spanish,Malay": 0.017045454545454544, + "Indonesian,Filipino,Spanish,Vietnamese": 0.017045454545454544, + "Indonesian,Filipino,Chinese,Malay": 0.028409090909090908, + "Indonesian,Filipino,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,Filipino,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Spanish,Chinese,Malay": 0.028409090909090908, + "Indonesian,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,Spanish,Malay,Vietnamese": 0.028409090909090908, + "Indonesian,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Chinese": 0.017045454545454544, + "English,Filipino,Spanish,Malay": 0.005681818181818182, + "English,Filipino,Spanish,Vietnamese": 0.017045454545454544, + "English,Filipino,Chinese,Malay": 0.005681818181818182, + "English,Filipino,Chinese,Vietnamese": 0.011363636363636364, + "English,Filipino,Malay,Vietnamese": 0.011363636363636364, + "English,Spanish,Chinese,Malay": 0.022727272727272728, + "English,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "English,Spanish,Malay,Vietnamese": 0.017045454545454544, + "English,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Filipino,Spanish,Chinese,Malay": 0.022727272727272728, + "Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "Filipino,Spanish,Malay,Vietnamese": 0.028409090909090908, + "Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.005681818181818182, + "Indonesian,English,Filipino,Spanish,Malay": 0.0, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.005681818181818182, + "Indonesian,English,Filipino,Chinese,Malay": 0.0, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese,Malay": 0.005681818181818182, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, + "English,Filipino,Spanish,Chinese,Malay": 0.0, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.0, + "English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, + "English,Filipino,Chinese,Malay,Vietnamese": 0.0, + "English,Spanish,Chinese,Malay,Vietnamese": 0.0, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 + }, + "6_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 + }, + "7_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 + } + }, + "AC3_2": 0.24984355109295978, + "AC3_3": 0.1020673755900859, + "AC3_4": 0.031040717107637145, + "AC3_5": 0.007985965644082193, + "AC3_6": 0.0016180714703611886, + "AC3_7": 0.0 } }, - "samsum": { + "sg_eval": { "prompt_1": { - "rouge1": 0.2531052878882486, - "rouge2": 0.08713672549211297, - "rougeL": 0.1948652378146507, - "avg_rouge": 0.1783690837316708 - }, - "prompt_2": { - "rouge1": 0.28780640303429517, - "rouge2": 0.11511978803154661, - "rougeL": 0.22377632714271273, - "avg_rouge": 0.2089008394028515 - }, - "prompt_3": { - "rouge1": 0.2784479525074015, - "rouge2": 0.11314851351502937, - "rougeL": 0.22014576539017125, - "avg_rouge": 0.20391407713753404 - }, - "prompt_4": { - "rouge1": 0.27468360705195427, - "rouge2": 0.10296466203487137, - "rougeL": 0.2097809862045213, - "avg_rouge": 0.19580975176378232 - }, - "prompt_5": { - "rouge1": 0.2886536120191834, - "rouge2": 0.11565508035194477, - "rougeL": 0.22552673587529226, - "avg_rouge": 0.20994514274880682 + "accuracy": 0.2524271844660194 } }, - "dialogsum": { + "cn_eval": { "prompt_1": { - "rouge1": 0.23432163845149853, - "rouge2": 0.06459547926897637, - "rougeL": 0.17291806311887278, - "avg_rouge": 0.1572783936131159 - }, - "prompt_2": { - "rouge1": 0.2303285886573876, - "rouge2": 0.06338585848004481, - "rougeL": 0.16962492626118594, - "avg_rouge": 0.15444645779953944 - }, - "prompt_3": { - "rouge1": 0.23375160347756882, - "rouge2": 0.06452620096536225, - "rougeL": 0.17235290582602503, - "avg_rouge": 0.15687690342298535 - }, - "prompt_4": { - "rouge1": 0.23351137969595284, - "rouge2": 0.06415520565021773, - "rougeL": 0.17206417674654728, - "avg_rouge": 0.15657692069757262 - }, - "prompt_5": { - "rouge1": 0.22567249318652557, - "rouge2": 0.06233832795677091, - "rougeL": 0.16601879419362936, - "avg_rouge": 0.15134320511230862 + "accuracy": 0.2857142857142857 } }, - "sst2": { + "us_eval": { "prompt_1": { - "accuracy": 0.8818807339449541 - }, - "prompt_2": { - "accuracy": 0.716743119266055 - }, - "prompt_3": { - "accuracy": 0.7603211009174312 - }, - "prompt_4": { - "accuracy": 0.8681192660550459 - }, - "prompt_5": { - "accuracy": 0.7947247706422018 + "accuracy": 0.2523364485981308 } }, - "cola": { + "ph_eval": { "prompt_1": { - "accuracy": 0.5656759348034516 - }, - "prompt_2": { - "accuracy": 0.49952061361457334 - }, - "prompt_3": { - "accuracy": 0.5474592521572388 - }, - "prompt_4": { - "accuracy": 0.5580057526366251 - }, - "prompt_5": { - "accuracy": 0.5081495685522531 + "accuracy": 0.21, + "category_acc": { + "brand": 0.4, + "demographics": 0.4, + "biology": 0.2, + "history": 0.13333333333333333, + "literature": 0.5, + "politics": 0.1, + "culture": 0.1, + "film": 0.0, + "law": 0.2, + "geography": 0.2 + } } }, - "qqp": { + "sing2eng": { "prompt_1": { - "accuracy": 0.5855 - }, - "prompt_2": { - "accuracy": 0.573 - }, - "prompt_3": { - "accuracy": 0.542 - }, - "prompt_4": { - "accuracy": 0.5995 - }, - "prompt_5": { - "accuracy": 0.6095 + "bleu_score": 0.008866968704698027 } }, - "mnli": { + "indommlu": { "prompt_1": { - "accuracy": 0.408 - }, - "prompt_2": { - "accuracy": 0.376 - }, - "prompt_3": { - "accuracy": 0.3945 - }, - "prompt_4": { - "accuracy": 0.373 - }, - "prompt_5": { - "accuracy": 0.373 + "accuracy": 0.23179117431070165, + "category_acc": { + "History": 0.1566265060240964, + "Geography": 0.18571428571428572, + "Lampungic": 0.2925170068027211, + "Social science": 0.24707846410684475, + "Balinese": 0.267515923566879, + "Makassarese": 0.23118279569892472, + "Banjarese": 0.2986111111111111, + "Chemistry": 0.2029197080291971, + "Biology": 0.18106508875739644, + "Science": 0.23426212590299278, + "Christian religion": 0.22885572139303484, + "Art": 0.24459234608985025, + "Islam religion": 0.24466571834992887, + "Hindu religion": 0.24666666666666667, + "Madurese": 0.21694915254237288, + "Sport": 0.2635135135135135, + "Indonesian language": 0.2425280199252802, + "Physics": 0.2383838383838384, + "Minangkabau culture": 0.23115577889447236, + "Dayak language": 0.21100917431192662, + "Sociology": 0.18548387096774194, + "Economy": 0.18442622950819673, + "Sundanese": 0.27398444252376836, + "Javanese": 0.25201612903225806, + "Civic education": 0.23032904148783978 + } } }, - "qnli": { + "flores_ind2eng": { "prompt_1": { - "accuracy": 0.525 - }, - "prompt_2": { - "accuracy": 0.5055 - }, - "prompt_3": { - "accuracy": 0.5365 - }, - "prompt_4": { - "accuracy": 0.555 - }, - "prompt_5": { - "accuracy": 0.5295 + "bleu_score": 0.009307104377338064 } }, - "wnli": { + "flores_vie2eng": { "prompt_1": { - "accuracy": 0.5774647887323944 - }, - "prompt_2": { - "accuracy": 0.5774647887323944 - }, - "prompt_3": { - "accuracy": 0.5492957746478874 - }, - "prompt_4": { - "accuracy": 0.4507042253521127 - }, - "prompt_5": { - "accuracy": 0.5633802816901409 + "bleu_score": 0.009288422226243908 } }, - "rte": { + "flores_zho2eng": { "prompt_1": { - "accuracy": 0.5451263537906137 - }, - "prompt_2": { - "accuracy": 0.5342960288808665 - }, - "prompt_3": { - "accuracy": 0.5306859205776173 - }, - "prompt_4": { - "accuracy": 0.555956678700361 - }, - "prompt_5": { - "accuracy": 0.4729241877256318 + "bleu_score": 0.009267355508947273 } }, - "mrpc": { - "prompt_1": { - "accuracy": 0.46568627450980393 - }, - "prompt_2": { - "accuracy": 0.6323529411764706 - }, - "prompt_3": { - "accuracy": 0.6544117647058824 - }, - "prompt_4": { - "accuracy": 0.5392156862745098 - }, - "prompt_5": { - "accuracy": 0.3284313725490196 - } - } - }, - "five_shot": { - "cross_xquad": { + "flores_zsm2eng": { "prompt_1": { - "overall_acc": 0.8584033613445379, - "language_acc": { - "English": 0.880672268907563, - "Chinese": 0.853781512605042, - "Spanish": 0.8815126050420168, - "Vietnamese": 0.8176470588235294 - }, - "consistency_score_2": 0.8092436974789915, - "consistency_score_3": 0.7247899159663866, - "consistency_score_4": 0.6621848739495798, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8201680672268907, - "English,Spanish": 0.8487394957983193, - "English,Vietnamese": 0.7756302521008404, - "Chinese,Spanish": 0.8252100840336134, - "Chinese,Vietnamese": 0.7924369747899159, - "Spanish,Vietnamese": 0.7932773109243697 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7546218487394958, - "English,Chinese,Vietnamese": 0.7084033613445379, - "English,Spanish,Vietnamese": 0.7201680672268908, - "Chinese,Spanish,Vietnamese": 0.7159663865546219 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.6621848739495798 - } - }, - "AC3_2": 0.8330989538173046, - "AC3_3": 0.7859584916272817, - "AC3_4": 0.7476339858469319 + "bleu_score": 0.009239626132587908 } }, - "cross_mmlu": { + "mmlu": { "prompt_1": { - "overall_acc": 0.42095238095238097, - "language_acc": { - "Spanish": 0.46, - "English": 0.44666666666666666, - "Chinese": 0.4066666666666667, - "Vietnamese": 0.4266666666666667, - "Filipino": 0.4533333333333333, - "Indonesian": 0.36666666666666664, - "Malay": 0.38666666666666666 - }, - "consistency_score_2": 0.39968253968253964, - "consistency_score_3": 0.19180952380952385, - "consistency_score_4": 0.10095238095238093, - "consistency_score_5": 0.05492063492063493, - "consistency_score_6": 0.029523809523809525, - "consistency_score_7": 0.013333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.3933333333333333, - "Spanish,Chinese": 0.42, - "Spanish,Vietnamese": 0.43333333333333335, - "Spanish,Filipino": 0.3466666666666667, - "Spanish,Indonesian": 0.36, - "Spanish,Malay": 0.44666666666666666, - "English,Chinese": 0.4533333333333333, - "English,Vietnamese": 0.35333333333333333, - "English,Filipino": 0.34, - "English,Indonesian": 0.4066666666666667, - "English,Malay": 0.38666666666666666, - "Chinese,Vietnamese": 0.44666666666666666, - "Chinese,Filipino": 0.37333333333333335, - "Chinese,Indonesian": 0.41333333333333333, - "Chinese,Malay": 0.4266666666666667, - "Vietnamese,Filipino": 0.38, - "Vietnamese,Indonesian": 0.38, - "Vietnamese,Malay": 0.5, - "Filipino,Indonesian": 0.38666666666666666, - "Filipino,Malay": 0.3333333333333333, - "Indonesian,Malay": 0.41333333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.20666666666666667, - "Spanish,English,Vietnamese": 0.16666666666666666, - "Spanish,English,Filipino": 0.13333333333333333, - "Spanish,English,Indonesian": 0.16666666666666666, - "Spanish,English,Malay": 0.2, - "Spanish,Chinese,Vietnamese": 0.24, - "Spanish,Chinese,Filipino": 0.17333333333333334, - "Spanish,Chinese,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Malay": 0.24, - "Spanish,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Filipino,Malay": 0.19333333333333333, - "Spanish,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese": 0.2, - "English,Chinese,Filipino": 0.17333333333333334, - "English,Chinese,Indonesian": 0.22666666666666666, - "English,Chinese,Malay": 0.21333333333333335, - "English,Vietnamese,Filipino": 0.14, - "English,Vietnamese,Indonesian": 0.17333333333333334, - "English,Vietnamese,Malay": 0.18, - "English,Filipino,Indonesian": 0.16, - "English,Filipino,Malay": 0.14666666666666667, - "English,Indonesian,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Filipino": 0.2, - "Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "Chinese,Vietnamese,Malay": 0.24666666666666667, - "Chinese,Filipino,Indonesian": 0.2, - "Chinese,Filipino,Malay": 0.18, - "Chinese,Indonesian,Malay": 0.20666666666666667, - "Vietnamese,Filipino,Indonesian": 0.16, - "Vietnamese,Filipino,Malay": 0.2, - "Vietnamese,Indonesian,Malay": 0.2, - "Filipino,Indonesian,Malay": 0.17333333333333334 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.11333333333333333, - "Spanish,English,Chinese,Filipino": 0.07333333333333333, - "Spanish,English,Chinese,Indonesian": 0.1, - "Spanish,English,Chinese,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.06, - "Spanish,English,Vietnamese,Indonesian": 0.07333333333333333, - "Spanish,English,Vietnamese,Malay": 0.09333333333333334, - "Spanish,English,Filipino,Indonesian": 0.07333333333333333, - "Spanish,English,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Malay": 0.16, - "Spanish,Chinese,Filipino,Indonesian": 0.12, - "Spanish,Chinese,Filipino,Malay": 0.11333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.11333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.08, - "English,Chinese,Vietnamese,Indonesian": 0.12, - "English,Chinese,Vietnamese,Malay": 0.10666666666666667, - "English,Chinese,Filipino,Indonesian": 0.10666666666666667, - "English,Chinese,Filipino,Malay": 0.08666666666666667, - "English,Chinese,Indonesian,Malay": 0.11333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "English,Vietnamese,Filipino,Malay": 0.07333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "English,Filipino,Indonesian,Malay": 0.08, - "Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.1, - "Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.04666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.06, - "Spanish,English,Chinese,Filipino,Indonesian": 0.05333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.06, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.02666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.04, - "Spanish,English,Filipino,Indonesian,Malay": 0.04, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.08, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.07333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.06, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.05333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.04, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.06666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.05333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.04, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.02, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.03333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.03333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.02666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334 - } - }, - "AC3_2": 0.41004181628972225, - "AC3_3": 0.2635368651227039, - "AC3_4": 0.162850191140156, - "AC3_5": 0.09716445882516146, - "AC3_6": 0.05517769051427899, - "AC3_7": 0.02584795321042244 + "accuracy": 0.2660443407234539 } }, - "cross_logiqa": { + "mmlu_full": { "prompt_1": { - "overall_acc": 0.4342532467532468, - "language_acc": { - "English": 0.5340909090909091, - "Filipino": 0.4034090909090909, - "Spanish": 0.4034090909090909, - "Malay": 0.42613636363636365, - "Chinese": 0.4715909090909091, - "Indonesian": 0.4090909090909091, - "Vietnamese": 0.39204545454545453 - }, - "consistency_score_2": 0.44588744588744594, - "consistency_score_3": 0.2503246753246754, - "consistency_score_4": 0.1602272727272727, - "consistency_score_5": 0.11201298701298701, - "consistency_score_6": 0.0844155844155844, - "consistency_score_7": 0.06818181818181818, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.38636363636363635, - "English,Spanish": 0.5284090909090909, - "English,Malay": 0.48863636363636365, - "English,Chinese": 0.45454545454545453, - "English,Indonesian": 0.4772727272727273, - "English,Vietnamese": 0.42045454545454547, - "Filipino,Spanish": 0.4034090909090909, - "Filipino,Malay": 0.5170454545454546, - "Filipino,Chinese": 0.35795454545454547, - "Filipino,Indonesian": 0.4431818181818182, - "Filipino,Vietnamese": 0.4715909090909091, - "Spanish,Malay": 0.4659090909090909, - "Spanish,Chinese": 0.4375, - "Spanish,Indonesian": 0.4715909090909091, - "Spanish,Vietnamese": 0.4034090909090909, - "Malay,Chinese": 0.42613636363636365, - "Malay,Indonesian": 0.5284090909090909, - "Malay,Vietnamese": 0.45454545454545453, - "Chinese,Indonesian": 0.42613636363636365, - "Chinese,Vietnamese": 0.3977272727272727, - "Indonesian,Vietnamese": 0.4034090909090909 - }, - "3_combine": { - "English,Filipino,Spanish": 0.23863636363636365, - "English,Filipino,Malay": 0.29545454545454547, - "English,Filipino,Chinese": 0.19318181818181818, - "English,Filipino,Indonesian": 0.22727272727272727, - "English,Filipino,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay": 0.32954545454545453, - "English,Spanish,Chinese": 0.26704545454545453, - "English,Spanish,Indonesian": 0.3068181818181818, - "English,Spanish,Vietnamese": 0.2556818181818182, - "English,Malay,Chinese": 0.26136363636363635, - "English,Malay,Indonesian": 0.3125, - "English,Malay,Vietnamese": 0.2727272727272727, - "English,Chinese,Indonesian": 0.2727272727272727, - "English,Chinese,Vietnamese": 0.21022727272727273, - "English,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay": 0.2784090909090909, - "Filipino,Spanish,Chinese": 0.20454545454545456, - "Filipino,Spanish,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Vietnamese": 0.25, - "Filipino,Malay,Chinese": 0.22727272727272727, - "Filipino,Malay,Indonesian": 0.3125, - "Filipino,Malay,Vietnamese": 0.3181818181818182, - "Filipino,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Indonesian,Vietnamese": 0.25, - "Spanish,Malay,Chinese": 0.23863636363636365, - "Spanish,Malay,Indonesian": 0.2897727272727273, - "Spanish,Malay,Vietnamese": 0.24431818181818182, - "Spanish,Chinese,Indonesian": 0.2556818181818182, - "Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Spanish,Indonesian,Vietnamese": 0.21022727272727273, - "Malay,Chinese,Indonesian": 0.26136363636363635, - "Malay,Chinese,Vietnamese": 0.2159090909090909, - "Malay,Indonesian,Vietnamese": 0.2784090909090909, - "Chinese,Indonesian,Vietnamese": 0.19886363636363635 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.20454545454545456, - "English,Filipino,Spanish,Chinese": 0.11931818181818182, - "English,Filipino,Spanish,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Chinese": 0.1590909090909091, - "English,Filipino,Malay,Indonesian": 0.19318181818181818, - "English,Filipino,Malay,Vietnamese": 0.19886363636363635, - "English,Filipino,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Chinese": 0.16477272727272727, - "English,Spanish,Malay,Indonesian": 0.2215909090909091, - "English,Spanish,Malay,Vietnamese": 0.19318181818181818, - "English,Spanish,Chinese,Indonesian": 0.18181818181818182, - "English,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "English,Malay,Chinese,Indonesian": 0.18181818181818182, - "English,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Malay,Indonesian,Vietnamese": 0.1875, - "English,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese": 0.1534090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.125, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.17045454545454544, - "Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - } - }, - "AC3_2": 0.4399934525472118, - "AC3_3": 0.31758051050305514, - "AC3_4": 0.23408408218097024, - "AC3_5": 0.17808899863589228, - "AC3_6": 0.14135316949493051, - "AC3_7": 0.1178587163813009 + "accuracy": 0.24640686449767607, + "category_acc": { + "high_school_european_history": 0.25609756097560976, + "business_ethics": 0.2222222222222222, + "clinical_knowledge": 0.3143939393939394, + "medical_genetics": 0.2222222222222222, + "high_school_us_history": 0.24630541871921183, + "high_school_physics": 0.30666666666666664, + "high_school_world_history": 0.2542372881355932, + "virology": 0.2606060606060606, + "high_school_microeconomics": 0.1940928270042194, + "econometrics": 0.23893805309734514, + "college_computer_science": 0.26262626262626265, + "high_school_biology": 0.24919093851132687, + "abstract_algebra": 0.29292929292929293, + "professional_accounting": 0.24555160142348753, + "philosophy": 0.25161290322580643, + "professional_medicine": 0.23247232472324722, + "nutrition": 0.24262295081967214, + "global_facts": 0.21212121212121213, + "machine_learning": 0.17117117117117117, + "security_studies": 0.2581967213114754, + "public_relations": 0.24770642201834864, + "professional_psychology": 0.23076923076923078, + "prehistory": 0.25077399380804954, + "anatomy": 0.29850746268656714, + "human_sexuality": 0.25384615384615383, + "college_medicine": 0.28488372093023256, + "high_school_government_and_politics": 0.21875, + "college_chemistry": 0.21212121212121213, + "logical_fallacies": 0.2654320987654321, + "high_school_geography": 0.28426395939086296, + "elementary_mathematics": 0.246684350132626, + "human_aging": 0.22072072072072071, + "college_mathematics": 0.30303030303030304, + "high_school_psychology": 0.27205882352941174, + "formal_logic": 0.32, + "high_school_statistics": 0.2372093023255814, + "international_law": 0.225, + "high_school_mathematics": 0.241635687732342, + "high_school_computer_science": 0.20202020202020202, + "conceptual_physics": 0.27350427350427353, + "miscellaneous": 0.23785166240409208, + "high_school_chemistry": 0.2524752475247525, + "marketing": 0.26609442060085836, + "professional_law": 0.23613829093281147, + "management": 0.2647058823529412, + "college_physics": 0.16831683168316833, + "jurisprudence": 0.19626168224299065, + "world_religions": 0.23529411764705882, + "sociology": 0.23, + "us_foreign_policy": 0.25252525252525254, + "high_school_macroeconomics": 0.23136246786632392, + "computer_security": 0.2727272727272727, + "moral_scenarios": 0.24049217002237136, + "moral_disputes": 0.24057971014492754, + "electrical_engineering": 0.2916666666666667, + "astronomy": 0.2185430463576159, + "college_biology": 0.2727272727272727 + } } }, - "sg_eval": { + "c_eval": { "prompt_1": { - "accuracy": 0.2815533980582524 + "accuracy": 0.24294205052005943 } }, - "cn_eval": { + "c_eval_full": { "prompt_1": { - "accuracy": 0.2761904761904762 + "accuracy": 0.2752179327521793, + "category_acc": { + "computer_network": 0.375, + "operating_system": 0.20833333333333334, + "computer_architecture": 0.19230769230769232, + "college_programming": 0.23809523809523808, + "college_physics": 0.2916666666666667, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.3333333333333333, + "probability_and_statistics": 0.30434782608695654, + "discrete_mathematics": 0.19047619047619047, + "electrical_engineer": 0.2857142857142857, + "metrology_engineer": 0.20689655172413793, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.20833333333333334, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.375, + "middle_school_mathematics": 0.20833333333333334, + "middle_school_biology": 0.23076923076923078, + "middle_school_physics": 0.16666666666666666, + "middle_school_chemistry": 0.28, + "veterinary_medicine": 0.4642857142857143, + "college_economics": 0.35, + "business_administration": 0.3157894736842105, + "marxism": 0.25, + "mao_zedong_thought": 0.2413793103448276, + "education_science": 0.3235294117647059, + "teacher_qualification": 0.42857142857142855, + "high_school_politics": 0.20833333333333334, + "high_school_geography": 0.20833333333333334, + "middle_school_politics": 0.38461538461538464, + "middle_school_geography": 0.11764705882352941, + "modern_chinese_history": 0.17857142857142858, + "ideological_and_moral_cultivation": 0.125, + "logic": 0.2962962962962963, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.2857142857142857, + "art_studies": 0.3157894736842105, + "professional_tour_guide": 0.3235294117647059, + "legal_professional": 0.17857142857142858, + "high_school_chinese": 0.08333333333333333, + "high_school_history": 0.32, + "middle_school_history": 0.2962962962962963, + "civil_servant": 0.2692307692307692, + "sports_science": 0.3333333333333333, + "plant_protection": 0.25925925925925924, + "basic_medicine": 0.20833333333333334, + "clinical_medicine": 0.2222222222222222, + "urban_and_rural_planner": 0.27450980392156865, + "accountant": 0.2962962962962963, + "fire_engineer": 0.2222222222222222, + "environmental_impact_assessment_engineer": 0.16666666666666666, + "tax_accountant": 0.3148148148148148, + "physician": 0.3148148148148148 + } } }, - "us_eval": { + "cmmlu": { "prompt_1": { - "accuracy": 0.42990654205607476 + "accuracy": 0.27956989247311825 } }, - "ph_eval": { + "cmmlu_full": { "prompt_1": { - "accuracy": 0.35, + "accuracy": 0.24987048868934553, "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.2, - "history": 0.3333333333333333, - "literature": 0.4, - "politics": 0.4, - "culture": 0.4, - "film": 0.3, - "law": 0.4, - "geography": 0.4 + "agronomy": 0.26627218934911245, + "anatomy": 0.22972972972972974, + "ancient_chinese": 0.21951219512195122, + "arts": 0.225, + "astronomy": 0.26666666666666666, + "business_ethics": 0.2727272727272727, + "chinese_civil_service_exam": 0.2875, + "chinese_driving_rule": 0.2366412213740458, + "chinese_food_culture": 0.2647058823529412, + "chinese_foreign_policy": 0.308411214953271, + "chinese_history": 0.2260061919504644, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.22346368715083798, + "clinical_knowledge": 0.26582278481012656, + "college_actuarial_science": 0.2169811320754717, + "college_education": 0.2523364485981308, + "college_engineering_hydrology": 0.27358490566037735, + "college_law": 0.17592592592592593, + "college_mathematics": 0.3142857142857143, + "college_medical_statistics": 0.24528301886792453, + "college_medicine": 0.20512820512820512, + "computer_science": 0.23039215686274508, + "computer_security": 0.1871345029239766, + "conceptual_physics": 0.2585034013605442, + "construction_project_management": 0.17985611510791366, + "economics": 0.1761006289308176, + "education": 0.22085889570552147, + "electrical_engineering": 0.26744186046511625, + "elementary_chinese": 0.25793650793650796, + "elementary_commonsense": 0.20202020202020202, + "elementary_information_and_technology": 0.3067226890756303, + "elementary_mathematics": 0.2391304347826087, + "ethnology": 0.21481481481481482, + "food_science": 0.3006993006993007, + "genetics": 0.24431818181818182, + "global_facts": 0.2550335570469799, + "high_school_biology": 0.2603550295857988, + "high_school_chemistry": 0.23484848484848486, + "high_school_geography": 0.2796610169491525, + "high_school_mathematics": 0.2865853658536585, + "high_school_physics": 0.2636363636363636, + "high_school_politics": 0.2727272727272727, + "human_sexuality": 0.23809523809523808, + "international_law": 0.2756756756756757, + "journalism": 0.23255813953488372, + "jurisprudence": 0.2773722627737226, + "legal_and_moral_basis": 0.29439252336448596, + "logical": 0.2764227642276423, + "machine_learning": 0.2786885245901639, + "management": 0.3238095238095238, + "marketing": 0.2611111111111111, + "marxist_theory": 0.2275132275132275, + "modern_chinese": 0.3275862068965517, + "nutrition": 0.2689655172413793, + "philosophy": 0.2761904761904762, + "professional_accounting": 0.19428571428571428, + "professional_law": 0.26540284360189575, + "professional_medicine": 0.2579787234042553, + "professional_psychology": 0.23275862068965517, + "public_relations": 0.20689655172413793, + "security_study": 0.23703703703703705, + "sociology": 0.2168141592920354, + "sports_science": 0.2787878787878788, + "traditional_chinese_medicine": 0.25405405405405407, + "virology": 0.2603550295857988, + "world_history": 0.2111801242236025, + "world_religions": 0.225 } } }, - "sing2eng": { + "zbench": { "prompt_1": { - "bleu_score": 0.08961631055433242 + "accuracy": 0.3939393939393939 } }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { + "ind_emotion": { "prompt_1": { - "bleu_score": 0.1141264375841478 + "accuracy": 0.17272727272727273 } }, - "flores_vie2eng": { + "ocnli": { "prompt_1": { - "bleu_score": 0.11416046378349158 + "accuracy": 0.32440677966101694 } }, - "flores_zho2eng": { + "c3": { "prompt_1": { - "bleu_score": 0.10438668987411628 + "accuracy": 0.2782348541510845 } }, - "flores_zsm2eng": { + "dream": { "prompt_1": { - "bleu_score": 0.11703502449197813 + "accuracy": 0.3390494855463008 } }, - "mmlu": { + "samsum": { "prompt_1": { - "accuracy": 0.45857642940490084 + "rouge1": 0.0895541253352484, + "rouge2": 0.0015213583740047467, + "rougeL": 0.0719862724420868, + "avg_rouge": 0.054353918717113314 } }, - "mmlu_full": { + "dialogsum": { "prompt_1": { - "accuracy": 0.4541294243832678, - "category_acc": { - "high_school_european_history": 0.6341463414634146, - "business_ethics": 0.4444444444444444, - "clinical_knowledge": 0.4810606060606061, - "medical_genetics": 0.46464646464646464, - "high_school_us_history": 0.625615763546798, - "high_school_physics": 0.28, - "high_school_world_history": 0.6016949152542372, - "virology": 0.3939393939393939, - "high_school_microeconomics": 0.48945147679324896, - "econometrics": 0.37168141592920356, - "college_computer_science": 0.40404040404040403, - "high_school_biology": 0.48220064724919093, - "abstract_algebra": 0.2727272727272727, - "professional_accounting": 0.42704626334519574, - "philosophy": 0.4967741935483871, - "professional_medicine": 0.5719557195571956, - "nutrition": 0.4786885245901639, - "global_facts": 0.31313131313131315, - "machine_learning": 0.34234234234234234, - "security_studies": 0.5368852459016393, - "public_relations": 0.43119266055045874, - "professional_psychology": 0.4795417348608838, - "prehistory": 0.5201238390092879, - "anatomy": 0.4253731343283582, - "human_sexuality": 0.6153846153846154, - "college_medicine": 0.46511627906976744, - "high_school_government_and_politics": 0.6302083333333334, - "college_chemistry": 0.36363636363636365, - "logical_fallacies": 0.5679012345679012, - "high_school_geography": 0.4365482233502538, - "elementary_mathematics": 0.35278514588859416, - "human_aging": 0.5135135135135135, - "college_mathematics": 0.29292929292929293, - "high_school_psychology": 0.53125, - "formal_logic": 0.344, - "high_school_statistics": 0.35348837209302325, - "international_law": 0.5083333333333333, - "high_school_mathematics": 0.2899628252788104, - "high_school_computer_science": 0.494949494949495, - "conceptual_physics": 0.4829059829059829, - "miscellaneous": 0.5127877237851662, - "high_school_chemistry": 0.39603960396039606, - "marketing": 0.6051502145922747, - "professional_law": 0.3842139595564253, - "management": 0.5196078431372549, - "college_physics": 0.3465346534653465, - "jurisprudence": 0.616822429906542, - "world_religions": 0.43529411764705883, - "sociology": 0.59, - "us_foreign_policy": 0.5353535353535354, - "high_school_macroeconomics": 0.4755784061696658, - "computer_security": 0.5050505050505051, - "moral_scenarios": 0.24049217002237136, - "moral_disputes": 0.518840579710145, - "electrical_engineering": 0.5, - "astronomy": 0.4304635761589404, - "college_biology": 0.5874125874125874 - } + "rouge1": 0.10633392487487775, + "rouge2": 0.0017154941667223626, + "rougeL": 0.08807783273920188, + "avg_rouge": 0.06537575059360066 } }, - "c_eval": { + "sst2": { "prompt_1": { - "accuracy": 0.3573551263001486 + "accuracy": 0.49426605504587157 } }, - "c_eval_full": { + "cola": { "prompt_1": { - "accuracy": 0.35678704856787047, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.375, - "computer_architecture": 0.5, - "college_programming": 0.4523809523809524, - "college_physics": 0.125, - "college_chemistry": 0.10344827586206896, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.21428571428571427, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.5, - "high_school_biology": 0.4166666666666667, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.3076923076923077, - "middle_school_physics": 0.5416666666666666, - "middle_school_chemistry": 0.6, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.36666666666666664, - "business_administration": 0.3684210526315789, - "marxism": 0.375, - "mao_zedong_thought": 0.5172413793103449, - "education_science": 0.3235294117647059, - "teacher_qualification": 0.46938775510204084, - "high_school_politics": 0.375, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.32142857142857145, - "ideological_and_moral_cultivation": 0.5, - "logic": 0.4074074074074074, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.6, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.3076923076923077, - "sports_science": 0.20833333333333334, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.375, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.4074074074074074, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.2777777777777778, - "physician": 0.3333333333333333 - } + "accuracy": 0.50143815915628 } }, - "cmmlu": { + "qqp": { "prompt_1": { - "accuracy": 0.4229390681003584 + "accuracy": 0.506 } }, - "cmmlu_full": { + "mnli": { "prompt_1": { - "accuracy": 0.39803142807805214, - "category_acc": { - "agronomy": 0.3254437869822485, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.2682926829268293, - "arts": 0.38125, - "astronomy": 0.3515151515151515, - "business_ethics": 0.49760765550239233, - "chinese_civil_service_exam": 0.3375, - "chinese_driving_rule": 0.5190839694656488, - "chinese_food_culture": 0.3382352941176471, - "chinese_foreign_policy": 0.4672897196261682, - "chinese_history": 0.42724458204334365, - "chinese_literature": 0.37745098039215685, - "chinese_teacher_qualification": 0.4692737430167598, - "clinical_knowledge": 0.379746835443038, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.48598130841121495, - "college_engineering_hydrology": 0.36792452830188677, - "college_law": 0.32407407407407407, - "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.42452830188679247, - "college_medicine": 0.31868131868131866, - "computer_science": 0.45098039215686275, - "computer_security": 0.49707602339181284, - "conceptual_physics": 0.35374149659863946, - "construction_project_management": 0.35251798561151076, - "economics": 0.4276729559748428, - "education": 0.4539877300613497, - "electrical_engineering": 0.4186046511627907, - "elementary_chinese": 0.27380952380952384, - "elementary_commonsense": 0.3787878787878788, - "elementary_information_and_technology": 0.5798319327731093, - "elementary_mathematics": 0.29130434782608694, - "ethnology": 0.34074074074074073, - "food_science": 0.5104895104895105, - "genetics": 0.39204545454545453, - "global_facts": 0.4429530201342282, - "high_school_biology": 0.30177514792899407, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.3559322033898305, - "high_school_mathematics": 0.3231707317073171, - "high_school_physics": 0.33636363636363636, - "high_school_politics": 0.34965034965034963, - "human_sexuality": 0.49206349206349204, - "international_law": 0.3621621621621622, - "journalism": 0.48255813953488375, - "jurisprudence": 0.39902676399026765, - "legal_and_moral_basis": 0.6401869158878505, - "logical": 0.44715447154471544, - "machine_learning": 0.5081967213114754, - "management": 0.4857142857142857, - "marketing": 0.48333333333333334, - "marxist_theory": 0.48148148148148145, - "modern_chinese": 0.33620689655172414, - "nutrition": 0.43448275862068964, - "philosophy": 0.42857142857142855, - "professional_accounting": 0.4057142857142857, - "professional_law": 0.2843601895734597, - "professional_medicine": 0.2872340425531915, - "professional_psychology": 0.4224137931034483, - "public_relations": 0.45977011494252873, - "security_study": 0.4888888888888889, - "sociology": 0.4557522123893805, - "sports_science": 0.42424242424242425, - "traditional_chinese_medicine": 0.31891891891891894, - "virology": 0.38461538461538464, - "world_history": 0.40372670807453415, - "world_religions": 0.4 - } + "accuracy": 0.2 } }, - "zbench": { + "qnli": { "prompt_1": { - "accuracy": 0.24242424242424243 + "accuracy": 0.5 } }, - "ind_emotion": { + "wnli": { "prompt_1": { - "accuracy": 0.47954545454545455 + "accuracy": 0.8 } }, - "ocnli": { + "rte": { "prompt_1": { - "accuracy": 0.3786440677966102 + "accuracy": 0.2 } }, - "c3": { + "mrpc": { "prompt_1": { - "accuracy": 0.6462228870605834 + "accuracy": 0.6 } + } + } + }, + "meta_llama_3_8b": { + "model_size": "8B", + "model_link": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", + "zero_shot": { + "cross_xquad": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cross_mmlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cross_logiqa": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "sg_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cn_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "us_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ph_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "sing2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "indommlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_ind2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_vie2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_zho2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_zsm2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mmlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mmlu_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c_eval_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cmmlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cmmlu_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "zbench": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ind_emotion": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ocnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c3": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.8368446839784419 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.2072998882301162, - "rouge2": 0.08540604530604064, - "rougeL": 0.1642875288787436, - "avg_rouge": 0.15233115413830015 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.18434813342233058, - "rouge2": 0.06512733850830763, - "rougeL": 0.1446562444989155, - "avg_rouge": 0.13137723880985122 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.5126146788990825 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.3700862895493768 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.5945 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.42 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.4885 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.4507042253521127 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.5812274368231047 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.5416666666666666 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 } - } - }, - "chatglm2_6b": { - "model_size": "6B", - "model_link": "https://huggingface.co/THUDM/chatglm2-6b", - "zero_shot": { + }, + "five_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.7756302521008404, - "language_acc": { - "English": 0.8596638655462185, - "Chinese": 0.838655462184874, - "Spanish": 0.8084033613445378, - "Vietnamese": 0.5957983193277311 - }, - "consistency_score_2": 0.707983193277311, - "consistency_score_3": 0.5865546218487394, - "consistency_score_4": 0.5058823529411764, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8361344537815126, - "English,Spanish": 0.8277310924369747, - "English,Vietnamese": 0.5966386554621849, - "Chinese,Spanish": 0.7966386554621848, - "Chinese,Vietnamese": 0.5941176470588235, - "Spanish,Vietnamese": 0.5966386554621849 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7403361344537815, - "English,Chinese,Vietnamese": 0.5403361344537815, - "English,Spanish,Vietnamese": 0.5394957983193277, - "Chinese,Spanish,Vietnamese": 0.5260504201680672 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5058823529411764 - } - }, - "AC3_2": 0.7402644999727128, - "AC3_3": 0.6679702849187443, - "AC3_4": 0.6123664416108461 - }, - "prompt_2": { - "overall_acc": 0.7743697478991597, - "language_acc": { - "English": 0.8697478991596639, - "Chinese": 0.8352941176470589, - "Spanish": 0.8092436974789916, - "Vietnamese": 0.5831932773109244 - }, - "consistency_score_2": 0.7054621848739494, - "consistency_score_3": 0.5796218487394957, - "consistency_score_4": 0.492436974789916, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8487394957983193, - "English,Spanish": 0.8428571428571429, - "English,Vietnamese": 0.5907563025210084, - "Chinese,Spanish": 0.7949579831932773, - "Chinese,Vietnamese": 0.5680672268907563, - "Spanish,Vietnamese": 0.5873949579831933 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7529411764705882, - "English,Chinese,Vietnamese": 0.5277310924369748, - "English,Spanish,Vietnamese": 0.5302521008403361, - "Chinese,Spanish,Vietnamese": 0.507563025210084 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.492436974789916 - } - }, - "AC3_2": 0.738311644880653, - "AC3_3": 0.6629902666550782, - "AC3_4": 0.6020307425977934 - }, - "prompt_3": { - "overall_acc": 0.7682773109243697, - "language_acc": { - "English": 0.8630252100840337, - "Chinese": 0.8285714285714286, - "Spanish": 0.8084033613445378, - "Vietnamese": 0.573109243697479 - }, - "consistency_score_2": 0.6945378151260503, - "consistency_score_3": 0.5653361344537815, - "consistency_score_4": 0.4815126050420168, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8285714285714286, - "English,Spanish": 0.8277310924369747, - "English,Vietnamese": 0.5714285714285714, - "Chinese,Spanish": 0.7924369747899159, - "Chinese,Vietnamese": 0.5722689075630252, - "Spanish,Vietnamese": 0.5747899159663865 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7361344537815127, - "English,Chinese,Vietnamese": 0.5117647058823529, - "English,Spanish,Vietnamese": 0.5117647058823529, - "Chinese,Spanish,Vietnamese": 0.5016806722689076 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.4815126050420168 - } - }, - "AC3_2": 0.7295489845590503, - "AC3_3": 0.6513655461696453, - "AC3_4": 0.5919958300547186 - }, - "prompt_4": { - "overall_acc": 0.7733193277310924, + "overall_acc": 0.9142857142857143, "language_acc": { - "English": 0.8705882352941177, - "Chinese": 0.8403361344537815, - "Spanish": 0.8084033613445378, - "Vietnamese": 0.5739495798319327 - }, - "consistency_score_2": 0.695518207282913, - "consistency_score_3": 0.5682773109243697, - "consistency_score_4": 0.4789915966386555, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8445378151260504, - "English,Spanish": 0.8378151260504202, - "English,Vietnamese": 0.5680672268907563, - "Chinese,Spanish": 0.7924369747899159, - "Chinese,Vietnamese": 0.553781512605042, - "Spanish,Vietnamese": 0.5764705882352941 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7495798319327731, - "English,Chinese,Vietnamese": 0.5100840336134453, - "English,Spanish,Vietnamese": 0.519327731092437, - "Chinese,Spanish,Vietnamese": 0.49411764705882355 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.4789915966386555 - } + "Spanish": 0.9117647058823529, + "English": 0.9302521008403362, + "Chinese": 0.9058823529411765, + "Vietnamese": 0.9092436974789916 }, - "AC3_2": 0.7323582896307325, - "AC3_3": 0.6551295901524221, - "AC3_4": 0.5915678802503263 - }, - "prompt_5": { - "overall_acc": 0.7615546218487395, - "language_acc": { - "English": 0.8672268907563025, - "Chinese": 0.8218487394957983, - "Spanish": 0.7949579831932773, - "Vietnamese": 0.5621848739495798 - }, - "consistency_score_2": 0.6775910364145658, - "consistency_score_3": 0.5409663865546218, - "consistency_score_4": 0.4495798319327731, + "consistency_score_2": 0.9068627450980392, + "consistency_score_3": 0.8649159663865547, + "consistency_score_4": 0.8369747899159664, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.8168067226890756, - "English,Spanish": 0.8050420168067227, - "English,Vietnamese": 0.5705882352941176, - "Chinese,Spanish": 0.7680672268907563, - "Chinese,Vietnamese": 0.5369747899159664, - "Spanish,Vietnamese": 0.5680672268907563 + "Spanish,English": 0.9294117647058824, + "Spanish,Chinese": 0.8907563025210085, + "Spanish,Vietnamese": 0.9042016806722689, + "English,Chinese": 0.9092436974789916, + "English,Vietnamese": 0.9159663865546218, + "Chinese,Vietnamese": 0.8915966386554622 }, "3_combine": { - "English,Chinese,Spanish": 0.7058823529411765, - "English,Chinese,Vietnamese": 0.4907563025210084, - "English,Spanish,Vietnamese": 0.49747899159663866, - "Chinese,Spanish,Vietnamese": 0.46974789915966386 + "Spanish,English,Chinese": 0.8680672268907563, + "Spanish,English,Vietnamese": 0.880672268907563, + "Spanish,Chinese,Vietnamese": 0.8487394957983193, + "English,Chinese,Vietnamese": 0.8621848739495799 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.4495798319327731 + "Spanish,English,Chinese,Vietnamese": 0.8369747899159664 } }, - "AC3_2": 0.7171234996347017, - "AC3_3": 0.6325816616483709, - "AC3_4": 0.5653866055929716 + "AC3_2": 0.9105591016957517, + "AC3_3": 0.8889158780263842, + "AC3_4": 0.8739237729141776 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.4, + "overall_acc": 0.5561904761904761, "language_acc": { - "Spanish": 0.4, - "English": 0.54, - "Chinese": 0.5333333333333333, - "Vietnamese": 0.3466666666666667, - "Filipino": 0.3466666666666667, - "Indonesian": 0.32666666666666666, - "Malay": 0.30666666666666664 + "Filipino": 0.5133333333333333, + "Vietnamese": 0.5066666666666667, + "Chinese": 0.5466666666666666, + "Spanish": 0.5666666666666667, + "Malay": 0.52, + "Indonesian": 0.5733333333333334, + "English": 0.6666666666666666 }, - "consistency_score_2": 0.47650793650793655, - "consistency_score_3": 0.2900952380952381, - "consistency_score_4": 0.19752380952380952, - "consistency_score_5": 0.14444444444444443, - "consistency_score_6": 0.11047619047619049, - "consistency_score_7": 0.08666666666666667, + "consistency_score_2": 0.6019047619047619, + "consistency_score_3": 0.4415238095238096, + "consistency_score_4": 0.35180952380952385, + "consistency_score_5": 0.29460317460317464, + "consistency_score_6": 0.2571428571428572, + "consistency_score_7": 0.23333333333333334, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5466666666666666, - "Spanish,Chinese": 0.54, - "Spanish,Vietnamese": 0.47333333333333333, - "Spanish,Filipino": 0.46, - "Spanish,Indonesian": 0.46, - "Spanish,Malay": 0.46, - "English,Chinese": 0.6533333333333333, - "English,Vietnamese": 0.47333333333333333, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.42, - "English,Malay": 0.4, - "Chinese,Vietnamese": 0.3466666666666667, - "Chinese,Filipino": 0.4, - "Chinese,Indonesian": 0.42, - "Chinese,Malay": 0.37333333333333335, - "Vietnamese,Filipino": 0.5133333333333333, - "Vietnamese,Indonesian": 0.5666666666666667, - "Vietnamese,Malay": 0.52, - "Filipino,Indonesian": 0.48, - "Filipino,Malay": 0.4533333333333333, - "Indonesian,Malay": 0.58 + "Filipino,Vietnamese": 0.5266666666666666, + "Filipino,Chinese": 0.5533333333333333, + "Filipino,Spanish": 0.6666666666666666, + "Filipino,Malay": 0.5533333333333333, + "Filipino,Indonesian": 0.6266666666666667, + "Filipino,English": 0.6466666666666666, + "Vietnamese,Chinese": 0.54, + "Vietnamese,Spanish": 0.6, + "Vietnamese,Malay": 0.6, + "Vietnamese,Indonesian": 0.58, + "Vietnamese,English": 0.6133333333333333, + "Chinese,Spanish": 0.5733333333333334, + "Chinese,Malay": 0.5533333333333333, + "Chinese,Indonesian": 0.56, + "Chinese,English": 0.58, + "Spanish,Malay": 0.6, + "Spanish,Indonesian": 0.64, + "Spanish,English": 0.7133333333333334, + "Malay,Indonesian": 0.6133333333333333, + "Malay,English": 0.6466666666666666, + "Indonesian,English": 0.6533333333333333 }, "3_combine": { - "Spanish,English,Chinese": 0.43333333333333335, - "Spanish,English,Vietnamese": 0.32, - "Spanish,English,Filipino": 0.30666666666666664, - "Spanish,English,Indonesian": 0.30666666666666664, - "Spanish,English,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese": 0.26, - "Spanish,Chinese,Filipino": 0.29333333333333333, - "Spanish,Chinese,Indonesian": 0.3, - "Spanish,Chinese,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,Vietnamese,Malay": 0.29333333333333333, - "Spanish,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Filipino,Malay": 0.28, - "Spanish,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese": 0.28, - "English,Chinese,Filipino": 0.31333333333333335, - "English,Chinese,Indonesian": 0.3, - "English,Chinese,Malay": 0.26, - "English,Vietnamese,Filipino": 0.3, - "English,Vietnamese,Indonesian": 0.29333333333333333, - "English,Vietnamese,Malay": 0.26, - "English,Filipino,Indonesian": 0.2733333333333333, - "English,Filipino,Malay": 0.26, - "English,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino": 0.22666666666666666, - "Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "Chinese,Vietnamese,Malay": 0.18666666666666668, - "Chinese,Filipino,Indonesian": 0.28, - "Chinese,Filipino,Malay": 0.23333333333333334, - "Chinese,Indonesian,Malay": 0.26, - "Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "Vietnamese,Filipino,Malay": 0.3, - "Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Filipino,Indonesian,Malay": 0.32666666666666666 + "Filipino,Vietnamese,Chinese": 0.36666666666666664, + "Filipino,Vietnamese,Spanish": 0.43333333333333335, + "Filipino,Vietnamese,Malay": 0.4, + "Filipino,Vietnamese,Indonesian": 0.4, + "Filipino,Vietnamese,English": 0.43333333333333335, + "Filipino,Chinese,Spanish": 0.43333333333333335, + "Filipino,Chinese,Malay": 0.38, + "Filipino,Chinese,Indonesian": 0.4, + "Filipino,Chinese,English": 0.42, + "Filipino,Spanish,Malay": 0.4666666666666667, + "Filipino,Spanish,Indonesian": 0.5, + "Filipino,Spanish,English": 0.5466666666666666, + "Filipino,Malay,Indonesian": 0.4266666666666667, + "Filipino,Malay,English": 0.4666666666666667, + "Filipino,Indonesian,English": 0.48, + "Vietnamese,Chinese,Spanish": 0.4066666666666667, + "Vietnamese,Chinese,Malay": 0.38666666666666666, + "Vietnamese,Chinese,Indonesian": 0.4066666666666667, + "Vietnamese,Chinese,English": 0.42, + "Vietnamese,Spanish,Malay": 0.44666666666666666, + "Vietnamese,Spanish,Indonesian": 0.44, + "Vietnamese,Spanish,English": 0.4866666666666667, + "Vietnamese,Malay,Indonesian": 0.44666666666666666, + "Vietnamese,Malay,English": 0.47333333333333333, + "Vietnamese,Indonesian,English": 0.4533333333333333, + "Chinese,Spanish,Malay": 0.4, + "Chinese,Spanish,Indonesian": 0.41333333333333333, + "Chinese,Spanish,English": 0.4666666666666667, + "Chinese,Malay,Indonesian": 0.41333333333333333, + "Chinese,Malay,English": 0.42, + "Chinese,Indonesian,English": 0.43333333333333335, + "Spanish,Malay,Indonesian": 0.46, + "Spanish,Malay,English": 0.5066666666666667, + "Spanish,Indonesian,English": 0.5266666666666666, + "Malay,Indonesian,English": 0.49333333333333335 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.22666666666666666, - "Spanish,English,Chinese,Filipino": 0.23333333333333334, - "Spanish,English,Chinese,Indonesian": 0.24, - "Spanish,English,Chinese,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.22, - "Spanish,English,Vietnamese,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Filipino,Malay": 0.2, - "Spanish,English,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Malay": 0.15333333333333332, - "Spanish,Chinese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.19333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.22, - "English,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "English,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "English,Chinese,Vietnamese,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Filipino,Malay": 0.17333333333333334, - "English,Chinese,Indonesian,Malay": 0.16666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "English,Filipino,Indonesian,Malay": 0.18, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Malay": 0.14, - "Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 + "Filipino,Vietnamese,Chinese,Spanish": 0.31333333333333335, + "Filipino,Vietnamese,Chinese,Malay": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.3, + "Filipino,Vietnamese,Chinese,English": 0.31333333333333335, + "Filipino,Vietnamese,Spanish,Malay": 0.3466666666666667, + "Filipino,Vietnamese,Spanish,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,Spanish,English": 0.38666666666666666, + "Filipino,Vietnamese,Malay,Indonesian": 0.3333333333333333, + "Filipino,Vietnamese,Malay,English": 0.35333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.35333333333333333, + "Filipino,Chinese,Spanish,Malay": 0.3333333333333333, + "Filipino,Chinese,Spanish,Indonesian": 0.3466666666666667, + "Filipino,Chinese,Spanish,English": 0.38, + "Filipino,Chinese,Malay,Indonesian": 0.32, + "Filipino,Chinese,Malay,English": 0.32, + "Filipino,Chinese,Indonesian,English": 0.34, + "Filipino,Spanish,Malay,Indonesian": 0.37333333333333335, + "Filipino,Spanish,Malay,English": 0.4066666666666667, + "Filipino,Spanish,Indonesian,English": 0.43333333333333335, + "Filipino,Malay,Indonesian,English": 0.37333333333333335, + "Vietnamese,Chinese,Spanish,Malay": 0.32, + "Vietnamese,Chinese,Spanish,Indonesian": 0.32, + "Vietnamese,Chinese,Spanish,English": 0.36, + "Vietnamese,Chinese,Malay,Indonesian": 0.3333333333333333, + "Vietnamese,Chinese,Malay,English": 0.34, + "Vietnamese,Chinese,Indonesian,English": 0.3466666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.38, + "Vietnamese,Spanish,Malay,English": 0.38, + "Vietnamese,Spanish,Indonesian,English": 0.3933333333333333, + "Vietnamese,Malay,Indonesian,English": 0.38666666666666666, + "Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, + "Chinese,Spanish,Malay,English": 0.34, + "Chinese,Spanish,Indonesian,English": 0.36, + "Chinese,Malay,Indonesian,English": 0.36, + "Spanish,Malay,Indonesian,English": 0.4 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.26, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.26, + "Filipino,Vietnamese,Chinese,Malay,English": 0.26, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.2733333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Spanish,Malay,English": 0.31333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.3333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.30666666666666664, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2866666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.2866666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.31333333333333335, + "Filipino,Chinese,Malay,Indonesian,English": 0.2866666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.34, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.28, + "Vietnamese,Chinese,Spanish,Malay,English": 0.2866666666666667, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.3, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.30666666666666664, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.34, + "Chinese,Spanish,Malay,Indonesian,English": 0.2866666666666667 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.24, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.26, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.24666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.29333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.26 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334 } }, - "AC3_2": 0.43491488586128285, - "AC3_3": 0.3362958873374122, - "AC3_4": 0.26445648704531727, - "AC3_5": 0.21224489792019988, - "AC3_6": 0.1731343283242927, - "AC3_7": 0.14246575339538375 - }, - "prompt_2": { - "overall_acc": 0.4, + "AC3_2": 0.5781453633585992, + "AC3_3": 0.49226785936628614, + "AC3_4": 0.4309980320581802, + "AC3_5": 0.3851826580926087, + "AC3_6": 0.35168952822708405, + "AC3_7": 0.3287494973447703 + } + }, + "cross_logiqa": { + "prompt_1": { + "overall_acc": 0.49512987012987014, "language_acc": { - "Spanish": 0.3933333333333333, - "English": 0.52, - "Chinese": 0.5266666666666666, - "Vietnamese": 0.37333333333333335, - "Filipino": 0.34, - "Indonesian": 0.3333333333333333, - "Malay": 0.31333333333333335 - }, - "consistency_score_2": 0.4676190476190477, - "consistency_score_3": 0.27638095238095245, - "consistency_score_4": 0.1862857142857143, - "consistency_score_5": 0.13714285714285712, - "consistency_score_6": 0.10666666666666667, - "consistency_score_7": 0.08666666666666667, + "Indonesian": 0.4431818181818182, + "English": 0.5625, + "Filipino": 0.42613636363636365, + "Spanish": 0.5170454545454546, + "Chinese": 0.5227272727272727, + "Malay": 0.4715909090909091, + "Vietnamese": 0.5227272727272727 + }, + "consistency_score_2": 0.5687229437229437, + "consistency_score_3": 0.39545454545454545, + "consistency_score_4": 0.3003246753246754, + "consistency_score_5": 0.24053030303030298, + "consistency_score_6": 0.2012987012987013, + "consistency_score_7": 0.17613636363636365, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.49333333333333335, - "Spanish,Chinese": 0.5133333333333333, - "Spanish,Vietnamese": 0.4666666666666667, - "Spanish,Filipino": 0.43333333333333335, - "Spanish,Indonesian": 0.47333333333333333, - "Spanish,Malay": 0.44666666666666666, - "English,Chinese": 0.6466666666666666, - "English,Vietnamese": 0.4666666666666667, - "English,Filipino": 0.4066666666666667, - "English,Indonesian": 0.3933333333333333, - "English,Malay": 0.38666666666666666, - "Chinese,Vietnamese": 0.4, - "Chinese,Filipino": 0.38666666666666666, - "Chinese,Indonesian": 0.4, - "Chinese,Malay": 0.41333333333333333, - "Vietnamese,Filipino": 0.46, - "Vietnamese,Indonesian": 0.5466666666666666, - "Vietnamese,Malay": 0.4866666666666667, - "Filipino,Indonesian": 0.49333333333333335, - "Filipino,Malay": 0.5333333333333333, - "Indonesian,Malay": 0.5733333333333334 + "Indonesian,English": 0.5454545454545454, + "Indonesian,Filipino": 0.5227272727272727, + "Indonesian,Spanish": 0.5738636363636364, + "Indonesian,Chinese": 0.5340909090909091, + "Indonesian,Malay": 0.6477272727272727, + "Indonesian,Vietnamese": 0.6363636363636364, + "English,Filipino": 0.48863636363636365, + "English,Spanish": 0.6136363636363636, + "English,Chinese": 0.5397727272727273, + "English,Malay": 0.6079545454545454, + "English,Vietnamese": 0.5681818181818182, + "Filipino,Spanish": 0.5284090909090909, + "Filipino,Chinese": 0.5170454545454546, + "Filipino,Malay": 0.5227272727272727, + "Filipino,Vietnamese": 0.5340909090909091, + "Spanish,Chinese": 0.5852272727272727, + "Spanish,Malay": 0.5795454545454546, + "Spanish,Vietnamese": 0.6136363636363636, + "Chinese,Malay": 0.5454545454545454, + "Chinese,Vietnamese": 0.6136363636363636, + "Malay,Vietnamese": 0.625 }, "3_combine": { - "Spanish,English,Chinese": 0.38, - "Spanish,English,Vietnamese": 0.2866666666666667, - "Spanish,English,Filipino": 0.23333333333333334, - "Spanish,English,Indonesian": 0.2733333333333333, - "Spanish,English,Malay": 0.26, - "Spanish,Chinese,Vietnamese": 0.25333333333333335, - "Spanish,Chinese,Filipino": 0.24666666666666667, - "Spanish,Chinese,Indonesian": 0.26, - "Spanish,Chinese,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,Vietnamese,Malay": 0.28, - "Spanish,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Filipino,Malay": 0.28, - "Spanish,Indonesian,Malay": 0.3, - "English,Chinese,Vietnamese": 0.31333333333333335, - "English,Chinese,Filipino": 0.28, - "English,Chinese,Indonesian": 0.2733333333333333, - "English,Chinese,Malay": 0.2866666666666667, - "English,Vietnamese,Filipino": 0.25333333333333335, - "English,Vietnamese,Indonesian": 0.2866666666666667, - "English,Vietnamese,Malay": 0.24666666666666667, - "English,Filipino,Indonesian": 0.25333333333333335, - "English,Filipino,Malay": 0.25333333333333335, - "English,Indonesian,Malay": 0.26, - "Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "Chinese,Vietnamese,Malay": 0.23333333333333334, - "Chinese,Filipino,Indonesian": 0.24666666666666667, - "Chinese,Filipino,Malay": 0.26666666666666666, - "Chinese,Indonesian,Malay": 0.25333333333333335, - "Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Vietnamese,Filipino,Malay": 0.30666666666666664, - "Vietnamese,Indonesian,Malay": 0.3466666666666667, - "Filipino,Indonesian,Malay": 0.36666666666666664 + "Indonesian,English,Filipino": 0.32954545454545453, + "Indonesian,English,Spanish": 0.4147727272727273, + "Indonesian,English,Chinese": 0.36363636363636365, + "Indonesian,English,Malay": 0.42613636363636365, + "Indonesian,English,Vietnamese": 0.4147727272727273, + "Indonesian,Filipino,Spanish": 0.375, + "Indonesian,Filipino,Chinese": 0.3181818181818182, + "Indonesian,Filipino,Malay": 0.4034090909090909, + "Indonesian,Filipino,Vietnamese": 0.4034090909090909, + "Indonesian,Spanish,Chinese": 0.4090909090909091, + "Indonesian,Spanish,Malay": 0.4375, + "Indonesian,Spanish,Vietnamese": 0.4431818181818182, + "Indonesian,Chinese,Malay": 0.39204545454545453, + "Indonesian,Chinese,Vietnamese": 0.42613636363636365, + "Indonesian,Malay,Vietnamese": 0.48863636363636365, + "English,Filipino,Spanish": 0.36363636363636365, + "English,Filipino,Chinese": 0.3409090909090909, + "English,Filipino,Malay": 0.35795454545454547, + "English,Filipino,Vietnamese": 0.3465909090909091, + "English,Spanish,Chinese": 0.4090909090909091, + "English,Spanish,Malay": 0.4375, + "English,Spanish,Vietnamese": 0.42045454545454547, + "English,Chinese,Malay": 0.38636363636363635, + "English,Chinese,Vietnamese": 0.3977272727272727, + "English,Malay,Vietnamese": 0.4375, + "Filipino,Spanish,Chinese": 0.35795454545454547, + "Filipino,Spanish,Malay": 0.375, + "Filipino,Spanish,Vietnamese": 0.3693181818181818, + "Filipino,Chinese,Malay": 0.3465909090909091, + "Filipino,Chinese,Vietnamese": 0.36363636363636365, + "Filipino,Malay,Vietnamese": 0.38636363636363635, + "Spanish,Chinese,Malay": 0.4034090909090909, + "Spanish,Chinese,Vietnamese": 0.4375, + "Spanish,Malay,Vietnamese": 0.4375, + "Chinese,Malay,Vietnamese": 0.42045454545454547 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.2, - "Spanish,English,Chinese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.16, - "Spanish,English,Vietnamese,Indonesian": 0.22, - "Spanish,English,Vietnamese,Malay": 0.18, - "Spanish,English,Filipino,Indonesian": 0.18, - "Spanish,English,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Indonesian,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Malay": 0.17333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Filipino,Malay": 0.18, - "Spanish,Chinese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.2, - "Spanish,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.16666666666666666, - "English,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "English,Chinese,Vietnamese,Malay": 0.18, - "English,Chinese,Filipino,Indonesian": 0.19333333333333333, - "English,Chinese,Filipino,Malay": 0.18666666666666668, - "English,Chinese,Indonesian,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.18, - "English,Vietnamese,Filipino,Malay": 0.18, - "English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.16, - "Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 + "Indonesian,English,Filipino,Spanish": 0.2727272727272727, + "Indonesian,English,Filipino,Chinese": 0.24431818181818182, + "Indonesian,English,Filipino,Malay": 0.29545454545454547, + "Indonesian,English,Filipino,Vietnamese": 0.2840909090909091, + "Indonesian,English,Spanish,Chinese": 0.3125, + "Indonesian,English,Spanish,Malay": 0.3409090909090909, + "Indonesian,English,Spanish,Vietnamese": 0.3409090909090909, + "Indonesian,English,Chinese,Malay": 0.2897727272727273, + "Indonesian,English,Chinese,Vietnamese": 0.3181818181818182, + "Indonesian,English,Malay,Vietnamese": 0.3522727272727273, + "Indonesian,Filipino,Spanish,Chinese": 0.2727272727272727, + "Indonesian,Filipino,Spanish,Malay": 0.3125, + "Indonesian,Filipino,Spanish,Vietnamese": 0.3125, + "Indonesian,Filipino,Chinese,Malay": 0.26704545454545453, + "Indonesian,Filipino,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,Filipino,Malay,Vietnamese": 0.32954545454545453, + "Indonesian,Spanish,Chinese,Malay": 0.3068181818181818, + "Indonesian,Spanish,Chinese,Vietnamese": 0.3352272727272727, + "Indonesian,Spanish,Malay,Vietnamese": 0.35795454545454547, + "Indonesian,Chinese,Malay,Vietnamese": 0.32954545454545453, + "English,Filipino,Spanish,Chinese": 0.26704545454545453, + "English,Filipino,Spanish,Malay": 0.2897727272727273, + "English,Filipino,Spanish,Vietnamese": 0.26704545454545453, + "English,Filipino,Chinese,Malay": 0.2556818181818182, + "English,Filipino,Chinese,Vietnamese": 0.2556818181818182, + "English,Filipino,Malay,Vietnamese": 0.2840909090909091, + "English,Spanish,Chinese,Malay": 0.3125, + "English,Spanish,Chinese,Vietnamese": 0.3181818181818182, + "English,Spanish,Malay,Vietnamese": 0.3409090909090909, + "English,Chinese,Malay,Vietnamese": 0.3181818181818182, + "Filipino,Spanish,Chinese,Malay": 0.2784090909090909, + "Filipino,Spanish,Chinese,Vietnamese": 0.2784090909090909, + "Filipino,Spanish,Malay,Vietnamese": 0.3068181818181818, + "Filipino,Chinese,Malay,Vietnamese": 0.26704545454545453, + "Spanish,Chinese,Malay,Vietnamese": 0.32386363636363635 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.14, - "Spanish,English,Chinese,Vietnamese,Malay": 0.14, - "Spanish,English,Chinese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.14, - "Spanish,English,Chinese,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,English,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.12, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 + "Indonesian,English,Filipino,Spanish,Chinese": 0.2159090909090909, + "Indonesian,English,Filipino,Spanish,Malay": 0.25, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.23863636363636365, + "Indonesian,English,Filipino,Chinese,Malay": 0.21022727272727273, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2159090909090909, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,English,Spanish,Chinese,Malay": 0.25, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2897727272727273, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.26136363636363635, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.23295454545454544, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2727272727272727, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.22727272727272727, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635, + "English,Filipino,Spanish,Chinese,Malay": 0.2159090909090909, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, + "English,Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, + "English,Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, + "English,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1875, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365 } }, - "AC3_2": 0.4311745334299964, - "AC3_3": 0.32689383267484673, - "AC3_4": 0.2541910330950454, - "AC3_5": 0.2042553191109099, - "AC3_6": 0.16842105259833795, - "AC3_7": 0.14246575339538375 - }, - "prompt_3": { - "overall_acc": 0.39619047619047626, - "language_acc": { - "Spanish": 0.4266666666666667, - "English": 0.56, - "Chinese": 0.5066666666666667, - "Vietnamese": 0.30666666666666664, - "Filipino": 0.32666666666666666, - "Indonesian": 0.35333333333333333, - "Malay": 0.29333333333333333 - }, - "consistency_score_2": 0.44349206349206355, - "consistency_score_3": 0.2537142857142857, - "consistency_score_4": 0.16666666666666666, - "consistency_score_5": 0.11936507936507937, - "consistency_score_6": 0.09142857142857143, - "consistency_score_7": 0.07333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.47333333333333333, - "Spanish,Chinese": 0.4666666666666667, - "Spanish,Vietnamese": 0.4066666666666667, - "Spanish,Filipino": 0.4266666666666667, - "Spanish,Indonesian": 0.54, - "Spanish,Malay": 0.44, - "English,Chinese": 0.5933333333333334, - "English,Vietnamese": 0.34, - "English,Filipino": 0.3466666666666667, - "English,Indonesian": 0.4266666666666667, - "English,Malay": 0.35333333333333333, - "Chinese,Vietnamese": 0.3466666666666667, - "Chinese,Filipino": 0.37333333333333335, - "Chinese,Indonesian": 0.4266666666666667, - "Chinese,Malay": 0.38666666666666666, - "Vietnamese,Filipino": 0.46, - "Vietnamese,Indonesian": 0.4666666666666667, - "Vietnamese,Malay": 0.48, - "Filipino,Indonesian": 0.5, - "Filipino,Malay": 0.48, - "Indonesian,Malay": 0.58 + "AC3_2": 0.5293809699466722, + "AC3_3": 0.43971431407252337, + "AC3_4": 0.37387357535718646, + "AC3_5": 0.3237737804234065, + "AC3_6": 0.2862289225514595, + "AC3_7": 0.2598384082277527 + } + }, + "sg_eval": { + "prompt_1": { + "accuracy": 0.6990291262135923 + } + }, + "cn_eval": { + "prompt_1": { + "accuracy": 0.42857142857142855 + } + }, + "us_eval": { + "prompt_1": { + "accuracy": 0.7102803738317757 + } + }, + "ph_eval": { + "prompt_1": { + "accuracy": 0.6, + "category_acc": { + "brand": 0.7, + "demographics": 0.2, + "biology": 0.7, + "history": 0.6666666666666666, + "literature": 0.4, + "politics": 0.9, + "culture": 0.7, + "film": 0.5, + "law": 0.4, + "geography": 0.6 + } + } + }, + "sing2eng": { + "prompt_1": { + "bleu_score": 0.35830522757252475 + } + }, + "indommlu": { + "prompt_1": { + "accuracy": 0.5063088323653114, + "category_acc": { + "History": 0.5040160642570282, + "Geography": 0.46122448979591835, + "Lampungic": 0.3469387755102041, + "Social science": 0.7395659432387313, + "Balinese": 0.30148619957537154, + "Makassarese": 0.3387096774193548, + "Banjarese": 0.4444444444444444, + "Chemistry": 0.35912408759124087, + "Biology": 0.49230769230769234, + "Science": 0.630546955624355, + "Christian religion": 0.6368159203980099, + "Art": 0.589018302828619, + "Islam religion": 0.6088193456614509, + "Hindu religion": 0.5066666666666667, + "Madurese": 0.31186440677966104, + "Sport": 0.5472972972972973, + "Indonesian language": 0.5585305105853051, + "Physics": 0.40606060606060607, + "Minangkabau culture": 0.4321608040201005, + "Dayak language": 0.3394495412844037, + "Sociology": 0.4879032258064516, + "Economy": 0.4651639344262295, + "Sundanese": 0.4373379429559205, + "Javanese": 0.39919354838709675, + "Civic education": 0.6051502145922747 + } + } + }, + "flores_ind2eng": { + "prompt_1": { + "bleu_score": 0.40201822328688375 + } + }, + "flores_vie2eng": { + "prompt_1": { + "bleu_score": 0.33775261020094416 + } + }, + "flores_zho2eng": { + "prompt_1": { + "bleu_score": 0.2651898327526456 + } + }, + "flores_zsm2eng": { + "prompt_1": { + "bleu_score": 0.41702568484678204 + } + }, + "mmlu": { + "prompt_1": { + "accuracy": 0.6114352392065344 + } + }, + "mmlu_full": { + "prompt_1": { + "accuracy": 0.6112263139077583, + "category_acc": { + "high_school_european_history": 0.7012195121951219, + "business_ethics": 0.6464646464646465, + "clinical_knowledge": 0.696969696969697, + "medical_genetics": 0.7575757575757576, + "high_school_us_history": 0.7733990147783252, + "high_school_physics": 0.4533333333333333, + "high_school_world_history": 0.8135593220338984, + "virology": 0.503030303030303, + "high_school_microeconomics": 0.729957805907173, + "econometrics": 0.45132743362831856, + "college_computer_science": 0.47474747474747475, + "high_school_biology": 0.7572815533980582, + "abstract_algebra": 0.37373737373737376, + "professional_accounting": 0.5124555160142349, + "philosophy": 0.6741935483870968, + "professional_medicine": 0.7047970479704797, + "nutrition": 0.7213114754098361, + "global_facts": 0.41414141414141414, + "machine_learning": 0.44144144144144143, + "security_studies": 0.6434426229508197, + "public_relations": 0.6697247706422018, + "professional_psychology": 0.6710310965630114, + "prehistory": 0.7089783281733746, + "anatomy": 0.664179104477612, + "human_sexuality": 0.7307692307692307, + "college_medicine": 0.627906976744186, + "high_school_government_and_politics": 0.859375, + "college_chemistry": 0.37373737373737376, + "logical_fallacies": 0.7283950617283951, + "high_school_geography": 0.7868020304568528, + "elementary_mathematics": 0.40848806366047746, + "human_aging": 0.6981981981981982, + "college_mathematics": 0.3333333333333333, + "high_school_psychology": 0.8051470588235294, + "formal_logic": 0.392, + "high_school_statistics": 0.49767441860465117, + "international_law": 0.7666666666666667, + "high_school_mathematics": 0.3754646840148699, + "high_school_computer_science": 0.6767676767676768, + "conceptual_physics": 0.5299145299145299, + "miscellaneous": 0.7928388746803069, + "high_school_chemistry": 0.5247524752475248, + "marketing": 0.8412017167381974, + "professional_law": 0.4474885844748858, + "management": 0.8333333333333334, + "college_physics": 0.4752475247524752, + "jurisprudence": 0.7383177570093458, + "world_religions": 0.8058823529411765, + "sociology": 0.8, + "us_foreign_policy": 0.8282828282828283, + "high_school_macroeconomics": 0.6041131105398457, + "computer_security": 0.7474747474747475, + "moral_scenarios": 0.2483221476510067, + "moral_disputes": 0.6753623188405797, + "electrical_engineering": 0.5902777777777778, + "astronomy": 0.695364238410596, + "college_biology": 0.7342657342657343 + } + } + }, + "c_eval": { + "prompt_1": { + "accuracy": 0.4799405646359584 + } + }, + "c_eval_full": { + "prompt_1": { + "accuracy": 0.4900373599003736, + "category_acc": { + "computer_network": 0.5416666666666666, + "operating_system": 0.5833333333333334, + "computer_architecture": 0.46153846153846156, + "college_programming": 0.47619047619047616, + "college_physics": 0.4583333333333333, + "college_chemistry": 0.3448275862068966, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.391304347826087, + "discrete_mathematics": 0.3333333333333333, + "electrical_engineer": 0.35714285714285715, + "metrology_engineer": 0.41379310344827586, + "high_school_mathematics": 0.21739130434782608, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.375, + "middle_school_mathematics": 0.375, + "middle_school_biology": 0.8076923076923077, + "middle_school_physics": 0.4583333333333333, + "middle_school_chemistry": 0.64, + "veterinary_medicine": 0.4642857142857143, + "college_economics": 0.43333333333333335, + "business_administration": 0.5526315789473685, + "marxism": 0.5, + "mao_zedong_thought": 0.5172413793103449, + "education_science": 0.5588235294117647, + "teacher_qualification": 0.6938775510204082, + "high_school_politics": 0.6666666666666666, + "high_school_geography": 0.625, + "middle_school_politics": 0.8076923076923077, + "middle_school_geography": 0.35294117647058826, + "modern_chinese_history": 0.5, + "ideological_and_moral_cultivation": 0.5833333333333334, + "logic": 0.5555555555555556, + "law": 0.5172413793103449, + "chinese_language_and_literature": 0.42857142857142855, + "art_studies": 0.5526315789473685, + "professional_tour_guide": 0.5, + "legal_professional": 0.5714285714285714, + "high_school_chinese": 0.3333333333333333, + "high_school_history": 0.56, + "middle_school_history": 0.7037037037037037, + "civil_servant": 0.4807692307692308, + "sports_science": 0.2916666666666667, + "plant_protection": 0.6666666666666666, + "basic_medicine": 0.375, + "clinical_medicine": 0.4444444444444444, + "urban_and_rural_planner": 0.5294117647058824, + "accountant": 0.5555555555555556, + "fire_engineer": 0.3055555555555556, + "environmental_impact_assessment_engineer": 0.5277777777777778, + "tax_accountant": 0.3888888888888889, + "physician": 0.5370370370370371 + } + } + }, + "cmmlu": { + "prompt_1": { + "accuracy": 0.5161290322580645 + } + }, + "cmmlu_full": { + "prompt_1": { + "accuracy": 0.4803142807805215, + "category_acc": { + "agronomy": 0.4556213017751479, + "anatomy": 0.32432432432432434, + "ancient_chinese": 0.29878048780487804, + "arts": 0.58125, + "astronomy": 0.2909090909090909, + "business_ethics": 0.5550239234449761, + "chinese_civil_service_exam": 0.49375, + "chinese_driving_rule": 0.648854961832061, + "chinese_food_culture": 0.38235294117647056, + "chinese_foreign_policy": 0.4953271028037383, + "chinese_history": 0.544891640866873, + "chinese_literature": 0.37254901960784315, + "chinese_teacher_qualification": 0.5865921787709497, + "clinical_knowledge": 0.4177215189873418, + "college_actuarial_science": 0.2358490566037736, + "college_education": 0.5514018691588785, + "college_engineering_hydrology": 0.44339622641509435, + "college_law": 0.39814814814814814, + "college_mathematics": 0.3619047619047619, + "college_medical_statistics": 0.5094339622641509, + "college_medicine": 0.38461538461538464, + "computer_science": 0.553921568627451, + "computer_security": 0.6257309941520468, + "conceptual_physics": 0.48299319727891155, + "construction_project_management": 0.3669064748201439, + "economics": 0.5723270440251572, + "education": 0.50920245398773, + "electrical_engineering": 0.4883720930232558, + "elementary_chinese": 0.3531746031746032, + "elementary_commonsense": 0.45454545454545453, + "elementary_information_and_technology": 0.7478991596638656, + "elementary_mathematics": 0.3, + "ethnology": 0.3925925925925926, + "food_science": 0.5874125874125874, + "genetics": 0.4147727272727273, + "global_facts": 0.5704697986577181, + "high_school_biology": 0.44970414201183434, + "high_school_chemistry": 0.3484848484848485, + "high_school_geography": 0.4576271186440678, + "high_school_mathematics": 0.27439024390243905, + "high_school_physics": 0.33636363636363636, + "high_school_politics": 0.45454545454545453, + "human_sexuality": 0.5079365079365079, + "international_law": 0.4486486486486487, + "journalism": 0.5116279069767442, + "jurisprudence": 0.5328467153284672, + "legal_and_moral_basis": 0.8130841121495327, + "logical": 0.5284552845528455, + "machine_learning": 0.5409836065573771, + "management": 0.580952380952381, + "marketing": 0.5555555555555556, + "marxist_theory": 0.544973544973545, + "modern_chinese": 0.3275862068965517, + "nutrition": 0.4482758620689655, + "philosophy": 0.5904761904761905, + "professional_accounting": 0.5085714285714286, + "professional_law": 0.3791469194312796, + "professional_medicine": 0.3723404255319149, + "professional_psychology": 0.4956896551724138, + "public_relations": 0.5114942528735632, + "security_study": 0.6148148148148148, + "sociology": 0.5353982300884956, + "sports_science": 0.4727272727272727, + "traditional_chinese_medicine": 0.2864864864864865, + "virology": 0.6035502958579881, + "world_history": 0.5838509316770186, + "world_religions": 0.61875 + } + } + }, + "zbench": { + "prompt_1": { + "accuracy": 0.36363636363636365 + } + }, + "ind_emotion": { + "prompt_1": { + "accuracy": 0.44545454545454544 + } + }, + "ocnli": { + "prompt_1": { + "accuracy": 0.40372881355932205 + } + }, + "c3": { + "prompt_1": { + "accuracy": 0.8130142109199701 + } + }, + "dream": { + "prompt_1": { + "accuracy": 0.8593826555609995 + } + }, + "samsum": { + "prompt_1": { + "rouge1": 0.46593725641472133, + "rouge2": 0.2267578192605436, + "rougeL": 0.3814517016374434, + "avg_rouge": 0.3580489257709028 + } + }, + "dialogsum": { + "prompt_1": { + "rouge1": 0.42405246791568507, + "rouge2": 0.17569185972744117, + "rougeL": 0.3491933150322079, + "avg_rouge": 0.31631254755844473 + } + }, + "sst2": { + "prompt_1": { + "accuracy": 0.7121559633027523 + } + }, + "cola": { + "prompt_1": { + "accuracy": 0.673058485139022 + } + }, + "qqp": { + "prompt_1": { + "accuracy": 0.573 + } + }, + "mnli": { + "prompt_1": { + "accuracy": 0.4 + } + }, + "qnli": { + "prompt_1": { + "accuracy": 0.6 + } + }, + "wnli": { + "prompt_1": { + "accuracy": 0.4 + } + }, + "rte": { + "prompt_1": { + "accuracy": 0.6 + } + }, + "mrpc": { + "prompt_1": { + "accuracy": 0.6 + } + } + } + }, + "mistral_7b_instruct_v0_2": { + "model_size": "7B", + "model_link": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2", + "zero_shot": { + "cross_xquad": { + "prompt_1": { + "overall_acc": 0.8600840336134454, + "language_acc": { + "Spanish": 0.8798319327731092, + "English": 0.8966386554621849, + "Chinese": 0.8369747899159664, + "Vietnamese": 0.826890756302521 + }, + "consistency_score_2": 0.8292717086834734, + "consistency_score_3": 0.7571428571428571, + "consistency_score_4": 0.7058823529411765, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.8932773109243698, + "Spanish,Chinese": 0.8285714285714286, + "Spanish,Vietnamese": 0.8134453781512605, + "English,Chinese": 0.8394957983193277, + "English,Vietnamese": 0.8117647058823529, + "Chinese,Vietnamese": 0.7890756302521008 }, "3_combine": { - "Spanish,English,Chinese": 0.3333333333333333, - "Spanish,English,Vietnamese": 0.2, - "Spanish,English,Filipino": 0.2, - "Spanish,English,Indonesian": 0.2866666666666667, - "Spanish,English,Malay": 0.24, - "Spanish,Chinese,Vietnamese": 0.20666666666666667, - "Spanish,Chinese,Filipino": 0.22, - "Spanish,Chinese,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Malay": 0.24, - "Spanish,Vietnamese,Filipino": 0.26, - "Spanish,Vietnamese,Indonesian": 0.28, - "Spanish,Vietnamese,Malay": 0.24666666666666667, - "Spanish,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Filipino,Malay": 0.25333333333333335, - "Spanish,Indonesian,Malay": 0.34, - "English,Chinese,Vietnamese": 0.22, - "English,Chinese,Filipino": 0.24666666666666667, - "English,Chinese,Indonesian": 0.2733333333333333, - "English,Chinese,Malay": 0.24666666666666667, - "English,Vietnamese,Filipino": 0.17333333333333334, - "English,Vietnamese,Indonesian": 0.21333333333333335, - "English,Vietnamese,Malay": 0.18666666666666668, - "English,Filipino,Indonesian": 0.22666666666666666, - "English,Filipino,Malay": 0.2, - "English,Indonesian,Malay": 0.2866666666666667, - "Chinese,Vietnamese,Filipino": 0.2, - "Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "Chinese,Vietnamese,Malay": 0.20666666666666667, - "Chinese,Filipino,Indonesian": 0.2733333333333333, - "Chinese,Filipino,Malay": 0.24, - "Chinese,Indonesian,Malay": 0.29333333333333333, - "Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Vietnamese,Filipino,Malay": 0.2866666666666667, - "Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Filipino,Indonesian,Malay": 0.34 + "Spanish,English,Chinese": 0.7907563025210084, + "Spanish,English,Vietnamese": 0.7714285714285715, + "Spanish,Chinese,Vietnamese": 0.7302521008403361, + "English,Chinese,Vietnamese": 0.7361344537815127 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.14, - "Spanish,English,Chinese,Filipino": 0.15333333333333332, - "Spanish,English,Chinese,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,English,Vietnamese,Malay": 0.12666666666666668, - "Spanish,English,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Filipino,Malay": 0.16666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.2, - "Spanish,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.18, - "Spanish,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "English,Chinese,Vietnamese,Malay": 0.12, - "English,Chinese,Filipino,Indonesian": 0.18666666666666668, - "English,Chinese,Filipino,Malay": 0.18, - "English,Chinese,Indonesian,Malay": 0.2, - "English,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "English,Vietnamese,Filipino,Malay": 0.12, - "English,Vietnamese,Indonesian,Malay": 0.16, - "English,Filipino,Indonesian,Malay": 0.18, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.08, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,English,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.16, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 + "Spanish,English,Chinese,Vietnamese": 0.7058823529411765 + } + }, + "AC3_2": 0.8443968765915703, + "AC3_3": 0.8053371932842782, + "AC3_4": 0.775391025692684 + }, + "prompt_2": { + "overall_acc": 0.8619747899159664, + "language_acc": { + "Spanish": 0.880672268907563, + "English": 0.9008403361344538, + "Chinese": 0.8478991596638655, + "Vietnamese": 0.8184873949579832 + }, + "consistency_score_2": 0.8357142857142857, + "consistency_score_3": 0.7651260504201681, + "consistency_score_4": 0.7184873949579832, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.8915966386554622, + "Spanish,Chinese": 0.8436974789915966, + "Spanish,Vietnamese": 0.8117647058823529, + "English,Chinese": 0.8546218487394958, + "English,Vietnamese": 0.8100840336134454, + "Chinese,Vietnamese": 0.8025210084033614 }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 + "3_combine": { + "Spanish,English,Chinese": 0.8033613445378152, + "Spanish,English,Vietnamese": 0.7663865546218488, + "Spanish,Chinese,Vietnamese": 0.7428571428571429, + "English,Chinese,Vietnamese": 0.7478991596638656 }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.7184873949579832 } }, - "AC3_2": 0.4185089566519321, - "AC3_3": 0.3093351196978393, - "AC3_4": 0.2346305696142647, - "AC3_5": 0.18345765889005478, - "AC3_6": 0.14857142854095984, - "AC3_7": 0.12375929679581868 + "AC3_2": 0.8486414340016675, + "AC3_3": 0.8106680915896736, + "AC3_4": 0.7837176076900385 }, - "prompt_4": { - "overall_acc": 0.4057142857142857, + "prompt_3": { + "overall_acc": 0.8609243697478992, "language_acc": { - "Spanish": 0.4266666666666667, - "English": 0.5933333333333334, - "Chinese": 0.5133333333333333, - "Vietnamese": 0.35333333333333333, - "Filipino": 0.3, - "Indonesian": 0.31333333333333335, - "Malay": 0.34 - }, - "consistency_score_2": 0.49555555555555564, - "consistency_score_3": 0.30476190476190473, - "consistency_score_4": 0.2064761904761905, - "consistency_score_5": 0.14634920634920634, - "consistency_score_6": 0.10666666666666667, - "consistency_score_7": 0.08, + "Spanish": 0.8815126050420168, + "English": 0.8983193277310925, + "Chinese": 0.8411764705882353, + "Vietnamese": 0.8226890756302521 + }, + "consistency_score_2": 0.8336134453781513, + "consistency_score_3": 0.7628151260504201, + "consistency_score_4": 0.7176470588235294, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5933333333333334, - "Spanish,Chinese": 0.5133333333333333, - "Spanish,Vietnamese": 0.4866666666666667, - "Spanish,Filipino": 0.5133333333333333, - "Spanish,Indonesian": 0.47333333333333333, - "Spanish,Malay": 0.47333333333333333, - "English,Chinese": 0.6266666666666667, - "English,Vietnamese": 0.4533333333333333, - "English,Filipino": 0.43333333333333335, - "English,Indonesian": 0.4266666666666667, - "English,Malay": 0.44666666666666666, - "Chinese,Vietnamese": 0.41333333333333333, - "Chinese,Filipino": 0.3933333333333333, - "Chinese,Indonesian": 0.3933333333333333, - "Chinese,Malay": 0.43333333333333335, - "Vietnamese,Filipino": 0.52, - "Vietnamese,Indonesian": 0.49333333333333335, - "Vietnamese,Malay": 0.58, - "Filipino,Indonesian": 0.5533333333333333, - "Filipino,Malay": 0.56, - "Indonesian,Malay": 0.6266666666666667 + "Spanish,English": 0.8890756302521008, + "Spanish,Chinese": 0.8369747899159664, + "Spanish,Vietnamese": 0.8176470588235294, + "English,Chinese": 0.8445378151260504, + "English,Vietnamese": 0.8117647058823529, + "Chinese,Vietnamese": 0.8016806722689076 }, "3_combine": { - "Spanish,English,Chinese": 0.4066666666666667, - "Spanish,English,Vietnamese": 0.32666666666666666, - "Spanish,English,Filipino": 0.32, - "Spanish,English,Indonesian": 0.31333333333333335, - "Spanish,English,Malay": 0.3333333333333333, - "Spanish,Chinese,Vietnamese": 0.26666666666666666, - "Spanish,Chinese,Filipino": 0.25333333333333335, - "Spanish,Chinese,Indonesian": 0.26666666666666666, - "Spanish,Chinese,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino": 0.3, - "Spanish,Vietnamese,Indonesian": 0.30666666666666664, - "Spanish,Vietnamese,Malay": 0.32, - "Spanish,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Filipino,Malay": 0.32666666666666666, - "Spanish,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese": 0.29333333333333333, - "English,Chinese,Filipino": 0.3, - "English,Chinese,Indonesian": 0.26666666666666666, - "English,Chinese,Malay": 0.3, - "English,Vietnamese,Filipino": 0.2733333333333333, - "English,Vietnamese,Indonesian": 0.26666666666666666, - "English,Vietnamese,Malay": 0.32, - "English,Filipino,Indonesian": 0.2733333333333333, - "English,Filipino,Malay": 0.28, - "English,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino": 0.24, - "Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "Chinese,Vietnamese,Malay": 0.28, - "Chinese,Filipino,Indonesian": 0.25333333333333335, - "Chinese,Filipino,Malay": 0.26666666666666666, - "Chinese,Indonesian,Malay": 0.2866666666666667, - "Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "Vietnamese,Filipino,Malay": 0.38, - "Vietnamese,Indonesian,Malay": 0.4, - "Filipino,Indonesian,Malay": 0.42 + "Spanish,English,Chinese": 0.7941176470588235, + "Spanish,English,Vietnamese": 0.7697478991596639, + "Spanish,Chinese,Vietnamese": 0.7436974789915967, + "English,Chinese,Vietnamese": 0.7436974789915967 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,English,Chinese,Filipino": 0.21333333333333335, - "Spanish,English,Chinese,Indonesian": 0.2, - "Spanish,English,Chinese,Malay": 0.22, - "Spanish,English,Vietnamese,Filipino": 0.2, - "Spanish,English,Vietnamese,Indonesian": 0.22, - "Spanish,English,Vietnamese,Malay": 0.23333333333333334, - "Spanish,English,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Filipino,Malay": 0.22, - "Spanish,English,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.14, - "Spanish,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.17333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.18, - "Spanish,Chinese,Filipino,Malay": 0.16666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.26, - "Spanish,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.16, - "English,Chinese,Vietnamese,Malay": 0.20666666666666667, - "English,Chinese,Filipino,Indonesian": 0.17333333333333334, - "English,Chinese,Filipino,Malay": 0.18666666666666668, - "English,Chinese,Indonesian,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "English,Vietnamese,Filipino,Malay": 0.22, - "English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.2, - "Chinese,Filipino,Indonesian,Malay": 0.2, - "Vietnamese,Filipino,Indonesian,Malay": 0.3 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.14, - "Spanish,English,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 + "Spanish,English,Chinese,Vietnamese": 0.7176470588235294 + } + }, + "AC3_2": 0.8470488219583601, + "AC3_3": 0.8089057798840994, + "AC3_4": 0.782783509947437 + }, + "prompt_4": { + "overall_acc": 0.8581932773109244, + "language_acc": { + "Spanish": 0.8865546218487395, + "English": 0.8899159663865546, + "Chinese": 0.8361344537815126, + "Vietnamese": 0.8201680672268907 + }, + "consistency_score_2": 0.8315126050420169, + "consistency_score_3": 0.759453781512605, + "consistency_score_4": 0.711764705882353, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.8865546218487395, + "Spanish,Chinese": 0.8420168067226891, + "Spanish,Vietnamese": 0.8176470588235294, + "English,Chinese": 0.8344537815126051, + "English,Vietnamese": 0.8050420168067227, + "Chinese,Vietnamese": 0.8033613445378152 }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 + "3_combine": { + "Spanish,English,Chinese": 0.7924369747899159, + "Spanish,English,Vietnamese": 0.7647058823529411, + "Spanish,Chinese,Vietnamese": 0.7436974789915967, + "English,Chinese,Vietnamese": 0.7369747899159664 }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.711764705882353 } }, - "AC3_2": 0.4461570975200636, - "AC3_3": 0.34806587509463255, - "AC3_4": 0.2736741043201284, - "AC3_5": 0.21510556144957627, - "AC3_6": 0.1689219330525338, - "AC3_7": 0.13364705879601385 + "AC3_2": 0.8446422955105344, + "AC3_3": 0.8058100512434309, + "AC3_4": 0.7781503608575436 }, "prompt_5": { - "overall_acc": 0.4152380952380953, + "overall_acc": 0.8556722689075631, "language_acc": { - "Spanish": 0.4066666666666667, - "English": 0.54, - "Chinese": 0.4866666666666667, - "Vietnamese": 0.38, - "Filipino": 0.3933333333333333, - "Indonesian": 0.35333333333333333, - "Malay": 0.3466666666666667 + "Spanish": 0.8798319327731092, + "English": 0.892436974789916, + "Chinese": 0.8285714285714286, + "Vietnamese": 0.8218487394957983 }, - "consistency_score_2": 0.4384126984126984, - "consistency_score_3": 0.24571428571428566, - "consistency_score_4": 0.1556190476190476, - "consistency_score_5": 0.10698412698412696, - "consistency_score_6": 0.07809523809523809, - "consistency_score_7": 0.06, + "consistency_score_2": 0.8245098039215687, + "consistency_score_3": 0.7510504201680672, + "consistency_score_4": 0.7008403361344537, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.52, - "Spanish,Chinese": 0.44, - "Spanish,Vietnamese": 0.42, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.46, - "Spanish,Malay": 0.42, - "English,Chinese": 0.5466666666666666, - "English,Vietnamese": 0.4266666666666667, - "English,Filipino": 0.43333333333333335, - "English,Indonesian": 0.4, - "English,Malay": 0.3933333333333333, - "Chinese,Vietnamese": 0.38, - "Chinese,Filipino": 0.41333333333333333, - "Chinese,Indonesian": 0.36666666666666664, - "Chinese,Malay": 0.3333333333333333, - "Vietnamese,Filipino": 0.4666666666666667, - "Vietnamese,Indonesian": 0.4533333333333333, - "Vietnamese,Malay": 0.4666666666666667, - "Filipino,Indonesian": 0.46, - "Filipino,Malay": 0.43333333333333335, - "Indonesian,Malay": 0.5333333333333333 + "Spanish,English": 0.8915966386554622, + "Spanish,Chinese": 0.8159663865546218, + "Spanish,Vietnamese": 0.8058823529411765, + "English,Chinese": 0.8302521008403362, + "English,Vietnamese": 0.8134453781512605, + "Chinese,Vietnamese": 0.7899159663865546 }, "3_combine": { - "Spanish,English,Chinese": 0.32, - "Spanish,English,Vietnamese": 0.25333333333333335, - "Spanish,English,Filipino": 0.2733333333333333, - "Spanish,English,Indonesian": 0.2866666666666667, - "Spanish,English,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese": 0.18, - "Spanish,Chinese,Filipino": 0.23333333333333334, - "Spanish,Chinese,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Malay": 0.2, - "Spanish,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Filipino,Malay": 0.24666666666666667, - "Spanish,Indonesian,Malay": 0.28, - "English,Chinese,Vietnamese": 0.26, - "English,Chinese,Filipino": 0.28, - "English,Chinese,Indonesian": 0.22, - "English,Chinese,Malay": 0.21333333333333335, - "English,Vietnamese,Filipino": 0.24, - "English,Vietnamese,Indonesian": 0.24, - "English,Vietnamese,Malay": 0.23333333333333334, - "English,Filipino,Indonesian": 0.25333333333333335, - "English,Filipino,Malay": 0.22, - "English,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino": 0.22, - "Chinese,Vietnamese,Indonesian": 0.2, - "Chinese,Vietnamese,Malay": 0.17333333333333334, - "Chinese,Filipino,Indonesian": 0.23333333333333334, - "Chinese,Filipino,Malay": 0.17333333333333334, - "Chinese,Indonesian,Malay": 0.21333333333333335, - "Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Vietnamese,Filipino,Malay": 0.25333333333333335, - "Vietnamese,Indonesian,Malay": 0.3, - "Filipino,Indonesian,Malay": 0.3 + "Spanish,English,Chinese": 0.7798319327731092, + "Spanish,English,Vietnamese": 0.7663865546218488, + "Spanish,Chinese,Vietnamese": 0.7260504201680672, + "English,Chinese,Vietnamese": 0.7319327731092437 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.15333333333333332, - "Spanish,English,Chinese,Filipino": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,English,Vietnamese,Malay": 0.16666666666666666, - "Spanish,English,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Filipino,Malay": 0.16, - "Spanish,English,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.1, - "Spanish,Chinese,Filipino,Indonesian": 0.16, - "Spanish,Chinese,Filipino,Malay": 0.12, - "Spanish,Chinese,Indonesian,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino": 0.16, - "English,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "English,Chinese,Vietnamese,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian": 0.16, - "English,Chinese,Filipino,Malay": 0.13333333333333333, - "English,Chinese,Indonesian,Malay": 0.12666666666666668, - "English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "English,Vietnamese,Indonesian,Malay": 0.16, - "English,Filipino,Indonesian,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.14, - "Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,English,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.1, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06 + "Spanish,English,Chinese,Vietnamese": 0.7008403361344537 } }, - "AC3_2": 0.4265108289004344, - "AC3_3": 0.30873610534645507, - "AC3_4": 0.22639274190863906, - "AC3_5": 0.17013402804673794, - "AC3_6": 0.13146534286726594, - "AC3_7": 0.10484969937673502 + "AC3_2": 0.8398020500577692, + "AC3_3": 0.7999551153645786, + "AC3_4": 0.77055545657774 } }, - "cross_logiqa": { + "cross_mmlu": { "prompt_1": { - "overall_acc": 0.34821428571428575, + "overall_acc": 0.4942857142857142, "language_acc": { - "English": 0.4715909090909091, - "Filipino": 0.2897727272727273, - "Spanish": 0.38636363636363635, - "Malay": 0.2840909090909091, - "Chinese": 0.4318181818181818, - "Indonesian": 0.2556818181818182, - "Vietnamese": 0.3181818181818182 + "Filipino": 0.4533333333333333, + "Vietnamese": 0.48, + "Chinese": 0.4666666666666667, + "Spanish": 0.5933333333333334, + "Malay": 0.36666666666666664, + "Indonesian": 0.4533333333333333, + "English": 0.6466666666666666 }, - "consistency_score_2": 0.4491341991341992, - "consistency_score_3": 0.25487012987012986, - "consistency_score_4": 0.16444805194805198, - "consistency_score_5": 0.11525974025974028, - "consistency_score_6": 0.0844155844155844, - "consistency_score_7": 0.0625, + "consistency_score_2": 0.4787301587301588, + "consistency_score_3": 0.29599999999999993, + "consistency_score_4": 0.20761904761904762, + "consistency_score_5": 0.15682539682539684, + "consistency_score_6": 0.1238095238095238, + "consistency_score_7": 0.1, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.36363636363636365, - "English,Spanish": 0.6534090909090909, - "English,Malay": 0.44886363636363635, - "English,Chinese": 0.6022727272727273, - "English,Indonesian": 0.3977272727272727, - "English,Vietnamese": 0.3693181818181818, - "Filipino,Spanish": 0.4375, - "Filipino,Malay": 0.48863636363636365, - "Filipino,Chinese": 0.35795454545454547, - "Filipino,Indonesian": 0.4659090909090909, - "Filipino,Vietnamese": 0.48863636363636365, - "Spanish,Malay": 0.4375, - "Spanish,Chinese": 0.5340909090909091, - "Spanish,Indonesian": 0.42045454545454547, - "Spanish,Vietnamese": 0.42045454545454547, - "Malay,Chinese": 0.39204545454545453, - "Malay,Indonesian": 0.5738636363636364, - "Malay,Vietnamese": 0.4943181818181818, - "Chinese,Indonesian": 0.32386363636363635, - "Chinese,Vietnamese": 0.3409090909090909, - "Indonesian,Vietnamese": 0.42045454545454547 + "Filipino,Vietnamese": 0.4066666666666667, + "Filipino,Chinese": 0.42, + "Filipino,Spanish": 0.44666666666666666, + "Filipino,Malay": 0.49333333333333335, + "Filipino,Indonesian": 0.41333333333333333, + "Filipino,English": 0.5333333333333333, + "Vietnamese,Chinese": 0.48, + "Vietnamese,Spanish": 0.48, + "Vietnamese,Malay": 0.52, + "Vietnamese,Indonesian": 0.44, + "Vietnamese,English": 0.4666666666666667, + "Chinese,Spanish": 0.49333333333333335, + "Chinese,Malay": 0.4533333333333333, + "Chinese,Indonesian": 0.4066666666666667, + "Chinese,English": 0.5066666666666667, + "Spanish,Malay": 0.5066666666666667, + "Spanish,Indonesian": 0.49333333333333335, + "Spanish,English": 0.5666666666666667, + "Malay,Indonesian": 0.5266666666666666, + "Malay,English": 0.49333333333333335, + "Indonesian,English": 0.5066666666666667 }, "3_combine": { - "English,Filipino,Spanish": 0.2784090909090909, - "English,Filipino,Malay": 0.23863636363636365, - "English,Filipino,Chinese": 0.23295454545454544, - "English,Filipino,Indonesian": 0.21022727272727273, - "English,Filipino,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay": 0.3465909090909091, - "English,Spanish,Chinese": 0.4375, - "English,Spanish,Indonesian": 0.3125, - "English,Spanish,Vietnamese": 0.30113636363636365, - "English,Malay,Chinese": 0.2897727272727273, - "English,Malay,Indonesian": 0.2727272727272727, - "English,Malay,Vietnamese": 0.25, - "English,Chinese,Indonesian": 0.2159090909090909, - "English,Chinese,Vietnamese": 0.23863636363636365, - "English,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay": 0.26704545454545453, - "Filipino,Spanish,Chinese": 0.2556818181818182, - "Filipino,Spanish,Indonesian": 0.25, - "Filipino,Spanish,Vietnamese": 0.26704545454545453, - "Filipino,Malay,Chinese": 0.20454545454545456, - "Filipino,Malay,Indonesian": 0.32386363636363635, - "Filipino,Malay,Vietnamese": 0.32386363636363635, - "Filipino,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Indonesian,Vietnamese": 0.2727272727272727, - "Spanish,Malay,Chinese": 0.25, - "Spanish,Malay,Indonesian": 0.2897727272727273, - "Spanish,Malay,Vietnamese": 0.26704545454545453, - "Spanish,Chinese,Indonesian": 0.2159090909090909, - "Spanish,Chinese,Vietnamese": 0.2215909090909091, - "Spanish,Indonesian,Vietnamese": 0.23295454545454544, - "Malay,Chinese,Indonesian": 0.21022727272727273, - "Malay,Chinese,Vietnamese": 0.19886363636363635, - "Malay,Indonesian,Vietnamese": 0.3068181818181818, - "Chinese,Indonesian,Vietnamese": 0.14204545454545456 + "Filipino,Vietnamese,Chinese": 0.24666666666666667, + "Filipino,Vietnamese,Spanish": 0.26, + "Filipino,Vietnamese,Malay": 0.2866666666666667, + "Filipino,Vietnamese,Indonesian": 0.22, + "Filipino,Vietnamese,English": 0.2866666666666667, + "Filipino,Chinese,Spanish": 0.26666666666666666, + "Filipino,Chinese,Malay": 0.26, + "Filipino,Chinese,Indonesian": 0.23333333333333334, + "Filipino,Chinese,English": 0.3, + "Filipino,Spanish,Malay": 0.31333333333333335, + "Filipino,Spanish,Indonesian": 0.25333333333333335, + "Filipino,Spanish,English": 0.32666666666666666, + "Filipino,Malay,Indonesian": 0.30666666666666664, + "Filipino,Malay,English": 0.32666666666666666, + "Filipino,Indonesian,English": 0.3, + "Vietnamese,Chinese,Spanish": 0.31333333333333335, + "Vietnamese,Chinese,Malay": 0.32666666666666666, + "Vietnamese,Chinese,Indonesian": 0.23333333333333334, + "Vietnamese,Chinese,English": 0.31333333333333335, + "Vietnamese,Spanish,Malay": 0.3333333333333333, + "Vietnamese,Spanish,Indonesian": 0.26666666666666666, + "Vietnamese,Spanish,English": 0.32, + "Vietnamese,Malay,Indonesian": 0.32666666666666666, + "Vietnamese,Malay,English": 0.31333333333333335, + "Vietnamese,Indonesian,English": 0.2733333333333333, + "Chinese,Spanish,Malay": 0.30666666666666664, + "Chinese,Spanish,Indonesian": 0.2733333333333333, + "Chinese,Spanish,English": 0.35333333333333333, + "Chinese,Malay,Indonesian": 0.2733333333333333, + "Chinese,Malay,English": 0.3, + "Chinese,Indonesian,English": 0.28, + "Spanish,Malay,Indonesian": 0.31333333333333335, + "Spanish,Malay,English": 0.35333333333333333, + "Spanish,Indonesian,English": 0.36, + "Malay,Indonesian,English": 0.34 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.19886363636363635, - "English,Filipino,Spanish,Chinese": 0.19318181818181818, - "English,Filipino,Spanish,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Vietnamese": 0.19886363636363635, - "English,Filipino,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Malay,Vietnamese": 0.17045454545454544, - "English,Filipino,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese": 0.23295454545454544, - "English,Spanish,Malay,Indonesian": 0.23295454545454544, - "English,Spanish,Malay,Vietnamese": 0.21022727272727273, - "English,Spanish,Chinese,Indonesian": 0.1875, - "English,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "English,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "Filipino,Spanish,Malay,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Malay,Vietnamese": 0.1875, - "Filipino,Spanish,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian": 0.11931818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 + "Filipino,Vietnamese,Chinese,Spanish": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Malay": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,English": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,Malay": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,English": 0.19333333333333333, + "Filipino,Vietnamese,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Malay,English": 0.22, + "Filipino,Vietnamese,Indonesian,English": 0.18, + "Filipino,Chinese,Spanish,Malay": 0.20666666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.16666666666666666, + "Filipino,Chinese,Spanish,English": 0.21333333333333335, + "Filipino,Chinese,Malay,Indonesian": 0.18666666666666668, + "Filipino,Chinese,Malay,English": 0.21333333333333335, + "Filipino,Chinese,Indonesian,English": 0.18666666666666668, + "Filipino,Spanish,Malay,Indonesian": 0.21333333333333335, + "Filipino,Spanish,Malay,English": 0.24, + "Filipino,Spanish,Indonesian,English": 0.22, + "Filipino,Malay,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Malay": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,English": 0.24666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.2, + "Vietnamese,Chinese,Malay,English": 0.23333333333333334, + "Vietnamese,Chinese,Indonesian,English": 0.18, + "Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Vietnamese,Spanish,Malay,English": 0.23333333333333334, + "Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, + "Vietnamese,Malay,Indonesian,English": 0.22666666666666666, + "Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Chinese,Spanish,Malay,English": 0.24, + "Chinese,Spanish,Indonesian,English": 0.22, + "Chinese,Malay,Indonesian,English": 0.22, + "Spanish,Malay,Indonesian,English": 0.26666666666666666 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Spanish,Malay,Chinese,Vietnamese": 0.125, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.16, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.16, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14, + "Filipino,Vietnamese,Spanish,Malay,English": 0.16, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.16, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Indonesian,English": 0.14666666666666667, + "Filipino,Chinese,Malay,Indonesian,English": 0.16, + "Filipino,Spanish,Malay,Indonesian,English": 0.18666666666666668, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,Malay,English": 0.19333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.15333333333333332, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.16, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Chinese,Spanish,Malay,Indonesian,English": 0.18 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 } }, - "AC3_2": 0.3922875563035084, - "AC3_3": 0.294318400258831, - "AC3_4": 0.22339523202011766, - "AC3_5": 0.1731923942583535, - "AC3_6": 0.1358885017107504, - "AC3_7": 0.10597826084376182 + "AC3_2": 0.48638359351794563, + "AC3_3": 0.370267534298773, + "AC3_4": 0.2924132583417477, + "AC3_5": 0.2381054537490407, + "AC3_6": 0.19801893019028305, + "AC3_7": 0.16634615381816292 }, "prompt_2": { - "overall_acc": 0.35227272727272724, + "overall_acc": 0.4933333333333333, "language_acc": { - "English": 0.4659090909090909, - "Filipino": 0.3068181818181818, - "Spanish": 0.39204545454545453, - "Malay": 0.2840909090909091, - "Chinese": 0.4602272727272727, - "Indonesian": 0.2727272727272727, - "Vietnamese": 0.2840909090909091 - }, - "consistency_score_2": 0.4439935064935066, - "consistency_score_3": 0.24983766233766233, - "consistency_score_4": 0.1589285714285714, - "consistency_score_5": 0.10714285714285716, - "consistency_score_6": 0.07224025974025973, - "consistency_score_7": 0.045454545454545456, - "detailed_consistency_score": { + "Filipino": 0.4666666666666667, + "Vietnamese": 0.47333333333333333, + "Chinese": 0.3933333333333333, + "Spanish": 0.56, + "Malay": 0.38, + "Indonesian": 0.5133333333333333, + "English": 0.6666666666666666 + }, + "consistency_score_2": 0.49523809523809537, + "consistency_score_3": 0.31752380952380943, + "consistency_score_4": 0.2287619047619048, + "consistency_score_5": 0.1736507936507937, + "consistency_score_6": 0.13523809523809524, + "consistency_score_7": 0.10666666666666667, + "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.3465909090909091, - "English,Spanish": 0.6193181818181818, - "English,Malay": 0.3465909090909091, - "English,Chinese": 0.5738636363636364, - "English,Indonesian": 0.38636363636363635, - "English,Vietnamese": 0.39204545454545453, - "Filipino,Spanish": 0.45454545454545453, - "Filipino,Malay": 0.5625, - "Filipino,Chinese": 0.3181818181818182, - "Filipino,Indonesian": 0.5625, - "Filipino,Vietnamese": 0.4602272727272727, - "Spanish,Malay": 0.3977272727272727, - "Spanish,Chinese": 0.5511363636363636, - "Spanish,Indonesian": 0.45454545454545453, - "Spanish,Vietnamese": 0.38636363636363635, - "Malay,Chinese": 0.32386363636363635, - "Malay,Indonesian": 0.5738636363636364, - "Malay,Vietnamese": 0.4943181818181818, - "Chinese,Indonesian": 0.3352272727272727, - "Chinese,Vietnamese": 0.3352272727272727, - "Indonesian,Vietnamese": 0.44886363636363635 + "Filipino,Vietnamese": 0.4, + "Filipino,Chinese": 0.38, + "Filipino,Spanish": 0.52, + "Filipino,Malay": 0.43333333333333335, + "Filipino,Indonesian": 0.44666666666666666, + "Filipino,English": 0.47333333333333333, + "Vietnamese,Chinese": 0.4266666666666667, + "Vietnamese,Spanish": 0.5266666666666666, + "Vietnamese,Malay": 0.5133333333333333, + "Vietnamese,Indonesian": 0.52, + "Vietnamese,English": 0.5, + "Chinese,Spanish": 0.5133333333333333, + "Chinese,Malay": 0.46, + "Chinese,Indonesian": 0.52, + "Chinese,English": 0.42, + "Spanish,Malay": 0.56, + "Spanish,Indonesian": 0.54, + "Spanish,English": 0.6533333333333333, + "Malay,Indonesian": 0.56, + "Malay,English": 0.47333333333333333, + "Indonesian,English": 0.56 }, "3_combine": { - "English,Filipino,Spanish": 0.2727272727272727, - "English,Filipino,Malay": 0.23295454545454544, - "English,Filipino,Chinese": 0.21022727272727273, - "English,Filipino,Indonesian": 0.23863636363636365, - "English,Filipino,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay": 0.26704545454545453, - "English,Spanish,Chinese": 0.42613636363636365, - "English,Spanish,Indonesian": 0.3125, - "English,Spanish,Vietnamese": 0.2784090909090909, - "English,Malay,Chinese": 0.19886363636363635, - "English,Malay,Indonesian": 0.22727272727272727, - "English,Malay,Vietnamese": 0.20454545454545456, - "English,Chinese,Indonesian": 0.23863636363636365, - "English,Chinese,Vietnamese": 0.23295454545454544, - "English,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay": 0.3125, - "Filipino,Spanish,Chinese": 0.25, - "Filipino,Spanish,Indonesian": 0.3068181818181818, - "Filipino,Spanish,Vietnamese": 0.25, - "Filipino,Malay,Chinese": 0.19886363636363635, - "Filipino,Malay,Indonesian": 0.3977272727272727, - "Filipino,Malay,Vietnamese": 0.3181818181818182, - "Filipino,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Chinese,Vietnamese": 0.17045454545454544, - "Filipino,Indonesian,Vietnamese": 0.3125, - "Spanish,Malay,Chinese": 0.2215909090909091, - "Spanish,Malay,Indonesian": 0.26704545454545453, - "Spanish,Malay,Vietnamese": 0.22727272727272727, - "Spanish,Chinese,Indonesian": 0.2556818181818182, - "Spanish,Chinese,Vietnamese": 0.23295454545454544, - "Spanish,Indonesian,Vietnamese": 0.22727272727272727, - "Malay,Chinese,Indonesian": 0.19318181818181818, - "Malay,Chinese,Vietnamese": 0.17613636363636365, - "Malay,Indonesian,Vietnamese": 0.30113636363636365, - "Chinese,Indonesian,Vietnamese": 0.16477272727272727 + "Filipino,Vietnamese,Chinese": 0.21333333333333335, + "Filipino,Vietnamese,Spanish": 0.2733333333333333, + "Filipino,Vietnamese,Malay": 0.26666666666666666, + "Filipino,Vietnamese,Indonesian": 0.28, + "Filipino,Vietnamese,English": 0.24666666666666667, + "Filipino,Chinese,Spanish": 0.2866666666666667, + "Filipino,Chinese,Malay": 0.24, + "Filipino,Chinese,Indonesian": 0.2733333333333333, + "Filipino,Chinese,English": 0.22666666666666666, + "Filipino,Spanish,Malay": 0.34, + "Filipino,Spanish,Indonesian": 0.32, + "Filipino,Spanish,English": 0.38, + "Filipino,Malay,Indonesian": 0.2866666666666667, + "Filipino,Malay,English": 0.2733333333333333, + "Filipino,Indonesian,English": 0.3, + "Vietnamese,Chinese,Spanish": 0.30666666666666664, + "Vietnamese,Chinese,Malay": 0.3, + "Vietnamese,Chinese,Indonesian": 0.30666666666666664, + "Vietnamese,Chinese,English": 0.26, + "Vietnamese,Spanish,Malay": 0.36, + "Vietnamese,Spanish,Indonesian": 0.36666666666666664, + "Vietnamese,Spanish,English": 0.38666666666666666, + "Vietnamese,Malay,Indonesian": 0.37333333333333335, + "Vietnamese,Malay,English": 0.32, + "Vietnamese,Indonesian,English": 0.38, + "Chinese,Spanish,Malay": 0.3466666666666667, + "Chinese,Spanish,Indonesian": 0.3466666666666667, + "Chinese,Spanish,English": 0.34, + "Chinese,Malay,Indonesian": 0.32666666666666666, + "Chinese,Malay,English": 0.28, + "Chinese,Indonesian,English": 0.31333333333333335, + "Spanish,Malay,Indonesian": 0.38666666666666666, + "Spanish,Malay,English": 0.4, + "Spanish,Indonesian,English": 0.43333333333333335, + "Malay,Indonesian,English": 0.37333333333333335 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.20454545454545456, - "English,Filipino,Spanish,Chinese": 0.17613636363636365, - "English,Filipino,Spanish,Indonesian": 0.20454545454545456, - "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, - "English,Filipino,Malay,Chinese": 0.13068181818181818, - "English,Filipino,Malay,Indonesian": 0.17045454545454544, - "English,Filipino,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Chinese,Vietnamese": 0.125, - "English,Filipino,Indonesian,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Chinese": 0.17045454545454544, - "English,Spanish,Malay,Indonesian": 0.1875, - "English,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Spanish,Chinese,Indonesian": 0.19886363636363635, - "English,Spanish,Chinese,Vietnamese": 0.18181818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "English,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "English,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese": 0.17045454545454544, - "Filipino,Spanish,Malay,Indonesian": 0.23295454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "Filipino,Malay,Chinese,Indonesian": 0.125, - "Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.125, - "Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 + "Filipino,Vietnamese,Chinese,Spanish": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Malay": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,Indonesian": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,English": 0.16, + "Filipino,Vietnamese,Spanish,Malay": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian": 0.22, + "Filipino,Vietnamese,Spanish,English": 0.22, + "Filipino,Vietnamese,Malay,Indonesian": 0.21333333333333335, + "Filipino,Vietnamese,Malay,English": 0.18666666666666668, + "Filipino,Vietnamese,Indonesian,English": 0.21333333333333335, + "Filipino,Chinese,Spanish,Malay": 0.21333333333333335, + "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, + "Filipino,Chinese,Spanish,English": 0.20666666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.2, + "Filipino,Chinese,Malay,English": 0.17333333333333334, + "Filipino,Chinese,Indonesian,English": 0.18666666666666668, + "Filipino,Spanish,Malay,Indonesian": 0.24, + "Filipino,Spanish,Malay,English": 0.26, + "Filipino,Spanish,Indonesian,English": 0.25333333333333335, + "Filipino,Malay,Indonesian,English": 0.20666666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,English": 0.23333333333333334, + "Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, + "Vietnamese,Chinese,Malay,English": 0.20666666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, + "Vietnamese,Spanish,Malay,Indonesian": 0.28, + "Vietnamese,Spanish,Malay,English": 0.2733333333333333, + "Vietnamese,Spanish,Indonesian,English": 0.31333333333333335, + "Vietnamese,Malay,Indonesian,English": 0.2733333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, + "Chinese,Spanish,Malay,English": 0.25333333333333335, + "Chinese,Spanish,Indonesian,English": 0.26, + "Chinese,Malay,Indonesian,English": 0.24, + "Spanish,Malay,Indonesian,English": 0.32 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.125, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.14, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.16, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.18, + "Filipino,Chinese,Spanish,Malay,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Malay,Indonesian,English": 0.14, + "Filipino,Spanish,Malay,Indonesian,English": 0.19333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2, + "Vietnamese,Chinese,Spanish,Malay,English": 0.18666666666666668, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.2, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.18, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.23333333333333334, + "Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.11333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, - "AC3_2": 0.392850523535132, - "AC3_3": 0.29234172401331143, - "AC3_4": 0.2190377897316839, - "AC3_5": 0.16431095402783905, - "AC3_6": 0.11989396833608892, - "AC3_7": 0.08051948049923562 + "AC3_2": 0.49428387920497774, + "AC3_3": 0.3863691174842586, + "AC3_4": 0.3125789149307736, + "AC3_5": 0.25688085035019004, + "AC3_6": 0.212282828249056, + "AC3_7": 0.17540740737817284 }, "prompt_3": { - "overall_acc": 0.3360389610389611, + "overall_acc": 0.4942857142857143, "language_acc": { - "English": 0.4431818181818182, - "Filipino": 0.2840909090909091, - "Spanish": 0.36363636363636365, - "Malay": 0.2897727272727273, - "Chinese": 0.44886363636363635, - "Indonesian": 0.26704545454545453, - "Vietnamese": 0.2556818181818182 + "Filipino": 0.4666666666666667, + "Vietnamese": 0.46, + "Chinese": 0.4066666666666667, + "Spanish": 0.58, + "Malay": 0.3933333333333333, + "Indonesian": 0.49333333333333335, + "English": 0.66 }, - "consistency_score_2": 0.4218073593073593, - "consistency_score_3": 0.22808441558441558, - "consistency_score_4": 0.14318181818181822, - "consistency_score_5": 0.09821428571428571, - "consistency_score_6": 0.0706168831168831, - "consistency_score_7": 0.05113636363636364, + "consistency_score_2": 0.4984126984126984, + "consistency_score_3": 0.32095238095238093, + "consistency_score_4": 0.23466666666666672, + "consistency_score_5": 0.18476190476190477, + "consistency_score_6": 0.15142857142857144, + "consistency_score_7": 0.12666666666666668, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.3409090909090909, - "English,Spanish": 0.6079545454545454, - "English,Malay": 0.35795454545454547, - "English,Chinese": 0.6079545454545454, - "English,Indonesian": 0.375, - "English,Vietnamese": 0.3181818181818182, - "Filipino,Spanish": 0.42045454545454547, - "Filipino,Malay": 0.5340909090909091, - "Filipino,Chinese": 0.32386363636363635, - "Filipino,Indonesian": 0.4431818181818182, - "Filipino,Vietnamese": 0.4659090909090909, - "Spanish,Malay": 0.375, - "Spanish,Chinese": 0.5056818181818182, - "Spanish,Indonesian": 0.3977272727272727, - "Spanish,Vietnamese": 0.32954545454545453, - "Malay,Chinese": 0.36363636363636365, - "Malay,Indonesian": 0.5454545454545454, - "Malay,Vietnamese": 0.4318181818181818, - "Chinese,Indonesian": 0.3352272727272727, - "Chinese,Vietnamese": 0.3181818181818182, - "Indonesian,Vietnamese": 0.4602272727272727 + "Filipino,Vietnamese": 0.46, + "Filipino,Chinese": 0.4066666666666667, + "Filipino,Spanish": 0.49333333333333335, + "Filipino,Malay": 0.5066666666666667, + "Filipino,Indonesian": 0.5133333333333333, + "Filipino,English": 0.49333333333333335, + "Vietnamese,Chinese": 0.4, + "Vietnamese,Spanish": 0.4866666666666667, + "Vietnamese,Malay": 0.54, + "Vietnamese,Indonesian": 0.5066666666666667, + "Vietnamese,English": 0.44666666666666666, + "Chinese,Spanish": 0.4866666666666667, + "Chinese,Malay": 0.5066666666666667, + "Chinese,Indonesian": 0.5333333333333333, + "Chinese,English": 0.44, + "Spanish,Malay": 0.5066666666666667, + "Spanish,Indonesian": 0.56, + "Spanish,English": 0.6733333333333333, + "Malay,Indonesian": 0.5066666666666667, + "Malay,English": 0.47333333333333333, + "Indonesian,English": 0.5266666666666666 }, "3_combine": { - "English,Filipino,Spanish": 0.26136363636363635, - "English,Filipino,Malay": 0.21022727272727273, - "English,Filipino,Chinese": 0.22727272727272727, - "English,Filipino,Indonesian": 0.19318181818181818, - "English,Filipino,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay": 0.2556818181818182, - "English,Spanish,Chinese": 0.4090909090909091, - "English,Spanish,Indonesian": 0.2727272727272727, - "English,Spanish,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese": 0.22727272727272727, - "English,Malay,Indonesian": 0.2215909090909091, - "English,Malay,Vietnamese": 0.1590909090909091, - "English,Chinese,Indonesian": 0.23295454545454544, - "English,Chinese,Vietnamese": 0.2159090909090909, - "English,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Malay": 0.26704545454545453, - "Filipino,Spanish,Chinese": 0.23863636363636365, - "Filipino,Spanish,Indonesian": 0.23863636363636365, - "Filipino,Spanish,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Chinese": 0.2159090909090909, - "Filipino,Malay,Indonesian": 0.3068181818181818, - "Filipino,Malay,Vietnamese": 0.2727272727272727, - "Filipino,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Chinese": 0.22727272727272727, - "Spanish,Malay,Indonesian": 0.24431818181818182, - "Spanish,Malay,Vietnamese": 0.17613636363636365, - "Spanish,Chinese,Indonesian": 0.23295454545454544, - "Spanish,Chinese,Vietnamese": 0.21022727272727273, - "Spanish,Indonesian,Vietnamese": 0.21022727272727273, - "Malay,Chinese,Indonesian": 0.20454545454545456, - "Malay,Chinese,Vietnamese": 0.16477272727272727, - "Malay,Indonesian,Vietnamese": 0.2897727272727273, - "Chinese,Indonesian,Vietnamese": 0.17045454545454544 + "Filipino,Vietnamese,Chinese": 0.22666666666666666, + "Filipino,Vietnamese,Spanish": 0.2866666666666667, + "Filipino,Vietnamese,Malay": 0.32, + "Filipino,Vietnamese,Indonesian": 0.3333333333333333, + "Filipino,Vietnamese,English": 0.2866666666666667, + "Filipino,Chinese,Spanish": 0.28, + "Filipino,Chinese,Malay": 0.28, + "Filipino,Chinese,Indonesian": 0.3, + "Filipino,Chinese,English": 0.26666666666666666, + "Filipino,Spanish,Malay": 0.32666666666666666, + "Filipino,Spanish,Indonesian": 0.3333333333333333, + "Filipino,Spanish,English": 0.38, + "Filipino,Malay,Indonesian": 0.3333333333333333, + "Filipino,Malay,English": 0.32666666666666666, + "Filipino,Indonesian,English": 0.34, + "Vietnamese,Chinese,Spanish": 0.26666666666666666, + "Vietnamese,Chinese,Malay": 0.31333333333333335, + "Vietnamese,Chinese,Indonesian": 0.2866666666666667, + "Vietnamese,Chinese,English": 0.24, + "Vietnamese,Spanish,Malay": 0.34, + "Vietnamese,Spanish,Indonesian": 0.3333333333333333, + "Vietnamese,Spanish,English": 0.35333333333333333, + "Vietnamese,Malay,Indonesian": 0.35333333333333333, + "Vietnamese,Malay,English": 0.30666666666666664, + "Vietnamese,Indonesian,English": 0.32, + "Chinese,Spanish,Malay": 0.32666666666666666, + "Chinese,Spanish,Indonesian": 0.35333333333333333, + "Chinese,Spanish,English": 0.36, + "Chinese,Malay,Indonesian": 0.3333333333333333, + "Chinese,Malay,English": 0.29333333333333333, + "Chinese,Indonesian,English": 0.32666666666666666, + "Spanish,Malay,Indonesian": 0.35333333333333333, + "Spanish,Malay,English": 0.3933333333333333, + "Spanish,Indonesian,English": 0.4266666666666667, + "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.1875, - "English,Filipino,Spanish,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese": 0.16477272727272727, - "English,Spanish,Malay,Indonesian": 0.17045454545454544, - "English,Spanish,Malay,Vietnamese": 0.11931818181818182, - "English,Spanish,Chinese,Indonesian": 0.1875, - "English,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "English,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Chinese": 0.16477272727272727, - "Filipino,Spanish,Malay,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.125, - "Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.125, - "Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 + "Filipino,Vietnamese,Chinese,Spanish": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Malay": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,English": 0.18, + "Filipino,Vietnamese,Spanish,Malay": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.24, + "Filipino,Vietnamese,Spanish,English": 0.24, + "Filipino,Vietnamese,Malay,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Malay,English": 0.23333333333333334, + "Filipino,Vietnamese,Indonesian,English": 0.24, + "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, + "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, + "Filipino,Chinese,Spanish,English": 0.23333333333333334, + "Filipino,Chinese,Malay,Indonesian": 0.21333333333333335, + "Filipino,Chinese,Malay,English": 0.22666666666666666, + "Filipino,Chinese,Indonesian,English": 0.22, + "Filipino,Spanish,Malay,Indonesian": 0.26, + "Filipino,Spanish,Malay,English": 0.2733333333333333, + "Filipino,Spanish,Indonesian,English": 0.28, + "Filipino,Malay,Indonesian,English": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Indonesian": 0.21333333333333335, + "Vietnamese,Chinese,Spanish,English": 0.2, + "Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, + "Vietnamese,Chinese,Malay,English": 0.2, + "Vietnamese,Chinese,Indonesian,English": 0.19333333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.25333333333333335, + "Vietnamese,Spanish,Malay,English": 0.25333333333333335, + "Vietnamese,Spanish,Indonesian,English": 0.26666666666666666, + "Vietnamese,Malay,Indonesian,English": 0.24666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.25333333333333335, + "Chinese,Spanish,Malay,English": 0.26666666666666666, + "Chinese,Spanish,Indonesian,English": 0.28, + "Chinese,Malay,Indonesian,English": 0.23333333333333334, + "Spanish,Malay,Indonesian,English": 0.29333333333333333 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.16, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16, + "Filipino,Vietnamese,Chinese,Malay,English": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Spanish,Malay,English": 0.19333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.19333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Filipino,Chinese,Spanish,Malay,English": 0.20666666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.19333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.18, + "Filipino,Spanish,Malay,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, + "Vietnamese,Chinese,Spanish,Malay,English": 0.18, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16666666666666666, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.16666666666666666, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, + "Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 } }, - "AC3_2": 0.37406978949704545, - "AC3_3": 0.2717322245617535, - "AC3_4": 0.20080376935942826, - "AC3_5": 0.15200267019196606, - "AC3_6": 0.11670814212854444, - "AC3_7": 0.08876500855340302 + "AC3_2": 0.4963406276625506, + "AC3_3": 0.3891922562940493, + "AC3_4": 0.31824405535221373, + "AC3_5": 0.2689801642561209, + "AC3_6": 0.23183312259367947, + "AC3_7": 0.20165644168531607 }, "prompt_4": { - "overall_acc": 0.3474025974025974, + "overall_acc": 0.4847619047619048, "language_acc": { - "English": 0.4602272727272727, - "Filipino": 0.2727272727272727, - "Spanish": 0.3522727272727273, - "Malay": 0.2727272727272727, - "Chinese": 0.4659090909090909, - "Indonesian": 0.2897727272727273, - "Vietnamese": 0.3181818181818182 + "Filipino": 0.4666666666666667, + "Vietnamese": 0.4266666666666667, + "Chinese": 0.4266666666666667, + "Spanish": 0.5333333333333333, + "Malay": 0.36666666666666664, + "Indonesian": 0.52, + "English": 0.6533333333333333 }, - "consistency_score_2": 0.4356060606060607, - "consistency_score_3": 0.24350649350649348, - "consistency_score_4": 0.1574675324675325, - "consistency_score_5": 0.11011904761904762, - "consistency_score_6": 0.08116883116883115, - "consistency_score_7": 0.0625, + "consistency_score_2": 0.4809523809523809, + "consistency_score_3": 0.29847619047619056, + "consistency_score_4": 0.2100952380952381, + "consistency_score_5": 0.1596825396825397, + "consistency_score_6": 0.12761904761904763, + "consistency_score_7": 0.10666666666666667, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.3181818181818182, - "English,Spanish": 0.6534090909090909, - "English,Malay": 0.42045454545454547, - "English,Chinese": 0.5795454545454546, - "English,Indonesian": 0.4034090909090909, - "English,Vietnamese": 0.375, - "Filipino,Spanish": 0.4090909090909091, - "Filipino,Malay": 0.4602272727272727, - "Filipino,Chinese": 0.30113636363636365, - "Filipino,Indonesian": 0.42045454545454547, - "Filipino,Vietnamese": 0.4375, - "Spanish,Malay": 0.4147727272727273, - "Spanish,Chinese": 0.4943181818181818, - "Spanish,Indonesian": 0.42045454545454547, - "Spanish,Vietnamese": 0.4659090909090909, - "Malay,Chinese": 0.35795454545454547, - "Malay,Indonesian": 0.5454545454545454, - "Malay,Vietnamese": 0.4943181818181818, - "Chinese,Indonesian": 0.3181818181818182, - "Chinese,Vietnamese": 0.32954545454545453, - "Indonesian,Vietnamese": 0.5284090909090909 + "Filipino,Vietnamese": 0.38666666666666666, + "Filipino,Chinese": 0.38, + "Filipino,Spanish": 0.5133333333333333, + "Filipino,Malay": 0.4266666666666667, + "Filipino,Indonesian": 0.4533333333333333, + "Filipino,English": 0.5066666666666667, + "Vietnamese,Chinese": 0.44, + "Vietnamese,Spanish": 0.47333333333333333, + "Vietnamese,Malay": 0.49333333333333335, + "Vietnamese,Indonesian": 0.5466666666666666, + "Vietnamese,English": 0.47333333333333333, + "Chinese,Spanish": 0.47333333333333333, + "Chinese,Malay": 0.41333333333333333, + "Chinese,Indonesian": 0.4533333333333333, + "Chinese,English": 0.48, + "Spanish,Malay": 0.4866666666666667, + "Spanish,Indonesian": 0.54, + "Spanish,English": 0.62, + "Malay,Indonesian": 0.5533333333333333, + "Malay,English": 0.4266666666666667, + "Indonesian,English": 0.56 }, "3_combine": { - "English,Filipino,Spanish": 0.26704545454545453, - "English,Filipino,Malay": 0.21022727272727273, - "English,Filipino,Chinese": 0.19886363636363635, - "English,Filipino,Indonesian": 0.18181818181818182, - "English,Filipino,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay": 0.3181818181818182, - "English,Spanish,Chinese": 0.4090909090909091, - "English,Spanish,Indonesian": 0.3068181818181818, - "English,Spanish,Vietnamese": 0.32954545454545453, - "English,Malay,Chinese": 0.2556818181818182, - "English,Malay,Indonesian": 0.2556818181818182, - "English,Malay,Vietnamese": 0.24431818181818182, - "English,Chinese,Indonesian": 0.23863636363636365, - "English,Chinese,Vietnamese": 0.23295454545454544, - "English,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay": 0.2556818181818182, - "Filipino,Spanish,Chinese": 0.20454545454545456, - "Filipino,Spanish,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Vietnamese": 0.25, - "Filipino,Malay,Chinese": 0.17613636363636365, - "Filipino,Malay,Indonesian": 0.2840909090909091, - "Filipino,Malay,Vietnamese": 0.2727272727272727, - "Filipino,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Chinese": 0.2215909090909091, - "Spanish,Malay,Indonesian": 0.26136363636363635, - "Spanish,Malay,Vietnamese": 0.2784090909090909, - "Spanish,Chinese,Indonesian": 0.2159090909090909, - "Spanish,Chinese,Vietnamese": 0.23863636363636365, - "Spanish,Indonesian,Vietnamese": 0.2897727272727273, - "Malay,Chinese,Indonesian": 0.19886363636363635, - "Malay,Chinese,Vietnamese": 0.1875, - "Malay,Indonesian,Vietnamese": 0.3409090909090909, - "Chinese,Indonesian,Vietnamese": 0.1875 + "Filipino,Vietnamese,Chinese": 0.23333333333333334, + "Filipino,Vietnamese,Spanish": 0.2733333333333333, + "Filipino,Vietnamese,Malay": 0.24, + "Filipino,Vietnamese,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,English": 0.2733333333333333, + "Filipino,Chinese,Spanish": 0.2733333333333333, + "Filipino,Chinese,Malay": 0.21333333333333335, + "Filipino,Chinese,Indonesian": 0.25333333333333335, + "Filipino,Chinese,English": 0.25333333333333335, + "Filipino,Spanish,Malay": 0.30666666666666664, + "Filipino,Spanish,Indonesian": 0.31333333333333335, + "Filipino,Spanish,English": 0.38666666666666666, + "Filipino,Malay,Indonesian": 0.2866666666666667, + "Filipino,Malay,English": 0.28, + "Filipino,Indonesian,English": 0.32, + "Vietnamese,Chinese,Spanish": 0.2733333333333333, + "Vietnamese,Chinese,Malay": 0.26666666666666666, + "Vietnamese,Chinese,Indonesian": 0.30666666666666664, + "Vietnamese,Chinese,English": 0.28, + "Vietnamese,Spanish,Malay": 0.32, + "Vietnamese,Spanish,Indonesian": 0.3333333333333333, + "Vietnamese,Spanish,English": 0.32666666666666666, + "Vietnamese,Malay,Indonesian": 0.35333333333333333, + "Vietnamese,Malay,English": 0.2733333333333333, + "Vietnamese,Indonesian,English": 0.35333333333333333, + "Chinese,Spanish,Malay": 0.2733333333333333, + "Chinese,Spanish,Indonesian": 0.2866666666666667, + "Chinese,Spanish,English": 0.34, + "Chinese,Malay,Indonesian": 0.2866666666666667, + "Chinese,Malay,English": 0.24666666666666667, + "Chinese,Indonesian,English": 0.29333333333333333, + "Spanish,Malay,Indonesian": 0.37333333333333335, + "Spanish,Malay,English": 0.3333333333333333, + "Spanish,Indonesian,English": 0.4, + "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.19318181818181818, - "English,Filipino,Spanish,Chinese": 0.17613636363636365, - "English,Filipino,Spanish,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Chinese": 0.13068181818181818, - "English,Filipino,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Chinese": 0.21022727272727273, - "English,Spanish,Malay,Indonesian": 0.19886363636363635, - "English,Spanish,Malay,Vietnamese": 0.21022727272727273, - "English,Spanish,Chinese,Indonesian": 0.1875, - "English,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "English,Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "English,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Malay,Indonesian,Vietnamese": 0.1875, - "English,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese": 0.13068181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Malay,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 + "Filipino,Vietnamese,Chinese,Spanish": 0.18, + "Filipino,Vietnamese,Chinese,Malay": 0.16, + "Filipino,Vietnamese,Chinese,Indonesian": 0.2, + "Filipino,Vietnamese,Chinese,English": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Malay": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,English": 0.22, + "Filipino,Vietnamese,Malay,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,Malay,English": 0.19333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.22666666666666666, + "Filipino,Chinese,Spanish,Malay": 0.18, + "Filipino,Chinese,Spanish,Indonesian": 0.19333333333333333, + "Filipino,Chinese,Spanish,English": 0.21333333333333335, + "Filipino,Chinese,Malay,Indonesian": 0.17333333333333334, + "Filipino,Chinese,Malay,English": 0.16666666666666666, + "Filipino,Chinese,Indonesian,English": 0.19333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.24, + "Filipino,Spanish,Malay,English": 0.24666666666666667, + "Filipino,Spanish,Indonesian,English": 0.26, + "Filipino,Malay,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish,Malay": 0.20666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.20666666666666667, + "Vietnamese,Chinese,Spanish,English": 0.20666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.20666666666666667, + "Vietnamese,Chinese,Malay,English": 0.18, + "Vietnamese,Chinese,Indonesian,English": 0.22, + "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, + "Vietnamese,Spanish,Malay,English": 0.23333333333333334, + "Vietnamese,Spanish,Indonesian,English": 0.25333333333333335, + "Vietnamese,Malay,Indonesian,English": 0.23333333333333334, + "Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, + "Chinese,Spanish,Malay,English": 0.2, + "Chinese,Spanish,Indonesian,English": 0.21333333333333335, + "Chinese,Malay,Indonesian,English": 0.19333333333333333, + "Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.125, - "English,Filipino,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.14, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.16, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Indonesian,English": 0.16, + "Filipino,Chinese,Malay,Indonesian,English": 0.14, + "Filipino,Spanish,Malay,Indonesian,English": 0.2, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16, + "Vietnamese,Chinese,Spanish,Malay,English": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.15333333333333332, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, + "Chinese,Spanish,Malay,Indonesian,English": 0.16 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, - "AC3_2": 0.38653640756513463, - "AC3_3": 0.28632082198665343, - "AC3_4": 0.21670772952654246, - "AC3_5": 0.16722987239734247, - "AC3_6": 0.1315918929248245, - "AC3_7": 0.10594059403356063 + "AC3_2": 0.4828496289534759, + "AC3_3": 0.36946590693819076, + "AC3_4": 0.29314275267293366, + "AC3_5": 0.24023176163290802, + "AC3_6": 0.20204695249607396, + "AC3_7": 0.1748577562774664 }, "prompt_5": { - "overall_acc": 0.3384740259740259, + "overall_acc": 0.46380952380952384, "language_acc": { - "English": 0.4431818181818182, - "Filipino": 0.2784090909090909, - "Spanish": 0.3465909090909091, - "Malay": 0.2840909090909091, - "Chinese": 0.42613636363636365, - "Indonesian": 0.2840909090909091, - "Vietnamese": 0.3068181818181818 + "Filipino": 0.41333333333333333, + "Vietnamese": 0.4666666666666667, + "Chinese": 0.42, + "Spanish": 0.49333333333333335, + "Malay": 0.3466666666666667, + "Indonesian": 0.4666666666666667, + "English": 0.64 }, - "consistency_score_2": 0.4015151515151516, - "consistency_score_3": 0.20422077922077925, - "consistency_score_4": 0.12012987012987011, - "consistency_score_5": 0.07602813852813853, - "consistency_score_6": 0.04951298701298702, - "consistency_score_7": 0.03409090909090909, + "consistency_score_2": 0.45682539682539686, + "consistency_score_3": 0.2666666666666666, + "consistency_score_4": 0.18095238095238095, + "consistency_score_5": 0.13714285714285715, + "consistency_score_6": 0.11333333333333333, + "consistency_score_7": 0.1, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.3465909090909091, - "English,Spanish": 0.5852272727272727, - "English,Malay": 0.3409090909090909, - "English,Chinese": 0.5454545454545454, - "English,Indonesian": 0.3806818181818182, - "English,Vietnamese": 0.375, - "Filipino,Spanish": 0.42045454545454547, - "Filipino,Malay": 0.4034090909090909, - "Filipino,Chinese": 0.3522727272727273, - "Filipino,Indonesian": 0.375, - "Filipino,Vietnamese": 0.3806818181818182, - "Spanish,Malay": 0.35795454545454547, - "Spanish,Chinese": 0.4715909090909091, - "Spanish,Indonesian": 0.4375, - "Spanish,Vietnamese": 0.35795454545454547, - "Malay,Chinese": 0.3125, - "Malay,Indonesian": 0.4772727272727273, - "Malay,Vietnamese": 0.4090909090909091, - "Chinese,Indonesian": 0.3181818181818182, - "Chinese,Vietnamese": 0.32386363636363635, - "Indonesian,Vietnamese": 0.4602272727272727 + "Filipino,Vietnamese": 0.41333333333333333, + "Filipino,Chinese": 0.36666666666666664, + "Filipino,Spanish": 0.46, + "Filipino,Malay": 0.43333333333333335, + "Filipino,Indonesian": 0.4666666666666667, + "Filipino,English": 0.4866666666666667, + "Vietnamese,Chinese": 0.4, + "Vietnamese,Spanish": 0.44666666666666666, + "Vietnamese,Malay": 0.42, + "Vietnamese,Indonesian": 0.44, + "Vietnamese,English": 0.44, + "Chinese,Spanish": 0.5133333333333333, + "Chinese,Malay": 0.37333333333333335, + "Chinese,Indonesian": 0.46, + "Chinese,English": 0.5066666666666667, + "Spanish,Malay": 0.43333333333333335, + "Spanish,Indonesian": 0.48, + "Spanish,English": 0.5933333333333334, + "Malay,Indonesian": 0.5133333333333333, + "Malay,English": 0.44, + "Indonesian,English": 0.5066666666666667 }, "3_combine": { - "English,Filipino,Spanish": 0.23863636363636365, - "English,Filipino,Malay": 0.16477272727272727, - "English,Filipino,Chinese": 0.19886363636363635, - "English,Filipino,Indonesian": 0.1590909090909091, - "English,Filipino,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay": 0.25, - "English,Spanish,Chinese": 0.3522727272727273, - "English,Spanish,Indonesian": 0.2727272727272727, - "English,Spanish,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese": 0.20454545454545456, - "English,Malay,Indonesian": 0.21022727272727273, - "English,Malay,Vietnamese": 0.19318181818181818, - "English,Chinese,Indonesian": 0.19886363636363635, - "English,Chinese,Vietnamese": 0.19886363636363635, - "English,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay": 0.20454545454545456, - "Filipino,Spanish,Chinese": 0.21022727272727273, - "Filipino,Spanish,Indonesian": 0.23295454545454544, - "Filipino,Spanish,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese": 0.14204545454545456, - "Filipino,Malay,Indonesian": 0.23295454545454544, - "Filipino,Malay,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Indonesian,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Chinese": 0.1875, - "Spanish,Malay,Indonesian": 0.23295454545454544, - "Spanish,Malay,Vietnamese": 0.1875, - "Spanish,Chinese,Indonesian": 0.2215909090909091, - "Spanish,Chinese,Vietnamese": 0.17613636363636365, - "Spanish,Indonesian,Vietnamese": 0.23295454545454544, - "Malay,Chinese,Indonesian": 0.16477272727272727, - "Malay,Chinese,Vietnamese": 0.14204545454545456, - "Malay,Indonesian,Vietnamese": 0.26136363636363635, - "Chinese,Indonesian,Vietnamese": 0.14204545454545456 + "Filipino,Vietnamese,Chinese": 0.2, + "Filipino,Vietnamese,Spanish": 0.24, + "Filipino,Vietnamese,Malay": 0.22, + "Filipino,Vietnamese,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,English": 0.24666666666666667, + "Filipino,Chinese,Spanish": 0.24666666666666667, + "Filipino,Chinese,Malay": 0.19333333333333333, + "Filipino,Chinese,Indonesian": 0.24666666666666667, + "Filipino,Chinese,English": 0.24, + "Filipino,Spanish,Malay": 0.25333333333333335, + "Filipino,Spanish,Indonesian": 0.29333333333333333, + "Filipino,Spanish,English": 0.34, + "Filipino,Malay,Indonesian": 0.31333333333333335, + "Filipino,Malay,English": 0.25333333333333335, + "Filipino,Indonesian,English": 0.29333333333333333, + "Vietnamese,Chinese,Spanish": 0.26, + "Vietnamese,Chinese,Malay": 0.2, + "Vietnamese,Chinese,Indonesian": 0.22666666666666666, + "Vietnamese,Chinese,English": 0.25333333333333335, + "Vietnamese,Spanish,Malay": 0.25333333333333335, + "Vietnamese,Spanish,Indonesian": 0.25333333333333335, + "Vietnamese,Spanish,English": 0.29333333333333333, + "Vietnamese,Malay,Indonesian": 0.26, + "Vietnamese,Malay,English": 0.24666666666666667, + "Vietnamese,Indonesian,English": 0.29333333333333333, + "Chinese,Spanish,Malay": 0.24, + "Chinese,Spanish,Indonesian": 0.3, + "Chinese,Spanish,English": 0.36666666666666664, + "Chinese,Malay,Indonesian": 0.26666666666666666, + "Chinese,Malay,English": 0.24, + "Chinese,Indonesian,English": 0.2866666666666667, + "Spanish,Malay,Indonesian": 0.2866666666666667, + "Spanish,Malay,English": 0.30666666666666664, + "Spanish,Indonesian,English": 0.36666666666666664, + "Malay,Indonesian,English": 0.30666666666666664 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.13636363636363635, - "English,Filipino,Spanish,Chinese": 0.13068181818181818, - "English,Filipino,Spanish,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese": 0.09090909090909091, - "English,Filipino,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Spanish,Malay,Indonesian": 0.17613636363636365, - "English,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Spanish,Chinese,Indonesian": 0.1590909090909091, - "English,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "English,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "English,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese": 0.10227272727272728, - "Filipino,Spanish,Malay,Indonesian": 0.1534090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Malay,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Chinese,Indonesian": 0.125, - "Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 + "Filipino,Vietnamese,Chinese,Spanish": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Malay": 0.12, + "Filipino,Vietnamese,Chinese,Indonesian": 0.14, + "Filipino,Vietnamese,Chinese,English": 0.14, + "Filipino,Vietnamese,Spanish,Malay": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.18, + "Filipino,Vietnamese,Spanish,English": 0.18666666666666668, + "Filipino,Vietnamese,Malay,Indonesian": 0.18, + "Filipino,Vietnamese,Malay,English": 0.14666666666666667, + "Filipino,Vietnamese,Indonesian,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Malay": 0.16, + "Filipino,Chinese,Spanish,Indonesian": 0.2, + "Filipino,Chinese,Spanish,English": 0.2, + "Filipino,Chinese,Malay,Indonesian": 0.18, + "Filipino,Chinese,Malay,English": 0.14, + "Filipino,Chinese,Indonesian,English": 0.17333333333333334, + "Filipino,Spanish,Malay,Indonesian": 0.21333333333333335, + "Filipino,Spanish,Malay,English": 0.21333333333333335, + "Filipino,Spanish,Indonesian,English": 0.24, + "Filipino,Malay,Indonesian,English": 0.20666666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,Indonesian": 0.18, + "Vietnamese,Chinese,Spanish,English": 0.2, + "Vietnamese,Chinese,Malay,Indonesian": 0.16, + "Vietnamese,Chinese,Malay,English": 0.16666666666666666, + "Vietnamese,Chinese,Indonesian,English": 0.18, + "Vietnamese,Spanish,Malay,Indonesian": 0.18, + "Vietnamese,Spanish,Malay,English": 0.18666666666666668, + "Vietnamese,Spanish,Indonesian,English": 0.2, + "Vietnamese,Malay,Indonesian,English": 0.20666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.18, + "Chinese,Spanish,Malay,English": 0.19333333333333333, + "Chinese,Spanish,Indonesian,English": 0.22666666666666666, + "Chinese,Malay,Indonesian,English": 0.17333333333333334, + "Spanish,Malay,Indonesian,English": 0.23333333333333334 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0625, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Malay,English": 0.1, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.11333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Malay,English": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14666666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.14, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.14, + "Filipino,Chinese,Spanish,Indonesian,English": 0.16, + "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Spanish,Malay,Indonesian,English": 0.18, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,Malay,English": 0.14, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.14, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, + "Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.1, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 } }, - "AC3_2": 0.36730929029438814, - "AC3_3": 0.2547414445688689, - "AC3_4": 0.17732444542737222, - "AC3_5": 0.12416605910673029, - "AC3_6": 0.08638876811336418, - "AC3_7": 0.06194295898515644 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.4174757281553398 - }, - "prompt_2": { - "accuracy": 0.42718446601941745 - }, - "prompt_3": { - "accuracy": 0.39805825242718446 - }, - "prompt_4": { - "accuracy": 0.42718446601941745 - }, - "prompt_5": { - "accuracy": 0.4368932038834951 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.4380952380952381 - }, - "prompt_2": { - "accuracy": 0.4857142857142857 - }, - "prompt_3": { - "accuracy": 0.4095238095238095 - }, - "prompt_4": { - "accuracy": 0.47619047619047616 - }, - "prompt_5": { - "accuracy": 0.45714285714285713 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.4205607476635514 - }, - "prompt_2": { - "accuracy": 0.4392523364485981 - }, - "prompt_3": { - "accuracy": 0.42990654205607476 - }, - "prompt_4": { - "accuracy": 0.4672897196261682 - }, - "prompt_5": { - "accuracy": 0.411214953271028 + "AC3_2": 0.46029096875131653, + "AC3_3": 0.33863537587715326, + "AC3_4": 0.26033621716437777, + "AC3_5": 0.21169119308227463, + "AC3_6": 0.18215621559000042, + "AC3_7": 0.16452702699784572 } }, - "ph_eval": { + "cross_logiqa": { "prompt_1": { - "accuracy": 0.31, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.4, - "history": 0.2, - "literature": 0.3, - "politics": 0.4, - "culture": 0.5, - "film": 0.1, - "law": 0.3, - "geography": 0.5 - } - }, - "prompt_2": { - "accuracy": 0.32, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.5, - "history": 0.2, - "literature": 0.2, - "politics": 0.5, - "culture": 0.4, - "film": 0.1, - "law": 0.3, - "geography": 0.5 - } - }, - "prompt_3": { - "accuracy": 0.31, - "category_acc": { - "brand": 0.3, - "demographics": 0.6, - "biology": 0.3, - "history": 0.13333333333333333, - "literature": 0.2, - "politics": 0.5, - "culture": 0.4, - "film": 0.2, - "law": 0.3, - "geography": 0.4 - } - }, - "prompt_4": { - "accuracy": 0.29, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.3, - "history": 0.13333333333333333, - "literature": 0.1, - "politics": 0.4, - "culture": 0.6, - "film": 0.1, - "law": 0.3, - "geography": 0.5 - } - }, - "prompt_5": { - "accuracy": 0.3, - "category_acc": { - "brand": 0.2, - "demographics": 0.4, - "biology": 0.5, - "history": 0.13333333333333333, - "literature": 0.3, - "politics": 0.3, - "culture": 0.6, - "film": 0.1, - "law": 0.3, - "geography": 0.3 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.20522658325519624 + "overall_acc": 0.4180194805194805, + "language_acc": { + "Indonesian": 0.4090909090909091, + "English": 0.5340909090909091, + "Filipino": 0.3409090909090909, + "Spanish": 0.39204545454545453, + "Chinese": 0.44886363636363635, + "Malay": 0.3806818181818182, + "Vietnamese": 0.42045454545454547 + }, + "consistency_score_2": 0.515151515151515, + "consistency_score_3": 0.33084415584415583, + "consistency_score_4": 0.23198051948051943, + "consistency_score_5": 0.17288961038961037, + "consistency_score_6": 0.13636363636363638, + "consistency_score_7": 0.11363636363636363, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.5568181818181818, + "Indonesian,Filipino": 0.5170454545454546, + "Indonesian,Spanish": 0.5681818181818182, + "Indonesian,Chinese": 0.4715909090909091, + "Indonesian,Malay": 0.5625, + "Indonesian,Vietnamese": 0.5454545454545454, + "English,Filipino": 0.4375, + "English,Spanish": 0.5965909090909091, + "English,Chinese": 0.5568181818181818, + "English,Malay": 0.5568181818181818, + "English,Vietnamese": 0.5340909090909091, + "Filipino,Spanish": 0.4659090909090909, + "Filipino,Chinese": 0.42613636363636365, + "Filipino,Malay": 0.4602272727272727, + "Filipino,Vietnamese": 0.45454545454545453, + "Spanish,Chinese": 0.5511363636363636, + "Spanish,Malay": 0.5568181818181818, + "Spanish,Vietnamese": 0.5284090909090909, + "Chinese,Malay": 0.48863636363636365, + "Chinese,Vietnamese": 0.4943181818181818, + "Malay,Vietnamese": 0.48863636363636365 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.3125, + "Indonesian,English,Spanish": 0.4090909090909091, + "Indonesian,English,Chinese": 0.3465909090909091, + "Indonesian,English,Malay": 0.38636363636363635, + "Indonesian,English,Vietnamese": 0.3693181818181818, + "Indonesian,Filipino,Spanish": 0.3409090909090909, + "Indonesian,Filipino,Chinese": 0.2727272727272727, + "Indonesian,Filipino,Malay": 0.32954545454545453, + "Indonesian,Filipino,Vietnamese": 0.32386363636363635, + "Indonesian,Spanish,Chinese": 0.32954545454545453, + "Indonesian,Spanish,Malay": 0.4034090909090909, + "Indonesian,Spanish,Vietnamese": 0.3806818181818182, + "Indonesian,Chinese,Malay": 0.3125, + "Indonesian,Chinese,Vietnamese": 0.3125, + "Indonesian,Malay,Vietnamese": 0.3522727272727273, + "English,Filipino,Spanish": 0.29545454545454547, + "English,Filipino,Chinese": 0.2897727272727273, + "English,Filipino,Malay": 0.2840909090909091, + "English,Filipino,Vietnamese": 0.29545454545454547, + "English,Spanish,Chinese": 0.39204545454545453, + "English,Spanish,Malay": 0.39204545454545453, + "English,Spanish,Vietnamese": 0.38636363636363635, + "English,Chinese,Malay": 0.3693181818181818, + "English,Chinese,Vietnamese": 0.3409090909090909, + "English,Malay,Vietnamese": 0.3465909090909091, + "Filipino,Spanish,Chinese": 0.2727272727272727, + "Filipino,Spanish,Malay": 0.2897727272727273, + "Filipino,Spanish,Vietnamese": 0.3125, + "Filipino,Chinese,Malay": 0.25, + "Filipino,Chinese,Vietnamese": 0.26704545454545453, + "Filipino,Malay,Vietnamese": 0.2897727272727273, + "Spanish,Chinese,Malay": 0.3465909090909091, + "Spanish,Chinese,Vietnamese": 0.3352272727272727, + "Spanish,Malay,Vietnamese": 0.3522727272727273, + "Chinese,Malay,Vietnamese": 0.2897727272727273 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.23295454545454544, + "Indonesian,English,Filipino,Chinese": 0.20454545454545456, + "Indonesian,English,Filipino,Malay": 0.2159090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.23295454545454544, + "Indonesian,English,Spanish,Chinese": 0.2784090909090909, + "Indonesian,English,Spanish,Malay": 0.2897727272727273, + "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, + "Indonesian,English,Chinese,Malay": 0.26136363636363635, + "Indonesian,English,Chinese,Vietnamese": 0.24431818181818182, + "Indonesian,English,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,Filipino,Spanish,Chinese": 0.19318181818181818, + "Indonesian,Filipino,Spanish,Malay": 0.25, + "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.19318181818181818, + "Indonesian,Filipino,Chinese,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Malay,Vietnamese": 0.22727272727272727, + "Indonesian,Spanish,Chinese,Malay": 0.24431818181818182, + "Indonesian,Spanish,Chinese,Vietnamese": 0.23863636363636365, + "Indonesian,Spanish,Malay,Vietnamese": 0.2897727272727273, + "Indonesian,Chinese,Malay,Vietnamese": 0.21022727272727273, + "English,Filipino,Spanish,Chinese": 0.19886363636363635, + "English,Filipino,Spanish,Malay": 0.20454545454545456, + "English,Filipino,Spanish,Vietnamese": 0.23863636363636365, + "English,Filipino,Chinese,Malay": 0.20454545454545456, + "English,Filipino,Chinese,Vietnamese": 0.19318181818181818, + "English,Filipino,Malay,Vietnamese": 0.19886363636363635, + "English,Spanish,Chinese,Malay": 0.2840909090909091, + "English,Spanish,Chinese,Vietnamese": 0.26704545454545453, + "English,Spanish,Malay,Vietnamese": 0.2784090909090909, + "English,Chinese,Malay,Vietnamese": 0.23863636363636365, + "Filipino,Spanish,Chinese,Malay": 0.17613636363636365, + "Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, + "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, + "Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, + "Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.1534090909090909, + "Indonesian,English,Filipino,Spanish,Malay": 0.17045454545454544, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.19318181818181818, + "Indonesian,English,Filipino,Chinese,Malay": 0.1590909090909091, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1590909090909091, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,English,Spanish,Chinese,Malay": 0.21022727272727273, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.20454545454545456, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, + "English,Filipino,Spanish,Chinese,Malay": 0.1534090909090909, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, + "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, + "English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, + "English,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 + }, + "6_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.125, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 + }, + "7_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 + } + }, + "AC3_2": 0.4615303513040302, + "AC3_3": 0.3693577720290219, + "AC3_4": 0.29837654224297583, + "AC3_5": 0.24461030037247625, + "AC3_6": 0.20564355114086408, + "AC3_7": 0.17869535041746457 }, "prompt_2": { - "bleu_score": 0.19516016031956118 + "overall_acc": 0.4480519480519481, + "language_acc": { + "Indonesian": 0.42613636363636365, + "English": 0.5681818181818182, + "Filipino": 0.3693181818181818, + "Spanish": 0.4375, + "Chinese": 0.4715909090909091, + "Malay": 0.42045454545454547, + "Vietnamese": 0.4431818181818182 + }, + "consistency_score_2": 0.525974025974026, + "consistency_score_3": 0.3459415584415584, + "consistency_score_4": 0.25129870129870124, + "consistency_score_5": 0.19345238095238096, + "consistency_score_6": 0.15584415584415587, + "consistency_score_7": 0.13068181818181818, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.5625, + "Indonesian,Filipino": 0.5738636363636364, + "Indonesian,Spanish": 0.625, + "Indonesian,Chinese": 0.5170454545454546, + "Indonesian,Malay": 0.6022727272727273, + "Indonesian,Vietnamese": 0.5568181818181818, + "English,Filipino": 0.44886363636363635, + "English,Spanish": 0.625, + "English,Chinese": 0.5795454545454546, + "English,Malay": 0.4943181818181818, + "English,Vietnamese": 0.4943181818181818, + "Filipino,Spanish": 0.4715909090909091, + "Filipino,Chinese": 0.45454545454545453, + "Filipino,Malay": 0.5056818181818182, + "Filipino,Vietnamese": 0.5113636363636364, + "Spanish,Chinese": 0.5227272727272727, + "Spanish,Malay": 0.5568181818181818, + "Spanish,Vietnamese": 0.5511363636363636, + "Chinese,Malay": 0.4318181818181818, + "Chinese,Vietnamese": 0.48863636363636365, + "Malay,Vietnamese": 0.4715909090909091 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.3522727272727273, + "Indonesian,English,Spanish": 0.4431818181818182, + "Indonesian,English,Chinese": 0.375, + "Indonesian,English,Malay": 0.3693181818181818, + "Indonesian,English,Vietnamese": 0.36363636363636365, + "Indonesian,Filipino,Spanish": 0.375, + "Indonesian,Filipino,Chinese": 0.3465909090909091, + "Indonesian,Filipino,Malay": 0.3806818181818182, + "Indonesian,Filipino,Vietnamese": 0.3806818181818182, + "Indonesian,Spanish,Chinese": 0.375, + "Indonesian,Spanish,Malay": 0.4318181818181818, + "Indonesian,Spanish,Vietnamese": 0.4090909090909091, + "Indonesian,Chinese,Malay": 0.3181818181818182, + "Indonesian,Chinese,Vietnamese": 0.3352272727272727, + "Indonesian,Malay,Vietnamese": 0.3522727272727273, + "English,Filipino,Spanish": 0.32386363636363635, + "English,Filipino,Chinese": 0.3181818181818182, + "English,Filipino,Malay": 0.29545454545454547, + "English,Filipino,Vietnamese": 0.3068181818181818, + "English,Spanish,Chinese": 0.4147727272727273, + "English,Spanish,Malay": 0.39204545454545453, + "English,Spanish,Vietnamese": 0.38636363636363635, + "English,Chinese,Malay": 0.3068181818181818, + "English,Chinese,Vietnamese": 0.3465909090909091, + "English,Malay,Vietnamese": 0.30113636363636365, + "Filipino,Spanish,Chinese": 0.2897727272727273, + "Filipino,Spanish,Malay": 0.32386363636363635, + "Filipino,Spanish,Vietnamese": 0.3465909090909091, + "Filipino,Chinese,Malay": 0.26704545454545453, + "Filipino,Chinese,Vietnamese": 0.29545454545454547, + "Filipino,Malay,Vietnamese": 0.29545454545454547, + "Spanish,Chinese,Malay": 0.3181818181818182, + "Spanish,Chinese,Vietnamese": 0.3465909090909091, + "Spanish,Malay,Vietnamese": 0.3465909090909091, + "Chinese,Malay,Vietnamese": 0.2784090909090909 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.2784090909090909, + "Indonesian,English,Filipino,Chinese": 0.26136363636363635, + "Indonesian,English,Filipino,Malay": 0.2556818181818182, + "Indonesian,English,Filipino,Vietnamese": 0.2556818181818182, + "Indonesian,English,Spanish,Chinese": 0.3068181818181818, + "Indonesian,English,Spanish,Malay": 0.3068181818181818, + "Indonesian,English,Spanish,Vietnamese": 0.30113636363636365, + "Indonesian,English,Chinese,Malay": 0.23863636363636365, + "Indonesian,English,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,English,Malay,Vietnamese": 0.23863636363636365, + "Indonesian,Filipino,Spanish,Chinese": 0.24431818181818182, + "Indonesian,Filipino,Spanish,Malay": 0.2840909090909091, + "Indonesian,Filipino,Spanish,Vietnamese": 0.2784090909090909, + "Indonesian,Filipino,Chinese,Malay": 0.22727272727272727, + "Indonesian,Filipino,Chinese,Vietnamese": 0.24431818181818182, + "Indonesian,Filipino,Malay,Vietnamese": 0.25, + "Indonesian,Spanish,Chinese,Malay": 0.25, + "Indonesian,Spanish,Chinese,Vietnamese": 0.26704545454545453, + "Indonesian,Spanish,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,Chinese,Malay,Vietnamese": 0.2159090909090909, + "English,Filipino,Spanish,Chinese": 0.24431818181818182, + "English,Filipino,Spanish,Malay": 0.24431818181818182, + "English,Filipino,Spanish,Vietnamese": 0.2556818181818182, + "English,Filipino,Chinese,Malay": 0.2159090909090909, + "English,Filipino,Chinese,Vietnamese": 0.23863636363636365, + "English,Filipino,Malay,Vietnamese": 0.20454545454545456, + "English,Spanish,Chinese,Malay": 0.26136363636363635, + "English,Spanish,Chinese,Vietnamese": 0.2840909090909091, + "English,Spanish,Malay,Vietnamese": 0.26136363636363635, + "English,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Filipino,Spanish,Chinese,Malay": 0.2159090909090909, + "Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, + "Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, + "Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, + "Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.21022727272727273, + "Indonesian,English,Filipino,Spanish,Malay": 0.2159090909090909, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2159090909090909, + "Indonesian,English,Filipino,Chinese,Malay": 0.1875, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.19886363636363635, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,English,Spanish,Chinese,Malay": 0.20454545454545456, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.22727272727272727, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.1875, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, + "English,Filipino,Spanish,Chinese,Malay": 0.1875, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, + "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, + "English,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, + "English,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 + }, + "6_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 + }, + "7_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 + } + }, + "AC3_2": 0.48389610384642395, + "AC3_3": 0.39043087351398426, + "AC3_4": 0.32199833587987, + "AC3_5": 0.2702295595168321, + "AC3_6": 0.2312526183110991, + "AC3_7": 0.20234604102075487 }, "prompt_3": { - "bleu_score": 0.2001082705254649 + "overall_acc": 0.4342532467532468, + "language_acc": { + "Indonesian": 0.4090909090909091, + "English": 0.5568181818181818, + "Filipino": 0.3465909090909091, + "Spanish": 0.44886363636363635, + "Chinese": 0.4602272727272727, + "Malay": 0.4090909090909091, + "Vietnamese": 0.4090909090909091 + }, + "consistency_score_2": 0.5300324675324676, + "consistency_score_3": 0.35698051948051945, + "consistency_score_4": 0.2662337662337662, + "consistency_score_5": 0.21049783549783546, + "consistency_score_6": 0.17370129870129872, + "consistency_score_7": 0.14772727272727273, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.5227272727272727, + "Indonesian,Filipino": 0.5681818181818182, + "Indonesian,Spanish": 0.6477272727272727, + "Indonesian,Chinese": 0.48295454545454547, + "Indonesian,Malay": 0.5795454545454546, + "Indonesian,Vietnamese": 0.5681818181818182, + "English,Filipino": 0.4318181818181818, + "English,Spanish": 0.6306818181818182, + "English,Chinese": 0.5454545454545454, + "English,Malay": 0.5397727272727273, + "English,Vietnamese": 0.4943181818181818, + "Filipino,Spanish": 0.4943181818181818, + "Filipino,Chinese": 0.44886363636363635, + "Filipino,Malay": 0.48295454545454547, + "Filipino,Vietnamese": 0.4715909090909091, + "Spanish,Chinese": 0.5170454545454546, + "Spanish,Malay": 0.6022727272727273, + "Spanish,Vietnamese": 0.5909090909090909, + "Chinese,Malay": 0.5170454545454546, + "Chinese,Vietnamese": 0.4943181818181818, + "Malay,Vietnamese": 0.5 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.3181818181818182, + "Indonesian,English,Spanish": 0.44886363636363635, + "Indonesian,English,Chinese": 0.32954545454545453, + "Indonesian,English,Malay": 0.3693181818181818, + "Indonesian,English,Vietnamese": 0.35795454545454547, + "Indonesian,Filipino,Spanish": 0.4034090909090909, + "Indonesian,Filipino,Chinese": 0.32386363636363635, + "Indonesian,Filipino,Malay": 0.3693181818181818, + "Indonesian,Filipino,Vietnamese": 0.36363636363636365, + "Indonesian,Spanish,Chinese": 0.3806818181818182, + "Indonesian,Spanish,Malay": 0.45454545454545453, + "Indonesian,Spanish,Vietnamese": 0.45454545454545453, + "Indonesian,Chinese,Malay": 0.3522727272727273, + "Indonesian,Chinese,Vietnamese": 0.32386363636363635, + "Indonesian,Malay,Vietnamese": 0.38636363636363635, + "English,Filipino,Spanish": 0.3352272727272727, + "English,Filipino,Chinese": 0.29545454545454547, + "English,Filipino,Malay": 0.2897727272727273, + "English,Filipino,Vietnamese": 0.2897727272727273, + "English,Spanish,Chinese": 0.39204545454545453, + "English,Spanish,Malay": 0.4375, + "English,Spanish,Vietnamese": 0.42045454545454547, + "English,Chinese,Malay": 0.36363636363636365, + "English,Chinese,Vietnamese": 0.32954545454545453, + "English,Malay,Vietnamese": 0.3409090909090909, + "Filipino,Spanish,Chinese": 0.29545454545454547, + "Filipino,Spanish,Malay": 0.3465909090909091, + "Filipino,Spanish,Vietnamese": 0.3522727272727273, + "Filipino,Chinese,Malay": 0.29545454545454547, + "Filipino,Chinese,Vietnamese": 0.29545454545454547, + "Filipino,Malay,Vietnamese": 0.3125, + "Spanish,Chinese,Malay": 0.3693181818181818, + "Spanish,Chinese,Vietnamese": 0.36363636363636365, + "Spanish,Malay,Vietnamese": 0.3977272727272727, + "Chinese,Malay,Vietnamese": 0.3352272727272727 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.2897727272727273, + "Indonesian,English,Filipino,Chinese": 0.2215909090909091, + "Indonesian,English,Filipino,Malay": 0.23295454545454544, + "Indonesian,English,Filipino,Vietnamese": 0.25, + "Indonesian,English,Spanish,Chinese": 0.3068181818181818, + "Indonesian,English,Spanish,Malay": 0.3352272727272727, + "Indonesian,English,Spanish,Vietnamese": 0.3409090909090909, + "Indonesian,English,Chinese,Malay": 0.26704545454545453, + "Indonesian,English,Chinese,Vietnamese": 0.23863636363636365, + "Indonesian,English,Malay,Vietnamese": 0.2727272727272727, + "Indonesian,Filipino,Spanish,Chinese": 0.2556818181818182, + "Indonesian,Filipino,Spanish,Malay": 0.29545454545454547, + "Indonesian,Filipino,Spanish,Vietnamese": 0.30113636363636365, + "Indonesian,Filipino,Chinese,Malay": 0.23863636363636365, + "Indonesian,Filipino,Chinese,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Malay,Vietnamese": 0.2727272727272727, + "Indonesian,Spanish,Chinese,Malay": 0.29545454545454547, + "Indonesian,Spanish,Chinese,Vietnamese": 0.2840909090909091, + "Indonesian,Spanish,Malay,Vietnamese": 0.32954545454545453, + "Indonesian,Chinese,Malay,Vietnamese": 0.26704545454545453, + "English,Filipino,Spanish,Chinese": 0.23295454545454544, + "English,Filipino,Spanish,Malay": 0.24431818181818182, + "English,Filipino,Spanish,Vietnamese": 0.2727272727272727, + "English,Filipino,Chinese,Malay": 0.2159090909090909, + "English,Filipino,Chinese,Vietnamese": 0.21022727272727273, + "English,Filipino,Malay,Vietnamese": 0.2159090909090909, + "English,Spanish,Chinese,Malay": 0.3125, + "English,Spanish,Chinese,Vietnamese": 0.2840909090909091, + "English,Spanish,Malay,Vietnamese": 0.3181818181818182, + "English,Chinese,Malay,Vietnamese": 0.2556818181818182, + "Filipino,Spanish,Chinese,Malay": 0.22727272727272727, + "Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, + "Filipino,Spanish,Malay,Vietnamese": 0.2556818181818182, + "Filipino,Chinese,Malay,Vietnamese": 0.22727272727272727, + "Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.20454545454545456, + "Indonesian,English,Filipino,Spanish,Malay": 0.2159090909090909, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.24431818181818182, + "Indonesian,English,Filipino,Chinese,Malay": 0.17613636363636365, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.17613636363636365, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.19318181818181818, + "Indonesian,English,Spanish,Chinese,Malay": 0.25, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.23295454545454544, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.19886363636363635, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, + "English,Filipino,Spanish,Chinese,Malay": 0.1875, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, + "English,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, + "English,Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, + "English,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1875 + }, + "6_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727 + }, + "7_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 + } + }, + "AC3_2": 0.47738614495027465, + "AC3_3": 0.39184361489113734, + "AC3_4": 0.3300928503238267, + "AC3_5": 0.2835493293509241, + "AC3_6": 0.2481447123896104, + "AC3_7": 0.2204577151897091 }, "prompt_4": { - "bleu_score": 0.18545912998416814 + "overall_acc": 0.4537337662337662, + "language_acc": { + "Indonesian": 0.4431818181818182, + "English": 0.5681818181818182, + "Filipino": 0.3409090909090909, + "Spanish": 0.4659090909090909, + "Chinese": 0.48863636363636365, + "Malay": 0.4090909090909091, + "Vietnamese": 0.4602272727272727 + }, + "consistency_score_2": 0.5454545454545454, + "consistency_score_3": 0.37775974025974035, + "consistency_score_4": 0.28847402597402594, + "consistency_score_5": 0.23214285714285715, + "consistency_score_6": 0.19318181818181818, + "consistency_score_7": 0.16477272727272727, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.5852272727272727, + "Indonesian,Filipino": 0.5852272727272727, + "Indonesian,Spanish": 0.6079545454545454, + "Indonesian,Chinese": 0.5568181818181818, + "Indonesian,Malay": 0.6136363636363636, + "Indonesian,Vietnamese": 0.6306818181818182, + "English,Filipino": 0.4431818181818182, + "English,Spanish": 0.6420454545454546, + "English,Chinese": 0.6022727272727273, + "English,Malay": 0.5397727272727273, + "English,Vietnamese": 0.5511363636363636, + "Filipino,Spanish": 0.4715909090909091, + "Filipino,Chinese": 0.45454545454545453, + "Filipino,Malay": 0.45454545454545453, + "Filipino,Vietnamese": 0.5, + "Spanish,Chinese": 0.5511363636363636, + "Spanish,Malay": 0.5681818181818182, + "Spanish,Vietnamese": 0.5511363636363636, + "Chinese,Malay": 0.4772727272727273, + "Chinese,Vietnamese": 0.5113636363636364, + "Malay,Vietnamese": 0.5568181818181818 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.3693181818181818, + "Indonesian,English,Spanish": 0.4715909090909091, + "Indonesian,English,Chinese": 0.4034090909090909, + "Indonesian,English,Malay": 0.4090909090909091, + "Indonesian,English,Vietnamese": 0.45454545454545453, + "Indonesian,Filipino,Spanish": 0.3806818181818182, + "Indonesian,Filipino,Chinese": 0.375, + "Indonesian,Filipino,Malay": 0.3806818181818182, + "Indonesian,Filipino,Vietnamese": 0.4090909090909091, + "Indonesian,Spanish,Chinese": 0.4090909090909091, + "Indonesian,Spanish,Malay": 0.44886363636363635, + "Indonesian,Spanish,Vietnamese": 0.44886363636363635, + "Indonesian,Chinese,Malay": 0.3806818181818182, + "Indonesian,Chinese,Vietnamese": 0.4090909090909091, + "Indonesian,Malay,Vietnamese": 0.4431818181818182, + "English,Filipino,Spanish": 0.3352272727272727, + "English,Filipino,Chinese": 0.3125, + "English,Filipino,Malay": 0.2897727272727273, + "English,Filipino,Vietnamese": 0.32954545454545453, + "English,Spanish,Chinese": 0.44886363636363635, + "English,Spanish,Malay": 0.42045454545454547, + "English,Spanish,Vietnamese": 0.4318181818181818, + "English,Chinese,Malay": 0.3465909090909091, + "English,Chinese,Vietnamese": 0.39204545454545453, + "English,Malay,Vietnamese": 0.3806818181818182, + "Filipino,Spanish,Chinese": 0.3125, + "Filipino,Spanish,Malay": 0.3181818181818182, + "Filipino,Spanish,Vietnamese": 0.32954545454545453, + "Filipino,Chinese,Malay": 0.2784090909090909, + "Filipino,Chinese,Vietnamese": 0.3125, + "Filipino,Malay,Vietnamese": 0.3181818181818182, + "Spanish,Chinese,Malay": 0.35795454545454547, + "Spanish,Chinese,Vietnamese": 0.375, + "Spanish,Malay,Vietnamese": 0.3977272727272727, + "Chinese,Malay,Vietnamese": 0.3409090909090909 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.30113636363636365, + "Indonesian,English,Filipino,Chinese": 0.2727272727272727, + "Indonesian,English,Filipino,Malay": 0.26704545454545453, + "Indonesian,English,Filipino,Vietnamese": 0.3068181818181818, + "Indonesian,English,Spanish,Chinese": 0.3522727272727273, + "Indonesian,English,Spanish,Malay": 0.3465909090909091, + "Indonesian,English,Spanish,Vietnamese": 0.3806818181818182, + "Indonesian,English,Chinese,Malay": 0.2840909090909091, + "Indonesian,English,Chinese,Vietnamese": 0.3352272727272727, + "Indonesian,English,Malay,Vietnamese": 0.32386363636363635, + "Indonesian,Filipino,Spanish,Chinese": 0.2840909090909091, + "Indonesian,Filipino,Spanish,Malay": 0.29545454545454547, + "Indonesian,Filipino,Spanish,Vietnamese": 0.29545454545454547, + "Indonesian,Filipino,Chinese,Malay": 0.26704545454545453, + "Indonesian,Filipino,Chinese,Vietnamese": 0.2897727272727273, + "Indonesian,Filipino,Malay,Vietnamese": 0.30113636363636365, + "Indonesian,Spanish,Chinese,Malay": 0.3125, + "Indonesian,Spanish,Chinese,Vietnamese": 0.32954545454545453, + "Indonesian,Spanish,Malay,Vietnamese": 0.3409090909090909, + "Indonesian,Chinese,Malay,Vietnamese": 0.30113636363636365, + "English,Filipino,Spanish,Chinese": 0.2556818181818182, + "English,Filipino,Spanish,Malay": 0.23863636363636365, + "English,Filipino,Spanish,Vietnamese": 0.2727272727272727, + "English,Filipino,Chinese,Malay": 0.2159090909090909, + "English,Filipino,Chinese,Vietnamese": 0.25, + "English,Filipino,Malay,Vietnamese": 0.23295454545454544, + "English,Spanish,Chinese,Malay": 0.30113636363636365, + "English,Spanish,Chinese,Vietnamese": 0.32954545454545453, + "English,Spanish,Malay,Vietnamese": 0.3125, + "English,Chinese,Malay,Vietnamese": 0.26704545454545453, + "Filipino,Spanish,Chinese,Malay": 0.23295454545454544, + "Filipino,Spanish,Chinese,Vietnamese": 0.24431818181818182, + "Filipino,Spanish,Malay,Vietnamese": 0.25, + "Filipino,Chinese,Malay,Vietnamese": 0.22727272727272727, + "Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.23863636363636365, + "Indonesian,English,Filipino,Spanish,Malay": 0.22727272727272727, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2556818181818182, + "Indonesian,English,Filipino,Chinese,Malay": 0.20454545454545456, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.23295454545454544, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.22727272727272727, + "Indonesian,English,Spanish,Chinese,Malay": 0.26136363636363635, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.29545454545454547, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2840909090909091, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.23863636363636365, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.22727272727272727, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, + "English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.2159090909090909, + "English,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, + "English,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, + "English,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635 + }, + "6_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.1875, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 + }, + "7_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727 + } + }, + "AC3_2": 0.4953843881049151, + "AC3_3": 0.41227585860952487, + "AC3_4": 0.352705556644256, + "AC3_5": 0.3071428570980761, + "AC3_6": 0.27098779509897997, + "AC3_7": 0.2417531614975395 }, "prompt_5": { - "bleu_score": 0.1504267975743821 + "overall_acc": 0.44642857142857145, + "language_acc": { + "Indonesian": 0.4318181818181818, + "English": 0.5625, + "Filipino": 0.3693181818181818, + "Spanish": 0.4431818181818182, + "Chinese": 0.45454545454545453, + "Malay": 0.42045454545454547, + "Vietnamese": 0.4431818181818182 + }, + "consistency_score_2": 0.5343614718614718, + "consistency_score_3": 0.3616883116883116, + "consistency_score_4": 0.26866883116883117, + "consistency_score_5": 0.21076839826839824, + "consistency_score_6": 0.17288961038961043, + "consistency_score_7": 0.14772727272727273, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.5454545454545454, + "Indonesian,Filipino": 0.5227272727272727, + "Indonesian,Spanish": 0.5852272727272727, + "Indonesian,Chinese": 0.5056818181818182, + "Indonesian,Malay": 0.5795454545454546, + "Indonesian,Vietnamese": 0.5170454545454546, + "English,Filipino": 0.4659090909090909, + "English,Spanish": 0.6477272727272727, + "English,Chinese": 0.6136363636363636, + "English,Malay": 0.5284090909090909, + "English,Vietnamese": 0.5, + "Filipino,Spanish": 0.5284090909090909, + "Filipino,Chinese": 0.4772727272727273, + "Filipino,Malay": 0.4659090909090909, + "Filipino,Vietnamese": 0.5056818181818182, + "Spanish,Chinese": 0.6420454545454546, + "Spanish,Malay": 0.5340909090909091, + "Spanish,Vietnamese": 0.5397727272727273, + "Chinese,Malay": 0.48863636363636365, + "Chinese,Vietnamese": 0.48863636363636365, + "Malay,Vietnamese": 0.5397727272727273 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.3409090909090909, + "Indonesian,English,Spanish": 0.4431818181818182, + "Indonesian,English,Chinese": 0.38636363636363635, + "Indonesian,English,Malay": 0.39204545454545453, + "Indonesian,English,Vietnamese": 0.36363636363636365, + "Indonesian,Filipino,Spanish": 0.3693181818181818, + "Indonesian,Filipino,Chinese": 0.3409090909090909, + "Indonesian,Filipino,Malay": 0.3352272727272727, + "Indonesian,Filipino,Vietnamese": 0.32386363636363635, + "Indonesian,Spanish,Chinese": 0.42045454545454547, + "Indonesian,Spanish,Malay": 0.4034090909090909, + "Indonesian,Spanish,Vietnamese": 0.375, + "Indonesian,Chinese,Malay": 0.32954545454545453, + "Indonesian,Chinese,Vietnamese": 0.32954545454545453, + "Indonesian,Malay,Vietnamese": 0.3693181818181818, + "English,Filipino,Spanish": 0.3806818181818182, + "English,Filipino,Chinese": 0.3409090909090909, + "English,Filipino,Malay": 0.29545454545454547, + "English,Filipino,Vietnamese": 0.3181818181818182, + "English,Spanish,Chinese": 0.5, + "English,Spanish,Malay": 0.3977272727272727, + "English,Spanish,Vietnamese": 0.4034090909090909, + "English,Chinese,Malay": 0.3465909090909091, + "English,Chinese,Vietnamese": 0.375, + "English,Malay,Vietnamese": 0.35795454545454547, + "Filipino,Spanish,Chinese": 0.375, + "Filipino,Spanish,Malay": 0.32386363636363635, + "Filipino,Spanish,Vietnamese": 0.3522727272727273, + "Filipino,Chinese,Malay": 0.2784090909090909, + "Filipino,Chinese,Vietnamese": 0.3125, + "Filipino,Malay,Vietnamese": 0.3181818181818182, + "Spanish,Chinese,Malay": 0.375, + "Spanish,Chinese,Vietnamese": 0.4090909090909091, + "Spanish,Malay,Vietnamese": 0.36363636363636365, + "Chinese,Malay,Vietnamese": 0.3125 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.29545454545454547, + "Indonesian,English,Filipino,Chinese": 0.26704545454545453, + "Indonesian,English,Filipino,Malay": 0.24431818181818182, + "Indonesian,English,Filipino,Vietnamese": 0.24431818181818182, + "Indonesian,English,Spanish,Chinese": 0.35795454545454547, + "Indonesian,English,Spanish,Malay": 0.3181818181818182, + "Indonesian,English,Spanish,Vietnamese": 0.30113636363636365, + "Indonesian,English,Chinese,Malay": 0.26704545454545453, + "Indonesian,English,Chinese,Vietnamese": 0.2840909090909091, + "Indonesian,English,Malay,Vietnamese": 0.2840909090909091, + "Indonesian,Filipino,Spanish,Chinese": 0.2840909090909091, + "Indonesian,Filipino,Spanish,Malay": 0.24431818181818182, + "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.2159090909090909, + "Indonesian,Filipino,Chinese,Vietnamese": 0.2215909090909091, + "Indonesian,Filipino,Malay,Vietnamese": 0.24431818181818182, + "Indonesian,Spanish,Chinese,Malay": 0.2840909090909091, + "Indonesian,Spanish,Chinese,Vietnamese": 0.29545454545454547, + "Indonesian,Spanish,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,Chinese,Malay,Vietnamese": 0.23863636363636365, + "English,Filipino,Spanish,Chinese": 0.30113636363636365, + "English,Filipino,Spanish,Malay": 0.25, + "English,Filipino,Spanish,Vietnamese": 0.2840909090909091, + "English,Filipino,Chinese,Malay": 0.22727272727272727, + "English,Filipino,Chinese,Vietnamese": 0.2556818181818182, + "English,Filipino,Malay,Vietnamese": 0.22727272727272727, + "English,Spanish,Chinese,Malay": 0.3125, + "English,Spanish,Chinese,Vietnamese": 0.3409090909090909, + "English,Spanish,Malay,Vietnamese": 0.2840909090909091, + "English,Chinese,Malay,Vietnamese": 0.2556818181818182, + "Filipino,Spanish,Chinese,Malay": 0.22727272727272727, + "Filipino,Spanish,Chinese,Vietnamese": 0.26704545454545453, + "Filipino,Spanish,Malay,Vietnamese": 0.25, + "Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, + "Spanish,Chinese,Malay,Vietnamese": 0.2840909090909091 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.24431818181818182, + "Indonesian,English,Filipino,Spanish,Malay": 0.20454545454545456, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2159090909090909, + "Indonesian,English,Filipino,Chinese,Malay": 0.1875, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.19886363636363635, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.19318181818181818, + "Indonesian,English,Spanish,Chinese,Malay": 0.25, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.26704545454545453, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091, + "English,Filipino,Spanish,Chinese,Malay": 0.20454545454545456, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, + "English,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, + "English,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, + "English,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1875 + }, + "6_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.1875, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365 + }, + "7_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 + } + }, + "AC3_2": 0.4864532019208453, + "AC3_3": 0.3996154507913924, + "AC3_4": 0.33545484023199346, + "AC3_5": 0.28634652703988495, + "AC3_6": 0.24925107653717393, + "AC3_7": 0.22199453548176265 } }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { + "sg_eval": { "prompt_1": { - "bleu_score": 0.11546307928194786 + "accuracy": 0.6699029126213593 }, "prompt_2": { - "bleu_score": 0.12504954338285795 + "accuracy": 0.6504854368932039 }, "prompt_3": { - "bleu_score": 0.12339784072132419 + "accuracy": 0.6116504854368932 }, "prompt_4": { - "bleu_score": 0.12146141845701107 + "accuracy": 0.6699029126213593 }, "prompt_5": { - "bleu_score": 0.10620991599630757 + "accuracy": 0.6407766990291263 } }, - "flores_vie2eng": { + "cn_eval": { "prompt_1": { - "bleu_score": 0.04168397392539289 + "accuracy": 0.3333333333333333 }, "prompt_2": { - "bleu_score": 0.04504363343396233 + "accuracy": 0.3333333333333333 }, "prompt_3": { - "bleu_score": 0.04548112556314333 + "accuracy": 0.3619047619047619 }, "prompt_4": { - "bleu_score": 0.044323869785541224 + "accuracy": 0.38095238095238093 }, "prompt_5": { - "bleu_score": 0.048254383763782764 + "accuracy": 0.4 } }, - "flores_zho2eng": { + "us_eval": { "prompt_1": { - "bleu_score": 0.15187617543799345 + "accuracy": 0.6261682242990654 }, "prompt_2": { - "bleu_score": 0.17634273934026473 + "accuracy": 0.5981308411214953 }, "prompt_3": { - "bleu_score": 0.1786268785905264 + "accuracy": 0.5981308411214953 }, "prompt_4": { - "bleu_score": 0.1834212260052482 + "accuracy": 0.6355140186915887 }, "prompt_5": { - "bleu_score": 0.18402241668611977 + "accuracy": 0.5887850467289719 } }, - "flores_zsm2eng": { + "ph_eval": { "prompt_1": { - "bleu_score": 0.06935625865149986 + "accuracy": 0.47, + "category_acc": { + "brand": 0.3, + "demographics": 0.0, + "biology": 0.4, + "history": 0.26666666666666666, + "literature": 0.4, + "politics": 0.7, + "culture": 0.8, + "film": 0.5, + "law": 0.6, + "geography": 0.6 + } }, "prompt_2": { - "bleu_score": 0.08045675082470904 - }, - "prompt_3": { - "bleu_score": 0.0742536152118873 - }, - "prompt_4": { - "bleu_score": 0.07402823250601796 - }, + "accuracy": 0.49, + "category_acc": { + "brand": 0.5, + "demographics": 0.4, + "biology": 0.3, + "history": 0.2, + "literature": 0.3, + "politics": 0.8, + "culture": 0.9, + "film": 0.5, + "law": 0.5, + "geography": 0.6 + } + }, + "prompt_3": { + "accuracy": 0.5, + "category_acc": { + "brand": 0.7, + "demographics": 0.4, + "biology": 0.5, + "history": 0.26666666666666666, + "literature": 0.3, + "politics": 0.8, + "culture": 0.7, + "film": 0.3, + "law": 0.5, + "geography": 0.6 + } + }, + "prompt_4": { + "accuracy": 0.49, + "category_acc": { + "brand": 0.7, + "demographics": 0.4, + "biology": 0.5, + "history": 0.26666666666666666, + "literature": 0.4, + "politics": 0.7, + "culture": 0.7, + "film": 0.4, + "law": 0.4, + "geography": 0.5 + } + }, "prompt_5": { - "bleu_score": 0.08573936332256699 + "accuracy": 0.48, + "category_acc": { + "brand": 0.3, + "demographics": 0.2, + "biology": 0.4, + "history": 0.3333333333333333, + "literature": 0.5, + "politics": 0.8, + "culture": 0.8, + "film": 0.4, + "law": 0.3, + "geography": 0.7 + } } }, - "mmlu": { + "sing2eng": { "prompt_1": { - "accuracy": 0.46557759626604434 + "bleu_score": 0.17850204847572934 }, "prompt_2": { - "accuracy": 0.456242707117853 + "bleu_score": 0.1808454080159326 }, "prompt_3": { - "accuracy": 0.45040840140023336 + "bleu_score": 0.18662234818554896 }, "prompt_4": { - "accuracy": 0.4515752625437573 + "bleu_score": 0.176343510769412 }, "prompt_5": { - "accuracy": 0.4539089848308051 + "bleu_score": 0.1324532828678051 } }, - "mmlu_full": { + "indommlu": { "prompt_1": { - "accuracy": 0.4406864497676082, + "accuracy": 0.42305894919554043, "category_acc": { - "high_school_european_history": 0.573170731707317, - "business_ethics": 0.5252525252525253, - "clinical_knowledge": 0.4734848484848485, - "medical_genetics": 0.5050505050505051, - "high_school_us_history": 0.6059113300492611, - "high_school_physics": 0.26666666666666666, - "high_school_world_history": 0.6779661016949152, - "virology": 0.3939393939393939, - "high_school_microeconomics": 0.4810126582278481, - "econometrics": 0.23008849557522124, - "college_computer_science": 0.35353535353535354, - "high_school_biology": 0.5210355987055016, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.3274021352313167, - "philosophy": 0.4838709677419355, - "professional_medicine": 0.3948339483394834, - "nutrition": 0.4885245901639344, - "global_facts": 0.25252525252525254, - "machine_learning": 0.4144144144144144, - "security_studies": 0.5286885245901639, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.40589198036006546, - "prehistory": 0.46439628482972134, - "anatomy": 0.3880597014925373, - "human_sexuality": 0.4846153846153846, - "college_medicine": 0.37790697674418605, - "high_school_government_and_politics": 0.5729166666666666, - "college_chemistry": 0.46464646464646464, - "logical_fallacies": 0.4691358024691358, - "high_school_geography": 0.5634517766497462, - "elementary_mathematics": 0.33156498673740054, - "human_aging": 0.481981981981982, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.5753676470588235, - "formal_logic": 0.304, - "high_school_statistics": 0.3302325581395349, - "international_law": 0.625, - "high_school_mathematics": 0.3048327137546468, - "high_school_computer_science": 0.43434343434343436, - "conceptual_physics": 0.36752136752136755, - "miscellaneous": 0.5997442455242967, - "high_school_chemistry": 0.400990099009901, - "marketing": 0.703862660944206, - "professional_law": 0.345075016307893, - "management": 0.6078431372549019, - "college_physics": 0.3069306930693069, - "jurisprudence": 0.5046728971962616, - "world_religions": 0.5058823529411764, - "sociology": 0.645, - "us_foreign_policy": 0.696969696969697, - "high_school_macroeconomics": 0.43958868894601544, - "computer_security": 0.494949494949495, - "moral_scenarios": 0.19239373601789708, - "moral_disputes": 0.5043478260869565, - "electrical_engineering": 0.4583333333333333, - "astronomy": 0.5033112582781457, - "college_biology": 0.4125874125874126 + "History": 0.39759036144578314, + "Geography": 0.38571428571428573, + "Lampungic": 0.29931972789115646, + "Social science": 0.5809682804674458, + "Balinese": 0.3205944798301486, + "Makassarese": 0.3817204301075269, + "Banjarese": 0.3611111111111111, + "Chemistry": 0.3021897810218978, + "Biology": 0.3940828402366864, + "Science": 0.5201238390092879, + "Christian religion": 0.48258706467661694, + "Art": 0.46589018302828616, + "Islam religion": 0.5049786628733998, + "Hindu religion": 0.4866666666666667, + "Madurese": 0.288135593220339, + "Sport": 0.4391891891891892, + "Indonesian language": 0.47665006226650064, + "Physics": 0.35555555555555557, + "Minangkabau culture": 0.3417085427135678, + "Dayak language": 0.23853211009174313, + "Sociology": 0.43548387096774194, + "Economy": 0.41188524590163933, + "Sundanese": 0.34831460674157305, + "Javanese": 0.33669354838709675, + "Civic education": 0.4721030042918455 } }, "prompt_2": { - "accuracy": 0.44790847336431894, + "accuracy": 0.4210561452700447, "category_acc": { - "high_school_european_history": 0.6158536585365854, - "business_ethics": 0.5353535353535354, - "clinical_knowledge": 0.4962121212121212, - "medical_genetics": 0.47474747474747475, - "high_school_us_history": 0.5763546798029556, - "high_school_physics": 0.31333333333333335, - "high_school_world_history": 0.6610169491525424, - "virology": 0.4303030303030303, - "high_school_microeconomics": 0.4810126582278481, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.35353535353535354, - "high_school_biology": 0.5275080906148867, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.3274021352313167, - "philosophy": 0.49032258064516127, - "professional_medicine": 0.3985239852398524, - "nutrition": 0.49508196721311476, - "global_facts": 0.30303030303030304, - "machine_learning": 0.3333333333333333, - "security_studies": 0.5614754098360656, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.4157119476268412, - "prehistory": 0.48606811145510836, - "anatomy": 0.3656716417910448, - "human_sexuality": 0.5, - "college_medicine": 0.38953488372093026, - "high_school_government_and_politics": 0.5677083333333334, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.46296296296296297, - "high_school_geography": 0.5685279187817259, - "elementary_mathematics": 0.3395225464190981, - "human_aging": 0.4774774774774775, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.59375, - "formal_logic": 0.368, - "high_school_statistics": 0.37209302325581395, - "international_law": 0.6166666666666667, - "high_school_mathematics": 0.30855018587360594, - "high_school_computer_science": 0.41414141414141414, - "conceptual_physics": 0.39316239316239315, - "miscellaneous": 0.618925831202046, - "high_school_chemistry": 0.400990099009901, - "marketing": 0.6995708154506438, - "professional_law": 0.34442270058708413, - "management": 0.6666666666666666, - "college_physics": 0.2871287128712871, - "jurisprudence": 0.5327102803738317, - "world_religions": 0.48823529411764705, - "sociology": 0.625, - "us_foreign_policy": 0.6868686868686869, - "high_school_macroeconomics": 0.4652956298200514, - "computer_security": 0.5555555555555556, - "moral_scenarios": 0.1935123042505593, - "moral_disputes": 0.5014492753623189, - "electrical_engineering": 0.4236111111111111, - "astronomy": 0.543046357615894, - "college_biology": 0.4405594405594406 + "History": 0.41365461847389556, + "Geography": 0.39387755102040817, + "Lampungic": 0.2653061224489796, + "Social science": 0.5876460767946577, + "Balinese": 0.3078556263269639, + "Makassarese": 0.3387096774193548, + "Banjarese": 0.3611111111111111, + "Chemistry": 0.2948905109489051, + "Biology": 0.39289940828402364, + "Science": 0.4984520123839009, + "Christian religion": 0.48756218905472637, + "Art": 0.4858569051580699, + "Islam religion": 0.4950213371266003, + "Hindu religion": 0.5, + "Madurese": 0.31186440677966104, + "Sport": 0.46621621621621623, + "Indonesian language": 0.4747820672478207, + "Physics": 0.3515151515151515, + "Minangkabau culture": 0.31155778894472363, + "Dayak language": 0.22018348623853212, + "Sociology": 0.41330645161290325, + "Economy": 0.4057377049180328, + "Sundanese": 0.35350043215211757, + "Javanese": 0.3387096774193548, + "Civic education": 0.47639484978540775 } }, "prompt_3": { - "accuracy": 0.4501251340722202, + "accuracy": 0.4227251485412911, "category_acc": { - "high_school_european_history": 0.6158536585365854, - "business_ethics": 0.5151515151515151, - "clinical_knowledge": 0.5, - "medical_genetics": 0.48484848484848486, - "high_school_us_history": 0.5960591133004927, - "high_school_physics": 0.31333333333333335, - "high_school_world_history": 0.6398305084745762, - "virology": 0.4484848484848485, - "high_school_microeconomics": 0.5021097046413502, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.29292929292929293, - "high_school_biology": 0.5275080906148867, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.35231316725978645, - "philosophy": 0.4935483870967742, - "professional_medicine": 0.4317343173431734, - "nutrition": 0.49836065573770494, - "global_facts": 0.36363636363636365, - "machine_learning": 0.36036036036036034, - "security_studies": 0.5737704918032787, - "public_relations": 0.5504587155963303, - "professional_psychology": 0.40916530278232405, - "prehistory": 0.5015479876160991, - "anatomy": 0.373134328358209, - "human_sexuality": 0.5153846153846153, - "college_medicine": 0.37790697674418605, - "high_school_government_and_politics": 0.5677083333333334, - "college_chemistry": 0.42424242424242425, - "logical_fallacies": 0.4691358024691358, - "high_school_geography": 0.5736040609137056, - "elementary_mathematics": 0.33156498673740054, - "human_aging": 0.481981981981982, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.6047794117647058, - "formal_logic": 0.32, - "high_school_statistics": 0.32558139534883723, - "international_law": 0.6333333333333333, - "high_school_mathematics": 0.26394052044609667, - "high_school_computer_science": 0.494949494949495, - "conceptual_physics": 0.41452991452991456, - "miscellaneous": 0.6086956521739131, - "high_school_chemistry": 0.4207920792079208, - "marketing": 0.6652360515021459, - "professional_law": 0.34637964774951074, - "management": 0.6470588235294118, - "college_physics": 0.26732673267326734, - "jurisprudence": 0.5233644859813084, - "world_religions": 0.5235294117647059, - "sociology": 0.62, - "us_foreign_policy": 0.6868686868686869, - "high_school_macroeconomics": 0.4498714652956298, - "computer_security": 0.5353535353535354, - "moral_scenarios": 0.2080536912751678, - "moral_disputes": 0.5014492753623189, - "electrical_engineering": 0.4652777777777778, - "astronomy": 0.5231788079470199, - "college_biology": 0.46853146853146854 + "History": 0.42369477911646586, + "Geography": 0.3816326530612245, + "Lampungic": 0.2653061224489796, + "Social science": 0.5876460767946577, + "Balinese": 0.37154989384288745, + "Makassarese": 0.3763440860215054, + "Banjarese": 0.3541666666666667, + "Chemistry": 0.28175182481751826, + "Biology": 0.36923076923076925, + "Science": 0.49019607843137253, + "Christian religion": 0.48756218905472637, + "Art": 0.46422628951747086, + "Islam religion": 0.5021337126600285, + "Hindu religion": 0.5066666666666667, + "Madurese": 0.3288135593220339, + "Sport": 0.4594594594594595, + "Indonesian language": 0.47322540473225405, + "Physics": 0.3797979797979798, + "Minangkabau culture": 0.2864321608040201, + "Dayak language": 0.3119266055045872, + "Sociology": 0.43548387096774194, + "Economy": 0.4016393442622951, + "Sundanese": 0.34485738980121, + "Javanese": 0.3467741935483871, + "Civic education": 0.4892703862660944 } }, "prompt_4": { - "accuracy": 0.44612084376117267, + "accuracy": 0.4135790106148608, "category_acc": { - "high_school_european_history": 0.6219512195121951, - "business_ethics": 0.5151515151515151, - "clinical_knowledge": 0.4696969696969697, - "medical_genetics": 0.494949494949495, - "high_school_us_history": 0.6108374384236454, - "high_school_physics": 0.3, - "high_school_world_history": 0.652542372881356, - "virology": 0.42424242424242425, - "high_school_microeconomics": 0.5189873417721519, - "econometrics": 0.23893805309734514, - "college_computer_science": 0.32323232323232326, - "high_school_biology": 0.540453074433657, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.3238434163701068, - "philosophy": 0.5032258064516129, - "professional_medicine": 0.4095940959409594, - "nutrition": 0.5016393442622951, - "global_facts": 0.31313131313131315, - "machine_learning": 0.36936936936936937, - "security_studies": 0.5491803278688525, - "public_relations": 0.5229357798165137, - "professional_psychology": 0.4238952536824877, - "prehistory": 0.46439628482972134, - "anatomy": 0.373134328358209, - "human_sexuality": 0.46153846153846156, - "college_medicine": 0.4011627906976744, - "high_school_government_and_politics": 0.5572916666666666, - "college_chemistry": 0.36363636363636365, - "logical_fallacies": 0.46296296296296297, - "high_school_geography": 0.5888324873096447, - "elementary_mathematics": 0.32891246684350134, - "human_aging": 0.44144144144144143, - "college_mathematics": 0.2727272727272727, - "high_school_psychology": 0.5845588235294118, - "formal_logic": 0.368, - "high_school_statistics": 0.35348837209302325, - "international_law": 0.6, - "high_school_mathematics": 0.25650557620817843, - "high_school_computer_science": 0.5050505050505051, - "conceptual_physics": 0.4017094017094017, - "miscellaneous": 0.612531969309463, - "high_school_chemistry": 0.41089108910891087, - "marketing": 0.7081545064377682, - "professional_law": 0.3431180691454664, - "management": 0.6568627450980392, - "college_physics": 0.297029702970297, - "jurisprudence": 0.5327102803738317, - "world_religions": 0.5058823529411764, - "sociology": 0.62, - "us_foreign_policy": 0.6767676767676768, - "high_school_macroeconomics": 0.44473007712082263, - "computer_security": 0.5656565656565656, - "moral_scenarios": 0.20469798657718122, - "moral_disputes": 0.5101449275362319, - "electrical_engineering": 0.4513888888888889, - "astronomy": 0.5364238410596026, - "college_biology": 0.4125874125874126 + "History": 0.40763052208835343, + "Geography": 0.3877551020408163, + "Lampungic": 0.2585034013605442, + "Social science": 0.5626043405676127, + "Balinese": 0.33970276008492567, + "Makassarese": 0.34408602150537637, + "Banjarese": 0.3402777777777778, + "Chemistry": 0.29781021897810217, + "Biology": 0.37514792899408284, + "Science": 0.5025799793601651, + "Christian religion": 0.48258706467661694, + "Art": 0.47420965058236275, + "Islam religion": 0.4907539118065434, + "Hindu religion": 0.4866666666666667, + "Madurese": 0.2983050847457627, + "Sport": 0.4797297297297297, + "Indonesian language": 0.462640099626401, + "Physics": 0.3515151515151515, + "Minangkabau culture": 0.32160804020100503, + "Dayak language": 0.25688073394495414, + "Sociology": 0.4274193548387097, + "Economy": 0.38729508196721313, + "Sundanese": 0.34312878133102853, + "Javanese": 0.31149193548387094, + "Civic education": 0.46924177396280403 } }, "prompt_5": { - "accuracy": 0.4401859134787272, + "accuracy": 0.4085052406702717, "category_acc": { - "high_school_european_history": 0.5853658536585366, - "business_ethics": 0.5353535353535354, - "clinical_knowledge": 0.4621212121212121, - "medical_genetics": 0.46464646464646464, - "high_school_us_history": 0.5665024630541872, - "high_school_physics": 0.2866666666666667, - "high_school_world_history": 0.6440677966101694, - "virology": 0.41818181818181815, - "high_school_microeconomics": 0.5274261603375527, - "econometrics": 0.20353982300884957, - "college_computer_science": 0.30303030303030304, - "high_school_biology": 0.540453074433657, - "abstract_algebra": 0.21212121212121213, - "professional_accounting": 0.3238434163701068, - "philosophy": 0.47096774193548385, - "professional_medicine": 0.3985239852398524, - "nutrition": 0.5311475409836065, - "global_facts": 0.35353535353535354, - "machine_learning": 0.3333333333333333, - "security_studies": 0.5327868852459017, - "public_relations": 0.5137614678899083, - "professional_psychology": 0.397708674304419, - "prehistory": 0.4984520123839009, - "anatomy": 0.3358208955223881, - "human_sexuality": 0.5384615384615384, - "college_medicine": 0.4186046511627907, - "high_school_government_and_politics": 0.5416666666666666, - "college_chemistry": 0.3939393939393939, - "logical_fallacies": 0.41358024691358025, - "high_school_geography": 0.5380710659898477, - "elementary_mathematics": 0.3952254641909814, - "human_aging": 0.4369369369369369, - "college_mathematics": 0.25252525252525254, - "high_school_psychology": 0.5974264705882353, - "formal_logic": 0.36, - "high_school_statistics": 0.32558139534883723, - "international_law": 0.6166666666666667, - "high_school_mathematics": 0.27137546468401486, - "high_school_computer_science": 0.46464646464646464, - "conceptual_physics": 0.3974358974358974, - "miscellaneous": 0.5882352941176471, - "high_school_chemistry": 0.40594059405940597, - "marketing": 0.6952789699570815, - "professional_law": 0.3378995433789954, - "management": 0.6176470588235294, - "college_physics": 0.31683168316831684, - "jurisprudence": 0.5514018691588785, - "world_religions": 0.4764705882352941, - "sociology": 0.58, - "us_foreign_policy": 0.6868686868686869, - "high_school_macroeconomics": 0.4498714652956298, - "computer_security": 0.5151515151515151, - "moral_scenarios": 0.20917225950782997, - "moral_disputes": 0.4985507246376812, - "electrical_engineering": 0.4513888888888889, - "astronomy": 0.5165562913907285, - "college_biology": 0.3916083916083916 + "History": 0.41566265060240964, + "Geography": 0.38571428571428573, + "Lampungic": 0.2925170068027211, + "Social science": 0.5809682804674458, + "Balinese": 0.34394904458598724, + "Makassarese": 0.3548387096774194, + "Banjarese": 0.3958333333333333, + "Chemistry": 0.2656934306569343, + "Biology": 0.37514792899408284, + "Science": 0.48297213622291024, + "Christian religion": 0.4527363184079602, + "Art": 0.46755407653910147, + "Islam religion": 0.47368421052631576, + "Hindu religion": 0.46, + "Madurese": 0.3016949152542373, + "Sport": 0.4391891891891892, + "Indonesian language": 0.4576587795765878, + "Physics": 0.33131313131313134, + "Minangkabau culture": 0.2914572864321608, + "Dayak language": 0.24770642201834864, + "Sociology": 0.4254032258064516, + "Economy": 0.3770491803278688, + "Sundanese": 0.3292999135695765, + "Javanese": 0.3316532258064516, + "Civic education": 0.46924177396280403 } } }, - "c_eval": { + "flores_ind2eng": { "prompt_1": { - "accuracy": 0.46285289747399705 + "bleu_score": 0.29083824388253576 }, "prompt_2": { - "accuracy": 0.5104011887072808 + "bleu_score": 0.30095965158253596 }, "prompt_3": { - "accuracy": 0.5185735512630015 + "bleu_score": 0.2998787922647266 }, "prompt_4": { - "accuracy": 0.4026745913818722 + "bleu_score": 0.2990516384145601 }, "prompt_5": { - "accuracy": 0.43016344725111444 + "bleu_score": 0.282716600632688 } }, - "c_eval_full": { + "flores_vie2eng": { "prompt_1": { - "accuracy": 0.48194271481942713, - "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.38095238095238093, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.5172413793103449, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.625, - "high_school_chemistry": 0.5416666666666666, - "high_school_biology": 0.6666666666666666, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.8076923076923077, - "middle_school_physics": 0.7083333333333334, - "middle_school_chemistry": 0.84, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.5, - "business_administration": 0.3684210526315789, - "marxism": 0.7083333333333334, - "mao_zedong_thought": 0.7241379310344828, - "education_science": 0.5882352941176471, - "teacher_qualification": 0.6938775510204082, - "high_school_politics": 0.5416666666666666, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.5, - "middle_school_geography": 0.6470588235294118, - "modern_chinese_history": 0.5, - "ideological_and_moral_cultivation": 0.6666666666666666, - "logic": 0.2222222222222222, - "law": 0.41379310344827586, - "chinese_language_and_literature": 0.4642857142857143, - "art_studies": 0.6842105263157895, - "professional_tour_guide": 0.5882352941176471, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.64, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.4230769230769231, - "sports_science": 0.375, - "plant_protection": 0.48148148148148145, - "basic_medicine": 0.5833333333333334, - "clinical_medicine": 0.5925925925925926, - "urban_and_rural_planner": 0.47058823529411764, - "accountant": 0.4444444444444444, - "fire_engineer": 0.4444444444444444, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.35185185185185186, - "physician": 0.5185185185185185 - } + "bleu_score": 0.21256255595737264 }, "prompt_2": { - "accuracy": 0.5193026151930261, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.40476190476190477, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.13043478260869565, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.5862068965517241, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.625, - "high_school_chemistry": 0.5, - "high_school_biology": 0.625, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.8461538461538461, - "middle_school_physics": 0.7083333333333334, - "middle_school_chemistry": 0.84, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.5, - "business_administration": 0.5, - "marxism": 0.7083333333333334, - "mao_zedong_thought": 0.8275862068965517, - "education_science": 0.6470588235294118, - "teacher_qualification": 0.6938775510204082, - "high_school_politics": 0.75, - "high_school_geography": 0.5833333333333334, - "middle_school_politics": 0.7307692307692307, - "middle_school_geography": 0.6470588235294118, - "modern_chinese_history": 0.6428571428571429, - "ideological_and_moral_cultivation": 0.6666666666666666, - "logic": 0.4444444444444444, - "law": 0.4482758620689655, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.6842105263157895, - "professional_tour_guide": 0.6176470588235294, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.68, - "middle_school_history": 0.7777777777777778, - "civil_servant": 0.46153846153846156, - "sports_science": 0.4166666666666667, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.5, - "clinical_medicine": 0.5185185185185185, - "urban_and_rural_planner": 0.5882352941176471, - "accountant": 0.48148148148148145, - "fire_engineer": 0.4166666666666667, - "environmental_impact_assessment_engineer": 0.3888888888888889, - "tax_accountant": 0.4074074074074074, - "physician": 0.5925925925925926 - } + "bleu_score": 0.22028609039927588 }, "prompt_3": { - "accuracy": 0.5062266500622665, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.4523809523809524, - "college_physics": 0.25, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.40476190476190477, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.5416666666666666, - "high_school_chemistry": 0.5416666666666666, - "high_school_biology": 0.5416666666666666, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.8846153846153846, - "middle_school_physics": 0.75, - "middle_school_chemistry": 0.92, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.5333333333333333, - "business_administration": 0.3684210526315789, - "marxism": 0.7083333333333334, - "mao_zedong_thought": 0.8275862068965517, - "education_science": 0.6176470588235294, - "teacher_qualification": 0.7142857142857143, - "high_school_politics": 0.7083333333333334, - "high_school_geography": 0.5, - "middle_school_politics": 0.6923076923076923, - "middle_school_geography": 0.7647058823529411, - "modern_chinese_history": 0.7142857142857143, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.4074074074074074, - "law": 0.4827586206896552, - "chinese_language_and_literature": 0.4642857142857143, - "art_studies": 0.6842105263157895, - "professional_tour_guide": 0.5588235294117647, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.56, - "middle_school_history": 0.7407407407407407, - "civil_servant": 0.46153846153846156, - "sports_science": 0.4583333333333333, - "plant_protection": 0.48148148148148145, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.48148148148148145, - "urban_and_rural_planner": 0.5882352941176471, - "accountant": 0.4074074074074074, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.42592592592592593, - "physician": 0.5185185185185185 - } + "bleu_score": 0.21794217050063686 }, "prompt_4": { - "accuracy": 0.40099626400996263, - "category_acc": { - "computer_network": 0.16666666666666666, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.30952380952380953, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.4482758620689655, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.08695652173913043, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.4482758620689655, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.5, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.6923076923076923, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.6, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.5, - "business_administration": 0.39473684210526316, - "marxism": 0.625, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.38235294117647056, - "teacher_qualification": 0.42857142857142855, - "high_school_politics": 0.5, - "high_school_geography": 0.5416666666666666, - "middle_school_politics": 0.5769230769230769, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.4583333333333333, - "logic": 0.3333333333333333, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.4642857142857143, - "art_studies": 0.47368421052631576, - "professional_tour_guide": 0.4411764705882353, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.44, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.4423076923076923, - "sports_science": 0.25, - "plant_protection": 0.37037037037037035, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.4117647058823529, - "accountant": 0.3888888888888889, - "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.3611111111111111, - "tax_accountant": 0.24074074074074073, - "physician": 0.48148148148148145 - } + "bleu_score": 0.21639144518942477 }, "prompt_5": { - "accuracy": 0.42341220423412207, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.47619047619047616, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.5, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.5416666666666666, - "high_school_chemistry": 0.375, - "high_school_biology": 0.5416666666666666, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.6538461538461539, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.48, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.36666666666666664, - "business_administration": 0.3684210526315789, - "marxism": 0.7083333333333334, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.4411764705882353, - "teacher_qualification": 0.4897959183673469, - "high_school_politics": 0.6666666666666666, - "high_school_geography": 0.5, - "middle_school_politics": 0.5, - "middle_school_geography": 0.7058823529411765, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.4166666666666667, - "logic": 0.4444444444444444, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.5789473684210527, - "professional_tour_guide": 0.6470588235294118, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.4, - "middle_school_history": 0.5555555555555556, - "civil_servant": 0.40384615384615385, - "sports_science": 0.2916666666666667, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.375, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.45098039215686275, - "accountant": 0.4444444444444444, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.2962962962962963, - "physician": 0.48148148148148145 - } + "bleu_score": 0.2165329134969405 } }, - "cmmlu": { + "flores_zho2eng": { "prompt_1": { - "accuracy": 0.5197132616487455 + "bleu_score": 0.1772473628232291 }, "prompt_2": { - "accuracy": 0.5555555555555556 + "bleu_score": 0.1790645913924739 }, "prompt_3": { - "accuracy": 0.5340501792114696 + "bleu_score": 0.17880842836525265 }, "prompt_4": { - "accuracy": 0.4910394265232975 + "bleu_score": 0.18272545312220376 }, "prompt_5": { - "accuracy": 0.4946236559139785 + "bleu_score": 0.17708566347520507 } }, - "cmmlu_full": { + "flores_zsm2eng": { "prompt_1": { - "accuracy": 0.4866171645657054, - "category_acc": { - "agronomy": 0.4260355029585799, - "anatomy": 0.3918918918918919, - "ancient_chinese": 0.25, - "arts": 0.65625, - "astronomy": 0.3151515151515151, - "business_ethics": 0.4688995215311005, - "chinese_civil_service_exam": 0.425, - "chinese_driving_rule": 0.6564885496183206, - "chinese_food_culture": 0.4485294117647059, - "chinese_foreign_policy": 0.5233644859813084, - "chinese_history": 0.5294117647058824, - "chinese_literature": 0.4117647058823529, - "chinese_teacher_qualification": 0.5251396648044693, - "clinical_knowledge": 0.41350210970464135, - "college_actuarial_science": 0.3584905660377358, - "college_education": 0.6448598130841121, - "college_engineering_hydrology": 0.42452830188679247, - "college_law": 0.4444444444444444, - "college_mathematics": 0.3523809523809524, - "college_medical_statistics": 0.3867924528301887, - "college_medicine": 0.4725274725274725, - "computer_science": 0.46568627450980393, - "computer_security": 0.6023391812865497, - "conceptual_physics": 0.6122448979591837, - "construction_project_management": 0.43884892086330934, - "economics": 0.4779874213836478, - "education": 0.5521472392638037, - "electrical_engineering": 0.45348837209302323, - "elementary_chinese": 0.39285714285714285, - "elementary_commonsense": 0.5202020202020202, - "elementary_information_and_technology": 0.634453781512605, - "elementary_mathematics": 0.4043478260869565, - "ethnology": 0.4666666666666667, - "food_science": 0.46153846153846156, - "genetics": 0.3806818181818182, - "global_facts": 0.5436241610738255, - "high_school_biology": 0.5739644970414202, - "high_school_chemistry": 0.4318181818181818, - "high_school_geography": 0.423728813559322, - "high_school_mathematics": 0.32926829268292684, - "high_school_physics": 0.509090909090909, - "high_school_politics": 0.4965034965034965, - "human_sexuality": 0.4603174603174603, - "international_law": 0.4486486486486487, - "journalism": 0.47093023255813954, - "jurisprudence": 0.49148418491484186, - "legal_and_moral_basis": 0.8130841121495327, - "logical": 0.43902439024390244, - "machine_learning": 0.36885245901639346, - "management": 0.6285714285714286, - "marketing": 0.5555555555555556, - "marxist_theory": 0.6031746031746031, - "modern_chinese": 0.3448275862068966, - "nutrition": 0.4827586206896552, - "philosophy": 0.5142857142857142, - "professional_accounting": 0.5942857142857143, - "professional_law": 0.35545023696682465, - "professional_medicine": 0.39095744680851063, - "professional_psychology": 0.5689655172413793, - "public_relations": 0.5, - "security_study": 0.5555555555555556, - "sociology": 0.5309734513274337, - "sports_science": 0.4909090909090909, - "traditional_chinese_medicine": 0.4918918918918919, - "virology": 0.4970414201183432, - "world_history": 0.546583850931677, - "world_religions": 0.58125 - } - }, - "prompt_2": { - "accuracy": 0.5107062683474357, - "category_acc": { - "agronomy": 0.4437869822485207, - "anatomy": 0.3918918918918919, - "ancient_chinese": 0.2804878048780488, - "arts": 0.68125, - "astronomy": 0.3575757575757576, - "business_ethics": 0.4784688995215311, - "chinese_civil_service_exam": 0.48125, - "chinese_driving_rule": 0.648854961832061, - "chinese_food_culture": 0.4485294117647059, - "chinese_foreign_policy": 0.5700934579439252, - "chinese_history": 0.628482972136223, - "chinese_literature": 0.4166666666666667, - "chinese_teacher_qualification": 0.6089385474860335, - "clinical_knowledge": 0.45147679324894513, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.6542056074766355, - "college_engineering_hydrology": 0.4056603773584906, - "college_law": 0.46296296296296297, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.4811320754716981, - "college_medicine": 0.4981684981684982, - "computer_science": 0.4852941176470588, - "computer_security": 0.5672514619883041, - "conceptual_physics": 0.6462585034013606, - "construction_project_management": 0.4676258992805755, - "economics": 0.4716981132075472, - "education": 0.588957055214724, - "electrical_engineering": 0.42441860465116277, - "elementary_chinese": 0.4722222222222222, - "elementary_commonsense": 0.5303030303030303, - "elementary_information_and_technology": 0.6680672268907563, - "elementary_mathematics": 0.39565217391304347, - "ethnology": 0.5185185185185185, - "food_science": 0.5524475524475524, - "genetics": 0.375, - "global_facts": 0.5637583892617449, - "high_school_biology": 0.6331360946745562, - "high_school_chemistry": 0.4318181818181818, - "high_school_geography": 0.5423728813559322, - "high_school_mathematics": 0.3170731707317073, - "high_school_physics": 0.4636363636363636, - "high_school_politics": 0.5314685314685315, - "human_sexuality": 0.5, - "international_law": 0.4540540540540541, - "journalism": 0.5348837209302325, - "jurisprudence": 0.5279805352798054, - "legal_and_moral_basis": 0.8551401869158879, - "logical": 0.45528455284552843, - "machine_learning": 0.4098360655737705, - "management": 0.6476190476190476, - "marketing": 0.5388888888888889, - "marxist_theory": 0.6031746031746031, - "modern_chinese": 0.41379310344827586, - "nutrition": 0.5103448275862069, - "philosophy": 0.580952380952381, - "professional_accounting": 0.5885714285714285, - "professional_law": 0.3981042654028436, - "professional_medicine": 0.375, - "professional_psychology": 0.5948275862068966, - "public_relations": 0.4942528735632184, - "security_study": 0.6444444444444445, - "sociology": 0.5796460176991151, - "sports_science": 0.49696969696969695, - "traditional_chinese_medicine": 0.4972972972972973, - "virology": 0.4911242603550296, - "world_history": 0.6024844720496895, - "world_religions": 0.59375 - } - }, - "prompt_3": { - "accuracy": 0.5040580210671732, - "category_acc": { - "agronomy": 0.44970414201183434, - "anatomy": 0.40540540540540543, - "ancient_chinese": 0.24390243902439024, - "arts": 0.68125, - "astronomy": 0.34545454545454546, - "business_ethics": 0.47368421052631576, - "chinese_civil_service_exam": 0.45625, - "chinese_driving_rule": 0.648854961832061, - "chinese_food_culture": 0.49264705882352944, - "chinese_foreign_policy": 0.5420560747663551, - "chinese_history": 0.6068111455108359, - "chinese_literature": 0.39215686274509803, - "chinese_teacher_qualification": 0.5921787709497207, - "clinical_knowledge": 0.4008438818565401, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.6542056074766355, - "college_engineering_hydrology": 0.41509433962264153, - "college_law": 0.48148148148148145, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.4528301886792453, - "college_medicine": 0.4835164835164835, - "computer_science": 0.47058823529411764, - "computer_security": 0.5614035087719298, - "conceptual_physics": 0.6598639455782312, - "construction_project_management": 0.4172661870503597, - "economics": 0.48427672955974843, - "education": 0.5828220858895705, - "electrical_engineering": 0.43023255813953487, - "elementary_chinese": 0.4444444444444444, - "elementary_commonsense": 0.5202020202020202, - "elementary_information_and_technology": 0.6512605042016807, - "elementary_mathematics": 0.4434782608695652, - "ethnology": 0.4962962962962963, - "food_science": 0.5104895104895105, - "genetics": 0.3693181818181818, - "global_facts": 0.5771812080536913, - "high_school_biology": 0.591715976331361, - "high_school_chemistry": 0.4696969696969697, - "high_school_geography": 0.5084745762711864, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.509090909090909, - "high_school_politics": 0.5804195804195804, - "human_sexuality": 0.5, - "international_law": 0.43783783783783786, - "journalism": 0.5406976744186046, - "jurisprudence": 0.5206812652068127, - "legal_and_moral_basis": 0.8691588785046729, - "logical": 0.43902439024390244, - "machine_learning": 0.4262295081967213, - "management": 0.6428571428571429, - "marketing": 0.5388888888888889, - "marxist_theory": 0.5608465608465608, - "modern_chinese": 0.41379310344827586, - "nutrition": 0.5655172413793104, - "philosophy": 0.5714285714285714, - "professional_accounting": 0.5828571428571429, - "professional_law": 0.3933649289099526, - "professional_medicine": 0.3617021276595745, - "professional_psychology": 0.5991379310344828, - "public_relations": 0.5287356321839081, - "security_study": 0.6074074074074074, - "sociology": 0.5575221238938053, - "sports_science": 0.4909090909090909, - "traditional_chinese_medicine": 0.4918918918918919, - "virology": 0.47337278106508873, - "world_history": 0.5838509316770186, - "world_religions": 0.5875 - } - }, - "prompt_4": { - "accuracy": 0.41711276118114315, - "category_acc": { - "agronomy": 0.34911242603550297, - "anatomy": 0.3918918918918919, - "ancient_chinese": 0.2926829268292683, - "arts": 0.50625, - "astronomy": 0.3333333333333333, - "business_ethics": 0.3923444976076555, - "chinese_civil_service_exam": 0.41875, - "chinese_driving_rule": 0.5725190839694656, - "chinese_food_culture": 0.4485294117647059, - "chinese_foreign_policy": 0.38317757009345793, - "chinese_history": 0.4117647058823529, - "chinese_literature": 0.28921568627450983, - "chinese_teacher_qualification": 0.5083798882681564, - "clinical_knowledge": 0.38396624472573837, - "college_actuarial_science": 0.19811320754716982, - "college_education": 0.48598130841121495, - "college_engineering_hydrology": 0.42452830188679247, - "college_law": 0.3611111111111111, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.3018867924528302, - "college_medicine": 0.40293040293040294, - "computer_science": 0.4950980392156863, - "computer_security": 0.43859649122807015, - "conceptual_physics": 0.4965986394557823, - "construction_project_management": 0.39568345323741005, - "economics": 0.4339622641509434, - "education": 0.3803680981595092, - "electrical_engineering": 0.4127906976744186, - "elementary_chinese": 0.35714285714285715, - "elementary_commonsense": 0.43434343434343436, - "elementary_information_and_technology": 0.5378151260504201, - "elementary_mathematics": 0.3739130434782609, - "ethnology": 0.3851851851851852, - "food_science": 0.5104895104895105, - "genetics": 0.4318181818181818, - "global_facts": 0.436241610738255, - "high_school_biology": 0.44970414201183434, - "high_school_chemistry": 0.3409090909090909, - "high_school_geography": 0.3813559322033898, - "high_school_mathematics": 0.3597560975609756, - "high_school_physics": 0.41818181818181815, - "high_school_politics": 0.40559440559440557, - "human_sexuality": 0.4603174603174603, - "international_law": 0.31351351351351353, - "journalism": 0.4186046511627907, - "jurisprudence": 0.34306569343065696, - "legal_and_moral_basis": 0.6915887850467289, - "logical": 0.4146341463414634, - "machine_learning": 0.4016393442622951, - "management": 0.5285714285714286, - "marketing": 0.5055555555555555, - "marxist_theory": 0.4708994708994709, - "modern_chinese": 0.3103448275862069, - "nutrition": 0.46206896551724136, - "philosophy": 0.4380952380952381, - "professional_accounting": 0.42857142857142855, - "professional_law": 0.35545023696682465, - "professional_medicine": 0.3191489361702128, - "professional_psychology": 0.5129310344827587, - "public_relations": 0.4425287356321839, - "security_study": 0.4666666666666667, - "sociology": 0.4690265486725664, - "sports_science": 0.4, - "traditional_chinese_medicine": 0.41081081081081083, - "virology": 0.41420118343195267, - "world_history": 0.4409937888198758, - "world_religions": 0.45625 - } - }, - "prompt_5": { - "accuracy": 0.43990675185632877, - "category_acc": { - "agronomy": 0.38461538461538464, - "anatomy": 0.35135135135135137, - "ancient_chinese": 0.2804878048780488, - "arts": 0.61875, - "astronomy": 0.3212121212121212, - "business_ethics": 0.4258373205741627, - "chinese_civil_service_exam": 0.35625, - "chinese_driving_rule": 0.5038167938931297, - "chinese_food_culture": 0.4411764705882353, - "chinese_foreign_policy": 0.4485981308411215, - "chinese_history": 0.4179566563467492, - "chinese_literature": 0.37745098039215685, - "chinese_teacher_qualification": 0.5083798882681564, - "clinical_knowledge": 0.4050632911392405, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.5981308411214953, - "college_engineering_hydrology": 0.32075471698113206, - "college_law": 0.37962962962962965, - "college_mathematics": 0.29523809523809524, - "college_medical_statistics": 0.330188679245283, - "college_medicine": 0.42857142857142855, - "computer_science": 0.47058823529411764, - "computer_security": 0.4444444444444444, - "conceptual_physics": 0.5170068027210885, - "construction_project_management": 0.3669064748201439, - "economics": 0.44654088050314467, - "education": 0.5521472392638037, - "electrical_engineering": 0.42441860465116277, - "elementary_chinese": 0.4087301587301587, - "elementary_commonsense": 0.4696969696969697, - "elementary_information_and_technology": 0.5756302521008403, - "elementary_mathematics": 0.4043478260869565, - "ethnology": 0.4666666666666667, - "food_science": 0.36363636363636365, - "genetics": 0.3693181818181818, - "global_facts": 0.5167785234899329, - "high_school_biology": 0.42011834319526625, - "high_school_chemistry": 0.2878787878787879, - "high_school_geography": 0.3305084745762712, - "high_school_mathematics": 0.38414634146341464, - "high_school_physics": 0.4636363636363636, - "high_school_politics": 0.3986013986013986, - "human_sexuality": 0.46825396825396826, - "international_law": 0.3621621621621622, - "journalism": 0.4418604651162791, - "jurisprudence": 0.45012165450121655, - "legal_and_moral_basis": 0.7383177570093458, - "logical": 0.4146341463414634, - "machine_learning": 0.36885245901639346, - "management": 0.5714285714285714, - "marketing": 0.5222222222222223, - "marxist_theory": 0.5079365079365079, - "modern_chinese": 0.29310344827586204, - "nutrition": 0.42758620689655175, - "philosophy": 0.44761904761904764, - "professional_accounting": 0.56, - "professional_law": 0.3744075829383886, - "professional_medicine": 0.31648936170212766, - "professional_psychology": 0.5301724137931034, - "public_relations": 0.5, - "security_study": 0.5333333333333333, - "sociology": 0.5398230088495575, - "sports_science": 0.4121212121212121, - "traditional_chinese_medicine": 0.42162162162162165, - "virology": 0.4319526627218935, - "world_history": 0.4658385093167702, - "world_religions": 0.54375 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.3939393939393939 - }, - "prompt_2": { - "accuracy": 0.36363636363636365 - }, - "prompt_3": { - "accuracy": 0.36363636363636365 - }, - "prompt_4": { - "accuracy": 0.36363636363636365 - }, - "prompt_5": { - "accuracy": 0.36363636363636365 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.19772727272727272 + "bleu_score": 0.28560213975589965 }, "prompt_2": { - "accuracy": 0.2545454545454545 + "bleu_score": 0.2917520756266462 }, "prompt_3": { - "accuracy": 0.20454545454545456 + "bleu_score": 0.2875298051907796 }, "prompt_4": { - "accuracy": 0.3477272727272727 + "bleu_score": 0.2897057158686528 }, "prompt_5": { - "accuracy": 0.3431818181818182 + "bleu_score": 0.2792743718264202 } }, - "ocnli": { + "mmlu": { "prompt_1": { - "accuracy": 0.3440677966101695 + "accuracy": 0.5787631271878646 }, "prompt_2": { - "accuracy": 0.3559322033898305 + "accuracy": 0.572928821470245 }, "prompt_3": { - "accuracy": 0.328135593220339 + "accuracy": 0.5927654609101517 }, "prompt_4": { - "accuracy": 0.3430508474576271 + "accuracy": 0.5869311551925321 }, "prompt_5": { - "accuracy": 0.3572881355932203 + "accuracy": 0.5799299883313885 } }, - "c3": { + "mmlu_full": { "prompt_1": { - "accuracy": 0.7400897531787585 + "accuracy": 0.5526635681086879, + "category_acc": { + "high_school_european_history": 0.6829268292682927, + "business_ethics": 0.6060606060606061, + "clinical_knowledge": 0.6439393939393939, + "medical_genetics": 0.6666666666666666, + "high_school_us_history": 0.7192118226600985, + "high_school_physics": 0.2866666666666667, + "high_school_world_history": 0.711864406779661, + "virology": 0.47878787878787876, + "high_school_microeconomics": 0.5780590717299579, + "econometrics": 0.3805309734513274, + "college_computer_science": 0.41414141414141414, + "high_school_biology": 0.7055016181229773, + "abstract_algebra": 0.26262626262626265, + "professional_accounting": 0.40213523131672596, + "philosophy": 0.6064516129032258, + "professional_medicine": 0.6125461254612546, + "nutrition": 0.5770491803278689, + "global_facts": 0.40404040404040403, + "machine_learning": 0.3963963963963964, + "security_studies": 0.5819672131147541, + "public_relations": 0.5504587155963303, + "professional_psychology": 0.5564648117839607, + "prehistory": 0.6253869969040248, + "anatomy": 0.5, + "human_sexuality": 0.6153846153846154, + "college_medicine": 0.5755813953488372, + "high_school_government_and_politics": 0.8125, + "college_chemistry": 0.42424242424242425, + "logical_fallacies": 0.654320987654321, + "high_school_geography": 0.6700507614213198, + "elementary_mathematics": 0.46684350132625996, + "human_aging": 0.6081081081081081, + "college_mathematics": 0.24242424242424243, + "high_school_psychology": 0.7665441176470589, + "formal_logic": 0.368, + "high_school_statistics": 0.46511627906976744, + "international_law": 0.7, + "high_school_mathematics": 0.26394052044609667, + "high_school_computer_science": 0.6363636363636364, + "conceptual_physics": 0.47863247863247865, + "miscellaneous": 0.7672634271099744, + "high_school_chemistry": 0.400990099009901, + "marketing": 0.8454935622317596, + "professional_law": 0.41030658838878015, + "management": 0.7745098039215687, + "college_physics": 0.39603960396039606, + "jurisprudence": 0.6448598130841121, + "world_religions": 0.7823529411764706, + "sociology": 0.765, + "us_foreign_policy": 0.7575757575757576, + "high_school_macroeconomics": 0.4884318766066838, + "computer_security": 0.6464646464646465, + "moral_scenarios": 0.29977628635346754, + "moral_disputes": 0.591304347826087, + "electrical_engineering": 0.5208333333333334, + "astronomy": 0.5960264900662252, + "college_biology": 0.6433566433566433 + } }, "prompt_2": { - "accuracy": 0.7266267763649963 + "accuracy": 0.5636753664640687, + "category_acc": { + "high_school_european_history": 0.7073170731707317, + "business_ethics": 0.6161616161616161, + "clinical_knowledge": 0.6590909090909091, + "medical_genetics": 0.6363636363636364, + "high_school_us_history": 0.7586206896551724, + "high_school_physics": 0.37333333333333335, + "high_school_world_history": 0.7372881355932204, + "virology": 0.48484848484848486, + "high_school_microeconomics": 0.6033755274261603, + "econometrics": 0.3805309734513274, + "college_computer_science": 0.41414141414141414, + "high_school_biology": 0.6699029126213593, + "abstract_algebra": 0.2727272727272727, + "professional_accounting": 0.4483985765124555, + "philosophy": 0.603225806451613, + "professional_medicine": 0.6162361623616236, + "nutrition": 0.5836065573770491, + "global_facts": 0.40404040404040403, + "machine_learning": 0.3963963963963964, + "security_studies": 0.6188524590163934, + "public_relations": 0.5963302752293578, + "professional_psychology": 0.5695581014729951, + "prehistory": 0.6377708978328174, + "anatomy": 0.5223880597014925, + "human_sexuality": 0.5923076923076923, + "college_medicine": 0.5755813953488372, + "high_school_government_and_politics": 0.8229166666666666, + "college_chemistry": 0.41414141414141414, + "logical_fallacies": 0.6790123456790124, + "high_school_geography": 0.6649746192893401, + "elementary_mathematics": 0.44562334217506633, + "human_aging": 0.6576576576576577, + "college_mathematics": 0.32323232323232326, + "high_school_psychology": 0.7628676470588235, + "formal_logic": 0.4, + "high_school_statistics": 0.4744186046511628, + "international_law": 0.725, + "high_school_mathematics": 0.2788104089219331, + "high_school_computer_science": 0.6262626262626263, + "conceptual_physics": 0.5213675213675214, + "miscellaneous": 0.7774936061381074, + "high_school_chemistry": 0.4405940594059406, + "marketing": 0.8540772532188842, + "professional_law": 0.41030658838878015, + "management": 0.7352941176470589, + "college_physics": 0.44554455445544555, + "jurisprudence": 0.6728971962616822, + "world_religions": 0.788235294117647, + "sociology": 0.74, + "us_foreign_policy": 0.7373737373737373, + "high_school_macroeconomics": 0.4961439588688946, + "computer_security": 0.6767676767676768, + "moral_scenarios": 0.31208053691275167, + "moral_disputes": 0.6144927536231884, + "electrical_engineering": 0.5208333333333334, + "astronomy": 0.6357615894039735, + "college_biology": 0.6503496503496503 + } }, "prompt_3": { - "accuracy": 0.7393418100224383 + "accuracy": 0.5614587057561673, + "category_acc": { + "high_school_european_history": 0.7134146341463414, + "business_ethics": 0.6262626262626263, + "clinical_knowledge": 0.6477272727272727, + "medical_genetics": 0.6666666666666666, + "high_school_us_history": 0.7635467980295566, + "high_school_physics": 0.35333333333333333, + "high_school_world_history": 0.7542372881355932, + "virology": 0.48484848484848486, + "high_school_microeconomics": 0.5991561181434599, + "econometrics": 0.35398230088495575, + "college_computer_science": 0.41414141414141414, + "high_school_biology": 0.6601941747572816, + "abstract_algebra": 0.2828282828282828, + "professional_accounting": 0.43416370106761565, + "philosophy": 0.6129032258064516, + "professional_medicine": 0.5830258302583026, + "nutrition": 0.6, + "global_facts": 0.37373737373737376, + "machine_learning": 0.3963963963963964, + "security_studies": 0.6475409836065574, + "public_relations": 0.6238532110091743, + "professional_psychology": 0.5695581014729951, + "prehistory": 0.6346749226006192, + "anatomy": 0.5671641791044776, + "human_sexuality": 0.5923076923076923, + "college_medicine": 0.5581395348837209, + "high_school_government_and_politics": 0.7864583333333334, + "college_chemistry": 0.3939393939393939, + "logical_fallacies": 0.654320987654321, + "high_school_geography": 0.6700507614213198, + "elementary_mathematics": 0.44562334217506633, + "human_aging": 0.6486486486486487, + "college_mathematics": 0.3333333333333333, + "high_school_psychology": 0.7738970588235294, + "formal_logic": 0.424, + "high_school_statistics": 0.4511627906976744, + "international_law": 0.7, + "high_school_mathematics": 0.26394052044609667, + "high_school_computer_science": 0.6565656565656566, + "conceptual_physics": 0.5085470085470085, + "miscellaneous": 0.7800511508951407, + "high_school_chemistry": 0.4504950495049505, + "marketing": 0.8283261802575107, + "professional_law": 0.4024787997390737, + "management": 0.7352941176470589, + "college_physics": 0.40594059405940597, + "jurisprudence": 0.616822429906542, + "world_religions": 0.8117647058823529, + "sociology": 0.79, + "us_foreign_policy": 0.7777777777777778, + "high_school_macroeconomics": 0.4987146529562982, + "computer_security": 0.6262626262626263, + "moral_scenarios": 0.28859060402684567, + "moral_disputes": 0.6521739130434783, + "electrical_engineering": 0.5277777777777778, + "astronomy": 0.6158940397350994, + "college_biology": 0.6713286713286714 + } }, "prompt_4": { - "accuracy": 0.712415856394914 + "accuracy": 0.5591705398641401, + "category_acc": { + "high_school_european_history": 0.6829268292682927, + "business_ethics": 0.6060606060606061, + "clinical_knowledge": 0.6590909090909091, + "medical_genetics": 0.6464646464646465, + "high_school_us_history": 0.7389162561576355, + "high_school_physics": 0.34, + "high_school_world_history": 0.7415254237288136, + "virology": 0.45454545454545453, + "high_school_microeconomics": 0.6244725738396625, + "econometrics": 0.3805309734513274, + "college_computer_science": 0.3838383838383838, + "high_school_biology": 0.686084142394822, + "abstract_algebra": 0.29292929292929293, + "professional_accounting": 0.4199288256227758, + "philosophy": 0.6290322580645161, + "professional_medicine": 0.5830258302583026, + "nutrition": 0.580327868852459, + "global_facts": 0.35353535353535354, + "machine_learning": 0.36936936936936937, + "security_studies": 0.6229508196721312, + "public_relations": 0.5779816513761468, + "professional_psychology": 0.5662847790507365, + "prehistory": 0.6439628482972136, + "anatomy": 0.582089552238806, + "human_sexuality": 0.6230769230769231, + "college_medicine": 0.5755813953488372, + "high_school_government_and_politics": 0.7864583333333334, + "college_chemistry": 0.3939393939393939, + "logical_fallacies": 0.691358024691358, + "high_school_geography": 0.7106598984771574, + "elementary_mathematics": 0.4217506631299735, + "human_aging": 0.6351351351351351, + "college_mathematics": 0.26262626262626265, + "high_school_psychology": 0.7702205882352942, + "formal_logic": 0.376, + "high_school_statistics": 0.44651162790697674, + "international_law": 0.725, + "high_school_mathematics": 0.275092936802974, + "high_school_computer_science": 0.6363636363636364, + "conceptual_physics": 0.5213675213675214, + "miscellaneous": 0.7672634271099744, + "high_school_chemistry": 0.44554455445544555, + "marketing": 0.8497854077253219, + "professional_law": 0.41030658838878015, + "management": 0.7352941176470589, + "college_physics": 0.3465346534653465, + "jurisprudence": 0.6542056074766355, + "world_religions": 0.8117647058823529, + "sociology": 0.76, + "us_foreign_policy": 0.7575757575757576, + "high_school_macroeconomics": 0.46786632390745503, + "computer_security": 0.7070707070707071, + "moral_scenarios": 0.29082774049217003, + "moral_disputes": 0.6376811594202898, + "electrical_engineering": 0.5486111111111112, + "astronomy": 0.6158940397350994, + "college_biology": 0.6713286713286714 + } }, "prompt_5": { - "accuracy": 0.6978309648466716 + "accuracy": 0.5610296746514122, + "category_acc": { + "high_school_european_history": 0.7012195121951219, + "business_ethics": 0.6060606060606061, + "clinical_knowledge": 0.6212121212121212, + "medical_genetics": 0.6565656565656566, + "high_school_us_history": 0.7487684729064039, + "high_school_physics": 0.3466666666666667, + "high_school_world_history": 0.7330508474576272, + "virology": 0.4666666666666667, + "high_school_microeconomics": 0.5864978902953587, + "econometrics": 0.40707964601769914, + "college_computer_science": 0.42424242424242425, + "high_school_biology": 0.6763754045307443, + "abstract_algebra": 0.30303030303030304, + "professional_accounting": 0.43416370106761565, + "philosophy": 0.6258064516129033, + "professional_medicine": 0.5793357933579336, + "nutrition": 0.5672131147540984, + "global_facts": 0.36363636363636365, + "machine_learning": 0.38738738738738737, + "security_studies": 0.5983606557377049, + "public_relations": 0.6238532110091743, + "professional_psychology": 0.5646481178396072, + "prehistory": 0.653250773993808, + "anatomy": 0.5223880597014925, + "human_sexuality": 0.6461538461538462, + "college_medicine": 0.5872093023255814, + "high_school_government_and_politics": 0.7916666666666666, + "college_chemistry": 0.37373737373737376, + "logical_fallacies": 0.7283950617283951, + "high_school_geography": 0.6751269035532995, + "elementary_mathematics": 0.4350132625994695, + "human_aging": 0.6306306306306306, + "college_mathematics": 0.23232323232323232, + "high_school_psychology": 0.7702205882352942, + "formal_logic": 0.416, + "high_school_statistics": 0.5162790697674419, + "international_law": 0.7, + "high_school_mathematics": 0.29739776951672864, + "high_school_computer_science": 0.6767676767676768, + "conceptual_physics": 0.5170940170940171, + "miscellaneous": 0.7634271099744245, + "high_school_chemistry": 0.44554455445544555, + "marketing": 0.8369098712446352, + "professional_law": 0.410958904109589, + "management": 0.7450980392156863, + "college_physics": 0.37623762376237624, + "jurisprudence": 0.6728971962616822, + "world_religions": 0.7823529411764706, + "sociology": 0.755, + "us_foreign_policy": 0.7777777777777778, + "high_school_macroeconomics": 0.5115681233933161, + "computer_security": 0.6666666666666666, + "moral_scenarios": 0.29194630872483224, + "moral_disputes": 0.6463768115942029, + "electrical_engineering": 0.5208333333333334, + "astronomy": 0.6225165562913907, + "college_biology": 0.6573426573426573 + } } }, - "dream": { + "c_eval": { "prompt_1": { - "accuracy": 0.7741303282704557 + "accuracy": 0.39821693907875183 }, "prompt_2": { - "accuracy": 0.7662910338069574 + "accuracy": 0.4049034175334324 }, "prompt_3": { - "accuracy": 0.7765801077902988 + "accuracy": 0.41233283803863297 }, "prompt_4": { - "accuracy": 0.7574718275355218 + "accuracy": 0.38558692421991086 }, "prompt_5": { - "accuracy": 0.7677609015188633 + "accuracy": 0.40936106983655274 } }, - "samsum": { + "c_eval_full": { "prompt_1": { - "rouge1": 0.3716455581819416, - "rouge2": 0.14990640406573963, - "rougeL": 0.2817151968446882, - "avg_rouge": 0.2677557196974565 - }, - "prompt_2": { - "rouge1": 0.38804537679124096, - "rouge2": 0.15986959424534522, - "rougeL": 0.294310560142838, - "avg_rouge": 0.28074184372647476 - }, - "prompt_3": { - "rouge1": 0.3784631340064383, - "rouge2": 0.14951110790554753, - "rougeL": 0.28648490833809026, - "avg_rouge": 0.271486383416692 - }, - "prompt_4": { - "rouge1": 0.362285669145153, - "rouge2": 0.1425448208824754, - "rougeL": 0.2732042149075602, - "avg_rouge": 0.25934490164506285 - }, - "prompt_5": { - "rouge1": 0.36637462739119897, - "rouge2": 0.14470147544431752, - "rougeL": 0.27952029090630093, - "avg_rouge": 0.2635321312472725 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.19179414847110124, - "rouge2": 0.059404576867587815, - "rougeL": 0.1458878528604547, - "avg_rouge": 0.13236219273304792 - }, - "prompt_2": { - "rouge1": 0.19528540665968133, - "rouge2": 0.059521449412275385, - "rougeL": 0.14838081172681936, - "avg_rouge": 0.1343958892662587 - }, - "prompt_3": { - "rouge1": 0.1942961219883703, - "rouge2": 0.06044368284411611, - "rougeL": 0.14807010143496818, - "avg_rouge": 0.13426996875581818 - }, - "prompt_4": { - "rouge1": 0.19111071803271812, - "rouge2": 0.05819351635276752, - "rougeL": 0.14487319627895726, - "avg_rouge": 0.13139247688814762 - }, - "prompt_5": { - "rouge1": 0.18950109298646187, - "rouge2": 0.05583987137944739, - "rougeL": 0.14633943166054145, - "avg_rouge": 0.1305601320088169 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.8967889908256881 - }, - "prompt_2": { - "accuracy": 0.8795871559633027 - }, - "prompt_3": { - "accuracy": 0.8577981651376146 - }, - "prompt_4": { - "accuracy": 0.8830275229357798 - }, - "prompt_5": { - "accuracy": 0.9277522935779816 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.6826462128475551 + "accuracy": 0.4078455790784558, + "category_acc": { + "computer_network": 0.4583333333333333, + "operating_system": 0.4583333333333333, + "computer_architecture": 0.5, + "college_programming": 0.5714285714285714, + "college_physics": 0.25, + "college_chemistry": 0.1724137931034483, + "advanced_mathematics": 0.16666666666666666, + "probability_and_statistics": 0.30434782608695654, + "discrete_mathematics": 0.3333333333333333, + "electrical_engineer": 0.2857142857142857, + "metrology_engineer": 0.3448275862068966, + "high_school_mathematics": 0.4782608695652174, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.2916666666666667, + "high_school_biology": 0.4583333333333333, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.6153846153846154, + "middle_school_physics": 0.5833333333333334, + "middle_school_chemistry": 0.4, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.36666666666666664, + "business_administration": 0.2894736842105263, + "marxism": 0.4583333333333333, + "mao_zedong_thought": 0.4827586206896552, + "education_science": 0.5882352941176471, + "teacher_qualification": 0.46938775510204084, + "high_school_politics": 0.5416666666666666, + "high_school_geography": 0.4583333333333333, + "middle_school_politics": 0.5384615384615384, + "middle_school_geography": 0.17647058823529413, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.6666666666666666, + "logic": 0.48148148148148145, + "law": 0.41379310344827586, + "chinese_language_and_literature": 0.2857142857142857, + "art_studies": 0.21052631578947367, + "professional_tour_guide": 0.38235294117647056, + "legal_professional": 0.42857142857142855, + "high_school_chinese": 0.4166666666666667, + "high_school_history": 0.56, + "middle_school_history": 0.5925925925925926, + "civil_servant": 0.40384615384615385, + "sports_science": 0.4166666666666667, + "plant_protection": 0.4444444444444444, + "basic_medicine": 0.3333333333333333, + "clinical_medicine": 0.3333333333333333, + "urban_and_rural_planner": 0.49019607843137253, + "accountant": 0.4074074074074074, + "fire_engineer": 0.3888888888888889, + "environmental_impact_assessment_engineer": 0.4444444444444444, + "tax_accountant": 0.4444444444444444, + "physician": 0.3148148148148148 + } }, "prompt_2": { - "accuracy": 0.6673058485139022 + "accuracy": 0.3991282689912827, + "category_acc": { + "computer_network": 0.5416666666666666, + "operating_system": 0.5833333333333334, + "computer_architecture": 0.38461538461538464, + "college_programming": 0.5476190476190477, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.25, + "probability_and_statistics": 0.13043478260869565, + "discrete_mathematics": 0.42857142857142855, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.3103448275862069, + "high_school_mathematics": 0.391304347826087, + "high_school_physics": 0.16666666666666666, + "high_school_chemistry": 0.20833333333333334, + "high_school_biology": 0.375, + "middle_school_mathematics": 0.5, + "middle_school_biology": 0.5, + "middle_school_physics": 0.5416666666666666, + "middle_school_chemistry": 0.56, + "veterinary_medicine": 0.42857142857142855, + "college_economics": 0.36666666666666664, + "business_administration": 0.2894736842105263, + "marxism": 0.5416666666666666, + "mao_zedong_thought": 0.4482758620689655, + "education_science": 0.4117647058823529, + "teacher_qualification": 0.5102040816326531, + "high_school_politics": 0.5, + "high_school_geography": 0.4583333333333333, + "middle_school_politics": 0.5769230769230769, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.35714285714285715, + "ideological_and_moral_cultivation": 0.625, + "logic": 0.4444444444444444, + "law": 0.41379310344827586, + "chinese_language_and_literature": 0.35714285714285715, + "art_studies": 0.2631578947368421, + "professional_tour_guide": 0.4411764705882353, + "legal_professional": 0.4642857142857143, + "high_school_chinese": 0.25, + "high_school_history": 0.52, + "middle_school_history": 0.48148148148148145, + "civil_servant": 0.4230769230769231, + "sports_science": 0.375, + "plant_protection": 0.4444444444444444, + "basic_medicine": 0.3333333333333333, + "clinical_medicine": 0.3333333333333333, + "urban_and_rural_planner": 0.49019607843137253, + "accountant": 0.46296296296296297, + "fire_engineer": 0.3611111111111111, + "environmental_impact_assessment_engineer": 0.3611111111111111, + "tax_accountant": 0.24074074074074073, + "physician": 0.2962962962962963 + } }, "prompt_3": { - "accuracy": 0.6807286673058485 + "accuracy": 0.4041095890410959, + "category_acc": { + "computer_network": 0.5833333333333334, + "operating_system": 0.5, + "computer_architecture": 0.5, + "college_programming": 0.5238095238095238, + "college_physics": 0.25, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.2857142857142857, + "metrology_engineer": 0.3448275862068966, + "high_school_mathematics": 0.391304347826087, + "high_school_physics": 0.16666666666666666, + "high_school_chemistry": 0.4166666666666667, + "high_school_biology": 0.4583333333333333, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.5384615384615384, + "middle_school_physics": 0.5, + "middle_school_chemistry": 0.44, + "veterinary_medicine": 0.32142857142857145, + "college_economics": 0.38333333333333336, + "business_administration": 0.3157894736842105, + "marxism": 0.4583333333333333, + "mao_zedong_thought": 0.4482758620689655, + "education_science": 0.5588235294117647, + "teacher_qualification": 0.5510204081632653, + "high_school_politics": 0.5416666666666666, + "high_school_geography": 0.2916666666666667, + "middle_school_politics": 0.5769230769230769, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.35714285714285715, + "ideological_and_moral_cultivation": 0.6666666666666666, + "logic": 0.4444444444444444, + "law": 0.3793103448275862, + "chinese_language_and_literature": 0.2857142857142857, + "art_studies": 0.21052631578947367, + "professional_tour_guide": 0.4411764705882353, + "legal_professional": 0.42857142857142855, + "high_school_chinese": 0.3333333333333333, + "high_school_history": 0.52, + "middle_school_history": 0.48148148148148145, + "civil_servant": 0.4423076923076923, + "sports_science": 0.4166666666666667, + "plant_protection": 0.4444444444444444, + "basic_medicine": 0.375, + "clinical_medicine": 0.37037037037037035, + "urban_and_rural_planner": 0.47058823529411764, + "accountant": 0.5, + "fire_engineer": 0.3611111111111111, + "environmental_impact_assessment_engineer": 0.4166666666666667, + "tax_accountant": 0.25925925925925924, + "physician": 0.2962962962962963 + } }, "prompt_4": { - "accuracy": 0.6999041227229147 + "accuracy": 0.3941469489414695, + "category_acc": { + "computer_network": 0.4166666666666667, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.5384615384615384, + "college_programming": 0.5476190476190477, + "college_physics": 0.25, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.3333333333333333, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.3103448275862069, + "high_school_mathematics": 0.2608695652173913, + "high_school_physics": 0.16666666666666666, + "high_school_chemistry": 0.25, + "high_school_biology": 0.375, + "middle_school_mathematics": 0.16666666666666666, + "middle_school_biology": 0.5, + "middle_school_physics": 0.5, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.32142857142857145, + "college_economics": 0.35, + "business_administration": 0.3157894736842105, + "marxism": 0.4166666666666667, + "mao_zedong_thought": 0.4482758620689655, + "education_science": 0.5588235294117647, + "teacher_qualification": 0.5306122448979592, + "high_school_politics": 0.625, + "high_school_geography": 0.4166666666666667, + "middle_school_politics": 0.5384615384615384, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.7083333333333334, + "logic": 0.4444444444444444, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.35714285714285715, + "art_studies": 0.3157894736842105, + "professional_tour_guide": 0.4117647058823529, + "legal_professional": 0.42857142857142855, + "high_school_chinese": 0.4166666666666667, + "high_school_history": 0.6, + "middle_school_history": 0.4074074074074074, + "civil_servant": 0.40384615384615385, + "sports_science": 0.375, + "plant_protection": 0.4074074074074074, + "basic_medicine": 0.375, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.5294117647058824, + "accountant": 0.3333333333333333, + "fire_engineer": 0.4166666666666667, + "environmental_impact_assessment_engineer": 0.3888888888888889, + "tax_accountant": 0.37037037037037035, + "physician": 0.3333333333333333 + } }, "prompt_5": { - "accuracy": 0.7583892617449665 + "accuracy": 0.40224159402241594, + "category_acc": { + "computer_network": 0.5, + "operating_system": 0.5, + "computer_architecture": 0.46153846153846156, + "college_programming": 0.5476190476190477, + "college_physics": 0.3333333333333333, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.3333333333333333, + "probability_and_statistics": 0.30434782608695654, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.2857142857142857, + "metrology_engineer": 0.3793103448275862, + "high_school_mathematics": 0.21739130434782608, + "high_school_physics": 0.20833333333333334, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.4166666666666667, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.6153846153846154, + "middle_school_physics": 0.4166666666666667, + "middle_school_chemistry": 0.4, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.35, + "business_administration": 0.3157894736842105, + "marxism": 0.4583333333333333, + "mao_zedong_thought": 0.5862068965517241, + "education_science": 0.5, + "teacher_qualification": 0.5714285714285714, + "high_school_politics": 0.5833333333333334, + "high_school_geography": 0.5416666666666666, + "middle_school_politics": 0.5, + "middle_school_geography": 0.35294117647058826, + "modern_chinese_history": 0.39285714285714285, + "ideological_and_moral_cultivation": 0.5833333333333334, + "logic": 0.4444444444444444, + "law": 0.41379310344827586, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.23684210526315788, + "professional_tour_guide": 0.38235294117647056, + "legal_professional": 0.4642857142857143, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.6, + "middle_school_history": 0.6296296296296297, + "civil_servant": 0.4423076923076923, + "sports_science": 0.375, + "plant_protection": 0.5185185185185185, + "basic_medicine": 0.3333333333333333, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.45098039215686275, + "accountant": 0.3888888888888889, + "fire_engineer": 0.3333333333333333, + "environmental_impact_assessment_engineer": 0.5277777777777778, + "tax_accountant": 0.25925925925925924, + "physician": 0.3148148148148148 + } } }, - "qqp": { + "cmmlu": { "prompt_1": { - "accuracy": 0.6475 + "accuracy": 0.37992831541218636 }, "prompt_2": { - "accuracy": 0.717 + "accuracy": 0.34408602150537637 }, "prompt_3": { - "accuracy": 0.676 + "accuracy": 0.34767025089605735 }, "prompt_4": { - "accuracy": 0.6715 + "accuracy": 0.36917562724014336 }, "prompt_5": { - "accuracy": 0.7015 + "accuracy": 0.3655913978494624 } }, - "mnli": { + "cmmlu_full": { "prompt_1": { - "accuracy": 0.594 + "accuracy": 0.41037817302711105, + "category_acc": { + "agronomy": 0.39644970414201186, + "anatomy": 0.27702702702702703, + "ancient_chinese": 0.2804878048780488, + "arts": 0.38125, + "astronomy": 0.34545454545454546, + "business_ethics": 0.48325358851674644, + "chinese_civil_service_exam": 0.3875, + "chinese_driving_rule": 0.5190839694656488, + "chinese_food_culture": 0.3014705882352941, + "chinese_foreign_policy": 0.4672897196261682, + "chinese_history": 0.4582043343653251, + "chinese_literature": 0.3333333333333333, + "chinese_teacher_qualification": 0.5027932960893855, + "clinical_knowledge": 0.34177215189873417, + "college_actuarial_science": 0.25471698113207547, + "college_education": 0.4766355140186916, + "college_engineering_hydrology": 0.32075471698113206, + "college_law": 0.25925925925925924, + "college_mathematics": 0.23809523809523808, + "college_medical_statistics": 0.41509433962264153, + "college_medicine": 0.3626373626373626, + "computer_science": 0.5049019607843137, + "computer_security": 0.5789473684210527, + "conceptual_physics": 0.41496598639455784, + "construction_project_management": 0.39568345323741005, + "economics": 0.46540880503144655, + "education": 0.43558282208588955, + "electrical_engineering": 0.436046511627907, + "elementary_chinese": 0.24603174603174602, + "elementary_commonsense": 0.41919191919191917, + "elementary_information_and_technology": 0.6176470588235294, + "elementary_mathematics": 0.3173913043478261, + "ethnology": 0.34074074074074073, + "food_science": 0.42657342657342656, + "genetics": 0.3977272727272727, + "global_facts": 0.3959731543624161, + "high_school_biology": 0.3609467455621302, + "high_school_chemistry": 0.3106060606060606, + "high_school_geography": 0.423728813559322, + "high_school_mathematics": 0.21951219512195122, + "high_school_physics": 0.33636363636363636, + "high_school_politics": 0.4405594405594406, + "human_sexuality": 0.4603174603174603, + "international_law": 0.3675675675675676, + "journalism": 0.436046511627907, + "jurisprudence": 0.45742092457420924, + "legal_and_moral_basis": 0.677570093457944, + "logical": 0.42276422764227645, + "machine_learning": 0.45081967213114754, + "management": 0.49523809523809526, + "marketing": 0.55, + "marxist_theory": 0.43386243386243384, + "modern_chinese": 0.31896551724137934, + "nutrition": 0.4068965517241379, + "philosophy": 0.4, + "professional_accounting": 0.45714285714285713, + "professional_law": 0.3459715639810427, + "professional_medicine": 0.30851063829787234, + "professional_psychology": 0.4267241379310345, + "public_relations": 0.4942528735632184, + "security_study": 0.42962962962962964, + "sociology": 0.4690265486725664, + "sports_science": 0.46060606060606063, + "traditional_chinese_medicine": 0.3081081081081081, + "virology": 0.42011834319526625, + "world_history": 0.5093167701863354, + "world_religions": 0.43125 + } }, "prompt_2": { - "accuracy": 0.5875 + "accuracy": 0.40614747021239855, + "category_acc": { + "agronomy": 0.3905325443786982, + "anatomy": 0.24324324324324326, + "ancient_chinese": 0.27439024390243905, + "arts": 0.375, + "astronomy": 0.36363636363636365, + "business_ethics": 0.5215311004784688, + "chinese_civil_service_exam": 0.3875, + "chinese_driving_rule": 0.5038167938931297, + "chinese_food_culture": 0.2867647058823529, + "chinese_foreign_policy": 0.5233644859813084, + "chinese_history": 0.4520123839009288, + "chinese_literature": 0.3284313725490196, + "chinese_teacher_qualification": 0.48044692737430167, + "clinical_knowledge": 0.31645569620253167, + "college_actuarial_science": 0.25471698113207547, + "college_education": 0.3925233644859813, + "college_engineering_hydrology": 0.33962264150943394, + "college_law": 0.25925925925925924, + "college_mathematics": 0.24761904761904763, + "college_medical_statistics": 0.37735849056603776, + "college_medicine": 0.38461538461538464, + "computer_science": 0.4950980392156863, + "computer_security": 0.5789473684210527, + "conceptual_physics": 0.41496598639455784, + "construction_project_management": 0.4244604316546763, + "economics": 0.4591194968553459, + "education": 0.4233128834355828, + "electrical_engineering": 0.4186046511627907, + "elementary_chinese": 0.28174603174603174, + "elementary_commonsense": 0.3787878787878788, + "elementary_information_and_technology": 0.6176470588235294, + "elementary_mathematics": 0.30869565217391304, + "ethnology": 0.35555555555555557, + "food_science": 0.42657342657342656, + "genetics": 0.3806818181818182, + "global_facts": 0.40939597315436244, + "high_school_biology": 0.40828402366863903, + "high_school_chemistry": 0.3333333333333333, + "high_school_geography": 0.3898305084745763, + "high_school_mathematics": 0.2926829268292683, + "high_school_physics": 0.33636363636363636, + "high_school_politics": 0.43356643356643354, + "human_sexuality": 0.48412698412698413, + "international_law": 0.34054054054054056, + "journalism": 0.436046511627907, + "jurisprudence": 0.43309002433090027, + "legal_and_moral_basis": 0.6542056074766355, + "logical": 0.43902439024390244, + "machine_learning": 0.4016393442622951, + "management": 0.5, + "marketing": 0.5555555555555556, + "marxist_theory": 0.42328042328042326, + "modern_chinese": 0.23275862068965517, + "nutrition": 0.41379310344827586, + "philosophy": 0.4, + "professional_accounting": 0.4514285714285714, + "professional_law": 0.36018957345971564, + "professional_medicine": 0.300531914893617, + "professional_psychology": 0.44396551724137934, + "public_relations": 0.47701149425287354, + "security_study": 0.4, + "sociology": 0.4557522123893805, + "sports_science": 0.4666666666666667, + "traditional_chinese_medicine": 0.3081081081081081, + "virology": 0.42011834319526625, + "world_history": 0.45962732919254656, + "world_religions": 0.3875 + } }, "prompt_3": { - "accuracy": 0.588 + "accuracy": 0.4114142635123467, + "category_acc": { + "agronomy": 0.40236686390532544, + "anatomy": 0.2972972972972973, + "ancient_chinese": 0.2682926829268293, + "arts": 0.3875, + "astronomy": 0.38181818181818183, + "business_ethics": 0.49282296650717705, + "chinese_civil_service_exam": 0.39375, + "chinese_driving_rule": 0.5648854961832062, + "chinese_food_culture": 0.29411764705882354, + "chinese_foreign_policy": 0.4485981308411215, + "chinese_history": 0.44891640866873067, + "chinese_literature": 0.3284313725490196, + "chinese_teacher_qualification": 0.49162011173184356, + "clinical_knowledge": 0.34177215189873417, + "college_actuarial_science": 0.32075471698113206, + "college_education": 0.45794392523364486, + "college_engineering_hydrology": 0.3490566037735849, + "college_law": 0.26851851851851855, + "college_mathematics": 0.23809523809523808, + "college_medical_statistics": 0.3867924528301887, + "college_medicine": 0.3626373626373626, + "computer_science": 0.47549019607843135, + "computer_security": 0.5555555555555556, + "conceptual_physics": 0.4557823129251701, + "construction_project_management": 0.43884892086330934, + "economics": 0.4591194968553459, + "education": 0.4233128834355828, + "electrical_engineering": 0.4418604651162791, + "elementary_chinese": 0.25396825396825395, + "elementary_commonsense": 0.3939393939393939, + "elementary_information_and_technology": 0.6554621848739496, + "elementary_mathematics": 0.28695652173913044, + "ethnology": 0.34814814814814815, + "food_science": 0.4195804195804196, + "genetics": 0.39204545454545453, + "global_facts": 0.42953020134228187, + "high_school_biology": 0.378698224852071, + "high_school_chemistry": 0.30303030303030304, + "high_school_geography": 0.3728813559322034, + "high_school_mathematics": 0.27439024390243905, + "high_school_physics": 0.2909090909090909, + "high_school_politics": 0.42657342657342656, + "human_sexuality": 0.46825396825396826, + "international_law": 0.3675675675675676, + "journalism": 0.4476744186046512, + "jurisprudence": 0.45255474452554745, + "legal_and_moral_basis": 0.6448598130841121, + "logical": 0.4715447154471545, + "machine_learning": 0.4262295081967213, + "management": 0.49523809523809526, + "marketing": 0.5666666666666667, + "marxist_theory": 0.42328042328042326, + "modern_chinese": 0.3017241379310345, + "nutrition": 0.4413793103448276, + "philosophy": 0.3904761904761905, + "professional_accounting": 0.4514285714285714, + "professional_law": 0.36492890995260663, + "professional_medicine": 0.300531914893617, + "professional_psychology": 0.41810344827586204, + "public_relations": 0.47701149425287354, + "security_study": 0.43703703703703706, + "sociology": 0.4823008849557522, + "sports_science": 0.4727272727272727, + "traditional_chinese_medicine": 0.34054054054054056, + "virology": 0.44970414201183434, + "world_history": 0.43478260869565216, + "world_religions": 0.40625 + } }, "prompt_4": { - "accuracy": 0.594 - }, - "prompt_5": { - "accuracy": 0.597 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.6355 + "accuracy": 0.40312553963046105, + "category_acc": { + "agronomy": 0.3727810650887574, + "anatomy": 0.25675675675675674, + "ancient_chinese": 0.25609756097560976, + "arts": 0.38125, + "astronomy": 0.3393939393939394, + "business_ethics": 0.5263157894736842, + "chinese_civil_service_exam": 0.3625, + "chinese_driving_rule": 0.5572519083969466, + "chinese_food_culture": 0.3088235294117647, + "chinese_foreign_policy": 0.5046728971962616, + "chinese_history": 0.4520123839009288, + "chinese_literature": 0.3235294117647059, + "chinese_teacher_qualification": 0.5195530726256983, + "clinical_knowledge": 0.35864978902953587, + "college_actuarial_science": 0.3018867924528302, + "college_education": 0.42990654205607476, + "college_engineering_hydrology": 0.3584905660377358, + "college_law": 0.2777777777777778, + "college_mathematics": 0.20952380952380953, + "college_medical_statistics": 0.3113207547169811, + "college_medicine": 0.37362637362637363, + "computer_science": 0.46568627450980393, + "computer_security": 0.5672514619883041, + "conceptual_physics": 0.40816326530612246, + "construction_project_management": 0.4316546762589928, + "economics": 0.4528301886792453, + "education": 0.4723926380368098, + "electrical_engineering": 0.42441860465116277, + "elementary_chinese": 0.2619047619047619, + "elementary_commonsense": 0.41414141414141414, + "elementary_information_and_technology": 0.634453781512605, + "elementary_mathematics": 0.29130434782608694, + "ethnology": 0.37037037037037035, + "food_science": 0.3916083916083916, + "genetics": 0.375, + "global_facts": 0.40939597315436244, + "high_school_biology": 0.3668639053254438, + "high_school_chemistry": 0.3106060606060606, + "high_school_geography": 0.3898305084745763, + "high_school_mathematics": 0.23170731707317074, + "high_school_physics": 0.3181818181818182, + "high_school_politics": 0.3706293706293706, + "human_sexuality": 0.5, + "international_law": 0.35135135135135137, + "journalism": 0.45930232558139533, + "jurisprudence": 0.45255474452554745, + "legal_and_moral_basis": 0.6542056074766355, + "logical": 0.4065040650406504, + "machine_learning": 0.39344262295081966, + "management": 0.47619047619047616, + "marketing": 0.5444444444444444, + "marxist_theory": 0.43915343915343913, + "modern_chinese": 0.29310344827586204, + "nutrition": 0.4, + "philosophy": 0.4, + "professional_accounting": 0.4114285714285714, + "professional_law": 0.3459715639810427, + "professional_medicine": 0.31117021276595747, + "professional_psychology": 0.40948275862068967, + "public_relations": 0.5057471264367817, + "security_study": 0.362962962962963, + "sociology": 0.4424778761061947, + "sports_science": 0.37575757575757573, + "traditional_chinese_medicine": 0.2648648648648649, + "virology": 0.40236686390532544, + "world_history": 0.5031055900621118, + "world_religions": 0.44375 + } + }, + "prompt_5": { + "accuracy": 0.40493869797962356, + "category_acc": { + "agronomy": 0.3668639053254438, + "anatomy": 0.32432432432432434, + "ancient_chinese": 0.25, + "arts": 0.3375, + "astronomy": 0.38181818181818183, + "business_ethics": 0.5358851674641149, + "chinese_civil_service_exam": 0.40625, + "chinese_driving_rule": 0.5419847328244275, + "chinese_food_culture": 0.3161764705882353, + "chinese_foreign_policy": 0.48598130841121495, + "chinese_history": 0.4551083591331269, + "chinese_literature": 0.3333333333333333, + "chinese_teacher_qualification": 0.48044692737430167, + "clinical_knowledge": 0.3291139240506329, + "college_actuarial_science": 0.2358490566037736, + "college_education": 0.4766355140186916, + "college_engineering_hydrology": 0.37735849056603776, + "college_law": 0.3148148148148148, + "college_mathematics": 0.2571428571428571, + "college_medical_statistics": 0.4339622641509434, + "college_medicine": 0.3333333333333333, + "computer_science": 0.49019607843137253, + "computer_security": 0.6023391812865497, + "conceptual_physics": 0.3469387755102041, + "construction_project_management": 0.43884892086330934, + "economics": 0.4591194968553459, + "education": 0.4294478527607362, + "electrical_engineering": 0.42441860465116277, + "elementary_chinese": 0.2777777777777778, + "elementary_commonsense": 0.3888888888888889, + "elementary_information_and_technology": 0.6134453781512605, + "elementary_mathematics": 0.32608695652173914, + "ethnology": 0.35555555555555557, + "food_science": 0.34965034965034963, + "genetics": 0.3465909090909091, + "global_facts": 0.3959731543624161, + "high_school_biology": 0.3609467455621302, + "high_school_chemistry": 0.24242424242424243, + "high_school_geography": 0.3983050847457627, + "high_school_mathematics": 0.24390243902439024, + "high_school_physics": 0.3, + "high_school_politics": 0.4405594405594406, + "human_sexuality": 0.4523809523809524, + "international_law": 0.3675675675675676, + "journalism": 0.42441860465116277, + "jurisprudence": 0.45255474452554745, + "legal_and_moral_basis": 0.719626168224299, + "logical": 0.36585365853658536, + "machine_learning": 0.4344262295081967, + "management": 0.5, + "marketing": 0.5666666666666667, + "marxist_theory": 0.43386243386243384, + "modern_chinese": 0.25862068965517243, + "nutrition": 0.43448275862068964, + "philosophy": 0.42857142857142855, + "professional_accounting": 0.46285714285714286, + "professional_law": 0.3127962085308057, + "professional_medicine": 0.28191489361702127, + "professional_psychology": 0.4224137931034483, + "public_relations": 0.4540229885057471, + "security_study": 0.37777777777777777, + "sociology": 0.4557522123893805, + "sports_science": 0.44242424242424244, + "traditional_chinese_medicine": 0.31351351351351353, + "virology": 0.4556213017751479, + "world_history": 0.4782608695652174, + "world_religions": 0.38125 + } + } + }, + "zbench": { + "prompt_1": { + "accuracy": 0.2727272727272727 }, "prompt_2": { - "accuracy": 0.6215 + "accuracy": 0.3939393939393939 }, "prompt_3": { - "accuracy": 0.6285 + "accuracy": 0.2727272727272727 }, "prompt_4": { - "accuracy": 0.5595 + "accuracy": 0.2727272727272727 }, "prompt_5": { - "accuracy": 0.6455 + "accuracy": 0.36363636363636365 } }, - "wnli": { + "ind_emotion": { "prompt_1": { - "accuracy": 0.4788732394366197 + "accuracy": 0.6113636363636363 }, "prompt_2": { - "accuracy": 0.5070422535211268 + "accuracy": 0.5772727272727273 }, "prompt_3": { - "accuracy": 0.49295774647887325 + "accuracy": 0.5681818181818182 }, "prompt_4": { - "accuracy": 0.5070422535211268 + "accuracy": 0.5568181818181818 }, "prompt_5": { - "accuracy": 0.5492957746478874 + "accuracy": 0.5681818181818182 } }, - "rte": { + "ocnli": { "prompt_1": { - "accuracy": 0.6389891696750902 + "accuracy": 0.488135593220339 }, "prompt_2": { - "accuracy": 0.7148014440433214 + "accuracy": 0.46576271186440676 }, "prompt_3": { - "accuracy": 0.7075812274368231 + "accuracy": 0.49254237288135594 }, "prompt_4": { - "accuracy": 0.6534296028880866 + "accuracy": 0.47016949152542376 }, "prompt_5": { - "accuracy": 0.7509025270758123 + "accuracy": 0.471864406779661 } }, - "mrpc": { + "c3": { "prompt_1": { - "accuracy": 0.4681372549019608 + "accuracy": 0.7554225878833208 }, "prompt_2": { - "accuracy": 0.6078431372549019 + "accuracy": 0.743455497382199 }, "prompt_3": { - "accuracy": 0.4730392156862745 + "accuracy": 0.7490650710545999 }, "prompt_4": { - "accuracy": 0.5049019607843137 + "accuracy": 0.7423335826477188 }, "prompt_5": { - "accuracy": 0.5563725490196079 + "accuracy": 0.74943904263276 } - } - }, - "five_shot": { - "cross_xquad": { + }, + "dream": { "prompt_1": { - "overall_acc": 0.7361344537815127, - "language_acc": { - "English": 0.8226890756302521, - "Chinese": 0.7941176470588235, - "Spanish": 0.7663865546218488, - "Vietnamese": 0.561344537815126 - }, - "consistency_score_2": 0.6815126050420167, - "consistency_score_3": 0.5512605042016807, - "consistency_score_4": 0.46554621848739497, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.7932773109243697, - "English,Spanish": 0.8058823529411765, - "English,Vietnamese": 0.5630252100840336, - "Chinese,Spanish": 0.7663865546218488, - "Chinese,Vietnamese": 0.5689075630252101, - "Spanish,Vietnamese": 0.5915966386554622 - }, - "3_combine": { - "English,Chinese,Spanish": 0.6974789915966386, - "English,Chinese,Vietnamese": 0.492436974789916, - "English,Spanish,Vietnamese": 0.5117647058823529, - "Chinese,Spanish,Vietnamese": 0.5033613445378151 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.46554621848739497 - } - }, - "AC3_2": 0.7077712412266679, - "AC3_3": 0.6304232397430315, - "AC3_4": 0.5703755067986302 + "accuracy": 0.8554630083292504 + }, + "prompt_2": { + "accuracy": 0.8637922586967173 + }, + "prompt_3": { + "accuracy": 0.855952964233219 + }, + "prompt_4": { + "accuracy": 0.8539931406173444 + }, + "prompt_5": { + "accuracy": 0.8564429201371877 } }, - "cross_mmlu": { + "samsum": { "prompt_1": { - "overall_acc": 0.4114285714285715, - "language_acc": { - "Spanish": 0.41333333333333333, - "English": 0.5666666666666667, - "Chinese": 0.5066666666666667, - "Vietnamese": 0.35333333333333333, - "Filipino": 0.38, - "Indonesian": 0.37333333333333335, - "Malay": 0.2866666666666667 - }, - "consistency_score_2": 0.48476190476190467, - "consistency_score_3": 0.2975238095238095, - "consistency_score_4": 0.2114285714285715, - "consistency_score_5": 0.16634920634920636, - "consistency_score_6": 0.13904761904761906, - "consistency_score_7": 0.12, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5866666666666667, - "Spanish,Chinese": 0.5266666666666666, - "Spanish,Vietnamese": 0.43333333333333335, - "Spanish,Filipino": 0.46, - "Spanish,Indonesian": 0.47333333333333333, - "Spanish,Malay": 0.44, - "English,Chinese": 0.6866666666666666, - "English,Vietnamese": 0.42, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.44, - "English,Malay": 0.36666666666666664, - "Chinese,Vietnamese": 0.4266666666666667, - "Chinese,Filipino": 0.43333333333333335, - "Chinese,Indonesian": 0.44666666666666666, - "Chinese,Malay": 0.3933333333333333, - "Vietnamese,Filipino": 0.5, - "Vietnamese,Indonesian": 0.48, - "Vietnamese,Malay": 0.54, - "Filipino,Indonesian": 0.54, - "Filipino,Malay": 0.54, - "Indonesian,Malay": 0.58 - }, - "3_combine": { - "Spanish,English,Chinese": 0.42, - "Spanish,English,Vietnamese": 0.3, - "Spanish,English,Filipino": 0.32666666666666666, - "Spanish,English,Indonesian": 0.32666666666666666, - "Spanish,English,Malay": 0.3, - "Spanish,Chinese,Vietnamese": 0.28, - "Spanish,Chinese,Filipino": 0.2866666666666667, - "Spanish,Chinese,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino": 0.2733333333333333, - "Spanish,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,Vietnamese,Malay": 0.2866666666666667, - "Spanish,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Filipino,Malay": 0.30666666666666664, - "Spanish,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese": 0.30666666666666664, - "English,Chinese,Filipino": 0.3333333333333333, - "English,Chinese,Indonesian": 0.32666666666666666, - "English,Chinese,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino": 0.25333333333333335, - "English,Vietnamese,Indonesian": 0.26, - "English,Vietnamese,Malay": 0.23333333333333334, - "English,Filipino,Indonesian": 0.31333333333333335, - "English,Filipino,Malay": 0.23333333333333334, - "English,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino": 0.25333333333333335, - "Chinese,Vietnamese,Indonesian": 0.26, - "Chinese,Vietnamese,Malay": 0.24, - "Chinese,Filipino,Indonesian": 0.29333333333333333, - "Chinese,Filipino,Malay": 0.25333333333333335, - "Chinese,Indonesian,Malay": 0.2733333333333333, - "Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "Vietnamese,Filipino,Malay": 0.35333333333333333, - "Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Filipino,Indonesian,Malay": 0.38 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,English,Chinese,Filipino": 0.24666666666666667, - "Spanish,English,Chinese,Indonesian": 0.24, - "Spanish,English,Chinese,Malay": 0.22, - "Spanish,English,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Filipino,Malay": 0.22, - "Spanish,English,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian": 0.2, - "Spanish,Chinese,Vietnamese,Malay": 0.19333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.2, - "Spanish,Chinese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.18, - "English,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "English,Chinese,Vietnamese,Malay": 0.17333333333333334, - "English,Chinese,Filipino,Indonesian": 0.23333333333333334, - "English,Chinese,Filipino,Malay": 0.17333333333333334, - "English,Chinese,Indonesian,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "English,Vietnamese,Filipino,Malay": 0.17333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian,Malay": 0.14, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - } - }, - "AC3_2": 0.44509488381248746, - "AC3_3": 0.3453258616232029, - "AC3_4": 0.27931847964060746, - "AC3_5": 0.23691051801237145, - "AC3_6": 0.20784972808878657, - "AC3_7": 0.18580645157793962 + "rouge1": 0.3166185955051697, + "rouge2": 0.10917539998518976, + "rougeL": 0.23837567739925294, + "avg_rouge": 0.22138989096320413 + }, + "prompt_2": { + "rouge1": 0.3461243017709684, + "rouge2": 0.1196525210050174, + "rougeL": 0.26254524986557515, + "avg_rouge": 0.24277402421385363 + }, + "prompt_3": { + "rouge1": 0.32016444453648835, + "rouge2": 0.10572427291148377, + "rougeL": 0.24004692556166563, + "avg_rouge": 0.22197854766987923 + }, + "prompt_4": { + "rouge1": 0.3245191995512825, + "rouge2": 0.10936588200960218, + "rougeL": 0.24459506910412812, + "avg_rouge": 0.22616005022167093 + }, + "prompt_5": { + "rouge1": 0.33512587005716804, + "rouge2": 0.10875270341184964, + "rougeL": 0.2504391336881377, + "avg_rouge": 0.23143923571905178 } }, - "cross_logiqa": { + "dialogsum": { "prompt_1": { - "overall_acc": 0.3262987012987013, - "language_acc": { - "English": 0.4431818181818182, - "Filipino": 0.2840909090909091, - "Spanish": 0.32386363636363635, - "Malay": 0.2556818181818182, - "Chinese": 0.45454545454545453, - "Indonesian": 0.25, - "Vietnamese": 0.2727272727272727 - }, - "consistency_score_2": 0.42397186147186144, - "consistency_score_3": 0.2222402597402598, - "consistency_score_4": 0.13344155844155844, - "consistency_score_5": 0.08685064935064937, - "consistency_score_6": 0.05844155844155845, - "consistency_score_7": 0.03977272727272727, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.25, - "English,Spanish": 0.5795454545454546, - "English,Malay": 0.3522727272727273, - "English,Chinese": 0.6534090909090909, - "English,Indonesian": 0.3352272727272727, - "English,Vietnamese": 0.3181818181818182, - "Filipino,Spanish": 0.375, - "Filipino,Malay": 0.5170454545454546, - "Filipino,Chinese": 0.26704545454545453, - "Filipino,Indonesian": 0.4943181818181818, - "Filipino,Vietnamese": 0.5227272727272727, - "Spanish,Malay": 0.38636363636363635, - "Spanish,Chinese": 0.4943181818181818, - "Spanish,Indonesian": 0.4318181818181818, - "Spanish,Vietnamese": 0.35795454545454547, - "Malay,Chinese": 0.3409090909090909, - "Malay,Indonesian": 0.5909090909090909, - "Malay,Vietnamese": 0.5227272727272727, - "Chinese,Indonesian": 0.3806818181818182, - "Chinese,Vietnamese": 0.29545454545454547, - "Indonesian,Vietnamese": 0.4375 - }, - "3_combine": { - "English,Filipino,Spanish": 0.1875, - "English,Filipino,Malay": 0.17045454545454544, - "English,Filipino,Chinese": 0.16477272727272727, - "English,Filipino,Indonesian": 0.1534090909090909, - "English,Filipino,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay": 0.25, - "English,Spanish,Chinese": 0.3977272727272727, - "English,Spanish,Indonesian": 0.25, - "English,Spanish,Vietnamese": 0.21022727272727273, - "English,Malay,Chinese": 0.22727272727272727, - "English,Malay,Indonesian": 0.23295454545454544, - "English,Malay,Vietnamese": 0.19318181818181818, - "English,Chinese,Indonesian": 0.23863636363636365, - "English,Chinese,Vietnamese": 0.19318181818181818, - "English,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay": 0.23863636363636365, - "Filipino,Spanish,Chinese": 0.18181818181818182, - "Filipino,Spanish,Indonesian": 0.23295454545454544, - "Filipino,Spanish,Vietnamese": 0.21022727272727273, - "Filipino,Malay,Chinese": 0.16477272727272727, - "Filipino,Malay,Indonesian": 0.3522727272727273, - "Filipino,Malay,Vietnamese": 0.3465909090909091, - "Filipino,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Indonesian,Vietnamese": 0.29545454545454547, - "Spanish,Malay,Chinese": 0.2215909090909091, - "Spanish,Malay,Indonesian": 0.2727272727272727, - "Spanish,Malay,Vietnamese": 0.23295454545454544, - "Spanish,Chinese,Indonesian": 0.25, - "Spanish,Chinese,Vietnamese": 0.17613636363636365, - "Spanish,Indonesian,Vietnamese": 0.2159090909090909, - "Malay,Chinese,Indonesian": 0.25, - "Malay,Chinese,Vietnamese": 0.17045454545454544, - "Malay,Indonesian,Vietnamese": 0.3181818181818182, - "Chinese,Indonesian,Vietnamese": 0.17045454545454544 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.13068181818181818, - "English,Filipino,Spanish,Chinese": 0.13068181818181818, - "English,Filipino,Spanish,Indonesian": 0.125, - "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese": 0.10795454545454546, - "English,Filipino,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese": 0.17045454545454544, - "English,Spanish,Malay,Indonesian": 0.17045454545454544, - "English,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Spanish,Chinese,Indonesian": 0.17613636363636365, - "English,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "English,Spanish,Indonesian,Vietnamese": 0.125, - "English,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese": 0.125, - "Filipino,Spanish,Malay,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.0625, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727 - } - }, - "AC3_2": 0.3687775440799962, - "AC3_3": 0.26439948031754196, - "AC3_4": 0.1894191613059261, - "AC3_5": 0.137186487349746, - "AC3_6": 0.09912871935611885, - "AC3_7": 0.07090304372181257 + "rouge1": 0.2242608916043325, + "rouge2": 0.05331708909380924, + "rougeL": 0.16398737459191393, + "avg_rouge": 0.14718845176335188 + }, + "prompt_2": { + "rouge1": 0.2234401281407043, + "rouge2": 0.05283218485911105, + "rougeL": 0.16327769653604746, + "avg_rouge": 0.1465166698452876 + }, + "prompt_3": { + "rouge1": 0.22427132271501854, + "rouge2": 0.052786421931518876, + "rougeL": 0.16357874102702746, + "avg_rouge": 0.14687882855785497 + }, + "prompt_4": { + "rouge1": 0.22333528000915123, + "rouge2": 0.05176884162561466, + "rougeL": 0.16264925111044223, + "avg_rouge": 0.14591779091506937 + }, + "prompt_5": { + "rouge1": 0.22259900342738578, + "rouge2": 0.052647787328002134, + "rougeL": 0.1627164107373179, + "avg_rouge": 0.14598773383090194 } }, - "sg_eval": { + "sst2": { "prompt_1": { - "accuracy": 0.4368932038834951 + "accuracy": 0.8864678899082569 + }, + "prompt_2": { + "accuracy": 0.8910550458715596 + }, + "prompt_3": { + "accuracy": 0.8830275229357798 + }, + "prompt_4": { + "accuracy": 0.8830275229357798 + }, + "prompt_5": { + "accuracy": 0.8589449541284404 } }, - "cn_eval": { + "cola": { + "prompt_1": { + "accuracy": 0.8092042186001918 + }, + "prompt_2": { + "accuracy": 0.8024928092042186 + }, + "prompt_3": { + "accuracy": 0.8082454458293384 + }, + "prompt_4": { + "accuracy": 0.8130393096836049 + }, + "prompt_5": { + "accuracy": 0.8130393096836049 + } + }, + "qqp": { + "prompt_1": { + "accuracy": 0.626 + }, + "prompt_2": { + "accuracy": 0.6235 + }, + "prompt_3": { + "accuracy": 0.653 + }, + "prompt_4": { + "accuracy": 0.612 + }, + "prompt_5": { + "accuracy": 0.6185 + } + }, + "mnli": { "prompt_1": { + "accuracy": 0.7 + }, + "prompt_2": { + "accuracy": 0.7 + }, + "prompt_3": { + "accuracy": 0.7 + }, + "prompt_4": { + "accuracy": 0.7 + }, + "prompt_5": { "accuracy": 0.6 } }, - "us_eval": { + "qnli": { "prompt_1": { - "accuracy": 0.4766355140186916 + "accuracy": 0.8 + }, + "prompt_2": { + "accuracy": 0.8 + }, + "prompt_3": { + "accuracy": 0.9 + }, + "prompt_4": { + "accuracy": 0.8 + }, + "prompt_5": { + "accuracy": 0.8 } }, - "ph_eval": { + "wnli": { "prompt_1": { - "accuracy": 0.31, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.4, - "history": 0.13333333333333333, - "literature": 0.1, - "politics": 0.4, - "culture": 0.4, - "film": 0.2, - "law": 0.5, - "geography": 0.6 - } + "accuracy": 0.7 + }, + "prompt_2": { + "accuracy": 0.8 + }, + "prompt_3": { + "accuracy": 0.9 + }, + "prompt_4": { + "accuracy": 0.5 + }, + "prompt_5": { + "accuracy": 0.8 } }, - "sing2eng": { + "rte": { + "prompt_1": { + "accuracy": 0.7 + }, + "prompt_2": { + "accuracy": 0.7 + }, + "prompt_3": { + "accuracy": 0.7 + }, + "prompt_4": { + "accuracy": 0.8 + }, + "prompt_5": { + "accuracy": 0.6 + } + }, + "mrpc": { "prompt_1": { - "bleu_score": 0.19877708184624676 + "accuracy": 0.8 + }, + "prompt_2": { + "accuracy": 0.9 + }, + "prompt_3": { + "accuracy": 0.9 + }, + "prompt_4": { + "accuracy": 0.6 + }, + "prompt_5": { + "accuracy": 0.6 } + } + }, + "five_shot": { + "cross_xquad": { + "prompt_1": -1 + }, + "cross_mmlu": { + "prompt_1": -1 + }, + "cross_logiqa": { + "prompt_1": -1 + }, + "sg_eval": { + "prompt_1": -1 + }, + "cn_eval": { + "prompt_1": -1 + }, + "us_eval": { + "prompt_1": -1 + }, + "ph_eval": { + "prompt_1": -1 + }, + "sing2eng": { + "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.11081706622935021 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.043712435371987976 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.18257251802391966 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.0921746857314546 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.4515752625437573 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.449767608151591, - "category_acc": { - "high_school_european_history": 0.6158536585365854, - "business_ethics": 0.5656565656565656, - "clinical_knowledge": 0.4810606060606061, - "medical_genetics": 0.4444444444444444, - "high_school_us_history": 0.5960591133004927, - "high_school_physics": 0.3, - "high_school_world_history": 0.6779661016949152, - "virology": 0.4303030303030303, - "high_school_microeconomics": 0.4936708860759494, - "econometrics": 0.3274336283185841, - "college_computer_science": 0.36363636363636365, - "high_school_biology": 0.5242718446601942, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.3807829181494662, - "philosophy": 0.5193548387096775, - "professional_medicine": 0.3837638376383764, - "nutrition": 0.49508196721311476, - "global_facts": 0.29292929292929293, - "machine_learning": 0.36036036036036034, - "security_studies": 0.5778688524590164, - "public_relations": 0.5871559633027523, - "professional_psychology": 0.43044189852700493, - "prehistory": 0.5046439628482973, - "anatomy": 0.3656716417910448, - "human_sexuality": 0.46153846153846156, - "college_medicine": 0.4069767441860465, - "high_school_government_and_politics": 0.546875, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.4382716049382716, - "high_school_geography": 0.5279187817258884, - "elementary_mathematics": 0.32625994694960214, - "human_aging": 0.46846846846846846, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.59375, - "formal_logic": 0.328, - "high_school_statistics": 0.34418604651162793, - "international_law": 0.6416666666666667, - "high_school_mathematics": 0.3048327137546468, - "high_school_computer_science": 0.43434343434343436, - "conceptual_physics": 0.4358974358974359, - "miscellaneous": 0.5971867007672634, - "high_school_chemistry": 0.400990099009901, - "marketing": 0.6909871244635193, - "professional_law": 0.3405088062622309, - "management": 0.6568627450980392, - "college_physics": 0.26732673267326734, - "jurisprudence": 0.48598130841121495, - "world_religions": 0.4764705882352941, - "sociology": 0.59, - "us_foreign_policy": 0.7373737373737373, - "high_school_macroeconomics": 0.4498714652956298, - "computer_security": 0.5555555555555556, - "moral_scenarios": 0.22595078299776286, - "moral_disputes": 0.5072463768115942, - "electrical_engineering": 0.4791666666666667, - "astronomy": 0.5496688741721855, - "college_biology": 0.42657342657342656 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.4836552748885587 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.5130759651307597, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.38095238095238093, - "college_physics": 0.4583333333333333, - "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.5833333333333334, - "high_school_chemistry": 0.5416666666666666, - "high_school_biology": 0.5833333333333334, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.8461538461538461, - "middle_school_physics": 0.75, - "middle_school_chemistry": 0.76, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.5333333333333333, - "business_administration": 0.5263157894736842, - "marxism": 0.5833333333333334, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.5294117647058824, - "teacher_qualification": 0.6938775510204082, - "high_school_politics": 0.6666666666666666, - "high_school_geography": 0.5833333333333334, - "middle_school_politics": 0.7692307692307693, - "middle_school_geography": 0.5882352941176471, - "modern_chinese_history": 0.6428571428571429, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.48148148148148145, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.5, - "art_studies": 0.6052631578947368, - "professional_tour_guide": 0.5882352941176471, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.56, - "middle_school_history": 0.7037037037037037, - "civil_servant": 0.4807692307692308, - "sports_science": 0.5, - "plant_protection": 0.37037037037037035, - "basic_medicine": 0.5, - "clinical_medicine": 0.5185185185185185, - "urban_and_rural_planner": 0.6666666666666666, - "accountant": 0.46296296296296297, - "fire_engineer": 0.5, - "environmental_impact_assessment_engineer": 0.5277777777777778, - "tax_accountant": 0.3888888888888889, - "physician": 0.5555555555555556 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.5161290322580645 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.4992229321360732, - "category_acc": { - "agronomy": 0.44970414201183434, - "anatomy": 0.3581081081081081, - "ancient_chinese": 0.24390243902439024, - "arts": 0.6375, - "astronomy": 0.3515151515151515, - "business_ethics": 0.49760765550239233, - "chinese_civil_service_exam": 0.5125, - "chinese_driving_rule": 0.6183206106870229, - "chinese_food_culture": 0.4338235294117647, - "chinese_foreign_policy": 0.5420560747663551, - "chinese_history": 0.6749226006191951, - "chinese_literature": 0.39215686274509803, - "chinese_teacher_qualification": 0.5865921787709497, - "clinical_knowledge": 0.4050632911392405, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.6728971962616822, - "college_engineering_hydrology": 0.37735849056603776, - "college_law": 0.4351851851851852, - "college_mathematics": 0.3333333333333333, - "college_medical_statistics": 0.42452830188679247, - "college_medicine": 0.43956043956043955, - "computer_science": 0.47549019607843135, - "computer_security": 0.5029239766081871, - "conceptual_physics": 0.6258503401360545, - "construction_project_management": 0.4028776978417266, - "economics": 0.4716981132075472, - "education": 0.5460122699386503, - "electrical_engineering": 0.45930232558139533, - "elementary_chinese": 0.45634920634920634, - "elementary_commonsense": 0.4696969696969697, - "elementary_information_and_technology": 0.6974789915966386, - "elementary_mathematics": 0.3782608695652174, - "ethnology": 0.43703703703703706, - "food_science": 0.5104895104895105, - "genetics": 0.42613636363636365, - "global_facts": 0.5771812080536913, - "high_school_biology": 0.6449704142011834, - "high_school_chemistry": 0.5227272727272727, - "high_school_geography": 0.5254237288135594, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.4818181818181818, - "high_school_politics": 0.5594405594405595, - "human_sexuality": 0.4444444444444444, - "international_law": 0.41081081081081083, - "journalism": 0.48255813953488375, - "jurisprudence": 0.5352798053527981, - "legal_and_moral_basis": 0.8644859813084113, - "logical": 0.43089430894308944, - "machine_learning": 0.4262295081967213, - "management": 0.6, - "marketing": 0.5555555555555556, - "marxist_theory": 0.6084656084656085, - "modern_chinese": 0.43103448275862066, - "nutrition": 0.4896551724137931, - "philosophy": 0.5714285714285714, - "professional_accounting": 0.6057142857142858, - "professional_law": 0.41706161137440756, - "professional_medicine": 0.35106382978723405, - "professional_psychology": 0.603448275862069, - "public_relations": 0.5689655172413793, - "security_study": 0.6, - "sociology": 0.5088495575221239, - "sports_science": 0.48484848484848486, - "traditional_chinese_medicine": 0.43783783783783786, - "virology": 0.4911242603550296, - "world_history": 0.5838509316770186, - "world_religions": 0.56875 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.3939393939393939 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.23863636363636365 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.38508474576271184 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.7243829468960359 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.7545320921117099 - } + "prompt_1": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.41638656088755, - "rouge2": 0.17606032194771565, - "rougeL": 0.3213118319389185, - "avg_rouge": 0.30458623825806136 - } + "prompt_1": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.34421334911769, - "rouge2": 0.12414175911419639, - "rougeL": 0.263637170837793, - "avg_rouge": 0.24399742635655977 - } + "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.8600917431192661 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.6836049856184084 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.6825 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.571 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.639 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.49295774647887325 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.7328519855595668 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.7230392156862745 - } + "prompt_1": -1 } } }, - "chatglm3_6b": { - "model_size": "6B", - "model_link": "https://huggingface.co/THUDM/chatglm3-6b", + "sailor_0_5b": { + "model_size": "0.5B", + "model_link": "https://huggingface.co/sail/Sailor-0.5B", "zero_shot": { + "cross_xquad": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cross_mmlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cross_logiqa": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "sg_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cn_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "us_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ph_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "sing2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "indommlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_ind2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_vie2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_zho2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_zsm2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mmlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mmlu_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c_eval_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cmmlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cmmlu_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "zbench": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ind_emotion": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ocnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c3": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "dream": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "samsum": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "dialogsum": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "sst2": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cola": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "qqp": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "qnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "wnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "rte": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mrpc": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + } + }, + "five_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.7903361344537816, - "language_acc": { - "English": 0.8840336134453781, - "Chinese": 0.8453781512605042, - "Spanish": 0.8294117647058824, - "Vietnamese": 0.6025210084033613 - }, - "consistency_score_2": 0.7212885154061625, - "consistency_score_3": 0.5997899159663865, - "consistency_score_4": 0.5134453781512605, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8403361344537815, - "English,Spanish": 0.8495798319327731, - "English,Vietnamese": 0.6176470588235294, - "Chinese,Spanish": 0.8109243697478992, - "Chinese,Vietnamese": 0.5941176470588235, - "Spanish,Vietnamese": 0.6151260504201681 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7579831932773109, - "English,Chinese,Vietnamese": 0.5487394957983194, - "English,Spanish,Vietnamese": 0.5596638655462185, - "Chinese,Spanish,Vietnamese": 0.5327731092436975 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5134453781512605 - } - }, - "AC3_2": 0.7542353547980567, - "AC3_3": 0.6820038276267885, - "AC3_4": 0.622488402338669 - }, - "prompt_2": { - "overall_acc": 0.7865546218487395, + "overall_acc": 0.2542016806722689, "language_acc": { - "English": 0.8865546218487395, - "Chinese": 0.8344537815126051, - "Spanish": 0.8226890756302521, - "Vietnamese": 0.6025210084033613 - }, - "consistency_score_2": 0.7184873949579832, - "consistency_score_3": 0.5964285714285714, - "consistency_score_4": 0.5084033613445378, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8420168067226891, - "English,Spanish": 0.846218487394958, - "English,Vietnamese": 0.6210084033613446, - "Chinese,Spanish": 0.7949579831932773, - "Chinese,Vietnamese": 0.5957983193277311, - "Spanish,Vietnamese": 0.6109243697478992 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7529411764705882, - "English,Chinese,Vietnamese": 0.5495798319327732, - "English,Spanish,Vietnamese": 0.5588235294117647, - "Chinese,Spanish,Vietnamese": 0.5243697478991597 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5084033613445378 - } - }, - "AC3_2": 0.7509817997051802, - "AC3_3": 0.678422777234424, - "AC3_4": 0.6176061598724722 - }, - "prompt_3": { - "overall_acc": 0.792016806722689, - "language_acc": { - "English": 0.888235294117647, - "Chinese": 0.8428571428571429, - "Spanish": 0.826890756302521, - "Vietnamese": 0.6100840336134454 - }, - "consistency_score_2": 0.7205882352941176, - "consistency_score_3": 0.6010504201680672, - "consistency_score_4": 0.5109243697478991, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8420168067226891, - "English,Spanish": 0.8478991596638655, - "English,Vietnamese": 0.6243697478991597, - "Chinese,Spanish": 0.8033613445378152, - "Chinese,Vietnamese": 0.5932773109243697, - "Spanish,Vietnamese": 0.6126050420168068 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7554621848739496, - "English,Chinese,Vietnamese": 0.553781512605042, - "English,Spanish,Vietnamese": 0.5630252100840336, - "Chinese,Spanish,Vietnamese": 0.5319327731092437 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5109243697478991 - } + "Spanish": 0.253781512605042, + "English": 0.23865546218487396, + "Chinese": 0.2647058823529412, + "Vietnamese": 0.2596638655462185 }, - "AC3_2": 0.7546160130220069, - "AC3_3": 0.6834444529847545, - "AC3_4": 0.6211495884559001 - }, - "prompt_4": { - "overall_acc": 0.8048319327731093, - "language_acc": { - "English": 0.8949579831932774, - "Chinese": 0.8571428571428571, - "Spanish": 0.8420168067226891, - "Vietnamese": 0.6252100840336134 - }, - "consistency_score_2": 0.7450980392156863, - "consistency_score_3": 0.6327731092436975, - "consistency_score_4": 0.5487394957983194, + "consistency_score_2": 0.2838935574229691, + "consistency_score_3": 0.0930672268907563, + "consistency_score_4": 0.03277310924369748, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.8647058823529412, - "English,Spanish": 0.8663865546218488, - "English,Vietnamese": 0.6512605042016807, - "Chinese,Spanish": 0.8260504201680672, - "Chinese,Vietnamese": 0.6184873949579832, - "Spanish,Vietnamese": 0.6436974789915967 + "Spanish,English": 0.3025210084033613, + "Spanish,Chinese": 0.2915966386554622, + "Spanish,Vietnamese": 0.2991596638655462, + "English,Chinese": 0.2504201680672269, + "English,Vietnamese": 0.2848739495798319, + "Chinese,Vietnamese": 0.2747899159663866 }, "3_combine": { - "English,Chinese,Spanish": 0.788235294117647, - "English,Chinese,Vietnamese": 0.5840336134453782, - "English,Spanish,Vietnamese": 0.5949579831932773, - "Chinese,Spanish,Vietnamese": 0.5638655462184874 + "Spanish,English,Chinese": 0.0865546218487395, + "Spanish,English,Vietnamese": 0.11176470588235295, + "Spanish,Chinese,Vietnamese": 0.09663865546218488, + "English,Chinese,Vietnamese": 0.0773109243697479 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5487394957983194 + "Spanish,English,Chinese,Vietnamese": 0.03277310924369748 } }, - "AC3_2": 0.7738139216693064, - "AC3_3": 0.7085061468198103, - "AC3_4": 0.6525596797293349 - }, - "prompt_5": { - "overall_acc": 0.7934873949579831, + "AC3_2": 0.2682284261462423, + "AC3_3": 0.13625087054493334, + "AC3_4": 0.05806070596054978 + } + }, + "cross_mmlu": { + "prompt_1": { + "overall_acc": 0.24761904761904763, "language_acc": { - "English": 0.8831932773109243, - "Chinese": 0.8403361344537815, - "Spanish": 0.8252100840336134, - "Vietnamese": 0.6252100840336134 + "Filipino": 0.28, + "Vietnamese": 0.24, + "Chinese": 0.26666666666666666, + "Spanish": 0.21333333333333335, + "Malay": 0.26, + "Indonesian": 0.23333333333333334, + "English": 0.24 }, - "consistency_score_2": 0.7249299719887955, - "consistency_score_3": 0.6058823529411764, - "consistency_score_4": 0.5201680672268908, + "consistency_score_2": 0.2911111111111111, + "consistency_score_3": 0.09828571428571428, + "consistency_score_4": 0.03619047619047618, + "consistency_score_5": 0.013333333333333332, + "consistency_score_6": 0.00380952380952381, + "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.8428571428571429, - "English,Spanish": 0.8420168067226891, - "English,Vietnamese": 0.6336134453781512, - "Chinese,Spanish": 0.8016806722689076, - "Chinese,Vietnamese": 0.6100840336134454, - "Spanish,Vietnamese": 0.619327731092437 - }, - "3_combine": { - "English,Chinese,Spanish": 0.753781512605042, - "English,Chinese,Vietnamese": 0.5621848739495798, - "English,Spanish,Vietnamese": 0.5689075630252101, - "Chinese,Spanish,Vietnamese": 0.5386554621848739 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5201680672268908 - } - }, - "AC3_2": 0.757660979759634, - "AC3_3": 0.6871093370050417, - "AC3_4": 0.6283943034595901 + "Filipino,Vietnamese": 0.36, + "Filipino,Chinese": 0.26666666666666666, + "Filipino,Spanish": 0.3, + "Filipino,Malay": 0.2866666666666667, + "Filipino,Indonesian": 0.28, + "Filipino,English": 0.2733333333333333, + "Vietnamese,Chinese": 0.31333333333333335, + "Vietnamese,Spanish": 0.24666666666666667, + "Vietnamese,Malay": 0.32, + "Vietnamese,Indonesian": 0.31333333333333335, + "Vietnamese,English": 0.2866666666666667, + "Chinese,Spanish": 0.3, + "Chinese,Malay": 0.24666666666666667, + "Chinese,Indonesian": 0.26, + "Chinese,English": 0.31333333333333335, + "Spanish,Malay": 0.30666666666666664, + "Spanish,Indonesian": 0.2733333333333333, + "Spanish,English": 0.2866666666666667, + "Malay,Indonesian": 0.30666666666666664, + "Malay,English": 0.31333333333333335, + "Indonesian,English": 0.26 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.1, + "Filipino,Vietnamese,Spanish": 0.08666666666666667, + "Filipino,Vietnamese,Malay": 0.12666666666666668, + "Filipino,Vietnamese,Indonesian": 0.12, + "Filipino,Vietnamese,English": 0.09333333333333334, + "Filipino,Chinese,Spanish": 0.08666666666666667, + "Filipino,Chinese,Malay": 0.08666666666666667, + "Filipino,Chinese,Indonesian": 0.06666666666666667, + "Filipino,Chinese,English": 0.10666666666666667, + "Filipino,Spanish,Malay": 0.09333333333333334, + "Filipino,Spanish,Indonesian": 0.07333333333333333, + "Filipino,Spanish,English": 0.08, + "Filipino,Malay,Indonesian": 0.10666666666666667, + "Filipino,Malay,English": 0.10666666666666667, + "Filipino,Indonesian,English": 0.08666666666666667, + "Vietnamese,Chinese,Spanish": 0.10666666666666667, + "Vietnamese,Chinese,Malay": 0.08666666666666667, + "Vietnamese,Chinese,Indonesian": 0.13333333333333333, + "Vietnamese,Chinese,English": 0.13333333333333333, + "Vietnamese,Spanish,Malay": 0.1, + "Vietnamese,Spanish,Indonesian": 0.08, + "Vietnamese,Spanish,English": 0.07333333333333333, + "Vietnamese,Malay,Indonesian": 0.14, + "Vietnamese,Malay,English": 0.1, + "Vietnamese,Indonesian,English": 0.10666666666666667, + "Chinese,Spanish,Malay": 0.09333333333333334, + "Chinese,Spanish,Indonesian": 0.08, + "Chinese,Spanish,English": 0.11333333333333333, + "Chinese,Malay,Indonesian": 0.06, + "Chinese,Malay,English": 0.12, + "Chinese,Indonesian,English": 0.08, + "Spanish,Malay,Indonesian": 0.1, + "Spanish,Malay,English": 0.12666666666666668, + "Spanish,Indonesian,English": 0.07333333333333333, + "Malay,Indonesian,English": 0.11333333333333333 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.03333333333333333, + "Filipino,Vietnamese,Chinese,Malay": 0.03333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.04, + "Filipino,Vietnamese,Chinese,English": 0.04, + "Filipino,Vietnamese,Spanish,Malay": 0.02666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian": 0.02, + "Filipino,Vietnamese,Spanish,English": 0.02, + "Filipino,Vietnamese,Malay,Indonesian": 0.06, + "Filipino,Vietnamese,Malay,English": 0.04, + "Filipino,Vietnamese,Indonesian,English": 0.04, + "Filipino,Chinese,Spanish,Malay": 0.02666666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.013333333333333334, + "Filipino,Chinese,Spanish,English": 0.03333333333333333, + "Filipino,Chinese,Malay,Indonesian": 0.02, + "Filipino,Chinese,Malay,English": 0.04666666666666667, + "Filipino,Chinese,Indonesian,English": 0.02, + "Filipino,Spanish,Malay,Indonesian": 0.03333333333333333, + "Filipino,Spanish,Malay,English": 0.02, + "Filipino,Spanish,Indonesian,English": 0.013333333333333334, + "Filipino,Malay,Indonesian,English": 0.05333333333333334, + "Vietnamese,Chinese,Spanish,Malay": 0.04, + "Vietnamese,Chinese,Spanish,Indonesian": 0.03333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.06, + "Vietnamese,Chinese,Malay,Indonesian": 0.04, + "Vietnamese,Chinese,Malay,English": 0.05333333333333334, + "Vietnamese,Chinese,Indonesian,English": 0.06, + "Vietnamese,Spanish,Malay,Indonesian": 0.04666666666666667, + "Vietnamese,Spanish,Malay,English": 0.03333333333333333, + "Vietnamese,Spanish,Indonesian,English": 0.02, + "Vietnamese,Malay,Indonesian,English": 0.04666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.02, + "Chinese,Spanish,Malay,English": 0.06666666666666667, + "Chinese,Spanish,Indonesian,English": 0.02666666666666667, + "Chinese,Malay,Indonesian,English": 0.04, + "Spanish,Malay,Indonesian,English": 0.04666666666666667 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.02, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.013333333333333334, + "Filipino,Vietnamese,Chinese,Malay,English": 0.02, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.013333333333333334, + "Filipino,Vietnamese,Spanish,Malay,English": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.0, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.02666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Chinese,Spanish,Malay,English": 0.013333333333333334, + "Filipino,Chinese,Spanish,Indonesian,English": 0.0, + "Filipino,Chinese,Malay,Indonesian,English": 0.013333333333333334, + "Filipino,Spanish,Malay,Indonesian,English": 0.006666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.013333333333333334, + "Vietnamese,Chinese,Spanish,Malay,English": 0.03333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.02, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.02666666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.013333333333333334, + "Chinese,Spanish,Malay,Indonesian,English": 0.02 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.0, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.006666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.0, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.006666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.0, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.0, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.013333333333333334 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.0 + } + }, + "AC3_2": 0.2676095069234156, + "AC3_3": 0.1407174323067073, + "AC3_4": 0.06315116648461995, + "AC3_5": 0.025304136243344523, + "AC3_6": 0.007503607500623115, + "AC3_7": 0.0 } }, - "cross_mmlu": { + "cross_logiqa": { "prompt_1": { - "overall_acc": 0.41523809523809524, + "overall_acc": 0.25081168831168826, "language_acc": { - "Spanish": 0.43333333333333335, - "English": 0.5933333333333334, - "Chinese": 0.5, - "Vietnamese": 0.37333333333333335, - "Filipino": 0.36, - "Indonesian": 0.34, - "Malay": 0.30666666666666664 - }, - "consistency_score_2": 0.43111111111111117, - "consistency_score_3": 0.23466666666666672, - "consistency_score_4": 0.14628571428571427, - "consistency_score_5": 0.1015873015873016, - "consistency_score_6": 0.07619047619047618, - "consistency_score_7": 0.06, + "Indonesian": 0.20454545454545456, + "English": 0.2840909090909091, + "Filipino": 0.2897727272727273, + "Spanish": 0.26704545454545453, + "Chinese": 0.23295454545454544, + "Malay": 0.2159090909090909, + "Vietnamese": 0.26136363636363635 + }, + "consistency_score_2": 0.28977272727272724, + "consistency_score_3": 0.09399350649350649, + "consistency_score_4": 0.032629870129870134, + "consistency_score_5": 0.011634199134199132, + "consistency_score_6": 0.004058441558441559, + "consistency_score_7": 0.0, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5733333333333334, - "Spanish,Chinese": 0.49333333333333335, - "Spanish,Vietnamese": 0.4066666666666667, - "Spanish,Filipino": 0.43333333333333335, - "Spanish,Indonesian": 0.4066666666666667, - "Spanish,Malay": 0.35333333333333333, - "English,Chinese": 0.6, - "English,Vietnamese": 0.4266666666666667, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.4066666666666667, - "English,Malay": 0.38, - "Chinese,Vietnamese": 0.31333333333333335, - "Chinese,Filipino": 0.42, - "Chinese,Indonesian": 0.36, - "Chinese,Malay": 0.32, - "Vietnamese,Filipino": 0.41333333333333333, - "Vietnamese,Indonesian": 0.43333333333333335, - "Vietnamese,Malay": 0.4533333333333333, - "Filipino,Indonesian": 0.42, - "Filipino,Malay": 0.44666666666666666, - "Indonesian,Malay": 0.5266666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.3933333333333333, - "Spanish,English,Vietnamese": 0.25333333333333335, - "Spanish,English,Filipino": 0.3, - "Spanish,English,Indonesian": 0.25333333333333335, - "Spanish,English,Malay": 0.24666666666666667, - "Spanish,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,Chinese,Filipino": 0.28, - "Spanish,Chinese,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino": 0.22666666666666666, - "Spanish,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,Vietnamese,Malay": 0.2, - "Spanish,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Filipino,Malay": 0.21333333333333335, - "Spanish,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese": 0.23333333333333334, - "English,Chinese,Filipino": 0.29333333333333333, - "English,Chinese,Indonesian": 0.26, - "English,Chinese,Malay": 0.25333333333333335, - "English,Vietnamese,Filipino": 0.23333333333333334, - "English,Vietnamese,Indonesian": 0.22, - "English,Vietnamese,Malay": 0.20666666666666667, - "English,Filipino,Indonesian": 0.23333333333333334, - "English,Filipino,Malay": 0.24, - "English,Indonesian,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian": 0.16, - "Chinese,Vietnamese,Malay": 0.17333333333333334, - "Chinese,Filipino,Indonesian": 0.20666666666666667, - "Chinese,Filipino,Malay": 0.2, - "Chinese,Indonesian,Malay": 0.20666666666666667, - "Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Vietnamese,Filipino,Malay": 0.22666666666666666, - "Vietnamese,Indonesian,Malay": 0.26, - "Filipino,Indonesian,Malay": 0.26666666666666666 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.17333333333333334, - "Spanish,English,Chinese,Filipino": 0.22666666666666666, - "Spanish,English,Chinese,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian": 0.14, - "Spanish,English,Vietnamese,Malay": 0.14666666666666667, - "Spanish,English,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Indonesian,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.12666666666666668, - "Spanish,Chinese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.14, - "Spanish,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.14, - "English,Chinese,Vietnamese,Indonesian": 0.12, - "English,Chinese,Vietnamese,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian": 0.14666666666666667, - "English,Chinese,Filipino,Malay": 0.16, - "English,Chinese,Indonesian,Malay": 0.16666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.14, - "English,Vietnamese,Filipino,Malay": 0.14, - "English,Vietnamese,Indonesian,Malay": 0.14, - "English,Filipino,Indonesian,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.12, - "Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,English,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 + "Indonesian,English": 0.25, + "Indonesian,Filipino": 0.26704545454545453, + "Indonesian,Spanish": 0.3125, + "Indonesian,Chinese": 0.22727272727272727, + "Indonesian,Malay": 0.2556818181818182, + "Indonesian,Vietnamese": 0.36363636363636365, + "English,Filipino": 0.32386363636363635, + "English,Spanish": 0.29545454545454547, + "English,Chinese": 0.23863636363636365, + "English,Malay": 0.3125, + "English,Vietnamese": 0.32386363636363635, + "Filipino,Spanish": 0.2784090909090909, + "Filipino,Chinese": 0.2897727272727273, + "Filipino,Malay": 0.2784090909090909, + "Filipino,Vietnamese": 0.2897727272727273, + "Spanish,Chinese": 0.26136363636363635, + "Spanish,Malay": 0.2897727272727273, + "Spanish,Vietnamese": 0.3465909090909091, + "Chinese,Malay": 0.30113636363636365, + "Chinese,Vietnamese": 0.26704545454545453, + "Malay,Vietnamese": 0.3125 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.08522727272727272, + "Indonesian,English,Spanish": 0.0625, + "Indonesian,English,Chinese": 0.045454545454545456, + "Indonesian,English,Malay": 0.09090909090909091, + "Indonesian,English,Vietnamese": 0.11363636363636363, + "Indonesian,Filipino,Spanish": 0.07954545454545454, + "Indonesian,Filipino,Chinese": 0.07386363636363637, + "Indonesian,Filipino,Malay": 0.07954545454545454, + "Indonesian,Filipino,Vietnamese": 0.10795454545454546, + "Indonesian,Spanish,Chinese": 0.09090909090909091, + "Indonesian,Spanish,Malay": 0.10227272727272728, + "Indonesian,Spanish,Vietnamese": 0.13068181818181818, + "Indonesian,Chinese,Malay": 0.045454545454545456, + "Indonesian,Chinese,Vietnamese": 0.09659090909090909, + "Indonesian,Malay,Vietnamese": 0.09659090909090909, + "English,Filipino,Spanish": 0.10795454545454546, + "English,Filipino,Chinese": 0.09659090909090909, + "English,Filipino,Malay": 0.11363636363636363, + "English,Filipino,Vietnamese": 0.125, + "English,Spanish,Chinese": 0.07954545454545454, + "English,Spanish,Malay": 0.09659090909090909, + "English,Spanish,Vietnamese": 0.10227272727272728, + "English,Chinese,Malay": 0.09659090909090909, + "English,Chinese,Vietnamese": 0.09659090909090909, + "English,Malay,Vietnamese": 0.11931818181818182, + "Filipino,Spanish,Chinese": 0.07954545454545454, + "Filipino,Spanish,Malay": 0.07386363636363637, + "Filipino,Spanish,Vietnamese": 0.09090909090909091, + "Filipino,Chinese,Malay": 0.10227272727272728, + "Filipino,Chinese,Vietnamese": 0.10227272727272728, + "Filipino,Malay,Vietnamese": 0.07954545454545454, + "Spanish,Chinese,Malay": 0.09090909090909091, + "Spanish,Chinese,Vietnamese": 0.09659090909090909, + "Spanish,Malay,Vietnamese": 0.125, + "Chinese,Malay,Vietnamese": 0.11363636363636363 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.028409090909090908, + "Indonesian,English,Filipino,Chinese": 0.022727272727272728, + "Indonesian,English,Filipino,Malay": 0.03409090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.03409090909090909, + "Indonesian,English,Spanish,Chinese": 0.022727272727272728, + "Indonesian,English,Spanish,Malay": 0.028409090909090908, + "Indonesian,English,Spanish,Vietnamese": 0.03409090909090909, + "Indonesian,English,Chinese,Malay": 0.017045454545454544, + "Indonesian,English,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Malay": 0.022727272727272728, + "Indonesian,Filipino,Spanish,Vietnamese": 0.028409090909090908, + "Indonesian,Filipino,Chinese,Malay": 0.028409090909090908, + "Indonesian,Filipino,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Malay,Vietnamese": 0.017045454545454544, + "Indonesian,Spanish,Chinese,Malay": 0.03409090909090909, + "Indonesian,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "Indonesian,Spanish,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Chinese": 0.03977272727272727, + "English,Filipino,Spanish,Malay": 0.03977272727272727, + "English,Filipino,Spanish,Vietnamese": 0.03977272727272727, + "English,Filipino,Chinese,Malay": 0.03977272727272727, + "English,Filipino,Chinese,Vietnamese": 0.05113636363636364, + "English,Filipino,Malay,Vietnamese": 0.03977272727272727, + "English,Spanish,Chinese,Malay": 0.028409090909090908, + "English,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "English,Spanish,Malay,Vietnamese": 0.03409090909090909, + "English,Chinese,Malay,Vietnamese": 0.05113636363636364, + "Filipino,Spanish,Chinese,Malay": 0.022727272727272728, + "Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Filipino,Spanish,Malay,Vietnamese": 0.028409090909090908, + "Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.017045454545454544, + "Indonesian,English,Filipino,Spanish,Malay": 0.011363636363636364, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.011363636363636364, + "Indonesian,English,Filipino,Chinese,Malay": 0.011363636363636364, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,English,Spanish,Chinese,Malay": 0.017045454545454544, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, + "English,Filipino,Spanish,Chinese,Malay": 0.011363636363636364, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, + "English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, + "English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.08, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.08, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.011363636363636364, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 } }, - "AC3_2": 0.42302575638912737, - "AC3_3": 0.29986713555370353, - "AC3_4": 0.21635200616422298, - "AC3_5": 0.1632385632069784, - "AC3_6": 0.12875599849724004, - "AC3_7": 0.10484969937673502 - }, - "prompt_2": { - "overall_acc": 0.3971428571428572, - "language_acc": { - "Spanish": 0.4666666666666667, - "English": 0.5533333333333333, - "Chinese": 0.4533333333333333, - "Vietnamese": 0.34, - "Filipino": 0.3333333333333333, - "Indonesian": 0.3466666666666667, - "Malay": 0.2866666666666667 - }, - "consistency_score_2": 0.43460317460317455, - "consistency_score_3": 0.2417142857142857, - "consistency_score_4": 0.15219047619047618, - "consistency_score_5": 0.10380952380952377, - "consistency_score_6": 0.07619047619047618, - "consistency_score_7": 0.06, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5733333333333334, - "Spanish,Chinese": 0.5533333333333333, - "Spanish,Vietnamese": 0.42, - "Spanish,Filipino": 0.4, - "Spanish,Indonesian": 0.4, - "Spanish,Malay": 0.32666666666666666, - "English,Chinese": 0.5533333333333333, - "English,Vietnamese": 0.41333333333333333, - "English,Filipino": 0.44, - "English,Indonesian": 0.47333333333333333, - "English,Malay": 0.4066666666666667, - "Chinese,Vietnamese": 0.37333333333333335, - "Chinese,Filipino": 0.42, - "Chinese,Indonesian": 0.4266666666666667, - "Chinese,Malay": 0.35333333333333333, - "Vietnamese,Filipino": 0.36666666666666664, - "Vietnamese,Indonesian": 0.34, - "Vietnamese,Malay": 0.46, - "Filipino,Indonesian": 0.44666666666666666, - "Filipino,Malay": 0.44, - "Indonesian,Malay": 0.54 - }, - "3_combine": { - "Spanish,English,Chinese": 0.38666666666666666, - "Spanish,English,Vietnamese": 0.2733333333333333, - "Spanish,English,Filipino": 0.26666666666666666, - "Spanish,English,Indonesian": 0.2866666666666667, - "Spanish,English,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese": 0.26, - "Spanish,Chinese,Filipino": 0.26, - "Spanish,Chinese,Indonesian": 0.26, - "Spanish,Chinese,Malay": 0.22, - "Spanish,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,Vietnamese,Indonesian": 0.2, - "Spanish,Vietnamese,Malay": 0.19333333333333333, - "Spanish,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Filipino,Malay": 0.17333333333333334, - "Spanish,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese": 0.26, - "English,Chinese,Filipino": 0.26666666666666666, - "English,Chinese,Indonesian": 0.30666666666666664, - "English,Chinese,Malay": 0.26, - "English,Vietnamese,Filipino": 0.21333333333333335, - "English,Vietnamese,Indonesian": 0.24, - "English,Vietnamese,Malay": 0.26, - "English,Filipino,Indonesian": 0.26, - "English,Filipino,Malay": 0.22666666666666666, - "English,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian": 0.22, - "Chinese,Vietnamese,Malay": 0.21333333333333335, - "Chinese,Filipino,Indonesian": 0.24, - "Chinese,Filipino,Malay": 0.18666666666666668, - "Chinese,Indonesian,Malay": 0.24666666666666667, - "Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Vietnamese,Filipino,Malay": 0.23333333333333334, - "Vietnamese,Indonesian,Malay": 0.25333333333333335, - "Filipino,Indonesian,Malay": 0.29333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.2, - "Spanish,English,Chinese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Indonesian": 0.21333333333333335, - "Spanish,English,Chinese,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.16, - "Spanish,English,Vietnamese,Malay": 0.15333333333333332, - "Spanish,English,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian": 0.14, - "Spanish,Chinese,Filipino,Malay": 0.10666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,Filipino,Indonesian,Malay": 0.12, - "English,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "English,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "English,Chinese,Vietnamese,Malay": 0.16666666666666666, - "English,Chinese,Filipino,Indonesian": 0.18666666666666668, - "English,Chinese,Filipino,Malay": 0.14, - "English,Chinese,Indonesian,Malay": 0.20666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "English,Vietnamese,Filipino,Malay": 0.16, - "English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "English,Filipino,Indonesian,Malay": 0.17333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,English,Vietnamese,Filipino,Malay": 0.08, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.08, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06 - } - }, - "AC3_2": 0.4150294437887056, - "AC3_3": 0.3005213390789498, - "AC3_4": 0.22005349708694788, - "AC3_5": 0.16459532859289036, - "AC3_6": 0.12785283124634322, - "AC3_7": 0.10424999997719532 - }, - "prompt_3": { - "overall_acc": 0.4095238095238095, - "language_acc": { - "Spanish": 0.44666666666666666, - "English": 0.58, - "Chinese": 0.49333333333333335, - "Vietnamese": 0.3466666666666667, - "Filipino": 0.35333333333333333, - "Indonesian": 0.36, - "Malay": 0.2866666666666667 - }, - "consistency_score_2": 0.4406349206349206, - "consistency_score_3": 0.24209523809523809, - "consistency_score_4": 0.15219047619047618, - "consistency_score_5": 0.10571428571428568, - "consistency_score_6": 0.0780952380952381, - "consistency_score_7": 0.06, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5733333333333334, - "Spanish,Chinese": 0.52, - "Spanish,Vietnamese": 0.42, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.44, - "Spanish,Malay": 0.32666666666666666, - "English,Chinese": 0.6, - "English,Vietnamese": 0.3933333333333333, - "English,Filipino": 0.42, - "English,Indonesian": 0.47333333333333333, - "English,Malay": 0.36666666666666664, - "Chinese,Vietnamese": 0.36666666666666664, - "Chinese,Filipino": 0.41333333333333333, - "Chinese,Indonesian": 0.41333333333333333, - "Chinese,Malay": 0.3466666666666667, - "Vietnamese,Filipino": 0.36666666666666664, - "Vietnamese,Indonesian": 0.5133333333333333, - "Vietnamese,Malay": 0.41333333333333333, - "Filipino,Indonesian": 0.48, - "Filipino,Malay": 0.4533333333333333, - "Indonesian,Malay": 0.5133333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.3933333333333333, - "Spanish,English,Vietnamese": 0.26, - "Spanish,English,Filipino": 0.26666666666666666, - "Spanish,English,Indonesian": 0.2866666666666667, - "Spanish,English,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese": 0.24, - "Spanish,Chinese,Filipino": 0.26, - "Spanish,Chinese,Indonesian": 0.26, - "Spanish,Chinese,Malay": 0.2, - "Spanish,Vietnamese,Filipino": 0.22, - "Spanish,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Malay": 0.19333333333333333, - "Spanish,Filipino,Indonesian": 0.24666666666666667, - "Spanish,Filipino,Malay": 0.19333333333333333, - "Spanish,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese": 0.28, - "English,Chinese,Filipino": 0.26, - "English,Chinese,Indonesian": 0.29333333333333333, - "English,Chinese,Malay": 0.23333333333333334, - "English,Vietnamese,Filipino": 0.20666666666666667, - "English,Vietnamese,Indonesian": 0.28, - "English,Vietnamese,Malay": 0.21333333333333335, - "English,Filipino,Indonesian": 0.24, - "English,Filipino,Malay": 0.19333333333333333, - "English,Indonesian,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian": 0.26, - "Chinese,Vietnamese,Malay": 0.18666666666666668, - "Chinese,Filipino,Indonesian": 0.22, - "Chinese,Filipino,Malay": 0.2, - "Chinese,Indonesian,Malay": 0.21333333333333335, - "Vietnamese,Filipino,Indonesian": 0.26, - "Vietnamese,Filipino,Malay": 0.21333333333333335, - "Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Filipino,Indonesian,Malay": 0.2866666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.2, - "Spanish,English,Chinese,Filipino": 0.19333333333333333, - "Spanish,English,Chinese,Indonesian": 0.2, - "Spanish,English,Chinese,Malay": 0.16, - "Spanish,English,Vietnamese,Filipino": 0.14, - "Spanish,English,Vietnamese,Indonesian": 0.18, - "Spanish,English,Vietnamese,Malay": 0.14666666666666667, - "Spanish,English,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Indonesian,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.12666666666666668, - "Spanish,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.14, - "English,Chinese,Vietnamese,Indonesian": 0.20666666666666667, - "English,Chinese,Vietnamese,Malay": 0.14666666666666667, - "English,Chinese,Filipino,Indonesian": 0.15333333333333332, - "English,Chinese,Filipino,Malay": 0.12666666666666668, - "English,Chinese,Indonesian,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.16, - "English,Vietnamese,Filipino,Malay": 0.13333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "English,Filipino,Indonesian,Malay": 0.14, - "Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Malay": 0.12, - "Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.12, - "Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.14, - "Spanish,English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Malay": 0.1, - "Spanish,English,Chinese,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.08, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.08, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06 - } - }, - "AC3_2": 0.42451011766406815, - "AC3_3": 0.3042997730633635, - "AC3_4": 0.22191218972619317, - "AC3_5": 0.16804858723963143, - "AC3_6": 0.131175595211194, - "AC3_7": 0.10466531437933091 - }, - "prompt_4": { - "overall_acc": 0.41619047619047617, - "language_acc": { - "Spanish": 0.44, - "English": 0.5666666666666667, - "Chinese": 0.49333333333333335, - "Vietnamese": 0.35333333333333333, - "Filipino": 0.36666666666666664, - "Indonesian": 0.35333333333333333, - "Malay": 0.34 - }, - "consistency_score_2": 0.4371428571428571, - "consistency_score_3": 0.24361904761904762, - "consistency_score_4": 0.15828571428571436, - "consistency_score_5": 0.11333333333333331, - "consistency_score_6": 0.08571428571428573, - "consistency_score_7": 0.06666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5933333333333334, - "Spanish,Chinese": 0.4866666666666667, - "Spanish,Vietnamese": 0.4, - "Spanish,Filipino": 0.4, - "Spanish,Indonesian": 0.44666666666666666, - "Spanish,Malay": 0.31333333333333335, - "English,Chinese": 0.58, - "English,Vietnamese": 0.4066666666666667, - "English,Filipino": 0.47333333333333333, - "English,Indonesian": 0.49333333333333335, - "English,Malay": 0.3933333333333333, - "Chinese,Vietnamese": 0.3333333333333333, - "Chinese,Filipino": 0.44666666666666666, - "Chinese,Indonesian": 0.37333333333333335, - "Chinese,Malay": 0.32, - "Vietnamese,Filipino": 0.43333333333333335, - "Vietnamese,Indonesian": 0.4533333333333333, - "Vietnamese,Malay": 0.4533333333333333, - "Filipino,Indonesian": 0.41333333333333333, - "Filipino,Malay": 0.4066666666666667, - "Indonesian,Malay": 0.56 - }, - "3_combine": { - "Spanish,English,Chinese": 0.38666666666666666, - "Spanish,English,Vietnamese": 0.28, - "Spanish,English,Filipino": 0.2733333333333333, - "Spanish,English,Indonesian": 0.3333333333333333, - "Spanish,English,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese": 0.22, - "Spanish,Chinese,Filipino": 0.23333333333333334, - "Spanish,Chinese,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino": 0.22, - "Spanish,Vietnamese,Indonesian": 0.24, - "Spanish,Vietnamese,Malay": 0.21333333333333335, - "Spanish,Filipino,Indonesian": 0.22, - "Spanish,Filipino,Malay": 0.16666666666666666, - "Spanish,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese": 0.26666666666666666, - "English,Chinese,Filipino": 0.3, - "English,Chinese,Indonesian": 0.28, - "English,Chinese,Malay": 0.22, - "English,Vietnamese,Filipino": 0.25333333333333335, - "English,Vietnamese,Indonesian": 0.2733333333333333, - "English,Vietnamese,Malay": 0.24, - "English,Filipino,Indonesian": 0.2733333333333333, - "English,Filipino,Malay": 0.22, - "English,Indonesian,Malay": 0.3, - "Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian": 0.2, - "Chinese,Vietnamese,Malay": 0.18666666666666668, - "Chinese,Filipino,Indonesian": 0.19333333333333333, - "Chinese,Filipino,Malay": 0.18, - "Chinese,Indonesian,Malay": 0.20666666666666667, - "Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Vietnamese,Filipino,Malay": 0.23333333333333334, - "Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Filipino,Indonesian,Malay": 0.2733333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.18666666666666668, - "Spanish,English,Chinese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Indonesian": 0.21333333333333335, - "Spanish,English,Chinese,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.16, - "Spanish,English,Vietnamese,Indonesian": 0.2, - "Spanish,English,Vietnamese,Malay": 0.17333333333333334, - "Spanish,English,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Indonesian,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.13333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.1, - "Spanish,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.18, - "English,Chinese,Vietnamese,Malay": 0.16, - "English,Chinese,Filipino,Indonesian": 0.16, - "English,Chinese,Filipino,Malay": 0.12, - "English,Chinese,Indonesian,Malay": 0.16666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Vietnamese,Filipino,Malay": 0.16666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "English,Filipino,Indonesian,Malay": 0.18, - "Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Chinese,Filipino,Indonesian,Malay": 0.12, - "Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Filipino,Malay": 0.08, - "Spanish,English,Chinese,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "English,Chinese,Filipino,Indonesian,Malay": 0.1, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 - } - }, - "AC3_2": 0.42640943872554027, - "AC3_3": 0.3073369624520456, - "AC3_4": 0.2293463432719379, - "AC3_5": 0.1781534771845813, - "AC3_6": 0.14215234478056876, - "AC3_7": 0.11492439182366786 - }, - "prompt_5": { - "overall_acc": 0.400952380952381, - "language_acc": { - "Spanish": 0.4266666666666667, - "English": 0.6, - "Chinese": 0.49333333333333335, - "Vietnamese": 0.28, - "Filipino": 0.32666666666666666, - "Indonesian": 0.36666666666666664, - "Malay": 0.31333333333333335 - }, - "consistency_score_2": 0.4276190476190476, - "consistency_score_3": 0.23638095238095239, - "consistency_score_4": 0.15295238095238092, - "consistency_score_5": 0.10920634920634921, - "consistency_score_6": 0.08285714285714285, - "consistency_score_7": 0.06666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5666666666666667, - "Spanish,Chinese": 0.5133333333333333, - "Spanish,Vietnamese": 0.36, - "Spanish,Filipino": 0.3933333333333333, - "Spanish,Indonesian": 0.4266666666666667, - "Spanish,Malay": 0.35333333333333333, - "English,Chinese": 0.5466666666666666, - "English,Vietnamese": 0.36, - "English,Filipino": 0.4266666666666667, - "English,Indonesian": 0.4666666666666667, - "English,Malay": 0.38, - "Chinese,Vietnamese": 0.35333333333333333, - "Chinese,Filipino": 0.41333333333333333, - "Chinese,Indonesian": 0.4066666666666667, - "Chinese,Malay": 0.32666666666666666, - "Vietnamese,Filipino": 0.3933333333333333, - "Vietnamese,Indonesian": 0.38666666666666666, - "Vietnamese,Malay": 0.44666666666666666, - "Filipino,Indonesian": 0.5, - "Filipino,Malay": 0.46, - "Indonesian,Malay": 0.5 - }, - "3_combine": { - "Spanish,English,Chinese": 0.37333333333333335, - "Spanish,English,Vietnamese": 0.22666666666666666, - "Spanish,English,Filipino": 0.26, - "Spanish,English,Indonesian": 0.32, - "Spanish,English,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese": 0.22, - "Spanish,Chinese,Filipino": 0.22666666666666666, - "Spanish,Chinese,Indonesian": 0.26666666666666666, - "Spanish,Chinese,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino": 0.2, - "Spanish,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,Vietnamese,Malay": 0.18, - "Spanish,Filipino,Indonesian": 0.26, - "Spanish,Filipino,Malay": 0.22, - "Spanish,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese": 0.22666666666666666, - "English,Chinese,Filipino": 0.25333333333333335, - "English,Chinese,Indonesian": 0.2866666666666667, - "English,Chinese,Malay": 0.22, - "English,Vietnamese,Filipino": 0.20666666666666667, - "English,Vietnamese,Indonesian": 0.20666666666666667, - "English,Vietnamese,Malay": 0.20666666666666667, - "English,Filipino,Indonesian": 0.26, - "English,Filipino,Malay": 0.22, - "English,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian": 0.22, - "Chinese,Vietnamese,Malay": 0.2, - "Chinese,Filipino,Indonesian": 0.25333333333333335, - "Chinese,Filipino,Malay": 0.19333333333333333, - "Chinese,Indonesian,Malay": 0.22666666666666666, - "Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Vietnamese,Filipino,Malay": 0.22666666666666666, - "Vietnamese,Indonesian,Malay": 0.24, - "Filipino,Indonesian,Malay": 0.30666666666666664 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.16666666666666666, - "Spanish,English,Chinese,Filipino": 0.16666666666666666, - "Spanish,English,Chinese,Indonesian": 0.22, - "Spanish,English,Chinese,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino": 0.14, - "Spanish,English,Vietnamese,Indonesian": 0.14, - "Spanish,English,Vietnamese,Malay": 0.13333333333333333, - "Spanish,English,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Filipino,Malay": 0.16, - "Spanish,English,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "English,Chinese,Vietnamese,Malay": 0.14666666666666667, - "English,Chinese,Filipino,Indonesian": 0.16666666666666666, - "English,Chinese,Filipino,Malay": 0.13333333333333333, - "English,Chinese,Indonesian,Malay": 0.18, - "English,Vietnamese,Filipino,Indonesian": 0.14, - "English,Vietnamese,Filipino,Malay": 0.13333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.16, - "English,Filipino,Indonesian,Malay": 0.16, - "Chinese,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.12, - "Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.14, - "Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,English,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,English,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 - } - }, - "AC3_2": 0.41385659546181974, - "AC3_3": 0.29741895079726977, - "AC3_4": 0.2214338114488552, - "AC3_5": 0.17165851776052624, - "AC3_6": 0.13733408321120905, - "AC3_7": 0.11432450778274798 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.3887987012987013, - "language_acc": { - "English": 0.48863636363636365, - "Filipino": 0.32386363636363635, - "Spanish": 0.38636363636363635, - "Malay": 0.3068181818181818, - "Chinese": 0.5511363636363636, - "Indonesian": 0.3465909090909091, - "Vietnamese": 0.3181818181818182 - }, - "consistency_score_2": 0.4494047619047619, - "consistency_score_3": 0.2613636363636364, - "consistency_score_4": 0.1728896103896104, - "consistency_score_5": 0.12418831168831167, - "consistency_score_6": 0.09577922077922077, - "consistency_score_7": 0.07954545454545454, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.42045454545454547, - "English,Spanish": 0.5511363636363636, - "English,Malay": 0.39204545454545453, - "English,Chinese": 0.6193181818181818, - "English,Indonesian": 0.4375, - "English,Vietnamese": 0.375, - "Filipino,Spanish": 0.4431818181818182, - "Filipino,Malay": 0.5113636363636364, - "Filipino,Chinese": 0.39204545454545453, - "Filipino,Indonesian": 0.5056818181818182, - "Filipino,Vietnamese": 0.4147727272727273, - "Spanish,Malay": 0.4375, - "Spanish,Chinese": 0.4659090909090909, - "Spanish,Indonesian": 0.5056818181818182, - "Spanish,Vietnamese": 0.38636363636363635, - "Malay,Chinese": 0.3693181818181818, - "Malay,Indonesian": 0.5852272727272727, - "Malay,Vietnamese": 0.4715909090909091, - "Chinese,Indonesian": 0.3806818181818182, - "Chinese,Vietnamese": 0.375, - "Indonesian,Vietnamese": 0.3977272727272727 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Malay": 0.25, - "English,Filipino,Chinese": 0.2784090909090909, - "English,Filipino,Indonesian": 0.2727272727272727, - "English,Filipino,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay": 0.26704545454545453, - "English,Spanish,Chinese": 0.3693181818181818, - "English,Spanish,Indonesian": 0.32386363636363635, - "English,Spanish,Vietnamese": 0.23295454545454544, - "English,Malay,Chinese": 0.26136363636363635, - "English,Malay,Indonesian": 0.3125, - "English,Malay,Vietnamese": 0.19886363636363635, - "English,Chinese,Indonesian": 0.30113636363636365, - "English,Chinese,Vietnamese": 0.26704545454545453, - "English,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Malay": 0.29545454545454547, - "Filipino,Spanish,Chinese": 0.23295454545454544, - "Filipino,Spanish,Indonesian": 0.32954545454545453, - "Filipino,Spanish,Vietnamese": 0.24431818181818182, - "Filipino,Malay,Chinese": 0.22727272727272727, - "Filipino,Malay,Indonesian": 0.3693181818181818, - "Filipino,Malay,Vietnamese": 0.26704545454545453, - "Filipino,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Indonesian,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Chinese": 0.2215909090909091, - "Spanish,Malay,Indonesian": 0.3181818181818182, - "Spanish,Malay,Vietnamese": 0.23863636363636365, - "Spanish,Chinese,Indonesian": 0.2556818181818182, - "Spanish,Chinese,Vietnamese": 0.21022727272727273, - "Spanish,Indonesian,Vietnamese": 0.23295454545454544, - "Malay,Chinese,Indonesian": 0.2556818181818182, - "Malay,Chinese,Vietnamese": 0.19886363636363635, - "Malay,Indonesian,Vietnamese": 0.30113636363636365, - "Chinese,Indonesian,Vietnamese": 0.19318181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.1875, - "English,Filipino,Spanish,Chinese": 0.1875, - "English,Filipino,Spanish,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Chinese": 0.16477272727272727, - "English,Filipino,Malay,Indonesian": 0.2215909090909091, - "English,Filipino,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Indonesian": 0.17045454545454544, - "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese": 0.18181818181818182, - "English,Spanish,Malay,Indonesian": 0.23863636363636365, - "English,Spanish,Malay,Vietnamese": 0.14204545454545456, - "English,Spanish,Chinese,Indonesian": 0.2215909090909091, - "English,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "English,Malay,Chinese,Indonesian": 0.21022727272727273, - "English,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Malay,Chinese": 0.1534090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.25, - "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "Filipino,Malay,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Chinese,Indonesian": 0.18181818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "English,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - } - }, - "AC3_2": 0.4169106796514511, - "AC3_3": 0.31259221422952843, - "AC3_4": 0.23934717734647282, - "AC3_5": 0.18824747242754825, - "AC3_6": 0.1536959689908847, - "AC3_7": 0.13207026939042357 - }, - "prompt_2": { - "overall_acc": 0.3896103896103895, - "language_acc": { - "English": 0.5170454545454546, - "Filipino": 0.35795454545454547, - "Spanish": 0.3806818181818182, - "Malay": 0.3409090909090909, - "Chinese": 0.5170454545454546, - "Indonesian": 0.3352272727272727, - "Vietnamese": 0.2784090909090909 - }, - "consistency_score_2": 0.4548160173160174, - "consistency_score_3": 0.2646103896103896, - "consistency_score_4": 0.1748376623376623, - "consistency_score_5": 0.12445887445887446, - "consistency_score_6": 0.09172077922077922, - "consistency_score_7": 0.06818181818181818, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4090909090909091, - "English,Spanish": 0.5568181818181818, - "English,Malay": 0.4090909090909091, - "English,Chinese": 0.6363636363636364, - "English,Indonesian": 0.3806818181818182, - "English,Vietnamese": 0.3693181818181818, - "Filipino,Spanish": 0.4431818181818182, - "Filipino,Malay": 0.5, - "Filipino,Chinese": 0.3977272727272727, - "Filipino,Indonesian": 0.5454545454545454, - "Filipino,Vietnamese": 0.4318181818181818, - "Spanish,Malay": 0.4772727272727273, - "Spanish,Chinese": 0.5340909090909091, - "Spanish,Indonesian": 0.5, - "Spanish,Vietnamese": 0.4034090909090909, - "Malay,Chinese": 0.375, - "Malay,Indonesian": 0.5625, - "Malay,Vietnamese": 0.45454545454545453, - "Chinese,Indonesian": 0.38636363636363635, - "Chinese,Vietnamese": 0.375, - "Indonesian,Vietnamese": 0.4034090909090909 - }, - "3_combine": { - "English,Filipino,Spanish": 0.29545454545454547, - "English,Filipino,Malay": 0.26704545454545453, - "English,Filipino,Chinese": 0.2840909090909091, - "English,Filipino,Indonesian": 0.2727272727272727, - "English,Filipino,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay": 0.30113636363636365, - "English,Spanish,Chinese": 0.4090909090909091, - "English,Spanish,Indonesian": 0.2897727272727273, - "English,Spanish,Vietnamese": 0.23863636363636365, - "English,Malay,Chinese": 0.26704545454545453, - "English,Malay,Indonesian": 0.2727272727272727, - "English,Malay,Vietnamese": 0.19886363636363635, - "English,Chinese,Indonesian": 0.2784090909090909, - "English,Chinese,Vietnamese": 0.25, - "English,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Malay": 0.3068181818181818, - "Filipino,Spanish,Chinese": 0.2556818181818182, - "Filipino,Spanish,Indonesian": 0.32954545454545453, - "Filipino,Spanish,Vietnamese": 0.24431818181818182, - "Filipino,Malay,Chinese": 0.23863636363636365, - "Filipino,Malay,Indonesian": 0.3693181818181818, - "Filipino,Malay,Vietnamese": 0.2727272727272727, - "Filipino,Chinese,Indonesian": 0.26704545454545453, - "Filipino,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Chinese": 0.26704545454545453, - "Spanish,Malay,Indonesian": 0.3352272727272727, - "Spanish,Malay,Vietnamese": 0.25, - "Spanish,Chinese,Indonesian": 0.2784090909090909, - "Spanish,Chinese,Vietnamese": 0.23295454545454544, - "Spanish,Indonesian,Vietnamese": 0.23295454545454544, - "Malay,Chinese,Indonesian": 0.24431818181818182, - "Malay,Chinese,Vietnamese": 0.19886363636363635, - "Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Chinese,Indonesian,Vietnamese": 0.18181818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.2159090909090909, - "English,Filipino,Spanish,Chinese": 0.19886363636363635, - "English,Filipino,Spanish,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese": 0.17613636363636365, - "English,Filipino,Malay,Indonesian": 0.2159090909090909, - "English,Filipino,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian": 0.1875, - "English,Filipino,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese": 0.2215909090909091, - "English,Spanish,Malay,Indonesian": 0.22727272727272727, - "English,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Spanish,Chinese,Indonesian": 0.2215909090909091, - "English,Spanish,Chinese,Vietnamese": 0.17045454545454544, - "English,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "English,Malay,Chinese,Indonesian": 0.19318181818181818, - "English,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese": 0.17613636363636365, - "Filipino,Spanish,Malay,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.1875, - "Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.1875, - "Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "English,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - } - }, - "AC3_2": 0.41969565194329067, - "AC3_3": 0.31516870221730175, - "AC3_4": 0.24136346824046565, - "AC3_5": 0.18865345177464848, - "AC3_6": 0.14848557847159297, - "AC3_7": 0.11605415858199924 - }, - "prompt_3": { - "overall_acc": 0.4034090909090909, - "language_acc": { - "English": 0.5056818181818182, - "Filipino": 0.39204545454545453, - "Spanish": 0.42613636363636365, - "Malay": 0.30113636363636365, - "Chinese": 0.5454545454545454, - "Indonesian": 0.32954545454545453, - "Vietnamese": 0.32386363636363635 - }, - "consistency_score_2": 0.45752164502164516, - "consistency_score_3": 0.2720779220779221, - "consistency_score_4": 0.1862012987012987, - "consistency_score_5": 0.1377164502164502, - "consistency_score_6": 0.10633116883116882, - "consistency_score_7": 0.08522727272727272, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4375, - "English,Spanish": 0.5454545454545454, - "English,Malay": 0.4034090909090909, - "English,Chinese": 0.625, - "English,Indonesian": 0.4090909090909091, - "English,Vietnamese": 0.3409090909090909, - "Filipino,Spanish": 0.4375, - "Filipino,Malay": 0.5170454545454546, - "Filipino,Chinese": 0.4431818181818182, - "Filipino,Indonesian": 0.4943181818181818, - "Filipino,Vietnamese": 0.48295454545454547, - "Spanish,Malay": 0.42045454545454547, - "Spanish,Chinese": 0.5397727272727273, - "Spanish,Indonesian": 0.4715909090909091, - "Spanish,Vietnamese": 0.4147727272727273, - "Malay,Chinese": 0.3693181818181818, - "Malay,Indonesian": 0.5738636363636364, - "Malay,Vietnamese": 0.4602272727272727, - "Chinese,Indonesian": 0.39204545454545453, - "Chinese,Vietnamese": 0.3977272727272727, - "Indonesian,Vietnamese": 0.4318181818181818 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Malay": 0.26704545454545453, - "English,Filipino,Chinese": 0.3125, - "English,Filipino,Indonesian": 0.2784090909090909, - "English,Filipino,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay": 0.2727272727272727, - "English,Spanish,Chinese": 0.4090909090909091, - "English,Spanish,Indonesian": 0.30113636363636365, - "English,Spanish,Vietnamese": 0.23863636363636365, - "English,Malay,Chinese": 0.26136363636363635, - "English,Malay,Indonesian": 0.2840909090909091, - "English,Malay,Vietnamese": 0.20454545454545456, - "English,Chinese,Indonesian": 0.2840909090909091, - "English,Chinese,Vietnamese": 0.2556818181818182, - "English,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay": 0.2784090909090909, - "Filipino,Spanish,Chinese": 0.2840909090909091, - "Filipino,Spanish,Indonesian": 0.3125, - "Filipino,Spanish,Vietnamese": 0.26136363636363635, - "Filipino,Malay,Chinese": 0.26704545454545453, - "Filipino,Malay,Indonesian": 0.3522727272727273, - "Filipino,Malay,Vietnamese": 0.30113636363636365, - "Filipino,Chinese,Indonesian": 0.2784090909090909, - "Filipino,Chinese,Vietnamese": 0.26704545454545453, - "Filipino,Indonesian,Vietnamese": 0.2897727272727273, - "Spanish,Malay,Chinese": 0.23863636363636365, - "Spanish,Malay,Indonesian": 0.29545454545454547, - "Spanish,Malay,Vietnamese": 0.2556818181818182, - "Spanish,Chinese,Indonesian": 0.2727272727272727, - "Spanish,Chinese,Vietnamese": 0.26136363636363635, - "Spanish,Indonesian,Vietnamese": 0.25, - "Malay,Chinese,Indonesian": 0.25, - "Malay,Chinese,Vietnamese": 0.23295454545454544, - "Malay,Indonesian,Vietnamese": 0.2897727272727273, - "Chinese,Indonesian,Vietnamese": 0.2159090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.21022727272727273, - "English,Filipino,Spanish,Chinese": 0.2159090909090909, - "English,Filipino,Spanish,Indonesian": 0.2215909090909091, - "English,Filipino,Spanish,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese": 0.19886363636363635, - "English,Filipino,Malay,Indonesian": 0.2215909090909091, - "English,Filipino,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian": 0.20454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.17045454545454544, - "English,Filipino,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese": 0.19318181818181818, - "English,Spanish,Malay,Indonesian": 0.2215909090909091, - "English,Spanish,Malay,Vietnamese": 0.16477272727272727, - "English,Spanish,Chinese,Indonesian": 0.2159090909090909, - "English,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "English,Malay,Chinese,Indonesian": 0.19886363636363635, - "English,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese": 0.1875, - "Filipino,Spanish,Malay,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.21022727272727273, - "Filipino,Spanish,Chinese,Vietnamese": 0.1875, - "Filipino,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.2159090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Chinese,Indonesian": 0.18181818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.18181818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese,Indonesian": 0.17045454545454544, - "English,Filipino,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.125, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.125, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 - } - }, - "AC3_2": 0.42876478481963354, - "AC3_3": 0.32497651346481304, - "AC3_4": 0.25479638061357374, - "AC3_5": 0.20533522723478131, - "AC3_6": 0.16830124490039347, - "AC3_7": 0.14072410145111608 - }, - "prompt_4": { - "overall_acc": 0.3758116883116883, - "language_acc": { - "English": 0.5056818181818182, - "Filipino": 0.3465909090909091, - "Spanish": 0.38636363636363635, - "Malay": 0.2784090909090909, - "Chinese": 0.5, - "Indonesian": 0.3181818181818182, - "Vietnamese": 0.29545454545454547 - }, - "consistency_score_2": 0.44453463203463217, - "consistency_score_3": 0.25681818181818183, - "consistency_score_4": 0.17386363636363636, - "consistency_score_5": 0.13041125541125542, - "consistency_score_6": 0.1055194805194805, - "consistency_score_7": 0.09090909090909091, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3693181818181818, - "English,Spanish": 0.5227272727272727, - "English,Malay": 0.4034090909090909, - "English,Chinese": 0.6079545454545454, - "English,Indonesian": 0.3693181818181818, - "English,Vietnamese": 0.3522727272727273, - "Filipino,Spanish": 0.4772727272727273, - "Filipino,Malay": 0.5, - "Filipino,Chinese": 0.375, - "Filipino,Indonesian": 0.5113636363636364, - "Filipino,Vietnamese": 0.48295454545454547, - "Spanish,Malay": 0.42045454545454547, - "Spanish,Chinese": 0.4772727272727273, - "Spanish,Indonesian": 0.4772727272727273, - "Spanish,Vietnamese": 0.42613636363636365, - "Malay,Chinese": 0.4034090909090909, - "Malay,Indonesian": 0.5340909090909091, - "Malay,Vietnamese": 0.44886363636363635, - "Chinese,Indonesian": 0.4034090909090909, - "Chinese,Vietnamese": 0.32954545454545453, - "Indonesian,Vietnamese": 0.4431818181818182 - }, - "3_combine": { - "English,Filipino,Spanish": 0.26136363636363635, - "English,Filipino,Malay": 0.23863636363636365, - "English,Filipino,Chinese": 0.2556818181818182, - "English,Filipino,Indonesian": 0.22727272727272727, - "English,Filipino,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay": 0.26136363636363635, - "English,Spanish,Chinese": 0.36363636363636365, - "English,Spanish,Indonesian": 0.26704545454545453, - "English,Spanish,Vietnamese": 0.26704545454545453, - "English,Malay,Chinese": 0.26704545454545453, - "English,Malay,Indonesian": 0.25, - "English,Malay,Vietnamese": 0.2159090909090909, - "English,Chinese,Indonesian": 0.26136363636363635, - "English,Chinese,Vietnamese": 0.2215909090909091, - "English,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.24431818181818182, - "Filipino,Spanish,Indonesian": 0.32954545454545453, - "Filipino,Spanish,Vietnamese": 0.2840909090909091, - "Filipino,Malay,Chinese": 0.23295454545454544, - "Filipino,Malay,Indonesian": 0.3181818181818182, - "Filipino,Malay,Vietnamese": 0.29545454545454547, - "Filipino,Chinese,Indonesian": 0.23863636363636365, - "Filipino,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Indonesian,Vietnamese": 0.2897727272727273, - "Spanish,Malay,Chinese": 0.22727272727272727, - "Spanish,Malay,Indonesian": 0.2840909090909091, - "Spanish,Malay,Vietnamese": 0.2556818181818182, - "Spanish,Chinese,Indonesian": 0.26704545454545453, - "Spanish,Chinese,Vietnamese": 0.23863636363636365, - "Spanish,Indonesian,Vietnamese": 0.29545454545454547, - "Malay,Chinese,Indonesian": 0.26136363636363635, - "Malay,Chinese,Vietnamese": 0.19318181818181818, - "Malay,Indonesian,Vietnamese": 0.2727272727272727, - "Chinese,Indonesian,Vietnamese": 0.19318181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.1875, - "English,Filipino,Spanish,Chinese": 0.17613636363636365, - "English,Filipino,Spanish,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, - "English,Filipino,Malay,Chinese": 0.17045454545454544, - "English,Filipino,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Malay,Vietnamese": 0.16477272727272727, - "English,Filipino,Chinese,Indonesian": 0.17045454545454544, - "English,Filipino,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese": 0.18181818181818182, - "English,Spanish,Malay,Indonesian": 0.19318181818181818, - "English,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Spanish,Chinese,Indonesian": 0.19318181818181818, - "English,Spanish,Chinese,Vietnamese": 0.18181818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Malay,Chinese,Indonesian": 0.1875, - "English,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Chinese": 0.1590909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.21022727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.1875, - "Filipino,Spanish,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.17613636363636365, - "Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Spanish,Malay,Chinese,Vietnamese": 0.125, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - } - }, - "AC3_2": 0.40729459354936404, - "AC3_3": 0.30512398810116126, - "AC3_4": 0.23774029421653528, - "AC3_5": 0.19363039415334138, - "AC3_6": 0.1647740960214819, - "AC3_7": 0.1464031620239675 - }, - "prompt_5": { - "overall_acc": 0.387987012987013, - "language_acc": { - "English": 0.5227272727272727, - "Filipino": 0.3465909090909091, - "Spanish": 0.4090909090909091, - "Malay": 0.32954545454545453, - "Chinese": 0.5340909090909091, - "Indonesian": 0.30113636363636365, - "Vietnamese": 0.2727272727272727 - }, - "consistency_score_2": 0.4429112554112553, - "consistency_score_3": 0.25340909090909086, - "consistency_score_4": 0.17012987012987016, - "consistency_score_5": 0.12824675324675322, - "consistency_score_6": 0.10470779220779221, - "consistency_score_7": 0.09090909090909091, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4034090909090909, - "English,Spanish": 0.5511363636363636, - "English,Malay": 0.4147727272727273, - "English,Chinese": 0.5965909090909091, - "English,Indonesian": 0.4090909090909091, - "English,Vietnamese": 0.3409090909090909, - "Filipino,Spanish": 0.4375, - "Filipino,Malay": 0.5681818181818182, - "Filipino,Chinese": 0.4090909090909091, - "Filipino,Indonesian": 0.48295454545454547, - "Filipino,Vietnamese": 0.42613636363636365, - "Spanish,Malay": 0.4147727272727273, - "Spanish,Chinese": 0.5340909090909091, - "Spanish,Indonesian": 0.42613636363636365, - "Spanish,Vietnamese": 0.3693181818181818, - "Malay,Chinese": 0.3977272727272727, - "Malay,Indonesian": 0.5454545454545454, - "Malay,Vietnamese": 0.4772727272727273, - "Chinese,Indonesian": 0.36363636363636365, - "Chinese,Vietnamese": 0.32386363636363635, - "Indonesian,Vietnamese": 0.4090909090909091 - }, - "3_combine": { - "English,Filipino,Spanish": 0.29545454545454547, - "English,Filipino,Malay": 0.2784090909090909, - "English,Filipino,Chinese": 0.2727272727272727, - "English,Filipino,Indonesian": 0.2556818181818182, - "English,Filipino,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay": 0.2556818181818182, - "English,Spanish,Chinese": 0.38636363636363635, - "English,Spanish,Indonesian": 0.26704545454545453, - "English,Spanish,Vietnamese": 0.2215909090909091, - "English,Malay,Chinese": 0.26704545454545453, - "English,Malay,Indonesian": 0.2897727272727273, - "English,Malay,Vietnamese": 0.21022727272727273, - "English,Chinese,Indonesian": 0.25, - "English,Chinese,Vietnamese": 0.21022727272727273, - "English,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.2784090909090909, - "Filipino,Spanish,Indonesian": 0.2784090909090909, - "Filipino,Spanish,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese": 0.2727272727272727, - "Filipino,Malay,Indonesian": 0.3522727272727273, - "Filipino,Malay,Vietnamese": 0.3068181818181818, - "Filipino,Chinese,Indonesian": 0.24431818181818182, - "Filipino,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Indonesian,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Chinese": 0.2556818181818182, - "Spanish,Malay,Indonesian": 0.2840909090909091, - "Spanish,Malay,Vietnamese": 0.23863636363636365, - "Spanish,Chinese,Indonesian": 0.2556818181818182, - "Spanish,Chinese,Vietnamese": 0.21022727272727273, - "Spanish,Indonesian,Vietnamese": 0.2159090909090909, - "Malay,Chinese,Indonesian": 0.2556818181818182, - "Malay,Chinese,Vietnamese": 0.19886363636363635, - "Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Chinese,Indonesian,Vietnamese": 0.16477272727272727 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.20454545454545456, - "English,Filipino,Spanish,Chinese": 0.21022727272727273, - "English,Filipino,Spanish,Indonesian": 0.19886363636363635, - "English,Filipino,Spanish,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Chinese": 0.19886363636363635, - "English,Filipino,Malay,Indonesian": 0.2215909090909091, - "English,Filipino,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Chinese,Indonesian": 0.17613636363636365, - "English,Filipino,Chinese,Vietnamese": 0.125, - "English,Filipino,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese": 0.1875, - "English,Spanish,Malay,Indonesian": 0.19318181818181818, - "English,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Spanish,Chinese,Indonesian": 0.19318181818181818, - "English,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "English,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "English,Malay,Chinese,Indonesian": 0.1875, - "English,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese": 0.19318181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Indonesian": 0.1875, - "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese,Indonesian": 0.2159090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.1875, - "Spanish,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.125, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - } - }, - "AC3_2": 0.4136338262346471, - "AC3_3": 0.3065794619986336, - "AC3_4": 0.2365389118892005, - "AC3_5": 0.19277342150966414, - "AC3_6": 0.1649104601817667, - "AC3_7": 0.1473035438829544 + "AC3_2": 0.26888820633846605, + "AC3_3": 0.1367419656217994, + "AC3_4": 0.05774702099984339, + "AC3_5": 0.022236912563491164, + "AC3_6": 0.007987633382588557, + "AC3_7": 0.0 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.5436893203883495 - }, - "prompt_2": { - "accuracy": 0.5533980582524272 - }, - "prompt_3": { - "accuracy": 0.5631067961165048 - }, - "prompt_4": { - "accuracy": 0.49514563106796117 - }, - "prompt_5": { - "accuracy": 0.5728155339805825 + "accuracy": 0.17475728155339806 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.5047619047619047 - }, - "prompt_2": { - "accuracy": 0.5523809523809524 - }, - "prompt_3": { - "accuracy": 0.5714285714285714 - }, - "prompt_4": { - "accuracy": 0.49523809523809526 - }, - "prompt_5": { - "accuracy": 0.5619047619047619 + "accuracy": 0.24761904761904763 } }, "us_eval": { "prompt_1": { - "accuracy": 0.5420560747663551 - }, - "prompt_2": { - "accuracy": 0.5233644859813084 - }, - "prompt_3": { - "accuracy": 0.514018691588785 - }, - "prompt_4": { - "accuracy": 0.5233644859813084 - }, - "prompt_5": { - "accuracy": 0.5233644859813084 + "accuracy": 0.2803738317757009 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.38, + "accuracy": 0.29, "category_acc": { - "brand": 0.5, + "brand": 0.2, "demographics": 0.4, - "biology": 0.3, + "biology": 0.2, "history": 0.5333333333333333, - "literature": 0.1, - "politics": 0.5, - "culture": 0.2, - "film": 0.3, - "law": 0.4, - "geography": 0.5 - } - }, - "prompt_2": { - "accuracy": 0.37, - "category_acc": { - "brand": 0.8, - "demographics": 0.0, - "biology": 0.3, - "history": 0.4666666666666667, - "literature": 0.1, - "politics": 0.3, - "culture": 0.2, - "film": 0.3, - "law": 0.5, - "geography": 0.5 - } - }, - "prompt_3": { - "accuracy": 0.33, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.3, - "history": 0.4, - "literature": 0.1, - "politics": 0.5, - "culture": 0.2, - "film": 0.2, - "law": 0.5, - "geography": 0.5 - } - }, - "prompt_4": { - "accuracy": 0.36, - "category_acc": { - "brand": 0.5, - "demographics": 0.4, - "biology": 0.3, - "history": 0.4, - "literature": 0.1, + "literature": 0.2, "politics": 0.5, "culture": 0.2, - "film": 0.3, - "law": 0.4, - "geography": 0.5 - } - }, - "prompt_5": { - "accuracy": 0.34, - "category_acc": { - "brand": 0.4, - "demographics": 0.0, - "biology": 0.3, - "history": 0.4, - "literature": 0.1, - "politics": 0.4, - "culture": 0.3, "film": 0.2, - "law": 0.5, - "geography": 0.6 + "law": 0.2, + "geography": 0.2 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.222894816768643 - }, - "prompt_2": { - "bleu_score": 0.2364511999378093 - }, - "prompt_3": { - "bleu_score": 0.2369226620931073 - }, - "prompt_4": { - "bleu_score": 0.22472289303028134 - }, - "prompt_5": { - "bleu_score": 0.18194199378880513 + "bleu_score": 0.11531135447491982 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "prompt_1": { + "accuracy": 0.23452833967554576, + "category_acc": { + "History": 0.19076305220883535, + "Geography": 0.21836734693877552, + "Lampungic": 0.2585034013605442, + "Social science": 0.25375626043405675, + "Balinese": 0.27176220806794055, + "Makassarese": 0.2903225806451613, + "Banjarese": 0.25, + "Chemistry": 0.2029197080291971, + "Biology": 0.20946745562130178, + "Science": 0.24561403508771928, + "Christian religion": 0.23880597014925373, + "Art": 0.2529118136439268, + "Islam religion": 0.23897581792318634, + "Hindu religion": 0.30666666666666664, + "Madurese": 0.2576271186440678, + "Sport": 0.17567567567567569, + "Indonesian language": 0.23225404732254049, + "Physics": 0.2222222222222222, + "Minangkabau culture": 0.2964824120603015, + "Dayak language": 0.22018348623853212, + "Sociology": 0.20161290322580644, + "Economy": 0.18442622950819673, + "Sundanese": 0.2549697493517718, + "Javanese": 0.25, + "Civic education": 0.23032904148783978 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.17768539550917986 - }, - "prompt_2": { - "bleu_score": 0.1754984806787498 - }, - "prompt_3": { - "bleu_score": 0.17216677837545769 - }, - "prompt_4": { - "bleu_score": 0.17111209487764534 - }, - "prompt_5": { - "bleu_score": 0.14720558888059931 + "bleu_score": 0.12903915185649562 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.06226873002502311 - }, - "prompt_2": { - "bleu_score": 0.06082405196483164 - }, - "prompt_3": { - "bleu_score": 0.06291051819127058 - }, - "prompt_4": { - "bleu_score": 0.05989807662961165 - }, - "prompt_5": { - "bleu_score": 0.06018465302671112 + "bleu_score": 0.12569739315639059 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.19873474182968318 - }, - "prompt_2": { - "bleu_score": 0.2008845653027542 - }, - "prompt_3": { - "bleu_score": 0.19867624577147552 - }, - "prompt_4": { - "bleu_score": 0.20084160441517682 - }, - "prompt_5": { - "bleu_score": 0.1978802328830003 + "bleu_score": 0.039003259240151175 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.13897959904603993 - }, - "prompt_2": { - "bleu_score": 0.1424665035266369 - }, - "prompt_3": { - "bleu_score": 0.14267640052669625 - }, - "prompt_4": { - "bleu_score": 0.1345614236508451 - }, - "prompt_5": { - "bleu_score": 0.13018872463189823 + "bleu_score": 0.13061525238014365 } }, "mmlu": { "prompt_1": { - "accuracy": 0.47724620770128356 - }, - "prompt_2": { - "accuracy": 0.49941656942823803 - }, - "prompt_3": { - "accuracy": 0.48191365227537925 - }, - "prompt_4": { - "accuracy": 0.4807467911318553 - }, - "prompt_5": { - "accuracy": 0.48191365227537925 + "accuracy": 0.2660443407234539 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.4811583839828388, + "accuracy": 0.24883804075795496, "category_acc": { - "high_school_european_history": 0.6463414634146342, - "business_ethics": 0.6464646464646465, - "clinical_knowledge": 0.5075757575757576, - "medical_genetics": 0.5454545454545454, - "high_school_us_history": 0.6403940886699507, - "high_school_physics": 0.34, - "high_school_world_history": 0.6610169491525424, - "virology": 0.43636363636363634, - "high_school_microeconomics": 0.48523206751054854, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.36363636363636365, - "high_school_biology": 0.5631067961165048, - "abstract_algebra": 0.36363636363636365, - "professional_accounting": 0.35587188612099646, - "philosophy": 0.4645161290322581, - "professional_medicine": 0.44649446494464945, - "nutrition": 0.5311475409836065, + "high_school_european_history": 0.2621951219512195, + "business_ethics": 0.29292929292929293, + "clinical_knowledge": 0.21212121212121213, + "medical_genetics": 0.23232323232323232, + "high_school_us_history": 0.28078817733990147, + "high_school_physics": 0.24666666666666667, + "high_school_world_history": 0.2457627118644068, + "virology": 0.24848484848484848, + "high_school_microeconomics": 0.25316455696202533, + "econometrics": 0.20353982300884957, + "college_computer_science": 0.29292929292929293, + "high_school_biology": 0.2297734627831715, + "abstract_algebra": 0.23232323232323232, + "professional_accounting": 0.2846975088967972, + "philosophy": 0.25483870967741934, + "professional_medicine": 0.22509225092250923, + "nutrition": 0.25245901639344265, "global_facts": 0.2727272727272727, - "machine_learning": 0.3963963963963964, - "security_studies": 0.5, - "public_relations": 0.5137614678899083, - "professional_psychology": 0.44844517184942717, - "prehistory": 0.48297213622291024, - "anatomy": 0.4552238805970149, - "human_sexuality": 0.5538461538461539, - "college_medicine": 0.4011627906976744, - "high_school_government_and_politics": 0.6979166666666666, - "college_chemistry": 0.29292929292929293, - "logical_fallacies": 0.5555555555555556, - "high_school_geography": 0.583756345177665, - "elementary_mathematics": 0.493368700265252, - "human_aging": 0.5315315315315315, - "college_mathematics": 0.25252525252525254, - "high_school_psychology": 0.6746323529411765, - "formal_logic": 0.256, - "high_school_statistics": 0.3813953488372093, - "international_law": 0.6083333333333333, - "high_school_mathematics": 0.35687732342007433, - "high_school_computer_science": 0.48484848484848486, - "conceptual_physics": 0.4444444444444444, - "miscellaneous": 0.6764705882352942, - "high_school_chemistry": 0.43564356435643564, - "marketing": 0.6351931330472103, - "professional_law": 0.3744292237442922, - "management": 0.6764705882352942, - "college_physics": 0.31683168316831684, - "jurisprudence": 0.5327102803738317, - "world_religions": 0.6470588235294118, - "sociology": 0.665, - "us_foreign_policy": 0.6767676767676768, - "high_school_macroeconomics": 0.46786632390745503, - "computer_security": 0.6363636363636364, - "moral_scenarios": 0.24272930648769575, - "moral_disputes": 0.5362318840579711, - "electrical_engineering": 0.4583333333333333, - "astronomy": 0.5165562913907285, - "college_biology": 0.46153846153846156 - } - }, - "prompt_2": { - "accuracy": 0.4967465141222739, - "category_acc": { - "high_school_european_history": 0.6585365853658537, - "business_ethics": 0.5757575757575758, - "clinical_knowledge": 0.5151515151515151, - "medical_genetics": 0.5151515151515151, - "high_school_us_history": 0.6699507389162561, - "high_school_physics": 0.3, - "high_school_world_history": 0.7203389830508474, - "virology": 0.42424242424242425, - "high_school_microeconomics": 0.5063291139240507, - "econometrics": 0.35398230088495575, - "college_computer_science": 0.3434343434343434, - "high_school_biology": 0.5825242718446602, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.3309608540925267, - "philosophy": 0.535483870967742, - "professional_medicine": 0.41697416974169743, - "nutrition": 0.5475409836065573, - "global_facts": 0.2828282828282828, - "machine_learning": 0.36036036036036034, - "security_studies": 0.5368852459016393, - "public_relations": 0.5321100917431193, - "professional_psychology": 0.4762684124386252, - "prehistory": 0.5263157894736842, - "anatomy": 0.4552238805970149, - "human_sexuality": 0.5615384615384615, - "college_medicine": 0.4883720930232558, - "high_school_government_and_politics": 0.7291666666666666, - "college_chemistry": 0.40404040404040403, - "logical_fallacies": 0.6172839506172839, - "high_school_geography": 0.6395939086294417, - "elementary_mathematics": 0.44562334217506633, - "human_aging": 0.5540540540540541, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.6838235294117647, - "formal_logic": 0.232, - "high_school_statistics": 0.4186046511627907, - "international_law": 0.6166666666666667, - "high_school_mathematics": 0.3680297397769517, - "high_school_computer_science": 0.47474747474747475, - "conceptual_physics": 0.4230769230769231, - "miscellaneous": 0.6982097186700768, - "high_school_chemistry": 0.4603960396039604, - "marketing": 0.7339055793991416, - "professional_law": 0.3926940639269406, - "management": 0.6862745098039216, - "college_physics": 0.31683168316831684, - "jurisprudence": 0.5794392523364486, - "world_religions": 0.6352941176470588, - "sociology": 0.695, - "us_foreign_policy": 0.7070707070707071, - "high_school_macroeconomics": 0.4961439588688946, - "computer_security": 0.6363636363636364, - "moral_scenarios": 0.23937360178970918, - "moral_disputes": 0.5681159420289855, - "electrical_engineering": 0.4513888888888889, - "astronomy": 0.4900662251655629, - "college_biology": 0.5034965034965035 - } - }, - "prompt_3": { - "accuracy": 0.4907400786557025, - "category_acc": { - "high_school_european_history": 0.6585365853658537, - "business_ethics": 0.5555555555555556, - "clinical_knowledge": 0.5416666666666666, - "medical_genetics": 0.5050505050505051, - "high_school_us_history": 0.6403940886699507, - "high_school_physics": 0.3333333333333333, - "high_school_world_history": 0.6694915254237288, - "virology": 0.45454545454545453, - "high_school_microeconomics": 0.5316455696202531, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.3333333333333333, - "high_school_biology": 0.6051779935275081, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.3416370106761566, - "philosophy": 0.5225806451612903, - "professional_medicine": 0.42435424354243545, - "nutrition": 0.5540983606557377, - "global_facts": 0.30303030303030304, - "machine_learning": 0.35135135135135137, - "security_studies": 0.5286885245901639, - "public_relations": 0.48623853211009177, - "professional_psychology": 0.4746317512274959, - "prehistory": 0.541795665634675, - "anatomy": 0.47761194029850745, - "human_sexuality": 0.5538461538461539, - "college_medicine": 0.47674418604651164, - "high_school_government_and_politics": 0.6979166666666666, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.6111111111111112, - "high_school_geography": 0.649746192893401, - "elementary_mathematics": 0.4350132625994695, - "human_aging": 0.5765765765765766, - "college_mathematics": 0.2727272727272727, - "high_school_psychology": 0.6875, - "formal_logic": 0.304, - "high_school_statistics": 0.39069767441860465, - "international_law": 0.5916666666666667, - "high_school_mathematics": 0.3420074349442379, - "high_school_computer_science": 0.46464646464646464, - "conceptual_physics": 0.44017094017094016, - "miscellaneous": 0.6764705882352942, - "high_school_chemistry": 0.4158415841584158, - "marketing": 0.7167381974248928, - "professional_law": 0.3822570123939987, - "management": 0.6862745098039216, - "college_physics": 0.3564356435643564, - "jurisprudence": 0.5794392523364486, - "world_religions": 0.6470588235294118, - "sociology": 0.695, - "us_foreign_policy": 0.7171717171717171, - "high_school_macroeconomics": 0.48586118251928023, - "computer_security": 0.6060606060606061, - "moral_scenarios": 0.2058165548098434, - "moral_disputes": 0.5449275362318841, - "electrical_engineering": 0.4583333333333333, - "astronomy": 0.5099337748344371, - "college_biology": 0.5104895104895105 - } - }, - "prompt_4": { - "accuracy": 0.49031104755094745, - "category_acc": { - "high_school_european_history": 0.6524390243902439, - "business_ethics": 0.6262626262626263, - "clinical_knowledge": 0.48863636363636365, - "medical_genetics": 0.46464646464646464, - "high_school_us_history": 0.6650246305418719, - "high_school_physics": 0.3333333333333333, - "high_school_world_history": 0.6949152542372882, - "virology": 0.45454545454545453, - "high_school_microeconomics": 0.5485232067510548, - "econometrics": 0.34513274336283184, - "college_computer_science": 0.36363636363636365, - "high_school_biology": 0.6213592233009708, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.3309608540925267, - "philosophy": 0.4838709677419355, - "professional_medicine": 0.42066420664206644, - "nutrition": 0.5081967213114754, - "global_facts": 0.30303030303030304, - "machine_learning": 0.3333333333333333, - "security_studies": 0.5040983606557377, - "public_relations": 0.5229357798165137, - "professional_psychology": 0.45990180032733224, - "prehistory": 0.5046439628482973, - "anatomy": 0.4626865671641791, - "human_sexuality": 0.5384615384615384, - "college_medicine": 0.47674418604651164, - "high_school_government_and_politics": 0.7135416666666666, - "college_chemistry": 0.32323232323232326, - "logical_fallacies": 0.6049382716049383, - "high_school_geography": 0.6192893401015228, - "elementary_mathematics": 0.5225464190981433, - "human_aging": 0.5630630630630631, - "college_mathematics": 0.32323232323232326, - "high_school_psychology": 0.6856617647058824, - "formal_logic": 0.352, - "high_school_statistics": 0.4372093023255814, - "international_law": 0.6083333333333333, - "high_school_mathematics": 0.3643122676579926, - "high_school_computer_science": 0.494949494949495, - "conceptual_physics": 0.42735042735042733, - "miscellaneous": 0.6879795396419437, - "high_school_chemistry": 0.4405940594059406, - "marketing": 0.7253218884120172, - "professional_law": 0.3835616438356164, - "management": 0.6862745098039216, - "college_physics": 0.25742574257425743, - "jurisprudence": 0.4953271028037383, - "world_religions": 0.6705882352941176, - "sociology": 0.655, - "us_foreign_policy": 0.7272727272727273, - "high_school_macroeconomics": 0.5012853470437018, - "computer_security": 0.5959595959595959, - "moral_scenarios": 0.2181208053691275, - "moral_disputes": 0.4956521739130435, - "electrical_engineering": 0.4166666666666667, - "astronomy": 0.5033112582781457, - "college_biology": 0.45454545454545453 - } - }, - "prompt_5": { - "accuracy": 0.49410082230961744, - "category_acc": { - "high_school_european_history": 0.6707317073170732, - "business_ethics": 0.5757575757575758, - "clinical_knowledge": 0.5227272727272727, - "medical_genetics": 0.5353535353535354, - "high_school_us_history": 0.6650246305418719, - "high_school_physics": 0.36, - "high_school_world_history": 0.7076271186440678, - "virology": 0.45454545454545453, - "high_school_microeconomics": 0.510548523206751, - "econometrics": 0.2920353982300885, - "college_computer_science": 0.40404040404040403, - "high_school_biology": 0.5922330097087378, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.3309608540925267, - "philosophy": 0.4967741935483871, - "professional_medicine": 0.4059040590405904, - "nutrition": 0.49508196721311476, - "global_facts": 0.31313131313131315, - "machine_learning": 0.36936936936936937, - "security_studies": 0.5327868852459017, - "public_relations": 0.4954128440366973, - "professional_psychology": 0.44844517184942717, - "prehistory": 0.5294117647058824, - "anatomy": 0.4925373134328358, - "human_sexuality": 0.5769230769230769, - "college_medicine": 0.47093023255813954, - "high_school_government_and_politics": 0.7083333333333334, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.6111111111111112, - "high_school_geography": 0.6091370558375635, - "elementary_mathematics": 0.5384615384615384, - "human_aging": 0.5540540540540541, - "college_mathematics": 0.25252525252525254, - "high_school_psychology": 0.6838235294117647, - "formal_logic": 0.344, - "high_school_statistics": 0.39069767441860465, - "international_law": 0.65, - "high_school_mathematics": 0.3903345724907063, - "high_school_computer_science": 0.5252525252525253, - "conceptual_physics": 0.4358974358974359, - "miscellaneous": 0.6892583120204604, - "high_school_chemistry": 0.4405940594059406, - "marketing": 0.7339055793991416, - "professional_law": 0.37116764514024786, - "management": 0.6666666666666666, - "college_physics": 0.36633663366336633, - "jurisprudence": 0.616822429906542, - "world_religions": 0.6588235294117647, - "sociology": 0.665, - "us_foreign_policy": 0.696969696969697, - "high_school_macroeconomics": 0.4832904884318766, - "computer_security": 0.6060606060606061, - "moral_scenarios": 0.23154362416107382, - "moral_disputes": 0.5391304347826087, - "electrical_engineering": 0.4375, - "astronomy": 0.4900662251655629, - "college_biology": 0.5244755244755245 + "machine_learning": 0.23423423423423423, + "security_studies": 0.21721311475409835, + "public_relations": 0.22018348623853212, + "professional_psychology": 0.27495908346972175, + "prehistory": 0.2260061919504644, + "anatomy": 0.22388059701492538, + "human_sexuality": 0.26153846153846155, + "college_medicine": 0.22674418604651161, + "high_school_government_and_politics": 0.2916666666666667, + "college_chemistry": 0.26262626262626265, + "logical_fallacies": 0.24074074074074073, + "high_school_geography": 0.23857868020304568, + "elementary_mathematics": 0.21220159151193635, + "human_aging": 0.2747747747747748, + "college_mathematics": 0.20202020202020202, + "high_school_psychology": 0.23897058823529413, + "formal_logic": 0.264, + "high_school_statistics": 0.32558139534883723, + "international_law": 0.26666666666666666, + "high_school_mathematics": 0.24907063197026022, + "high_school_computer_science": 0.30303030303030304, + "conceptual_physics": 0.24358974358974358, + "miscellaneous": 0.22506393861892582, + "high_school_chemistry": 0.2524752475247525, + "marketing": 0.33476394849785407, + "professional_law": 0.23222439660795824, + "management": 0.2647058823529412, + "college_physics": 0.16831683168316833, + "jurisprudence": 0.205607476635514, + "world_religions": 0.3235294117647059, + "sociology": 0.24, + "us_foreign_policy": 0.3333333333333333, + "high_school_macroeconomics": 0.2390745501285347, + "computer_security": 0.2222222222222222, + "moral_scenarios": 0.2483221476510067, + "moral_disputes": 0.23478260869565218, + "electrical_engineering": 0.2708333333333333, + "astronomy": 0.33112582781456956, + "college_biology": 0.2867132867132867 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.5393759286775631 - }, - "prompt_2": { - "accuracy": 0.524517087667162 - }, - "prompt_3": { - "accuracy": 0.5185735512630015 - }, - "prompt_4": { - "accuracy": 0.5029717682020802 - }, - "prompt_5": { - "accuracy": 0.5 + "accuracy": 0.23402674591381872 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.5155666251556662, + "accuracy": 0.24221668742216687, "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.375, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.42857142857142855, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.375, + "computer_network": 0.2916666666666667, + "operating_system": 0.25, + "computer_architecture": 0.2692307692307692, + "college_programming": 0.21428571428571427, + "college_physics": 0.08333333333333333, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.4782608695652174, - "high_school_physics": 0.5416666666666666, - "high_school_chemistry": 0.4166666666666667, - "high_school_biology": 0.4166666666666667, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.6923076923076923, - "middle_school_physics": 0.6666666666666666, - "middle_school_chemistry": 0.88, - "veterinary_medicine": 0.5357142857142857, - "college_economics": 0.26666666666666666, - "business_administration": 0.47368421052631576, - "marxism": 0.75, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.6470588235294118, - "teacher_qualification": 0.7959183673469388, - "high_school_politics": 0.7083333333333334, - "high_school_geography": 0.625, - "middle_school_politics": 0.8846153846153846, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.7142857142857143, - "ideological_and_moral_cultivation": 0.6666666666666666, - "logic": 0.48148148148148145, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.6071428571428571, - "art_studies": 0.6052631578947368, - "professional_tour_guide": 0.5882352941176471, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.72, - "middle_school_history": 0.8518518518518519, - "civil_servant": 0.5, - "sports_science": 0.4166666666666667, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.48148148148148145, - "urban_and_rural_planner": 0.5882352941176471, - "accountant": 0.48148148148148145, - "fire_engineer": 0.4444444444444444, - "environmental_impact_assessment_engineer": 0.5, - "tax_accountant": 0.5370370370370371, - "physician": 0.4074074074074074 - } - }, - "prompt_2": { - "accuracy": 0.523038605230386, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.40476190476190477, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.4827586206896552, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.5862068965517241, - "high_school_mathematics": 0.43478260869565216, - "high_school_physics": 0.5416666666666666, - "high_school_chemistry": 0.5, - "high_school_biology": 0.5833333333333334, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.7307692307692307, - "middle_school_physics": 0.75, - "middle_school_chemistry": 0.76, - "veterinary_medicine": 0.42857142857142855, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.16666666666666666, + "metrology_engineer": 0.2413793103448276, + "high_school_mathematics": 0.391304347826087, + "high_school_physics": 0.125, + "high_school_chemistry": 0.25, + "high_school_biology": 0.375, + "middle_school_mathematics": 0.3333333333333333, + "middle_school_biology": 0.23076923076923078, + "middle_school_physics": 0.25, + "middle_school_chemistry": 0.28, + "veterinary_medicine": 0.17857142857142858, "college_economics": 0.3333333333333333, - "business_administration": 0.5263157894736842, - "marxism": 0.7083333333333334, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.6176470588235294, - "teacher_qualification": 0.7959183673469388, - "high_school_politics": 0.9166666666666666, - "high_school_geography": 0.5, - "middle_school_politics": 0.8461538461538461, - "middle_school_geography": 0.5294117647058824, - "modern_chinese_history": 0.5714285714285714, - "ideological_and_moral_cultivation": 0.7083333333333334, - "logic": 0.4074074074074074, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.7142857142857143, - "art_studies": 0.5789473684210527, - "professional_tour_guide": 0.6176470588235294, - "legal_professional": 0.5714285714285714, - "high_school_chinese": 0.4583333333333333, - "high_school_history": 0.6, - "middle_school_history": 0.8148148148148148, - "civil_servant": 0.5384615384615384, - "sports_science": 0.5, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.375, - "clinical_medicine": 0.48148148148148145, - "urban_and_rural_planner": 0.6078431372549019, - "accountant": 0.48148148148148145, - "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.5277777777777778, - "tax_accountant": 0.48148148148148145, - "physician": 0.4444444444444444 - } - }, - "prompt_3": { - "accuracy": 0.5186799501867995, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.5, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.6206896551724138, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.625, - "high_school_chemistry": 0.5416666666666666, - "high_school_biology": 0.4583333333333333, - "middle_school_mathematics": 0.4583333333333333, - "middle_school_biology": 0.7307692307692307, - "middle_school_physics": 0.6666666666666666, - "middle_school_chemistry": 0.88, - "veterinary_medicine": 0.5, - "college_economics": 0.3, - "business_administration": 0.5, - "marxism": 0.75, - "mao_zedong_thought": 0.7241379310344828, - "education_science": 0.5588235294117647, - "teacher_qualification": 0.7551020408163265, - "high_school_politics": 0.75, - "high_school_geography": 0.5833333333333334, - "middle_school_politics": 0.8076923076923077, - "middle_school_geography": 0.5882352941176471, - "modern_chinese_history": 0.6071428571428571, - "ideological_and_moral_cultivation": 0.7083333333333334, - "logic": 0.4074074074074074, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.6071428571428571, - "art_studies": 0.5789473684210527, - "professional_tour_guide": 0.5, - "legal_professional": 0.4642857142857143, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.72, - "middle_school_history": 0.8148148148148148, - "civil_servant": 0.46153846153846156, - "sports_science": 0.5833333333333334, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.375, - "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.6078431372549019, - "accountant": 0.5, - "fire_engineer": 0.4166666666666667, - "environmental_impact_assessment_engineer": 0.5, - "tax_accountant": 0.48148148148148145, - "physician": 0.46296296296296297 + "business_administration": 0.21052631578947367, + "marxism": 0.25, + "mao_zedong_thought": 0.2413793103448276, + "education_science": 0.17647058823529413, + "teacher_qualification": 0.2857142857142857, + "high_school_politics": 0.3333333333333333, + "high_school_geography": 0.375, + "middle_school_politics": 0.4230769230769231, + "middle_school_geography": 0.11764705882352941, + "modern_chinese_history": 0.10714285714285714, + "ideological_and_moral_cultivation": 0.3333333333333333, + "logic": 0.18518518518518517, + "law": 0.2413793103448276, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.3157894736842105, + "professional_tour_guide": 0.35294117647058826, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.25, + "high_school_history": 0.2, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.23076923076923078, + "sports_science": 0.25, + "plant_protection": 0.25925925925925924, + "basic_medicine": 0.041666666666666664, + "clinical_medicine": 0.2222222222222222, + "urban_and_rural_planner": 0.21568627450980393, + "accountant": 0.2962962962962963, + "fire_engineer": 0.1388888888888889, + "environmental_impact_assessment_engineer": 0.19444444444444445, + "tax_accountant": 0.18518518518518517, + "physician": 0.12962962962962962 } - }, - "prompt_4": { - "accuracy": 0.5043586550435866, + } + }, + "cmmlu": { + "prompt_1": { + "accuracy": 0.27598566308243727 + } + }, + "cmmlu_full": { + "prompt_1": { + "accuracy": 0.2515973061647384, "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.5476190476190477, - "college_physics": 0.4583333333333333, - "college_chemistry": 0.4827586206896552, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.09523809523809523, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.6206896551724138, - "high_school_mathematics": 0.43478260869565216, - "high_school_physics": 0.7083333333333334, - "high_school_chemistry": 0.4166666666666667, - "high_school_biology": 0.5, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.6923076923076923, - "middle_school_physics": 0.7083333333333334, - "middle_school_chemistry": 0.72, - "veterinary_medicine": 0.5, - "college_economics": 0.31666666666666665, - "business_administration": 0.5, - "marxism": 0.7083333333333334, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.6470588235294118, - "teacher_qualification": 0.6938775510204082, - "high_school_politics": 0.625, - "high_school_geography": 0.625, - "middle_school_politics": 0.8076923076923077, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.7142857142857143, - "ideological_and_moral_cultivation": 0.6666666666666666, - "logic": 0.4074074074074074, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.5714285714285714, - "art_studies": 0.47368421052631576, - "professional_tour_guide": 0.5588235294117647, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.64, - "middle_school_history": 0.8148148148148148, - "civil_servant": 0.46153846153846156, - "sports_science": 0.5, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.5490196078431373, - "accountant": 0.4444444444444444, - "fire_engineer": 0.4444444444444444, - "environmental_impact_assessment_engineer": 0.5833333333333334, - "tax_accountant": 0.48148148148148145, - "physician": 0.4444444444444444 - } - }, - "prompt_5": { - "accuracy": 0.48692403486924035, - "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.375, - "computer_architecture": 0.5, - "college_programming": 0.40476190476190477, - "college_physics": 0.25, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.625, - "high_school_chemistry": 0.375, - "high_school_biology": 0.5416666666666666, - "middle_school_mathematics": 0.5, - "middle_school_biology": 0.7692307692307693, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.6, - "veterinary_medicine": 0.5, - "college_economics": 0.35, - "business_administration": 0.47368421052631576, - "marxism": 0.5833333333333334, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.6176470588235294, - "teacher_qualification": 0.7142857142857143, - "high_school_politics": 0.7083333333333334, - "high_school_geography": 0.625, - "middle_school_politics": 0.7692307692307693, - "middle_school_geography": 0.5882352941176471, - "modern_chinese_history": 0.6071428571428571, - "ideological_and_moral_cultivation": 0.75, - "logic": 0.4074074074074074, - "law": 0.4482758620689655, - "chinese_language_and_literature": 0.6428571428571429, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.6176470588235294, - "legal_professional": 0.4642857142857143, - "high_school_chinese": 0.375, - "high_school_history": 0.6, - "middle_school_history": 0.7037037037037037, - "civil_servant": 0.40384615384615385, - "sports_science": 0.4583333333333333, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.5882352941176471, - "accountant": 0.46296296296296297, - "fire_engineer": 0.4444444444444444, - "environmental_impact_assessment_engineer": 0.5277777777777778, - "tax_accountant": 0.5, - "physician": 0.4444444444444444 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.48028673835125446 - }, - "prompt_2": { - "accuracy": 0.4910394265232975 - }, - "prompt_3": { - "accuracy": 0.5268817204301075 - }, - "prompt_4": { - "accuracy": 0.4910394265232975 - }, - "prompt_5": { - "accuracy": 0.5161290322580645 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.5250388533931963, - "category_acc": { - "agronomy": 0.48520710059171596, - "anatomy": 0.43243243243243246, - "ancient_chinese": 0.3231707317073171, - "arts": 0.65625, - "astronomy": 0.3393939393939394, - "business_ethics": 0.47368421052631576, - "chinese_civil_service_exam": 0.54375, - "chinese_driving_rule": 0.732824427480916, - "chinese_food_culture": 0.5073529411764706, - "chinese_foreign_policy": 0.6915887850467289, - "chinese_history": 0.7151702786377709, - "chinese_literature": 0.4166666666666667, - "chinese_teacher_qualification": 0.6815642458100558, - "clinical_knowledge": 0.459915611814346, + "agronomy": 0.2485207100591716, + "anatomy": 0.22972972972972974, + "ancient_chinese": 0.25, + "arts": 0.30625, + "astronomy": 0.24242424242424243, + "business_ethics": 0.28708133971291866, + "chinese_civil_service_exam": 0.2875, + "chinese_driving_rule": 0.2748091603053435, + "chinese_food_culture": 0.18382352941176472, + "chinese_foreign_policy": 0.27102803738317754, + "chinese_history": 0.2260061919504644, + "chinese_literature": 0.22058823529411764, + "chinese_teacher_qualification": 0.24022346368715083, + "clinical_knowledge": 0.25738396624472576, "college_actuarial_science": 0.18867924528301888, - "college_education": 0.6448598130841121, - "college_engineering_hydrology": 0.4056603773584906, - "college_law": 0.4074074074074074, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.3867924528301887, - "college_medicine": 0.46153846153846156, - "computer_science": 0.4852941176470588, - "computer_security": 0.6491228070175439, - "conceptual_physics": 0.5986394557823129, - "construction_project_management": 0.4172661870503597, - "economics": 0.4716981132075472, - "education": 0.5705521472392638, - "electrical_engineering": 0.47093023255813954, - "elementary_chinese": 0.49603174603174605, - "elementary_commonsense": 0.5353535353535354, - "elementary_information_and_technology": 0.6848739495798319, - "elementary_mathematics": 0.3695652173913043, - "ethnology": 0.5481481481481482, - "food_science": 0.5384615384615384, - "genetics": 0.3522727272727273, - "global_facts": 0.5973154362416108, - "high_school_biology": 0.5266272189349113, - "high_school_chemistry": 0.4318181818181818, - "high_school_geography": 0.5254237288135594, - "high_school_mathematics": 0.3353658536585366, - "high_school_physics": 0.44545454545454544, - "high_school_politics": 0.6153846153846154, - "human_sexuality": 0.5158730158730159, - "international_law": 0.3891891891891892, - "journalism": 0.5290697674418605, - "jurisprudence": 0.5620437956204379, - "legal_and_moral_basis": 0.8691588785046729, - "logical": 0.4878048780487805, - "machine_learning": 0.4262295081967213, - "management": 0.680952380952381, - "marketing": 0.6444444444444445, - "marxist_theory": 0.671957671957672, - "modern_chinese": 0.47413793103448276, - "nutrition": 0.5379310344827586, - "philosophy": 0.6095238095238096, - "professional_accounting": 0.5828571428571429, - "professional_law": 0.3886255924170616, - "professional_medicine": 0.3723404255319149, - "professional_psychology": 0.6724137931034483, - "public_relations": 0.5459770114942529, - "security_study": 0.5777777777777777, - "sociology": 0.584070796460177, - "sports_science": 0.4666666666666667, - "traditional_chinese_medicine": 0.5081081081081081, - "virology": 0.4970414201183432, - "world_history": 0.6832298136645962, - "world_religions": 0.6125 - } - }, - "prompt_2": { - "accuracy": 0.5285788292177517, - "category_acc": { - "agronomy": 0.46745562130177515, - "anatomy": 0.41216216216216217, - "ancient_chinese": 0.3597560975609756, - "arts": 0.64375, - "astronomy": 0.3696969696969697, - "business_ethics": 0.4688995215311005, - "chinese_civil_service_exam": 0.55625, - "chinese_driving_rule": 0.732824427480916, - "chinese_food_culture": 0.5, - "chinese_foreign_policy": 0.6355140186915887, - "chinese_history": 0.7306501547987616, - "chinese_literature": 0.45098039215686275, - "chinese_teacher_qualification": 0.7094972067039106, - "clinical_knowledge": 0.4430379746835443, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.6822429906542056, - "college_engineering_hydrology": 0.4056603773584906, - "college_law": 0.48148148148148145, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.4056603773584906, - "college_medicine": 0.4432234432234432, - "computer_science": 0.4803921568627451, - "computer_security": 0.6432748538011696, - "conceptual_physics": 0.6190476190476191, - "construction_project_management": 0.4172661870503597, - "economics": 0.5283018867924528, - "education": 0.6196319018404908, - "electrical_engineering": 0.47093023255813954, - "elementary_chinese": 0.5396825396825397, - "elementary_commonsense": 0.5202020202020202, - "elementary_information_and_technology": 0.6932773109243697, + "college_education": 0.308411214953271, + "college_engineering_hydrology": 0.2358490566037736, + "college_law": 0.26851851851851855, + "college_mathematics": 0.13333333333333333, + "college_medical_statistics": 0.22641509433962265, + "college_medicine": 0.26373626373626374, + "computer_science": 0.20588235294117646, + "computer_security": 0.23976608187134502, + "conceptual_physics": 0.25170068027210885, + "construction_project_management": 0.2733812949640288, + "economics": 0.2389937106918239, + "education": 0.2331288343558282, + "electrical_engineering": 0.2441860465116279, + "elementary_chinese": 0.24206349206349206, + "elementary_commonsense": 0.2828282828282828, + "elementary_information_and_technology": 0.25630252100840334, "elementary_mathematics": 0.28695652173913044, - "ethnology": 0.5407407407407407, - "food_science": 0.5454545454545454, - "genetics": 0.4090909090909091, - "global_facts": 0.5838926174496645, - "high_school_biology": 0.5325443786982249, - "high_school_chemistry": 0.4318181818181818, - "high_school_geography": 0.5423728813559322, - "high_school_mathematics": 0.3780487804878049, - "high_school_physics": 0.4090909090909091, - "high_school_politics": 0.6573426573426573, - "human_sexuality": 0.5079365079365079, - "international_law": 0.41621621621621624, - "journalism": 0.5290697674418605, - "jurisprudence": 0.5450121654501217, - "legal_and_moral_basis": 0.8691588785046729, - "logical": 0.45528455284552843, - "machine_learning": 0.45081967213114754, - "management": 0.6761904761904762, - "marketing": 0.6222222222222222, - "marxist_theory": 0.6402116402116402, - "modern_chinese": 0.4396551724137931, - "nutrition": 0.593103448275862, - "philosophy": 0.6190476190476191, - "professional_accounting": 0.5885714285714285, - "professional_law": 0.3696682464454976, - "professional_medicine": 0.3829787234042553, - "professional_psychology": 0.6767241379310345, - "public_relations": 0.5632183908045977, - "security_study": 0.5851851851851851, - "sociology": 0.5575221238938053, - "sports_science": 0.4666666666666667, - "traditional_chinese_medicine": 0.5081081081081081, - "virology": 0.5088757396449705, - "world_history": 0.6832298136645962, - "world_religions": 0.5875 - } - }, - "prompt_3": { - "accuracy": 0.5316007597996891, - "category_acc": { - "agronomy": 0.4556213017751479, - "anatomy": 0.41216216216216217, - "ancient_chinese": 0.31097560975609756, - "arts": 0.675, - "astronomy": 0.3939393939393939, - "business_ethics": 0.48325358851674644, - "chinese_civil_service_exam": 0.575, - "chinese_driving_rule": 0.7404580152671756, - "chinese_food_culture": 0.45588235294117646, - "chinese_foreign_policy": 0.6728971962616822, - "chinese_history": 0.718266253869969, - "chinese_literature": 0.4411764705882353, - "chinese_teacher_qualification": 0.6703910614525139, - "clinical_knowledge": 0.4345991561181435, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.6822429906542056, - "college_engineering_hydrology": 0.4339622641509434, - "college_law": 0.3888888888888889, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.36792452830188677, - "college_medicine": 0.45054945054945056, - "computer_science": 0.5098039215686274, - "computer_security": 0.6432748538011696, - "conceptual_physics": 0.6326530612244898, - "construction_project_management": 0.43884892086330934, - "economics": 0.5094339622641509, - "education": 0.6196319018404908, - "electrical_engineering": 0.48255813953488375, - "elementary_chinese": 0.48412698412698413, - "elementary_commonsense": 0.5454545454545454, - "elementary_information_and_technology": 0.7226890756302521, - "elementary_mathematics": 0.3217391304347826, - "ethnology": 0.5407407407407407, - "food_science": 0.5174825174825175, - "genetics": 0.39204545454545453, - "global_facts": 0.5234899328859061, - "high_school_biology": 0.5502958579881657, - "high_school_chemistry": 0.4696969696969697, - "high_school_geography": 0.5677966101694916, - "high_school_mathematics": 0.3475609756097561, - "high_school_physics": 0.4727272727272727, - "high_school_politics": 0.6503496503496503, - "human_sexuality": 0.5158730158730159, - "international_law": 0.44324324324324327, - "journalism": 0.5465116279069767, - "jurisprudence": 0.5790754257907542, - "legal_and_moral_basis": 0.8738317757009346, - "logical": 0.4959349593495935, - "machine_learning": 0.4672131147540984, - "management": 0.6952380952380952, - "marketing": 0.6388888888888888, - "marxist_theory": 0.6507936507936508, - "modern_chinese": 0.4396551724137931, - "nutrition": 0.5793103448275863, - "philosophy": 0.638095238095238, - "professional_accounting": 0.5771428571428572, - "professional_law": 0.4312796208530806, - "professional_medicine": 0.39893617021276595, - "professional_psychology": 0.6551724137931034, - "public_relations": 0.5747126436781609, - "security_study": 0.6074074074074074, - "sociology": 0.5663716814159292, - "sports_science": 0.46060606060606063, - "traditional_chinese_medicine": 0.4486486486486487, - "virology": 0.5029585798816568, - "world_history": 0.6832298136645962, - "world_religions": 0.6 - } - }, - "prompt_4": { - "accuracy": 0.5044897254360214, - "category_acc": { - "agronomy": 0.4556213017751479, - "anatomy": 0.39864864864864863, - "ancient_chinese": 0.34146341463414637, - "arts": 0.6125, - "astronomy": 0.3333333333333333, - "business_ethics": 0.45933014354066987, - "chinese_civil_service_exam": 0.5375, - "chinese_driving_rule": 0.732824427480916, - "chinese_food_culture": 0.4485294117647059, - "chinese_foreign_policy": 0.6635514018691588, - "chinese_history": 0.6625386996904025, - "chinese_literature": 0.4166666666666667, - "chinese_teacher_qualification": 0.6256983240223464, - "clinical_knowledge": 0.39662447257383965, - "college_actuarial_science": 0.18867924528301888, - "college_education": 0.6542056074766355, - "college_engineering_hydrology": 0.4339622641509434, - "college_law": 0.42592592592592593, - "college_mathematics": 0.2857142857142857, - "college_medical_statistics": 0.3584905660377358, - "college_medicine": 0.4065934065934066, - "computer_science": 0.49019607843137253, - "computer_security": 0.5906432748538012, - "conceptual_physics": 0.5102040816326531, - "construction_project_management": 0.45323741007194246, - "economics": 0.5094339622641509, - "education": 0.6196319018404908, - "electrical_engineering": 0.4476744186046512, - "elementary_chinese": 0.4365079365079365, - "elementary_commonsense": 0.48484848484848486, - "elementary_information_and_technology": 0.6848739495798319, - "elementary_mathematics": 0.3347826086956522, - "ethnology": 0.5555555555555556, - "food_science": 0.48951048951048953, - "genetics": 0.3806818181818182, - "global_facts": 0.5302013422818792, - "high_school_biology": 0.5088757396449705, - "high_school_chemistry": 0.44696969696969696, - "high_school_geography": 0.5254237288135594, - "high_school_mathematics": 0.31097560975609756, - "high_school_physics": 0.509090909090909, - "high_school_politics": 0.6293706293706294, - "human_sexuality": 0.49206349206349204, - "international_law": 0.40540540540540543, - "journalism": 0.5465116279069767, - "jurisprudence": 0.5085158150851582, - "legal_and_moral_basis": 0.8504672897196262, - "logical": 0.4715447154471545, - "machine_learning": 0.45081967213114754, - "management": 0.6476190476190476, - "marketing": 0.5722222222222222, - "marxist_theory": 0.6402116402116402, - "modern_chinese": 0.4224137931034483, - "nutrition": 0.5448275862068965, - "philosophy": 0.5714285714285714, - "professional_accounting": 0.5714285714285714, - "professional_law": 0.33649289099526064, - "professional_medicine": 0.3670212765957447, - "professional_psychology": 0.6336206896551724, - "public_relations": 0.5287356321839081, - "security_study": 0.6, - "sociology": 0.5398230088495575, - "sports_science": 0.45454545454545453, - "traditional_chinese_medicine": 0.4486486486486487, - "virology": 0.5029585798816568, - "world_history": 0.6708074534161491, - "world_religions": 0.6125 - } - }, - "prompt_5": { - "accuracy": 0.49361077534104647, - "category_acc": { - "agronomy": 0.46153846153846156, - "anatomy": 0.38513513513513514, - "ancient_chinese": 0.3048780487804878, - "arts": 0.6375, - "astronomy": 0.3878787878787879, - "business_ethics": 0.44019138755980863, - "chinese_civil_service_exam": 0.5375, - "chinese_driving_rule": 0.7404580152671756, - "chinese_food_culture": 0.49264705882352944, - "chinese_foreign_policy": 0.6261682242990654, - "chinese_history": 0.631578947368421, - "chinese_literature": 0.4166666666666667, - "chinese_teacher_qualification": 0.6145251396648045, - "clinical_knowledge": 0.4219409282700422, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.6635514018691588, - "college_engineering_hydrology": 0.4528301886792453, - "college_law": 0.39814814814814814, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.37735849056603776, - "college_medicine": 0.4065934065934066, - "computer_science": 0.4264705882352941, - "computer_security": 0.5497076023391813, - "conceptual_physics": 0.5714285714285714, - "construction_project_management": 0.41007194244604317, - "economics": 0.4276729559748428, - "education": 0.5644171779141104, - "electrical_engineering": 0.4883720930232558, - "elementary_chinese": 0.46825396825396826, - "elementary_commonsense": 0.4393939393939394, - "elementary_information_and_technology": 0.6428571428571429, - "elementary_mathematics": 0.3391304347826087, - "ethnology": 0.5111111111111111, - "food_science": 0.5244755244755245, - "genetics": 0.38636363636363635, - "global_facts": 0.5436241610738255, - "high_school_biology": 0.47928994082840237, - "high_school_chemistry": 0.3333333333333333, - "high_school_geography": 0.4576271186440678, - "high_school_mathematics": 0.34146341463414637, - "high_school_physics": 0.39090909090909093, - "high_school_politics": 0.5804195804195804, - "human_sexuality": 0.5158730158730159, - "international_law": 0.3891891891891892, - "journalism": 0.5058139534883721, - "jurisprudence": 0.5206812652068127, - "legal_and_moral_basis": 0.8457943925233645, - "logical": 0.45528455284552843, - "machine_learning": 0.4262295081967213, - "management": 0.6047619047619047, - "marketing": 0.6222222222222222, - "marxist_theory": 0.5502645502645502, - "modern_chinese": 0.39655172413793105, - "nutrition": 0.496551724137931, - "philosophy": 0.6095238095238096, - "professional_accounting": 0.5828571428571429, - "professional_law": 0.4075829383886256, - "professional_medicine": 0.35638297872340424, - "professional_psychology": 0.5603448275862069, - "public_relations": 0.5057471264367817, - "security_study": 0.5259259259259259, - "sociology": 0.5575221238938053, - "sports_science": 0.46060606060606063, - "traditional_chinese_medicine": 0.4594594594594595, - "virology": 0.46745562130177515, - "world_history": 0.6521739130434783, - "world_religions": 0.6125 + "ethnology": 0.2518518518518518, + "food_science": 0.23776223776223776, + "genetics": 0.26136363636363635, + "global_facts": 0.19463087248322147, + "high_school_biology": 0.21301775147928995, + "high_school_chemistry": 0.22727272727272727, + "high_school_geography": 0.2627118644067797, + "high_school_mathematics": 0.2073170731707317, + "high_school_physics": 0.3, + "high_school_politics": 0.23776223776223776, + "human_sexuality": 0.21428571428571427, + "international_law": 0.22702702702702704, + "journalism": 0.27325581395348836, + "jurisprudence": 0.22871046228710462, + "legal_and_moral_basis": 0.2757009345794392, + "logical": 0.21951219512195122, + "machine_learning": 0.22131147540983606, + "management": 0.2761904761904762, + "marketing": 0.24444444444444444, + "marxist_theory": 0.291005291005291, + "modern_chinese": 0.2413793103448276, + "nutrition": 0.2, + "philosophy": 0.18095238095238095, + "professional_accounting": 0.3028571428571429, + "professional_law": 0.23222748815165878, + "professional_medicine": 0.23138297872340424, + "professional_psychology": 0.31896551724137934, + "public_relations": 0.28160919540229884, + "security_study": 0.2740740740740741, + "sociology": 0.3008849557522124, + "sports_science": 0.32727272727272727, + "traditional_chinese_medicine": 0.20540540540540542, + "virology": 0.30177514792899407, + "world_history": 0.2857142857142857, + "world_religions": 0.3125 } } }, "zbench": { "prompt_1": { - "accuracy": 0.3333333333333333 - }, - "prompt_2": { - "accuracy": 0.3939393939393939 - }, - "prompt_3": { - "accuracy": 0.42424242424242425 - }, - "prompt_4": { - "accuracy": 0.3333333333333333 - }, - "prompt_5": { - "accuracy": 0.36363636363636365 + "accuracy": 0.18181818181818182 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.3181818181818182 - }, - "prompt_2": { - "accuracy": 0.37272727272727274 - }, - "prompt_3": { - "accuracy": 0.3386363636363636 - }, - "prompt_4": { - "accuracy": 0.39545454545454545 - }, - "prompt_5": { - "accuracy": 0.3159090909090909 + "accuracy": 0.16590909090909092 } }, "ocnli": { "prompt_1": { - "accuracy": 0.3888135593220339 - }, - "prompt_2": { - "accuracy": 0.38745762711864407 - }, - "prompt_3": { - "accuracy": 0.39322033898305087 - }, - "prompt_4": { - "accuracy": 0.39152542372881355 - }, - "prompt_5": { - "accuracy": 0.38813559322033897 + "accuracy": 0.3311864406779661 } }, "c3": { "prompt_1": { - "accuracy": 0.7898279730740464 - }, - "prompt_2": { - "accuracy": 0.7797307404637247 - }, - "prompt_3": { - "accuracy": 0.7890800299177263 - }, - "prompt_4": { - "accuracy": 0.7789827973074046 - }, - "prompt_5": { - "accuracy": 0.7621540762902019 + "accuracy": 0.2629020194465221 } }, "dream": { "prompt_1": { - "accuracy": 0.8108770210681039 - }, - "prompt_2": { - "accuracy": 0.8049975502204801 - }, - "prompt_3": { - "accuracy": 0.8201861832435081 - }, - "prompt_4": { - "accuracy": 0.8064674179323861 - }, - "prompt_5": { - "accuracy": 0.8059774620284175 + "accuracy": 0.329250367466928 } }, "samsum": { "prompt_1": { - "rouge1": 0.373140639549932, - "rouge2": 0.15701621462169868, - "rougeL": 0.28951053468500426, - "avg_rouge": 0.2732224629522116 - }, - "prompt_2": { - "rouge1": 0.3943490230577903, - "rouge2": 0.16615934787060646, - "rougeL": 0.30572336761968233, - "avg_rouge": 0.2887439128493597 - }, - "prompt_3": { - "rouge1": 0.3781293472327215, - "rouge2": 0.1539935037937406, - "rougeL": 0.2915377314690962, - "avg_rouge": 0.2745535274985194 - }, - "prompt_4": { - "rouge1": 0.3705780377474849, - "rouge2": 0.15047971950430872, - "rougeL": 0.2838871599833531, - "avg_rouge": 0.2683149724117156 - }, - "prompt_5": { - "rouge1": 0.3790697118655646, - "rouge2": 0.15690679582444042, - "rougeL": 0.2915156132325537, - "avg_rouge": 0.27583070697418627 + "rouge1": 0.12898713332403366, + "rouge2": 0.04349609409834999, + "rougeL": 0.10177675146188521, + "avg_rouge": 0.09141999296142295 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.19267615599451351, - "rouge2": 0.06026782298868, - "rougeL": 0.1465858402073522, - "avg_rouge": 0.13317660639684858 - }, - "prompt_2": { - "rouge1": 0.19122438331820696, - "rouge2": 0.0596186248631677, - "rougeL": 0.14548272315260302, - "avg_rouge": 0.1321085771113259 - }, - "prompt_3": { - "rouge1": 0.19129865844661617, - "rouge2": 0.0598746158944248, - "rougeL": 0.14568006773712652, - "avg_rouge": 0.13228444735938916 - }, - "prompt_4": { - "rouge1": 0.19109117820835103, - "rouge2": 0.059441499902232534, - "rougeL": 0.14537531866001915, - "avg_rouge": 0.13196933225686758 - }, - "prompt_5": { - "rouge1": 0.19131225228663346, - "rouge2": 0.05928232884166233, - "rougeL": 0.1458946687910772, - "avg_rouge": 0.13216308330645768 + "rouge1": 0.13775292661101599, + "rouge2": 0.04111655319766571, + "rougeL": 0.10976116581390764, + "avg_rouge": 0.09621021520752977 } }, "sst2": { "prompt_1": { - "accuracy": 0.8555045871559633 - }, - "prompt_2": { - "accuracy": 0.8314220183486238 - }, - "prompt_3": { - "accuracy": 0.8165137614678899 - }, - "prompt_4": { - "accuracy": 0.8337155963302753 - }, - "prompt_5": { - "accuracy": 0.8876146788990825 + "accuracy": 0.5080275229357798 } }, "cola": { "prompt_1": { - "accuracy": 0.6049856184084372 - }, - "prompt_2": { - "accuracy": 0.5714285714285714 - }, - "prompt_3": { - "accuracy": 0.6212847555129435 - }, - "prompt_4": { - "accuracy": 0.6116970278044104 - }, - "prompt_5": { - "accuracy": 0.6347075743048898 + "accuracy": 0.5148609779482263 } }, "qqp": { "prompt_1": { - "accuracy": 0.544 - }, - "prompt_2": { - "accuracy": 0.576 - }, - "prompt_3": { - "accuracy": 0.5285 - }, - "prompt_4": { - "accuracy": 0.5405 - }, - "prompt_5": { - "accuracy": 0.5695 + "accuracy": 0.498 } }, "mnli": { "prompt_1": { - "accuracy": 0.5475 - }, - "prompt_2": { - "accuracy": 0.5445 - }, - "prompt_3": { - "accuracy": 0.5465 - }, - "prompt_4": { - "accuracy": 0.5245 - }, - "prompt_5": { - "accuracy": 0.5105 + "accuracy": 0.2 } }, "qnli": { "prompt_1": { - "accuracy": 0.6845 - }, - "prompt_2": { - "accuracy": 0.693 - }, - "prompt_3": { - "accuracy": 0.6585 - }, - "prompt_4": { - "accuracy": 0.7035 - }, - "prompt_5": { - "accuracy": 0.6885 + "accuracy": 0.6 } }, "wnli": { "prompt_1": { - "accuracy": 0.5633802816901409 - }, - "prompt_2": { - "accuracy": 0.5352112676056338 - }, - "prompt_3": { - "accuracy": 0.5070422535211268 - }, - "prompt_4": { - "accuracy": 0.49295774647887325 - }, - "prompt_5": { - "accuracy": 0.5492957746478874 + "accuracy": 0.3 } }, "rte": { "prompt_1": { - "accuracy": 0.6064981949458483 - }, - "prompt_2": { - "accuracy": 0.6389891696750902 - }, - "prompt_3": { - "accuracy": 0.6064981949458483 - }, - "prompt_4": { - "accuracy": 0.5740072202166066 - }, - "prompt_5": { - "accuracy": 0.5740072202166066 + "accuracy": 0.1 } }, "mrpc": { "prompt_1": { - "accuracy": 0.46568627450980393 - }, - "prompt_2": { - "accuracy": 0.4803921568627451 - }, - "prompt_3": { - "accuracy": 0.49754901960784315 - }, - "prompt_4": { - "accuracy": 0.4877450980392157 - }, - "prompt_5": { - "accuracy": 0.46568627450980393 + "accuracy": 0.7 } } - }, - "five_shot": { + } + }, + "sailor_1_8b": { + "model_size": "1.8B", + "model_link": "https://huggingface.co/sail/Sailor-1.8B", + "zero_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.7607142857142857, - "language_acc": { - "English": 0.8588235294117647, - "Chinese": 0.826890756302521, - "Spanish": 0.7756302521008404, - "Vietnamese": 0.5815126050420169 - }, - "consistency_score_2": 0.6893557422969189, - "consistency_score_3": 0.5592436974789916, - "consistency_score_4": 0.46974789915966386, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.826890756302521, - "English,Spanish": 0.8016806722689076, - "English,Vietnamese": 0.5764705882352941, - "Chinese,Spanish": 0.7714285714285715, - "Chinese,Vietnamese": 0.5865546218487395, - "Spanish,Vietnamese": 0.573109243697479 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7142857142857143, - "English,Chinese,Vietnamese": 0.519327731092437, - "English,Spanish,Vietnamese": 0.5058823529411764, - "Chinese,Spanish,Vietnamese": 0.49747899159663866 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.46974789915966386 - } - }, - "AC3_2": 0.7232792222972915, - "AC3_3": 0.6446033513983808, - "AC3_4": 0.5808288006552519 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.4104761904761905, - "language_acc": { - "Spanish": 0.4866666666666667, - "English": 0.56, - "Chinese": 0.5266666666666666, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.30666666666666664, - "Indonesian": 0.36666666666666664, - "Malay": 0.34 - }, - "consistency_score_2": 0.4409523809523809, - "consistency_score_3": 0.252, - "consistency_score_4": 0.1676190476190476, - "consistency_score_5": 0.12253968253968256, - "consistency_score_6": 0.09428571428571429, - "consistency_score_7": 0.07333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5933333333333334, - "Spanish,Chinese": 0.58, - "Spanish,Vietnamese": 0.36666666666666664, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.4666666666666667, - "Spanish,Malay": 0.38666666666666666, - "English,Chinese": 0.6666666666666666, - "English,Vietnamese": 0.38666666666666666, - "English,Filipino": 0.43333333333333335, - "English,Indonesian": 0.41333333333333333, - "English,Malay": 0.38, - "Chinese,Vietnamese": 0.34, - "Chinese,Filipino": 0.3933333333333333, - "Chinese,Indonesian": 0.4266666666666667, - "Chinese,Malay": 0.36, - "Vietnamese,Filipino": 0.44666666666666666, - "Vietnamese,Indonesian": 0.41333333333333333, - "Vietnamese,Malay": 0.41333333333333333, - "Filipino,Indonesian": 0.47333333333333333, - "Filipino,Malay": 0.4266666666666667, - "Indonesian,Malay": 0.4533333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.46, - "Spanish,English,Vietnamese": 0.2733333333333333, - "Spanish,English,Filipino": 0.3, - "Spanish,English,Indonesian": 0.30666666666666664, - "Spanish,English,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese": 0.25333333333333335, - "Spanish,Chinese,Filipino": 0.28, - "Spanish,Chinese,Indonesian": 0.3, - "Spanish,Chinese,Malay": 0.24, - "Spanish,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,Vietnamese,Indonesian": 0.24, - "Spanish,Vietnamese,Malay": 0.2, - "Spanish,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Filipino,Malay": 0.20666666666666667, - "Spanish,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese": 0.26666666666666666, - "English,Chinese,Filipino": 0.32666666666666666, - "English,Chinese,Indonesian": 0.32666666666666666, - "English,Chinese,Malay": 0.2733333333333333, - "English,Vietnamese,Filipino": 0.22666666666666666, - "English,Vietnamese,Indonesian": 0.22666666666666666, - "English,Vietnamese,Malay": 0.2, - "English,Filipino,Indonesian": 0.24, - "English,Filipino,Malay": 0.21333333333333335, - "English,Indonesian,Malay": 0.21333333333333335, - "Chinese,Vietnamese,Filipino": 0.2, - "Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "Chinese,Vietnamese,Malay": 0.18, - "Chinese,Filipino,Indonesian": 0.24, - "Chinese,Filipino,Malay": 0.18666666666666668, - "Chinese,Indonesian,Malay": 0.22666666666666666, - "Vietnamese,Filipino,Indonesian": 0.26, - "Vietnamese,Filipino,Malay": 0.22, - "Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,English,Chinese,Filipino": 0.24, - "Spanish,English,Chinese,Indonesian": 0.26, - "Spanish,English,Chinese,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,English,Vietnamese,Indonesian": 0.2, - "Spanish,English,Vietnamese,Malay": 0.14666666666666667, - "Spanish,English,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Indonesian,Malay": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.18, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Filipino,Malay": 0.14, - "Spanish,Chinese,Indonesian,Malay": 0.18, - "Spanish,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino": 0.16, - "English,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "English,Chinese,Vietnamese,Malay": 0.14, - "English,Chinese,Filipino,Indonesian": 0.2, - "English,Chinese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Indonesian,Malay": 0.17333333333333334, - "English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "English,Vietnamese,Filipino,Malay": 0.12, - "English,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "English,Filipino,Indonesian,Malay": 0.14, - "Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.16 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.12, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.12, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 - } - }, - "AC3_2": 0.42516885048804204, - "AC3_3": 0.3122829211719527, - "AC3_4": 0.23803561618224928, - "AC3_5": 0.1887359255438757, - "AC3_6": 0.15334770886449853, - "AC3_7": 0.12443569551233777 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.3782467532467532, - "language_acc": { - "English": 0.5113636363636364, - "Filipino": 0.3125, - "Spanish": 0.42045454545454547, - "Malay": 0.2727272727272727, - "Chinese": 0.5340909090909091, - "Indonesian": 0.3068181818181818, - "Vietnamese": 0.2897727272727273 - }, - "consistency_score_2": 0.46482683982683975, - "consistency_score_3": 0.2732142857142857, - "consistency_score_4": 0.18279220779220776, - "consistency_score_5": 0.13474025974025972, - "consistency_score_6": 0.10714285714285714, - "consistency_score_7": 0.09090909090909091, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4147727272727273, - "English,Spanish": 0.5965909090909091, - "English,Malay": 0.3806818181818182, - "English,Chinese": 0.6988636363636364, - "English,Indonesian": 0.4034090909090909, - "English,Vietnamese": 0.39204545454545453, - "Filipino,Spanish": 0.4772727272727273, - "Filipino,Malay": 0.5113636363636364, - "Filipino,Chinese": 0.45454545454545453, - "Filipino,Indonesian": 0.48295454545454547, - "Filipino,Vietnamese": 0.44886363636363635, - "Spanish,Malay": 0.4318181818181818, - "Spanish,Chinese": 0.5852272727272727, - "Spanish,Indonesian": 0.48863636363636365, - "Spanish,Vietnamese": 0.4431818181818182, - "Malay,Chinese": 0.4034090909090909, - "Malay,Indonesian": 0.4772727272727273, - "Malay,Vietnamese": 0.3977272727272727, - "Chinese,Indonesian": 0.4375, - "Chinese,Vietnamese": 0.42045454545454547, - "Indonesian,Vietnamese": 0.4147727272727273 - }, - "3_combine": { - "English,Filipino,Spanish": 0.3068181818181818, - "English,Filipino,Malay": 0.23295454545454544, - "English,Filipino,Chinese": 0.3352272727272727, - "English,Filipino,Indonesian": 0.22727272727272727, - "English,Filipino,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay": 0.26704545454545453, - "English,Spanish,Chinese": 0.4772727272727273, - "English,Spanish,Indonesian": 0.29545454545454547, - "English,Spanish,Vietnamese": 0.2840909090909091, - "English,Malay,Chinese": 0.2897727272727273, - "English,Malay,Indonesian": 0.23295454545454544, - "English,Malay,Vietnamese": 0.17613636363636365, - "English,Chinese,Indonesian": 0.32954545454545453, - "English,Chinese,Vietnamese": 0.3068181818181818, - "English,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.32954545454545453, - "Filipino,Spanish,Indonesian": 0.2897727272727273, - "Filipino,Spanish,Vietnamese": 0.2784090909090909, - "Filipino,Malay,Chinese": 0.26704545454545453, - "Filipino,Malay,Indonesian": 0.30113636363636365, - "Filipino,Malay,Vietnamese": 0.2727272727272727, - "Filipino,Chinese,Indonesian": 0.25, - "Filipino,Chinese,Vietnamese": 0.2556818181818182, - "Filipino,Indonesian,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Chinese": 0.2727272727272727, - "Spanish,Malay,Indonesian": 0.2727272727272727, - "Spanish,Malay,Vietnamese": 0.22727272727272727, - "Spanish,Chinese,Indonesian": 0.3068181818181818, - "Spanish,Chinese,Vietnamese": 0.3068181818181818, - "Spanish,Indonesian,Vietnamese": 0.26136363636363635, - "Malay,Chinese,Indonesian": 0.2556818181818182, - "Malay,Chinese,Vietnamese": 0.19886363636363635, - "Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Chinese,Indonesian,Vietnamese": 0.23863636363636365 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.19318181818181818, - "English,Filipino,Spanish,Chinese": 0.26704545454545453, - "English,Filipino,Spanish,Indonesian": 0.1875, - "English,Filipino,Spanish,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese": 0.19886363636363635, - "English,Filipino,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Indonesian": 0.19318181818181818, - "English,Filipino,Chinese,Vietnamese": 0.19886363636363635, - "English,Filipino,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Chinese": 0.22727272727272727, - "English,Spanish,Malay,Indonesian": 0.17613636363636365, - "English,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Spanish,Chinese,Indonesian": 0.25, - "English,Spanish,Chinese,Vietnamese": 0.24431818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "English,Malay,Chinese,Indonesian": 0.19318181818181818, - "English,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Malay,Indonesian,Vietnamese": 0.125, - "English,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Malay,Chinese": 0.20454545454545456, - "Filipino,Spanish,Malay,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "Filipino,Malay,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Spanish,Malay,Chinese,Indonesian": 0.17613636363636365, - "Spanish,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.1875, - "Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - } - }, - "AC3_2": 0.41709109242672887, - "AC3_3": 0.3172635363152596, - "AC3_4": 0.2464732893981078, - "AC3_5": 0.19869924375544673, - "AC3_6": 0.16698518868991685, - "AC3_7": 0.1465869770055592 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.5533980582524272 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.5333333333333333 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.48598130841121495 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.36, - "category_acc": { - "brand": 0.5, - "demographics": 0.0, - "biology": 0.4, - "history": 0.3333333333333333, - "literature": 0.2, - "politics": 0.5, - "culture": 0.4, - "film": 0.2, - "law": 0.3, - "geography": 0.6 - } - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.24969772942264867 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "indommlu": { - "prompt_1": -1 + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.15866609653199967 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.05360963983497839 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.2027770960151152 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.14042845422999728 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.485414235705951 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.47765462996067215, - "category_acc": { - "high_school_european_history": 0.6646341463414634, - "business_ethics": 0.5858585858585859, - "clinical_knowledge": 0.5075757575757576, - "medical_genetics": 0.5252525252525253, - "high_school_us_history": 0.6305418719211823, - "high_school_physics": 0.3333333333333333, - "high_school_world_history": 0.673728813559322, - "virology": 0.44242424242424244, - "high_school_microeconomics": 0.48945147679324896, - "econometrics": 0.20353982300884957, - "college_computer_science": 0.36363636363636365, - "high_school_biology": 0.598705501618123, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.36298932384341637, - "philosophy": 0.535483870967742, - "professional_medicine": 0.41697416974169743, - "nutrition": 0.5180327868852459, - "global_facts": 0.35353535353535354, - "machine_learning": 0.42342342342342343, - "security_studies": 0.5942622950819673, - "public_relations": 0.5321100917431193, - "professional_psychology": 0.46317512274959083, - "prehistory": 0.47678018575851394, - "anatomy": 0.4925373134328358, - "human_sexuality": 0.5692307692307692, - "college_medicine": 0.45348837209302323, - "high_school_government_and_politics": 0.6354166666666666, - "college_chemistry": 0.41414141414141414, - "logical_fallacies": 0.5864197530864198, - "high_school_geography": 0.6142131979695431, - "elementary_mathematics": 0.3660477453580902, - "human_aging": 0.545045045045045, - "college_mathematics": 0.20202020202020202, - "high_school_psychology": 0.6727941176470589, - "formal_logic": 0.264, - "high_school_statistics": 0.3953488372093023, - "international_law": 0.5833333333333334, - "high_school_mathematics": 0.30111524163568776, - "high_school_computer_science": 0.41414141414141414, - "conceptual_physics": 0.4188034188034188, - "miscellaneous": 0.6624040920716112, - "high_school_chemistry": 0.3910891089108911, - "marketing": 0.7553648068669528, - "professional_law": 0.3496412263535551, - "management": 0.6372549019607843, - "college_physics": 0.2871287128712871, - "jurisprudence": 0.5700934579439252, - "world_religions": 0.5941176470588235, - "sociology": 0.69, - "us_foreign_policy": 0.7070707070707071, - "high_school_macroeconomics": 0.4704370179948586, - "computer_security": 0.5454545454545454, - "moral_scenarios": 0.24496644295302014, - "moral_disputes": 0.5072463768115942, - "electrical_engineering": 0.4375, - "astronomy": 0.5364238410596026, - "college_biology": 0.46153846153846156 - } - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.4962852897473997 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.5049813200498132, - "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.5, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.47619047619047616, - "college_physics": 0.375, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.5217391304347826, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.08695652173913043, - "high_school_physics": 0.5416666666666666, - "high_school_chemistry": 0.5, - "high_school_biology": 0.5416666666666666, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.6923076923076923, - "middle_school_physics": 0.6666666666666666, - "middle_school_chemistry": 0.84, - "veterinary_medicine": 0.4642857142857143, - "college_economics": 0.3, - "business_administration": 0.42105263157894735, - "marxism": 0.6666666666666666, - "mao_zedong_thought": 0.8275862068965517, - "education_science": 0.6470588235294118, - "teacher_qualification": 0.8163265306122449, - "high_school_politics": 0.75, - "high_school_geography": 0.4583333333333333, - "middle_school_politics": 0.8076923076923077, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.6428571428571429, - "ideological_and_moral_cultivation": 0.75, - "logic": 0.4074074074074074, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.5714285714285714, - "art_studies": 0.5, - "professional_tour_guide": 0.5588235294117647, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.72, - "middle_school_history": 0.7037037037037037, - "civil_servant": 0.4807692307692308, - "sports_science": 0.375, - "plant_protection": 0.37037037037037035, - "basic_medicine": 0.375, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.5882352941176471, - "accountant": 0.46296296296296297, - "fire_engineer": 0.5277777777777778, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.48148148148148145, - "physician": 0.5185185185185185 - } - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.4731182795698925 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.5147642894146088, - "category_acc": { - "agronomy": 0.5088757396449705, - "anatomy": 0.40540540540540543, - "ancient_chinese": 0.3048780487804878, - "arts": 0.65625, - "astronomy": 0.38181818181818183, - "business_ethics": 0.5406698564593302, - "chinese_civil_service_exam": 0.5625, - "chinese_driving_rule": 0.7175572519083969, - "chinese_food_culture": 0.4338235294117647, - "chinese_foreign_policy": 0.6261682242990654, - "chinese_history": 0.6718266253869969, - "chinese_literature": 0.44607843137254904, - "chinese_teacher_qualification": 0.6703910614525139, - "clinical_knowledge": 0.4219409282700422, - "college_actuarial_science": 0.22641509433962265, - "college_education": 0.7009345794392523, - "college_engineering_hydrology": 0.42452830188679247, - "college_law": 0.3888888888888889, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.44339622641509435, - "college_medicine": 0.43956043956043955, - "computer_science": 0.45098039215686275, - "computer_security": 0.6081871345029239, - "conceptual_physics": 0.5918367346938775, - "construction_project_management": 0.38848920863309355, - "economics": 0.4716981132075472, - "education": 0.588957055214724, - "electrical_engineering": 0.436046511627907, - "elementary_chinese": 0.4642857142857143, - "elementary_commonsense": 0.5303030303030303, - "elementary_information_and_technology": 0.6932773109243697, - "elementary_mathematics": 0.3521739130434783, - "ethnology": 0.4962962962962963, - "food_science": 0.5524475524475524, - "genetics": 0.4090909090909091, - "global_facts": 0.5503355704697986, - "high_school_biology": 0.5207100591715976, - "high_school_chemistry": 0.4318181818181818, - "high_school_geography": 0.4830508474576271, - "high_school_mathematics": 0.3231707317073171, - "high_school_physics": 0.4090909090909091, - "high_school_politics": 0.6293706293706294, - "human_sexuality": 0.49206349206349204, - "international_law": 0.42162162162162165, - "journalism": 0.5465116279069767, - "jurisprudence": 0.5474452554744526, - "legal_and_moral_basis": 0.8644859813084113, - "logical": 0.5040650406504065, - "machine_learning": 0.45081967213114754, - "management": 0.6428571428571429, - "marketing": 0.6055555555555555, - "marxist_theory": 0.6455026455026455, - "modern_chinese": 0.31896551724137934, - "nutrition": 0.5517241379310345, - "philosophy": 0.6095238095238096, - "professional_accounting": 0.5885714285714285, - "professional_law": 0.3696682464454976, - "professional_medicine": 0.38563829787234044, - "professional_psychology": 0.6206896551724138, - "public_relations": 0.5057471264367817, - "security_study": 0.5925925925925926, - "sociology": 0.5530973451327433, - "sports_science": 0.5393939393939394, - "traditional_chinese_medicine": 0.4648648648648649, - "virology": 0.47337278106508873, - "world_history": 0.6708074534161491, - "world_religions": 0.575 - } - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.3333333333333333 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.3840909090909091 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.39559322033898303 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.7722513089005235 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.773640372366487 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.4260141130393391, - "rouge2": 0.18166009337951242, - "rougeL": 0.3322785533926477, - "avg_rouge": 0.3133175866038331 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.3755416873674651, - "rouge2": 0.13946795529199787, - "rougeL": 0.29193455454618034, - "avg_rouge": 0.2689813990685478 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.8497706422018348 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.6749760306807286 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.5755 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.4465 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.6125 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.4507042253521127 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.5703971119133574 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.6176470588235294 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 } - } - }, - "sailor_0_5b_chat": { - "model_size": "0.5B", - "model_link": "https://huggingface.co/sail/Sailor-0.5B-Chat", - "zero_shot": { + }, + "five_shot": { "cross_xquad": { + "prompt_1": -1 + }, + "cross_mmlu": { "prompt_1": { - "overall_acc": 0.3403361344537815, + "overall_acc": 0.26, "language_acc": { - "English": 0.3663865546218487, - "Chinese": 0.3277310924369748, - "Spanish": 0.32941176470588235, - "Vietnamese": 0.3378151260504202 + "Filipino": 0.20666666666666667, + "Vietnamese": 0.26666666666666666, + "Chinese": 0.30666666666666664, + "Spanish": 0.25333333333333335, + "Malay": 0.26, + "Indonesian": 0.28, + "English": 0.24666666666666667 }, - "consistency_score_2": 0.3952380952380952, - "consistency_score_3": 0.18823529411764706, - "consistency_score_4": 0.10084033613445378, + "consistency_score_2": 0.3946031746031746, + "consistency_score_3": 0.18895238095238098, + "consistency_score_4": 0.10114285714285713, + "consistency_score_5": 0.056507936507936514, + "consistency_score_6": 0.030476190476190473, + "consistency_score_7": 0.013333333333333334, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.3949579831932773, - "English,Spanish": 0.4394957983193277, - "English,Vietnamese": 0.4008403361344538, - "Chinese,Spanish": 0.3495798319327731, - "Chinese,Vietnamese": 0.3226890756302521, - "Spanish,Vietnamese": 0.4638655462184874 + "Filipino,Vietnamese": 0.36, + "Filipino,Chinese": 0.31333333333333335, + "Filipino,Spanish": 0.3333333333333333, + "Filipino,Malay": 0.4066666666666667, + "Filipino,Indonesian": 0.41333333333333333, + "Filipino,English": 0.34, + "Vietnamese,Chinese": 0.5, + "Vietnamese,Spanish": 0.37333333333333335, + "Vietnamese,Malay": 0.36666666666666664, + "Vietnamese,Indonesian": 0.42, + "Vietnamese,English": 0.44, + "Chinese,Spanish": 0.32666666666666666, + "Chinese,Malay": 0.3933333333333333, + "Chinese,Indonesian": 0.44666666666666666, + "Chinese,English": 0.44666666666666666, + "Spanish,Malay": 0.34, + "Spanish,Indonesian": 0.35333333333333333, + "Spanish,English": 0.3466666666666667, + "Malay,Indonesian": 0.4666666666666667, + "Malay,English": 0.4666666666666667, + "Indonesian,English": 0.43333333333333335 }, "3_combine": { - "English,Chinese,Spanish": 0.18571428571428572, - "English,Chinese,Vietnamese": 0.16302521008403362, - "English,Spanish,Vietnamese": 0.23361344537815126, - "Chinese,Spanish,Vietnamese": 0.17058823529411765 + "Filipino,Vietnamese,Chinese": 0.18666666666666668, + "Filipino,Vietnamese,Spanish": 0.13333333333333333, + "Filipino,Vietnamese,Malay": 0.19333333333333333, + "Filipino,Vietnamese,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,English": 0.16666666666666666, + "Filipino,Chinese,Spanish": 0.09333333333333334, + "Filipino,Chinese,Malay": 0.16666666666666666, + "Filipino,Chinese,Indonesian": 0.18, + "Filipino,Chinese,English": 0.13333333333333333, + "Filipino,Spanish,Malay": 0.15333333333333332, + "Filipino,Spanish,Indonesian": 0.15333333333333332, + "Filipino,Spanish,English": 0.12, + "Filipino,Malay,Indonesian": 0.22, + "Filipino,Malay,English": 0.18666666666666668, + "Filipino,Indonesian,English": 0.18, + "Vietnamese,Chinese,Spanish": 0.2, + "Vietnamese,Chinese,Malay": 0.24, + "Vietnamese,Chinese,Indonesian": 0.26, + "Vietnamese,Chinese,English": 0.2733333333333333, + "Vietnamese,Spanish,Malay": 0.16666666666666666, + "Vietnamese,Spanish,Indonesian": 0.18, + "Vietnamese,Spanish,English": 0.16, + "Vietnamese,Malay,Indonesian": 0.23333333333333334, + "Vietnamese,Malay,English": 0.22, + "Vietnamese,Indonesian,English": 0.24, + "Chinese,Spanish,Malay": 0.16, + "Chinese,Spanish,Indonesian": 0.16, + "Chinese,Spanish,English": 0.16, + "Chinese,Malay,Indonesian": 0.24666666666666667, + "Chinese,Malay,English": 0.23333333333333334, + "Chinese,Indonesian,English": 0.22666666666666666, + "Spanish,Malay,Indonesian": 0.18666666666666668, + "Spanish,Malay,English": 0.18, + "Spanish,Indonesian,English": 0.16666666666666666, + "Malay,Indonesian,English": 0.26666666666666666 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.10084033613445378 - } - }, - "AC3_2": 0.36573822212414425, - "AC3_3": 0.24240157107543453, - "AC3_4": 0.15558223285789197 - }, - "prompt_2": { - "overall_acc": 0.32941176470588235, - "language_acc": { - "English": 0.3495798319327731, - "Chinese": 0.292436974789916, - "Spanish": 0.3403361344537815, - "Vietnamese": 0.3352941176470588 - }, - "consistency_score_2": 0.4761904761904762, - "consistency_score_3": 0.2857142857142857, - "consistency_score_4": 0.1899159663865546, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.5361344537815126, - "English,Spanish": 0.5218487394957984, - "English,Vietnamese": 0.4504201680672269, - "Chinese,Spanish": 0.4823529411764706, - "Chinese,Vietnamese": 0.4, - "Spanish,Vietnamese": 0.46638655462184875 - }, - "3_combine": { - "English,Chinese,Spanish": 0.33361344537815124, - "English,Chinese,Vietnamese": 0.26974789915966385, - "English,Spanish,Vietnamese": 0.2915966386554622, - "Chinese,Spanish,Vietnamese": 0.24789915966386555 + "Filipino,Vietnamese,Chinese,Spanish": 0.06, + "Filipino,Vietnamese,Chinese,Malay": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,English": 0.1, + "Filipino,Vietnamese,Spanish,Malay": 0.08666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian": 0.08, + "Filipino,Vietnamese,Spanish,English": 0.06, + "Filipino,Vietnamese,Malay,Indonesian": 0.11333333333333333, + "Filipino,Vietnamese,Malay,English": 0.11333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.10666666666666667, + "Filipino,Chinese,Spanish,Malay": 0.06666666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.05333333333333334, + "Filipino,Chinese,Spanish,English": 0.04, + "Filipino,Chinese,Malay,Indonesian": 0.11333333333333333, + "Filipino,Chinese,Malay,English": 0.08666666666666667, + "Filipino,Chinese,Indonesian,English": 0.07333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.08, + "Filipino,Spanish,Malay,English": 0.08, + "Filipino,Spanish,Indonesian,English": 0.07333333333333333, + "Filipino,Malay,Indonesian,English": 0.12, + "Vietnamese,Chinese,Spanish,Malay": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.11333333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.15333333333333332, + "Vietnamese,Chinese,Malay,English": 0.15333333333333332, + "Vietnamese,Chinese,Indonesian,English": 0.14666666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.10666666666666667, + "Vietnamese,Spanish,Malay,English": 0.1, + "Vietnamese,Spanish,Indonesian,English": 0.09333333333333334, + "Vietnamese,Malay,Indonesian,English": 0.14666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, + "Chinese,Spanish,Malay,English": 0.10666666666666667, + "Chinese,Spanish,Indonesian,English": 0.08666666666666667, + "Chinese,Malay,Indonesian,English": 0.14, + "Spanish,Malay,Indonesian,English": 0.11333333333333333 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.05333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.03333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.03333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.06666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.06, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.04666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.05333333333333334, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.04, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.07333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.04, + "Filipino,Chinese,Spanish,Malay,English": 0.04, + "Filipino,Chinese,Spanish,Indonesian,English": 0.02, + "Filipino,Chinese,Malay,Indonesian,English": 0.05333333333333334, + "Filipino,Spanish,Malay,Indonesian,English": 0.05333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08666666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.08, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.06, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.08666666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.06666666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.02666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.03333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.013333333333333334, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.04, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.03333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.02, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.04666666666666667 }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.1899159663865546 + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.013333333333333334 } }, - "AC3_2": 0.38942976351216047, - "AC3_3": 0.30601092891200093, - "AC3_4": 0.24092899291001338 - }, - "prompt_3": { - "overall_acc": 0.32752100840336135, + "AC3_2": 0.31346265756608116, + "AC3_3": 0.21885447598010244, + "AC3_4": 0.14563291135207948, + "AC3_5": 0.0928385155173078, + "AC3_6": 0.05455737703039828, + "AC3_7": 0.0253658536492564 + } + }, + "cross_logiqa": { + "prompt_1": { + "overall_acc": 0.25487012987012986, "language_acc": { - "English": 0.3403361344537815, - "Chinese": 0.33277310924369746, - "Spanish": 0.31512605042016806, - "Vietnamese": 0.32184873949579834 + "Indonesian": 0.2556818181818182, + "English": 0.24431818181818182, + "Filipino": 0.23863636363636365, + "Spanish": 0.26704545454545453, + "Chinese": 0.26704545454545453, + "Malay": 0.2556818181818182, + "Vietnamese": 0.2556818181818182 }, - "consistency_score_2": 0.4327731092436975, - "consistency_score_3": 0.22584033613445376, - "consistency_score_4": 0.1319327731092437, + "consistency_score_2": 0.5722402597402597, + "consistency_score_3": 0.39074675324675323, + "consistency_score_4": 0.27516233766233766, + "consistency_score_5": 0.19318181818181818, + "consistency_score_6": 0.13392857142857145, + "consistency_score_7": 0.09090909090909091, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.46554621848739497, - "English,Spanish": 0.4831932773109244, - "English,Vietnamese": 0.4336134453781513, - "Chinese,Spanish": 0.3865546218487395, - "Chinese,Vietnamese": 0.3546218487394958, - "Spanish,Vietnamese": 0.473109243697479 + "Indonesian,English": 0.7045454545454546, + "Indonesian,Filipino": 0.4034090909090909, + "Indonesian,Spanish": 0.5454545454545454, + "Indonesian,Chinese": 0.625, + "Indonesian,Malay": 0.6534090909090909, + "Indonesian,Vietnamese": 0.6704545454545454, + "English,Filipino": 0.4602272727272727, + "English,Spanish": 0.5795454545454546, + "English,Chinese": 0.6534090909090909, + "English,Malay": 0.7329545454545454, + "English,Vietnamese": 0.75, + "Filipino,Spanish": 0.4318181818181818, + "Filipino,Chinese": 0.4090909090909091, + "Filipino,Malay": 0.39204545454545453, + "Filipino,Vietnamese": 0.4090909090909091, + "Spanish,Chinese": 0.5056818181818182, + "Spanish,Malay": 0.5113636363636364, + "Spanish,Vietnamese": 0.5568181818181818, + "Chinese,Malay": 0.625, + "Chinese,Vietnamese": 0.6704545454545454, + "Malay,Vietnamese": 0.7272727272727273 }, "3_combine": { - "English,Chinese,Spanish": 0.23865546218487396, - "English,Chinese,Vietnamese": 0.2042016806722689, - "English,Spanish,Vietnamese": 0.2647058823529412, - "Chinese,Spanish,Vietnamese": 0.1957983193277311 + "Indonesian,English,Filipino": 0.3181818181818182, + "Indonesian,English,Spanish": 0.4318181818181818, + "Indonesian,English,Chinese": 0.5170454545454546, + "Indonesian,English,Malay": 0.5568181818181818, + "Indonesian,English,Vietnamese": 0.5681818181818182, + "Indonesian,Filipino,Spanish": 0.24431818181818182, + "Indonesian,Filipino,Chinese": 0.2784090909090909, + "Indonesian,Filipino,Malay": 0.2727272727272727, + "Indonesian,Filipino,Vietnamese": 0.2897727272727273, + "Indonesian,Spanish,Chinese": 0.3693181818181818, + "Indonesian,Spanish,Malay": 0.3806818181818182, + "Indonesian,Spanish,Vietnamese": 0.4147727272727273, + "Indonesian,Chinese,Malay": 0.4772727272727273, + "Indonesian,Chinese,Vietnamese": 0.5, + "Indonesian,Malay,Vietnamese": 0.5340909090909091, + "English,Filipino,Spanish": 0.2840909090909091, + "English,Filipino,Chinese": 0.30113636363636365, + "English,Filipino,Malay": 0.3352272727272727, + "English,Filipino,Vietnamese": 0.3409090909090909, + "English,Spanish,Chinese": 0.39204545454545453, + "English,Spanish,Malay": 0.42613636363636365, + "English,Spanish,Vietnamese": 0.4602272727272727, + "English,Chinese,Malay": 0.5284090909090909, + "English,Chinese,Vietnamese": 0.5511363636363636, + "English,Malay,Vietnamese": 0.6136363636363636, + "Filipino,Spanish,Chinese": 0.23863636363636365, + "Filipino,Spanish,Malay": 0.22727272727272727, + "Filipino,Spanish,Vietnamese": 0.2556818181818182, + "Filipino,Chinese,Malay": 0.26704545454545453, + "Filipino,Chinese,Vietnamese": 0.30113636363636365, + "Filipino,Malay,Vietnamese": 0.30113636363636365, + "Spanish,Chinese,Malay": 0.35795454545454547, + "Spanish,Chinese,Vietnamese": 0.4034090909090909, + "Spanish,Malay,Vietnamese": 0.4090909090909091, + "Chinese,Malay,Vietnamese": 0.5284090909090909 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.1319327731092437 - } - }, - "AC3_2": 0.37286171717169564, - "AC3_3": 0.26733871220196415, - "AC3_4": 0.18809619865801921 - }, - "prompt_4": { - "overall_acc": 0.31953781512605045, - "language_acc": { - "English": 0.3411764705882353, - "Chinese": 0.32184873949579834, - "Spanish": 0.3092436974789916, - "Vietnamese": 0.3058823529411765 - }, - "consistency_score_2": 0.42478991596638654, - "consistency_score_3": 0.21974789915966386, - "consistency_score_4": 0.12436974789915967, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.4689075630252101, - "English,Spanish": 0.44789915966386556, - "English,Vietnamese": 0.3949579831932773, - "Chinese,Spanish": 0.41092436974789914, - "Chinese,Vietnamese": 0.3865546218487395, - "Spanish,Vietnamese": 0.4394957983193277 - }, - "3_combine": { - "English,Chinese,Spanish": 0.24285714285714285, - "English,Chinese,Vietnamese": 0.20336134453781513, - "English,Spanish,Vietnamese": 0.22268907563025211, - "Chinese,Spanish,Vietnamese": 0.21008403361344538 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.12436974789915967 - } - }, - "AC3_2": 0.36472224786794816, - "AC3_3": 0.2604102489753573, - "AC3_4": 0.17905005822244008 - }, - "prompt_5": { - "overall_acc": 0.33466386554621846, - "language_acc": { - "English": 0.326890756302521, - "Chinese": 0.3403361344537815, - "Spanish": 0.3420168067226891, - "Vietnamese": 0.32941176470588235 - }, - "consistency_score_2": 0.43067226890756305, - "consistency_score_3": 0.22983193277310926, - "consistency_score_4": 0.1403361344537815, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.43529411764705883, - "English,Spanish": 0.42773109243697477, - "English,Vietnamese": 0.39663865546218485, - "Chinese,Spanish": 0.43445378151260505, - "Chinese,Vietnamese": 0.4092436974789916, - "Spanish,Vietnamese": 0.48067226890756304 - }, - "3_combine": { - "English,Chinese,Spanish": 0.23697478991596638, - "English,Chinese,Vietnamese": 0.20672268907563024, - "English,Spanish,Vietnamese": 0.23277310924369748, - "Chinese,Spanish,Vietnamese": 0.24285714285714285 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.1403361344537815 - } - }, - "AC3_2": 0.37664612916832624, - "AC3_3": 0.2725137840288089, - "AC3_4": 0.19774919251464806 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.24761904761904763, - "language_acc": { - "Spanish": 0.26, - "English": 0.22666666666666666, - "Chinese": 0.26666666666666666, - "Vietnamese": 0.2733333333333333, - "Filipino": 0.24666666666666667, - "Indonesian": 0.24, - "Malay": 0.22 - }, - "consistency_score_2": 0.5384126984126982, - "consistency_score_3": 0.34780952380952385, - "consistency_score_4": 0.24533333333333332, - "consistency_score_5": 0.18222222222222223, - "consistency_score_6": 0.13904761904761906, - "consistency_score_7": 0.10666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.41333333333333333, - "Spanish,Chinese": 0.5533333333333333, - "Spanish,Vietnamese": 0.5066666666666667, - "Spanish,Filipino": 0.62, - "Spanish,Indonesian": 0.56, - "Spanish,Malay": 0.5266666666666666, - "English,Chinese": 0.49333333333333335, - "English,Vietnamese": 0.48, - "English,Filipino": 0.47333333333333333, - "English,Indonesian": 0.4533333333333333, - "English,Malay": 0.48, - "Chinese,Vietnamese": 0.44666666666666666, - "Chinese,Filipino": 0.5666666666666667, - "Chinese,Indonesian": 0.49333333333333335, - "Chinese,Malay": 0.5066666666666667, - "Vietnamese,Filipino": 0.5466666666666666, - "Vietnamese,Indonesian": 0.5933333333333334, - "Vietnamese,Malay": 0.6266666666666667, - "Filipino,Indonesian": 0.62, - "Filipino,Malay": 0.6466666666666666, - "Indonesian,Malay": 0.7 - }, - "3_combine": { - "Spanish,English,Chinese": 0.2733333333333333, - "Spanish,English,Vietnamese": 0.24, - "Spanish,English,Filipino": 0.2866666666666667, - "Spanish,English,Indonesian": 0.2733333333333333, - "Spanish,English,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese": 0.29333333333333333, - "Spanish,Chinese,Filipino": 0.4066666666666667, - "Spanish,Chinese,Indonesian": 0.34, - "Spanish,Chinese,Malay": 0.35333333333333333, - "Spanish,Vietnamese,Filipino": 0.36666666666666664, - "Spanish,Vietnamese,Indonesian": 0.37333333333333335, - "Spanish,Vietnamese,Malay": 0.37333333333333335, - "Spanish,Filipino,Indonesian": 0.42, - "Spanish,Filipino,Malay": 0.42, - "Spanish,Indonesian,Malay": 0.42, - "English,Chinese,Vietnamese": 0.26, - "English,Chinese,Filipino": 0.32, - "English,Chinese,Indonesian": 0.2733333333333333, - "English,Chinese,Malay": 0.29333333333333333, - "English,Vietnamese,Filipino": 0.29333333333333333, - "English,Vietnamese,Indonesian": 0.30666666666666664, - "English,Vietnamese,Malay": 0.3333333333333333, - "English,Filipino,Indonesian": 0.31333333333333335, - "English,Filipino,Malay": 0.34, - "English,Indonesian,Malay": 0.35333333333333333, - "Chinese,Vietnamese,Filipino": 0.3333333333333333, - "Chinese,Vietnamese,Indonesian": 0.32666666666666666, - "Chinese,Vietnamese,Malay": 0.3466666666666667, - "Chinese,Filipino,Indonesian": 0.37333333333333335, - "Chinese,Filipino,Malay": 0.4, - "Chinese,Indonesian,Malay": 0.38666666666666666, - "Vietnamese,Filipino,Indonesian": 0.4066666666666667, - "Vietnamese,Filipino,Malay": 0.44, - "Vietnamese,Indonesian,Malay": 0.48, - "Filipino,Indonesian,Malay": 0.5 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.16, - "Spanish,English,Chinese,Filipino": 0.21333333333333335, - "Spanish,English,Chinese,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino": 0.18, - "Spanish,English,Vietnamese,Indonesian": 0.18, - "Spanish,English,Vietnamese,Malay": 0.16666666666666666, - "Spanish,English,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Filipino,Malay": 0.2, - "Spanish,English,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.24666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.28, - "Spanish,Chinese,Filipino,Malay": 0.3, - "Spanish,Chinese,Indonesian,Malay": 0.28, - "Spanish,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,Vietnamese,Filipino,Malay": 0.32, - "Spanish,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Spanish,Filipino,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.2, - "English,Chinese,Vietnamese,Malay": 0.2, - "English,Chinese,Filipino,Indonesian": 0.22666666666666666, - "English,Chinese,Filipino,Malay": 0.23333333333333334, - "English,Chinese,Indonesian,Malay": 0.23333333333333334, - "English,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "English,Vietnamese,Filipino,Malay": 0.24, - "English,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "English,Filipino,Indonesian,Malay": 0.2733333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian,Malay": 0.36 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16, - "Spanish,English,Chinese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.2, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - } - }, - "AC3_2": 0.3392260942717905, - "AC3_3": 0.28928495257609965, - "AC3_4": 0.24647089124314897, - "AC3_5": 0.20994583943909434, - "AC3_6": 0.17809054651739506, - "AC3_7": 0.14910394261024396 - }, - "prompt_2": { - "overall_acc": 0.260952380952381, - "language_acc": { - "Spanish": 0.21333333333333335, - "English": 0.32, - "Chinese": 0.26, - "Vietnamese": 0.26666666666666666, - "Filipino": 0.25333333333333335, - "Indonesian": 0.28, - "Malay": 0.23333333333333334 - }, - "consistency_score_2": 0.5273015873015874, - "consistency_score_3": 0.3259047619047619, - "consistency_score_4": 0.2196190476190476, - "consistency_score_5": 0.15619047619047619, - "consistency_score_6": 0.11523809523809524, - "consistency_score_7": 0.08666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5066666666666667, - "Spanish,Chinese": 0.44666666666666666, - "Spanish,Vietnamese": 0.48, - "Spanish,Filipino": 0.5666666666666667, - "Spanish,Indonesian": 0.6266666666666667, - "Spanish,Malay": 0.6133333333333333, - "English,Chinese": 0.6, - "English,Vietnamese": 0.44, - "English,Filipino": 0.46, - "English,Indonesian": 0.4866666666666667, - "English,Malay": 0.46, - "Chinese,Vietnamese": 0.43333333333333335, - "Chinese,Filipino": 0.4, - "Chinese,Indonesian": 0.44, - "Chinese,Malay": 0.5, - "Vietnamese,Filipino": 0.54, - "Vietnamese,Indonesian": 0.5666666666666667, - "Vietnamese,Malay": 0.5266666666666666, - "Filipino,Indonesian": 0.6733333333333333, - "Filipino,Malay": 0.6266666666666667, - "Indonesian,Malay": 0.68 - }, - "3_combine": { - "Spanish,English,Chinese": 0.32, - "Spanish,English,Vietnamese": 0.28, - "Spanish,English,Filipino": 0.30666666666666664, - "Spanish,English,Indonesian": 0.34, - "Spanish,English,Malay": 0.3333333333333333, - "Spanish,Chinese,Vietnamese": 0.23333333333333334, - "Spanish,Chinese,Filipino": 0.24666666666666667, - "Spanish,Chinese,Indonesian": 0.28, - "Spanish,Chinese,Malay": 0.32666666666666666, - "Spanish,Vietnamese,Filipino": 0.32, - "Spanish,Vietnamese,Indonesian": 0.35333333333333333, - "Spanish,Vietnamese,Malay": 0.32666666666666666, - "Spanish,Filipino,Indonesian": 0.4533333333333333, - "Spanish,Filipino,Malay": 0.4266666666666667, - "Spanish,Indonesian,Malay": 0.47333333333333333, - "English,Chinese,Vietnamese": 0.26666666666666666, - "English,Chinese,Filipino": 0.26, - "English,Chinese,Indonesian": 0.3, - "English,Chinese,Malay": 0.31333333333333335, - "English,Vietnamese,Filipino": 0.26666666666666666, - "English,Vietnamese,Indonesian": 0.3, - "English,Vietnamese,Malay": 0.25333333333333335, - "English,Filipino,Indonesian": 0.3466666666666667, - "English,Filipino,Malay": 0.31333333333333335, - "English,Indonesian,Malay": 0.34, - "Chinese,Vietnamese,Filipino": 0.24, - "Chinese,Vietnamese,Indonesian": 0.28, - "Chinese,Vietnamese,Malay": 0.28, - "Chinese,Filipino,Indonesian": 0.3, - "Chinese,Filipino,Malay": 0.3, - "Chinese,Indonesian,Malay": 0.34, - "Vietnamese,Filipino,Indonesian": 0.4066666666666667, - "Vietnamese,Filipino,Malay": 0.36666666666666664, - "Vietnamese,Indonesian,Malay": 0.4, - "Filipino,Indonesian,Malay": 0.5133333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.17333333333333334, - "Spanish,English,Chinese,Filipino": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian": 0.22, - "Spanish,English,Chinese,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Malay": 0.18666666666666668, - "Spanish,English,Filipino,Indonesian": 0.24, - "Spanish,English,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino": 0.16, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Malay": 0.18666666666666668, - "Spanish,Chinese,Filipino,Indonesian": 0.2, - "Spanish,Chinese,Filipino,Malay": 0.22, - "Spanish,Chinese,Indonesian,Malay": 0.24, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.28, - "Spanish,Filipino,Indonesian,Malay": 0.38, - "English,Chinese,Vietnamese,Filipino": 0.16, - "English,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "English,Chinese,Vietnamese,Malay": 0.16666666666666666, - "English,Chinese,Filipino,Indonesian": 0.20666666666666667, - "English,Chinese,Filipino,Malay": 0.19333333333333333, - "English,Chinese,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.22, - "English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "English,Filipino,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.18, - "Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Chinese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Vietnamese,Filipino,Indonesian,Malay": 0.31333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.14, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 - } - }, - "AC3_2": 0.34912759138696514, - "AC3_3": 0.28983416016451957, - "AC3_4": 0.2385082006940046, - "AC3_5": 0.19541639482160328, - "AC3_6": 0.15987462322453, - "AC3_7": 0.1301187214237558 - }, - "prompt_3": { - "overall_acc": 0.23904761904761904, - "language_acc": { - "Spanish": 0.2, - "English": 0.2866666666666667, - "Chinese": 0.24, - "Vietnamese": 0.22666666666666666, - "Filipino": 0.2, - "Indonesian": 0.28, - "Malay": 0.24 - }, - "consistency_score_2": 0.5285714285714287, - "consistency_score_3": 0.3323809523809524, - "consistency_score_4": 0.2316190476190477, - "consistency_score_5": 0.1746031746031746, - "consistency_score_6": 0.13904761904761903, - "consistency_score_7": 0.11333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.58, - "Spanish,Chinese": 0.5133333333333333, - "Spanish,Vietnamese": 0.5733333333333334, - "Spanish,Filipino": 0.6, - "Spanish,Indonesian": 0.5933333333333334, - "Spanish,Malay": 0.62, - "English,Chinese": 0.5466666666666666, - "English,Vietnamese": 0.44, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.4533333333333333, - "English,Malay": 0.44666666666666666, - "Chinese,Vietnamese": 0.4066666666666667, - "Chinese,Filipino": 0.47333333333333333, - "Chinese,Indonesian": 0.4866666666666667, - "Chinese,Malay": 0.4, - "Vietnamese,Filipino": 0.5466666666666666, - "Vietnamese,Indonesian": 0.5466666666666666, - "Vietnamese,Malay": 0.5333333333333333, - "Filipino,Indonesian": 0.5933333333333334, - "Filipino,Malay": 0.6066666666666667, - "Indonesian,Malay": 0.6733333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.36, - "Spanish,English,Vietnamese": 0.3333333333333333, - "Spanish,English,Filipino": 0.3466666666666667, - "Spanish,English,Indonesian": 0.35333333333333333, - "Spanish,English,Malay": 0.36, - "Spanish,Chinese,Vietnamese": 0.29333333333333333, - "Spanish,Chinese,Filipino": 0.32666666666666666, - "Spanish,Chinese,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino": 0.3933333333333333, - "Spanish,Vietnamese,Indonesian": 0.3933333333333333, - "Spanish,Vietnamese,Malay": 0.3933333333333333, - "Spanish,Filipino,Indonesian": 0.42, - "Spanish,Filipino,Malay": 0.44, - "Spanish,Indonesian,Malay": 0.4666666666666667, - "English,Chinese,Vietnamese": 0.26, - "English,Chinese,Filipino": 0.2866666666666667, - "English,Chinese,Indonesian": 0.3, - "English,Chinese,Malay": 0.24, - "English,Vietnamese,Filipino": 0.2733333333333333, - "English,Vietnamese,Indonesian": 0.2733333333333333, - "English,Vietnamese,Malay": 0.26, - "English,Filipino,Indonesian": 0.32, - "English,Filipino,Malay": 0.30666666666666664, - "English,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino": 0.26, - "Chinese,Vietnamese,Indonesian": 0.26, - "Chinese,Vietnamese,Malay": 0.21333333333333335, - "Chinese,Filipino,Indonesian": 0.31333333333333335, - "Chinese,Filipino,Malay": 0.28, - "Chinese,Indonesian,Malay": 0.31333333333333335, - "Vietnamese,Filipino,Indonesian": 0.3933333333333333, - "Vietnamese,Filipino,Malay": 0.36666666666666664, - "Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Filipino,Indonesian,Malay": 0.4533333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.22, - "Spanish,English,Chinese,Filipino": 0.21333333333333335, - "Spanish,English,Chinese,Indonesian": 0.24, - "Spanish,English,Chinese,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.24, - "Spanish,English,Vietnamese,Malay": 0.22666666666666666, - "Spanish,English,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.22, - "Spanish,Chinese,Vietnamese,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.23333333333333334, - "Spanish,Chinese,Indonesian,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "English,Chinese,Vietnamese,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian": 0.20666666666666667, - "English,Chinese,Filipino,Malay": 0.16666666666666666, - "English,Chinese,Indonesian,Malay": 0.2, - "English,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Vietnamese,Filipino,Malay": 0.20666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "English,Filipino,Indonesian,Malay": 0.26, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2, - "Chinese,Vietnamese,Filipino,Malay": 0.16, - "Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.16, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.16, - "Spanish,English,Chinese,Vietnamese,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - } - }, - "AC3_2": 0.329209500134355, - "AC3_3": 0.2780920634433974, - "AC3_4": 0.23527471040076503, - "AC3_5": 0.201805357550892, - "AC3_6": 0.1758234376409151, - "AC3_7": 0.15376576572212952 - }, - "prompt_4": { - "overall_acc": 0.24571428571428572, - "language_acc": { - "Spanish": 0.26, - "English": 0.24, - "Chinese": 0.26666666666666666, - "Vietnamese": 0.25333333333333335, - "Filipino": 0.23333333333333334, - "Indonesian": 0.22666666666666666, - "Malay": 0.24 - }, - "consistency_score_2": 0.5482539682539682, - "consistency_score_3": 0.3506666666666667, - "consistency_score_4": 0.24533333333333332, - "consistency_score_5": 0.18253968253968259, - "consistency_score_6": 0.14190476190476192, - "consistency_score_7": 0.11333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6, - "Spanish,Chinese": 0.56, - "Spanish,Vietnamese": 0.6466666666666666, - "Spanish,Filipino": 0.5533333333333333, - "Spanish,Indonesian": 0.5733333333333334, - "Spanish,Malay": 0.5, - "English,Chinese": 0.6066666666666667, - "English,Vietnamese": 0.5, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.44666666666666666, - "English,Malay": 0.42, - "Chinese,Vietnamese": 0.5066666666666667, - "Chinese,Filipino": 0.5066666666666667, - "Chinese,Indonesian": 0.5133333333333333, - "Chinese,Malay": 0.5266666666666666, - "Vietnamese,Filipino": 0.56, - "Vietnamese,Indonesian": 0.5933333333333334, - "Vietnamese,Malay": 0.49333333333333335, - "Filipino,Indonesian": 0.62, - "Filipino,Malay": 0.6333333333333333, - "Indonesian,Malay": 0.6866666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.41333333333333333, - "Spanish,English,Vietnamese": 0.4, - "Spanish,English,Filipino": 0.36, - "Spanish,English,Indonesian": 0.3466666666666667, - "Spanish,English,Malay": 0.3, - "Spanish,Chinese,Vietnamese": 0.38666666666666666, - "Spanish,Chinese,Filipino": 0.3466666666666667, - "Spanish,Chinese,Indonesian": 0.3466666666666667, - "Spanish,Chinese,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino": 0.4, - "Spanish,Vietnamese,Indonesian": 0.42, - "Spanish,Vietnamese,Malay": 0.3466666666666667, - "Spanish,Filipino,Indonesian": 0.4, - "Spanish,Filipino,Malay": 0.36, - "Spanish,Indonesian,Malay": 0.4066666666666667, - "English,Chinese,Vietnamese": 0.32, - "English,Chinese,Filipino": 0.32, - "English,Chinese,Indonesian": 0.32, - "English,Chinese,Malay": 0.31333333333333335, - "English,Vietnamese,Filipino": 0.2866666666666667, - "English,Vietnamese,Indonesian": 0.29333333333333333, - "English,Vietnamese,Malay": 0.24, - "English,Filipino,Indonesian": 0.30666666666666664, - "English,Filipino,Malay": 0.3, - "English,Indonesian,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Filipino": 0.31333333333333335, - "Chinese,Vietnamese,Indonesian": 0.32666666666666666, - "Chinese,Vietnamese,Malay": 0.29333333333333333, - "Chinese,Filipino,Indonesian": 0.35333333333333333, - "Chinese,Filipino,Malay": 0.36666666666666664, - "Chinese,Indonesian,Malay": 0.38666666666666666, - "Vietnamese,Filipino,Indonesian": 0.41333333333333333, - "Vietnamese,Filipino,Malay": 0.36, - "Vietnamese,Indonesian,Malay": 0.41333333333333333, - "Filipino,Indonesian,Malay": 0.4866666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.28, - "Spanish,English,Chinese,Filipino": 0.26, - "Spanish,English,Chinese,Indonesian": 0.25333333333333335, - "Spanish,English,Chinese,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.24666666666666667, - "Spanish,English,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian": 0.26, - "Spanish,English,Filipino,Malay": 0.24, - "Spanish,English,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino": 0.24, - "Spanish,Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.21333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Chinese,Filipino,Malay": 0.25333333333333335, - "Spanish,Chinese,Indonesian,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Vietnamese,Filipino,Malay": 0.26, - "Spanish,Vietnamese,Indonesian,Malay": 0.3, - "Spanish,Filipino,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "English,Chinese,Vietnamese,Indonesian": 0.21333333333333335, - "English,Chinese,Vietnamese,Malay": 0.18666666666666668, - "English,Chinese,Filipino,Indonesian": 0.22666666666666666, - "English,Chinese,Filipino,Malay": 0.22666666666666666, - "English,Chinese,Indonesian,Malay": 0.24, - "English,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "English,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.24, - "Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.32 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,English,Chinese,Vietnamese,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian": 0.2, - "Spanish,English,Chinese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - } - }, - "AC3_2": 0.339343119852159, - "AC3_3": 0.2889556051894916, - "AC3_4": 0.24552366170329715, - "AC3_5": 0.20946733024866301, - "AC3_6": 0.17990873986232614, - "AC3_7": 0.15511936335202245 - }, - "prompt_5": { - "overall_acc": 0.24571428571428572, - "language_acc": { - "Spanish": 0.26666666666666666, - "English": 0.22, - "Chinese": 0.15333333333333332, - "Vietnamese": 0.22, - "Filipino": 0.24666666666666667, - "Indonesian": 0.30666666666666664, - "Malay": 0.30666666666666664 - }, - "consistency_score_2": 0.5546031746031745, - "consistency_score_3": 0.3620952380952381, - "consistency_score_4": 0.2567619047619048, - "consistency_score_5": 0.19365079365079368, - "consistency_score_6": 0.15333333333333332, - "consistency_score_7": 0.12666666666666668, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6266666666666667, - "Spanish,Chinese": 0.5466666666666666, - "Spanish,Vietnamese": 0.52, - "Spanish,Filipino": 0.5466666666666666, - "Spanish,Indonesian": 0.5666666666666667, - "Spanish,Malay": 0.58, - "English,Chinese": 0.52, - "English,Vietnamese": 0.5, - "English,Filipino": 0.5466666666666666, - "English,Indonesian": 0.5333333333333333, - "English,Malay": 0.52, - "Chinese,Vietnamese": 0.5, - "Chinese,Filipino": 0.5, - "Chinese,Indonesian": 0.4866666666666667, - "Chinese,Malay": 0.52, - "Vietnamese,Filipino": 0.5266666666666666, - "Vietnamese,Indonesian": 0.6066666666666667, - "Vietnamese,Malay": 0.6133333333333333, - "Filipino,Indonesian": 0.6, - "Filipino,Malay": 0.5933333333333334, - "Indonesian,Malay": 0.6933333333333334 - }, - "3_combine": { - "Spanish,English,Chinese": 0.38, - "Spanish,English,Vietnamese": 0.35333333333333333, - "Spanish,English,Filipino": 0.38666666666666666, - "Spanish,English,Indonesian": 0.4, - "Spanish,English,Malay": 0.3933333333333333, - "Spanish,Chinese,Vietnamese": 0.3333333333333333, - "Spanish,Chinese,Filipino": 0.32666666666666666, - "Spanish,Chinese,Indonesian": 0.3466666666666667, - "Spanish,Chinese,Malay": 0.36666666666666664, - "Spanish,Vietnamese,Filipino": 0.31333333333333335, - "Spanish,Vietnamese,Indonesian": 0.36666666666666664, - "Spanish,Vietnamese,Malay": 0.36666666666666664, - "Spanish,Filipino,Indonesian": 0.37333333333333335, - "Spanish,Filipino,Malay": 0.38, - "Spanish,Indonesian,Malay": 0.43333333333333335, - "English,Chinese,Vietnamese": 0.3, - "English,Chinese,Filipino": 0.32, - "English,Chinese,Indonesian": 0.3, - "English,Chinese,Malay": 0.30666666666666664, - "English,Vietnamese,Filipino": 0.30666666666666664, - "English,Vietnamese,Indonesian": 0.3466666666666667, - "English,Vietnamese,Malay": 0.3466666666666667, - "English,Filipino,Indonesian": 0.37333333333333335, - "English,Filipino,Malay": 0.37333333333333335, - "English,Indonesian,Malay": 0.4, - "Chinese,Vietnamese,Filipino": 0.3, - "Chinese,Vietnamese,Indonesian": 0.3466666666666667, - "Chinese,Vietnamese,Malay": 0.36, - "Chinese,Filipino,Indonesian": 0.34, - "Chinese,Filipino,Malay": 0.35333333333333333, - "Chinese,Indonesian,Malay": 0.38, - "Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "Vietnamese,Filipino,Malay": 0.38, - "Vietnamese,Indonesian,Malay": 0.47333333333333333, - "Filipino,Indonesian,Malay": 0.46 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.25333333333333335, - "Spanish,English,Chinese,Filipino": 0.24, - "Spanish,English,Chinese,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Malay": 0.26, - "Spanish,English,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,English,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,English,Vietnamese,Malay": 0.2733333333333333, - "Spanish,English,Filipino,Indonesian": 0.2733333333333333, - "Spanish,English,Filipino,Malay": 0.2866666666666667, - "Spanish,English,Indonesian,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.26, - "Spanish,Chinese,Indonesian,Malay": 0.2866666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.3, - "English,Chinese,Vietnamese,Filipino": 0.2, - "English,Chinese,Vietnamese,Indonesian": 0.22, - "English,Chinese,Vietnamese,Malay": 0.23333333333333334, - "English,Chinese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Filipino,Malay": 0.23333333333333334, - "English,Chinese,Indonesian,Malay": 0.23333333333333334, - "English,Vietnamese,Filipino,Indonesian": 0.24, - "English,Vietnamese,Filipino,Malay": 0.25333333333333335, - "English,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "English,Filipino,Indonesian,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.28, - "Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.2, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "English,Chinese,Vietnamese,Filipino,Malay": 0.18, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "English,Chinese,Filipino,Indonesian,Malay": 0.18, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - } - }, - "AC3_2": 0.34054966845634316, - "AC3_3": 0.29276268070573924, - "AC3_4": 0.25111664676037837, - "AC3_5": 0.2165978529647401, - "AC3_6": 0.188830548878694, - "AC3_7": 0.16716112527480328 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.26136363636363635, - "language_acc": { - "English": 0.29545454545454547, - "Filipino": 0.2215909090909091, - "Spanish": 0.2784090909090909, - "Malay": 0.23863636363636365, - "Chinese": 0.26136363636363635, - "Indonesian": 0.26704545454545453, - "Vietnamese": 0.26704545454545453 - }, - "consistency_score_2": 0.3831168831168831, - "consistency_score_3": 0.18831168831168832, - "consistency_score_4": 0.10503246753246752, - "consistency_score_5": 0.06493506493506493, - "consistency_score_6": 0.044642857142857144, - "consistency_score_7": 0.03409090909090909, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.42045454545454547, - "English,Spanish": 0.4659090909090909, - "English,Malay": 0.32954545454545453, - "English,Chinese": 0.3181818181818182, - "English,Indonesian": 0.3465909090909091, - "English,Vietnamese": 0.4147727272727273, - "Filipino,Spanish": 0.5568181818181818, - "Filipino,Malay": 0.4431818181818182, - "Filipino,Chinese": 0.24431818181818182, - "Filipino,Indonesian": 0.45454545454545453, - "Filipino,Vietnamese": 0.5113636363636364, - "Spanish,Malay": 0.3806818181818182, - "Spanish,Chinese": 0.24431818181818182, - "Spanish,Indonesian": 0.48863636363636365, - "Spanish,Vietnamese": 0.48295454545454547, - "Malay,Chinese": 0.19318181818181818, - "Malay,Indonesian": 0.45454545454545453, - "Malay,Vietnamese": 0.4147727272727273, - "Chinese,Indonesian": 0.18181818181818182, - "Chinese,Vietnamese": 0.21022727272727273, - "Indonesian,Vietnamese": 0.48863636363636365 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Malay": 0.1875, - "English,Filipino,Chinese": 0.14204545454545456, - "English,Filipino,Indonesian": 0.2159090909090909, - "English,Filipino,Vietnamese": 0.2556818181818182, - "English,Spanish,Malay": 0.19318181818181818, - "English,Spanish,Chinese": 0.125, - "English,Spanish,Indonesian": 0.24431818181818182, - "English,Spanish,Vietnamese": 0.2727272727272727, - "English,Malay,Chinese": 0.09090909090909091, - "English,Malay,Indonesian": 0.17045454545454544, - "English,Malay,Vietnamese": 0.17613636363636365, - "English,Chinese,Indonesian": 0.10227272727272728, - "English,Chinese,Vietnamese": 0.13636363636363635, - "English,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay": 0.25, - "Filipino,Spanish,Chinese": 0.14204545454545456, - "Filipino,Spanish,Indonesian": 0.30113636363636365, - "Filipino,Spanish,Vietnamese": 0.3181818181818182, - "Filipino,Malay,Chinese": 0.11931818181818182, - "Filipino,Malay,Indonesian": 0.23863636363636365, - "Filipino,Malay,Vietnamese": 0.26136363636363635, - "Filipino,Chinese,Indonesian": 0.125, - "Filipino,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Chinese": 0.09659090909090909, - "Spanish,Malay,Indonesian": 0.22727272727272727, - "Spanish,Malay,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Indonesian": 0.11363636363636363, - "Spanish,Chinese,Vietnamese": 0.10795454545454546, - "Spanish,Indonesian,Vietnamese": 0.3068181818181818, - "Malay,Chinese,Indonesian": 0.08522727272727272, - "Malay,Chinese,Vietnamese": 0.09659090909090909, - "Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Chinese,Indonesian,Vietnamese": 0.09659090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.125, - "English,Filipino,Spanish,Chinese": 0.09090909090909091, - "English,Filipino,Spanish,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Chinese": 0.07386363636363637, - "English,Filipino,Malay,Indonesian": 0.11363636363636363, - "English,Filipino,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Chinese": 0.056818181818181816, - "English,Spanish,Malay,Indonesian": 0.13068181818181818, - "English,Spanish,Malay,Vietnamese": 0.125, - "English,Spanish,Chinese,Indonesian": 0.07386363636363637, - "English,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "English,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "English,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Malay,Chinese,Vietnamese": 0.0625, - "English,Malay,Indonesian,Vietnamese": 0.125, - "English,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay,Chinese": 0.07954545454545454, - "Filipino,Spanish,Malay,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Malay,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Indonesian": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.0625, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Indonesian": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - } - }, - "AC3_2": 0.310739638147342, - "AC3_3": 0.21890383979378758, - "AC3_4": 0.14984694078964983, - "AC3_5": 0.10402532787404449, - "AC3_6": 0.07625994692468109, - "AC3_7": 0.06031468529427112 - }, - "prompt_2": { - "overall_acc": 0.2743506493506493, - "language_acc": { - "English": 0.26136363636363635, - "Filipino": 0.26704545454545453, - "Spanish": 0.30113636363636365, - "Malay": 0.2727272727272727, - "Chinese": 0.26136363636363635, - "Indonesian": 0.2556818181818182, - "Vietnamese": 0.30113636363636365 - }, - "consistency_score_2": 0.3633658008658009, - "consistency_score_3": 0.14935064935064934, - "consistency_score_4": 0.06396103896103895, - "consistency_score_5": 0.02651515151515152, - "consistency_score_6": 0.008928571428571428, - "consistency_score_7": 0.0, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.23863636363636365, - "English,Spanish": 0.5227272727272727, - "English,Malay": 0.2840909090909091, - "English,Chinese": 0.5738636363636364, - "English,Indonesian": 0.32386363636363635, - "English,Vietnamese": 0.2556818181818182, - "Filipino,Spanish": 0.3522727272727273, - "Filipino,Malay": 0.4715909090909091, - "Filipino,Chinese": 0.05113636363636364, - "Filipino,Indonesian": 0.5056818181818182, - "Filipino,Vietnamese": 0.5397727272727273, - "Spanish,Malay": 0.38636363636363635, - "Spanish,Chinese": 0.45454545454545453, - "Spanish,Indonesian": 0.4375, - "Spanish,Vietnamese": 0.4147727272727273, - "Malay,Chinese": 0.16477272727272727, - "Malay,Indonesian": 0.42613636363636365, - "Malay,Vietnamese": 0.42045454545454547, - "Chinese,Indonesian": 0.17045454545454544, - "Chinese,Vietnamese": 0.16477272727272727, - "Indonesian,Vietnamese": 0.4715909090909091 - }, - "3_combine": { - "English,Filipino,Spanish": 0.13068181818181818, - "English,Filipino,Malay": 0.125, - "English,Filipino,Chinese": 0.011363636363636364, - "English,Filipino,Indonesian": 0.1534090909090909, - "English,Filipino,Vietnamese": 0.125, - "English,Spanish,Malay": 0.18181818181818182, - "English,Spanish,Chinese": 0.3181818181818182, - "English,Spanish,Indonesian": 0.2159090909090909, - "English,Spanish,Vietnamese": 0.1875, - "English,Malay,Chinese": 0.08522727272727272, - "English,Malay,Indonesian": 0.1590909090909091, - "English,Malay,Vietnamese": 0.10795454545454546, - "English,Chinese,Indonesian": 0.10795454545454546, - "English,Chinese,Vietnamese": 0.10227272727272728, - "English,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay": 0.1875, - "Filipino,Spanish,Chinese": 0.028409090909090908, - "Filipino,Spanish,Indonesian": 0.23863636363636365, - "Filipino,Spanish,Vietnamese": 0.24431818181818182, - "Filipino,Malay,Chinese": 0.03409090909090909, - "Filipino,Malay,Indonesian": 0.2556818181818182, - "Filipino,Malay,Vietnamese": 0.26704545454545453, - "Filipino,Chinese,Indonesian": 0.03977272727272727, - "Filipino,Chinese,Vietnamese": 0.03977272727272727, - "Filipino,Indonesian,Vietnamese": 0.3068181818181818, - "Spanish,Malay,Chinese": 0.10227272727272728, - "Spanish,Malay,Indonesian": 0.21022727272727273, - "Spanish,Malay,Vietnamese": 0.1875, - "Spanish,Chinese,Indonesian": 0.125, - "Spanish,Chinese,Vietnamese": 0.11931818181818182, - "Spanish,Indonesian,Vietnamese": 0.26704545454545453, - "Malay,Chinese,Indonesian": 0.07386363636363637, - "Malay,Chinese,Vietnamese": 0.045454545454545456, - "Malay,Indonesian,Vietnamese": 0.22727272727272727, - "Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.06818181818181818, - "English,Filipino,Spanish,Chinese": 0.011363636363636364, - "English,Filipino,Spanish,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese": 0.005681818181818182, - "English,Filipino,Malay,Indonesian": 0.08522727272727272, - "English,Filipino,Malay,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese": 0.0625, - "English,Spanish,Malay,Indonesian": 0.10227272727272728, - "English,Spanish,Malay,Vietnamese": 0.07954545454545454, - "English,Spanish,Chinese,Indonesian": 0.09090909090909091, - "English,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "English,Spanish,Indonesian,Vietnamese": 0.125, - "English,Malay,Chinese,Indonesian": 0.03409090909090909, - "English,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Malay,Chinese": 0.017045454545454544, - "Filipino,Spanish,Malay,Indonesian": 0.125, - "Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.022727272727272728, - "Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian": 0.022727272727272728, - "Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Vietnamese": 0.05113636363636364, - "English,Filipino,Spanish,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.0, - "English,Filipino,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.045454545454545456, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 - } - }, - "AC3_2": 0.3126456699364829, - "AC3_3": 0.19341195198699349, - "AC3_4": 0.1037371936777397, - "AC3_5": 0.04835676911407789, - "AC3_6": 0.017294310268147924, - "AC3_7": 0.0 - }, - "prompt_3": { - "overall_acc": 0.26379870129870137, - "language_acc": { - "English": 0.2556818181818182, - "Filipino": 0.26136363636363635, - "Spanish": 0.24431818181818182, - "Malay": 0.2840909090909091, - "Chinese": 0.2840909090909091, - "Indonesian": 0.26704545454545453, - "Vietnamese": 0.25 - }, - "consistency_score_2": 0.3958333333333333, - "consistency_score_3": 0.19902597402597402, - "consistency_score_4": 0.1136363636363636, - "consistency_score_5": 0.07007575757575758, - "consistency_score_6": 0.046266233766233775, - "consistency_score_7": 0.03409090909090909, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.5113636363636364, - "English,Malay": 0.3068181818181818, - "English,Chinese": 0.4431818181818182, - "English,Indonesian": 0.36363636363636365, - "English,Vietnamese": 0.4034090909090909, - "Filipino,Spanish": 0.4772727272727273, - "Filipino,Malay": 0.5056818181818182, - "Filipino,Chinese": 0.16477272727272727, - "Filipino,Indonesian": 0.5284090909090909, - "Filipino,Vietnamese": 0.4943181818181818, - "Spanish,Malay": 0.4375, - "Spanish,Chinese": 0.2897727272727273, - "Spanish,Indonesian": 0.4715909090909091, - "Spanish,Vietnamese": 0.5, - "Malay,Chinese": 0.1875, - "Malay,Indonesian": 0.45454545454545453, - "Malay,Vietnamese": 0.4375, - "Chinese,Indonesian": 0.22727272727272727, - "Chinese,Vietnamese": 0.26704545454545453, - "Indonesian,Vietnamese": 0.5056818181818182 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2556818181818182, - "English,Filipino,Malay": 0.19318181818181818, - "English,Filipino,Chinese": 0.09659090909090909, - "English,Filipino,Indonesian": 0.21022727272727273, - "English,Filipino,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay": 0.25, - "English,Spanish,Chinese": 0.19318181818181818, - "English,Spanish,Indonesian": 0.2727272727272727, - "English,Spanish,Vietnamese": 0.3125, - "English,Malay,Chinese": 0.10227272727272728, - "English,Malay,Indonesian": 0.19886363636363635, - "English,Malay,Vietnamese": 0.18181818181818182, - "English,Chinese,Indonesian": 0.11363636363636363, - "English,Chinese,Vietnamese": 0.14204545454545456, - "English,Indonesian,Vietnamese": 0.25, - "Filipino,Spanish,Malay": 0.2784090909090909, - "Filipino,Spanish,Chinese": 0.10795454545454546, - "Filipino,Spanish,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Vietnamese": 0.2784090909090909, - "Filipino,Malay,Chinese": 0.10227272727272728, - "Filipino,Malay,Indonesian": 0.2840909090909091, - "Filipino,Malay,Vietnamese": 0.2784090909090909, - "Filipino,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Indonesian,Vietnamese": 0.29545454545454547, - "Spanish,Malay,Chinese": 0.125, - "Spanish,Malay,Indonesian": 0.26704545454545453, - "Spanish,Malay,Vietnamese": 0.25, - "Spanish,Chinese,Indonesian": 0.13068181818181818, - "Spanish,Chinese,Vietnamese": 0.16477272727272727, - "Spanish,Indonesian,Vietnamese": 0.3068181818181818, - "Malay,Chinese,Indonesian": 0.10227272727272728, - "Malay,Chinese,Vietnamese": 0.11363636363636363, - "Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Chinese,Indonesian,Vietnamese": 0.13636363636363635 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17045454545454544, - "English,Filipino,Spanish,Chinese": 0.07954545454545454, - "English,Filipino,Spanish,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Chinese": 0.0625, - "English,Filipino,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese": 0.08522727272727272, - "English,Spanish,Malay,Indonesian": 0.16477272727272727, - "English,Spanish,Malay,Vietnamese": 0.1534090909090909, - "English,Spanish,Chinese,Indonesian": 0.09659090909090909, - "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "English,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay,Chinese": 0.08522727272727272, - "Filipino,Spanish,Malay,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "Filipino,Malay,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Malay,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "Spanish,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.0625, - "English,Filipino,Spanish,Malay,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian": 0.0625, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.125, - "English,Filipino,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Indonesian": 0.0625, - "English,Spanish,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.0625, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - } - }, - "AC3_2": 0.3166017226755472, - "AC3_3": 0.22687983705529904, - "AC3_4": 0.15884652977218594, - "AC3_5": 0.11073559743257652, - "AC3_6": 0.07872526685432722, - "AC3_7": 0.0603789942824311 - }, - "prompt_4": { - "overall_acc": 0.2646103896103896, - "language_acc": { - "English": 0.29545454545454547, - "Filipino": 0.2784090909090909, - "Spanish": 0.2556818181818182, - "Malay": 0.2556818181818182, - "Chinese": 0.23295454545454544, - "Indonesian": 0.2556818181818182, - "Vietnamese": 0.2784090909090909 - }, - "consistency_score_2": 0.44101731601731603, - "consistency_score_3": 0.23782467532467527, - "consistency_score_4": 0.13896103896103895, - "consistency_score_5": 0.08306277056277056, - "consistency_score_6": 0.04707792207792207, - "consistency_score_7": 0.022727272727272728, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.375, - "English,Spanish": 0.48295454545454547, - "English,Malay": 0.3181818181818182, - "English,Chinese": 0.44886363636363635, - "English,Indonesian": 0.3806818181818182, - "English,Vietnamese": 0.45454545454545453, - "Filipino,Spanish": 0.5965909090909091, - "Filipino,Malay": 0.5681818181818182, - "Filipino,Chinese": 0.20454545454545456, - "Filipino,Indonesian": 0.6306818181818182, - "Filipino,Vietnamese": 0.5795454545454546, - "Spanish,Malay": 0.4772727272727273, - "Spanish,Chinese": 0.36363636363636365, - "Spanish,Indonesian": 0.5568181818181818, - "Spanish,Vietnamese": 0.5284090909090909, - "Malay,Chinese": 0.17613636363636365, - "Malay,Indonesian": 0.5852272727272727, - "Malay,Vietnamese": 0.5, - "Chinese,Indonesian": 0.18181818181818182, - "Chinese,Vietnamese": 0.2727272727272727, - "Indonesian,Vietnamese": 0.5795454545454546 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2840909090909091, - "English,Filipino,Malay": 0.22727272727272727, - "English,Filipino,Chinese": 0.09659090909090909, - "English,Filipino,Indonesian": 0.2784090909090909, - "English,Filipino,Vietnamese": 0.2784090909090909, - "English,Spanish,Malay": 0.2215909090909091, - "English,Spanish,Chinese": 0.2215909090909091, - "English,Spanish,Indonesian": 0.2840909090909091, - "English,Spanish,Vietnamese": 0.3068181818181818, - "English,Malay,Chinese": 0.10227272727272728, - "English,Malay,Indonesian": 0.22727272727272727, - "English,Malay,Vietnamese": 0.23295454545454544, - "English,Chinese,Indonesian": 0.09659090909090909, - "English,Chinese,Vietnamese": 0.16477272727272727, - "English,Indonesian,Vietnamese": 0.2840909090909091, - "Filipino,Spanish,Malay": 0.375, - "Filipino,Spanish,Chinese": 0.1534090909090909, - "Filipino,Spanish,Indonesian": 0.4147727272727273, - "Filipino,Spanish,Vietnamese": 0.3693181818181818, - "Filipino,Malay,Chinese": 0.11931818181818182, - "Filipino,Malay,Indonesian": 0.4034090909090909, - "Filipino,Malay,Vietnamese": 0.36363636363636365, - "Filipino,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Indonesian,Vietnamese": 0.4147727272727273, - "Spanish,Malay,Chinese": 0.11931818181818182, - "Spanish,Malay,Indonesian": 0.3465909090909091, - "Spanish,Malay,Vietnamese": 0.2897727272727273, - "Spanish,Chinese,Indonesian": 0.13636363636363635, - "Spanish,Chinese,Vietnamese": 0.18181818181818182, - "Spanish,Indonesian,Vietnamese": 0.375, - "Malay,Chinese,Indonesian": 0.09090909090909091, - "Malay,Chinese,Vietnamese": 0.11363636363636363, - "Malay,Indonesian,Vietnamese": 0.35795454545454547, - "Chinese,Indonesian,Vietnamese": 0.11931818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.1875, - "English,Filipino,Spanish,Chinese": 0.08522727272727272, - "English,Filipino,Spanish,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Vietnamese": 0.20454545454545456, - "English,Filipino,Malay,Chinese": 0.0625, - "English,Filipino,Malay,Indonesian": 0.1875, - "English,Filipino,Malay,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay,Chinese": 0.06818181818181818, - "English,Spanish,Malay,Indonesian": 0.17613636363636365, - "English,Spanish,Malay,Vietnamese": 0.17045454545454544, - "English,Spanish,Chinese,Indonesian": 0.07954545454545454, - "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Spanish,Indonesian,Vietnamese": 0.21022727272727273, - "English,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese": 0.09090909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.2897727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.25, - "Filipino,Spanish,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Filipino,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "Spanish,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Indonesian": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.0625, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 - } - }, - "AC3_2": 0.330762986966112, - "AC3_3": 0.2505035351871183, - "AC3_4": 0.18222565844760094, - "AC3_5": 0.12643640402649675, - "AC3_6": 0.0799343885024929, - "AC3_7": 0.041859270658262025 - }, - "prompt_5": { - "overall_acc": 0.2719155844155844, - "language_acc": { - "English": 0.3125, - "Filipino": 0.25, - "Spanish": 0.2840909090909091, - "Malay": 0.25, - "Chinese": 0.2784090909090909, - "Indonesian": 0.2727272727272727, - "Vietnamese": 0.2556818181818182 - }, - "consistency_score_2": 0.4507575757575757, - "consistency_score_3": 0.2506493506493507, - "consistency_score_4": 0.15227272727272725, - "consistency_score_5": 0.09442640692640693, - "consistency_score_6": 0.05600649350649351, - "consistency_score_7": 0.028409090909090908, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.39204545454545453, - "English,Spanish": 0.5170454545454546, - "English,Malay": 0.30113636363636365, - "English,Chinese": 0.4375, - "English,Indonesian": 0.35795454545454547, - "English,Vietnamese": 0.3806818181818182, - "Filipino,Spanish": 0.5625, - "Filipino,Malay": 0.5511363636363636, - "Filipino,Chinese": 0.32386363636363635, - "Filipino,Indonesian": 0.5681818181818182, - "Filipino,Vietnamese": 0.5454545454545454, - "Spanish,Malay": 0.45454545454545453, - "Spanish,Chinese": 0.3977272727272727, - "Spanish,Indonesian": 0.5511363636363636, - "Spanish,Vietnamese": 0.5852272727272727, - "Malay,Chinese": 0.21022727272727273, - "Malay,Indonesian": 0.5681818181818182, - "Malay,Vietnamese": 0.5284090909090909, - "Chinese,Indonesian": 0.29545454545454547, - "Chinese,Vietnamese": 0.3522727272727273, - "Indonesian,Vietnamese": 0.5852272727272727 - }, - "3_combine": { - "English,Filipino,Spanish": 0.32386363636363635, - "English,Filipino,Malay": 0.2159090909090909, - "English,Filipino,Chinese": 0.1590909090909091, - "English,Filipino,Indonesian": 0.26704545454545453, - "English,Filipino,Vietnamese": 0.26704545454545453, - "English,Spanish,Malay": 0.23295454545454544, - "English,Spanish,Chinese": 0.2215909090909091, - "English,Spanish,Indonesian": 0.2897727272727273, - "English,Spanish,Vietnamese": 0.30113636363636365, - "English,Malay,Chinese": 0.10795454545454546, - "English,Malay,Indonesian": 0.21022727272727273, - "English,Malay,Vietnamese": 0.19318181818181818, - "English,Chinese,Indonesian": 0.1534090909090909, - "English,Chinese,Vietnamese": 0.17613636363636365, - "English,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Malay": 0.3125, - "Filipino,Spanish,Chinese": 0.2159090909090909, - "Filipino,Spanish,Indonesian": 0.375, - "Filipino,Spanish,Vietnamese": 0.375, - "Filipino,Malay,Chinese": 0.14772727272727273, - "Filipino,Malay,Indonesian": 0.36363636363636365, - "Filipino,Malay,Vietnamese": 0.3352272727272727, - "Filipino,Chinese,Indonesian": 0.21022727272727273, - "Filipino,Chinese,Vietnamese": 0.21022727272727273, - "Filipino,Indonesian,Vietnamese": 0.3806818181818182, - "Spanish,Malay,Chinese": 0.11931818181818182, - "Spanish,Malay,Indonesian": 0.32386363636363635, - "Spanish,Malay,Vietnamese": 0.32386363636363635, - "Spanish,Chinese,Indonesian": 0.2159090909090909, - "Spanish,Chinese,Vietnamese": 0.2215909090909091, - "Spanish,Indonesian,Vietnamese": 0.4090909090909091, - "Malay,Chinese,Indonesian": 0.14204545454545456, - "Malay,Chinese,Vietnamese": 0.14204545454545456, - "Malay,Indonesian,Vietnamese": 0.3693181818181818, - "Chinese,Indonesian,Vietnamese": 0.19886363636363635 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.1875, - "English,Filipino,Spanish,Chinese": 0.125, - "English,Filipino,Spanish,Indonesian": 0.23295454545454544, - "English,Filipino,Spanish,Vietnamese": 0.22727272727272727, - "English,Filipino,Malay,Chinese": 0.07386363636363637, - "English,Filipino,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Chinese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian": 0.17045454545454544, - "English,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Spanish,Chinese,Indonesian": 0.11931818181818182, - "English,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.2159090909090909, - "English,Malay,Chinese,Indonesian": 0.07386363636363637, - "English,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese": 0.10795454545454546, - "Filipino,Spanish,Malay,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2784090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.11931818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Malay,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Spanish,Malay,Indonesian,Vietnamese": 0.2556818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 + "Indonesian,English,Filipino,Spanish": 0.21022727272727273, + "Indonesian,English,Filipino,Chinese": 0.23295454545454544, + "Indonesian,English,Filipino,Malay": 0.23863636363636365, + "Indonesian,English,Filipino,Vietnamese": 0.24431818181818182, + "Indonesian,English,Spanish,Chinese": 0.3125, + "Indonesian,English,Spanish,Malay": 0.32954545454545453, + "Indonesian,English,Spanish,Vietnamese": 0.36363636363636365, + "Indonesian,English,Chinese,Malay": 0.4147727272727273, + "Indonesian,English,Chinese,Vietnamese": 0.42613636363636365, + "Indonesian,English,Malay,Vietnamese": 0.4659090909090909, + "Indonesian,Filipino,Spanish,Chinese": 0.17613636363636365, + "Indonesian,Filipino,Spanish,Malay": 0.16477272727272727, + "Indonesian,Filipino,Spanish,Vietnamese": 0.19318181818181818, + "Indonesian,Filipino,Chinese,Malay": 0.19886363636363635, + "Indonesian,Filipino,Chinese,Vietnamese": 0.23863636363636365, + "Indonesian,Filipino,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Spanish,Chinese,Malay": 0.2727272727272727, + "Indonesian,Spanish,Chinese,Vietnamese": 0.3125, + "Indonesian,Spanish,Malay,Vietnamese": 0.3125, + "Indonesian,Chinese,Malay,Vietnamese": 0.4090909090909091, + "English,Filipino,Spanish,Chinese": 0.19318181818181818, + "English,Filipino,Spanish,Malay": 0.20454545454545456, + "English,Filipino,Spanish,Vietnamese": 0.2215909090909091, + "English,Filipino,Chinese,Malay": 0.2215909090909091, + "English,Filipino,Chinese,Vietnamese": 0.2556818181818182, + "English,Filipino,Malay,Vietnamese": 0.26704545454545453, + "English,Spanish,Chinese,Malay": 0.3068181818181818, + "English,Spanish,Chinese,Vietnamese": 0.3409090909090909, + "English,Spanish,Malay,Vietnamese": 0.36363636363636365, + "English,Chinese,Malay,Vietnamese": 0.4659090909090909, + "Filipino,Spanish,Chinese,Malay": 0.1534090909090909, + "Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, + "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, + "Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Spanish,Chinese,Malay,Vietnamese": 0.3181818181818182 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "English,Filipino,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian": 0.0625, - "English,Spanish,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 + "Indonesian,English,Filipino,Spanish,Chinese": 0.1534090909090909, + "Indonesian,English,Filipino,Spanish,Malay": 0.14772727272727273, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.17045454545454544, + "Indonesian,English,Filipino,Chinese,Malay": 0.17045454545454544, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.19318181818181818, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,English,Spanish,Chinese,Malay": 0.23863636363636365, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.35795454545454547, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.125, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182, + "English,Filipino,Spanish,Chinese,Malay": 0.13636363636363635, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, + "English,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, + "English,Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, + "English,Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 } }, - "AC3_2": 0.3392073108002082, - "AC3_3": 0.26084974355844714, - "AC3_4": 0.19522144517542286, - "AC3_5": 0.14017509443889897, - "AC3_6": 0.09288205603442472, - "AC3_7": 0.05144348892635966 + "AC3_2": 0.35266622480219295, + "AC3_3": 0.30851013452470777, + "AC3_4": 0.264627791760101, + "AC3_5": 0.21977931483895832, + "AC3_6": 0.1755890247087898, + "AC3_7": 0.13401621848450304 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.24271844660194175 - }, - "prompt_2": { - "accuracy": 0.27184466019417475 - }, - "prompt_3": { - "accuracy": 0.24271844660194175 - }, - "prompt_4": { - "accuracy": 0.2524271844660194 - }, - "prompt_5": { - "accuracy": 0.30097087378640774 + "accuracy": 0.23300970873786409 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.1619047619047619 - }, - "prompt_2": { - "accuracy": 0.2 - }, - "prompt_3": { - "accuracy": 0.2571428571428571 - }, - "prompt_4": { - "accuracy": 0.22857142857142856 - }, - "prompt_5": { - "accuracy": 0.18095238095238095 + "accuracy": 0.2857142857142857 } }, "us_eval": { "prompt_1": { - "accuracy": 0.2803738317757009 - }, - "prompt_2": { - "accuracy": 0.3177570093457944 - }, - "prompt_3": { - "accuracy": 0.27102803738317754 - }, - "prompt_4": { - "accuracy": 0.29906542056074764 - }, - "prompt_5": { - "accuracy": 0.2523364485981308 + "accuracy": 0.32710280373831774 } }, "ph_eval": { "prompt_1": { "accuracy": 0.22, "category_acc": { - "brand": 0.1, - "demographics": 0.4, - "biology": 0.3, - "history": 0.3333333333333333, + "brand": 0.0, + "demographics": 0.2, + "biology": 0.4, + "history": 0.06666666666666667, "literature": 0.2, - "politics": 0.0, - "culture": 0.3, - "film": 0.2, - "law": 0.0, - "geography": 0.4 - } - }, - "prompt_2": { - "accuracy": 0.16, - "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.2, - "history": 0.0, - "literature": 0.2, - "politics": 0.1, - "culture": 0.2, - "film": 0.3, - "law": 0.0, - "geography": 0.4 - } - }, - "prompt_3": { - "accuracy": 0.2, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.2, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.1, - "culture": 0.2, - "film": 0.3, - "law": 0.0, - "geography": 0.2 - } - }, - "prompt_4": { - "accuracy": 0.24, - "category_acc": { - "brand": 0.1, - "demographics": 0.4, - "biology": 0.3, - "history": 0.26666666666666666, - "literature": 0.4, - "politics": 0.2, - "culture": 0.4, - "film": 0.2, - "law": 0.0, - "geography": 0.2 - } - }, - "prompt_5": { - "accuracy": 0.26, - "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.3, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.0, - "culture": 0.5, - "film": 0.3, - "law": 0.1, + "politics": 0.3, + "culture": 0.1, + "film": 0.5, + "law": 0.3, "geography": 0.2 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.17178621629731683 - }, - "prompt_2": { - "bleu_score": 0.1805825644970613 - }, - "prompt_3": { - "bleu_score": 0.1638182494134646 - }, - "prompt_4": { - "bleu_score": 0.15986162389926584 - }, - "prompt_5": { - "bleu_score": 0.1446974926792948 + "bleu_score": 0.24634338960756805 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "prompt_1": { + "accuracy": 0.2410708324988317, + "category_acc": { + "History": 0.20883534136546184, + "Geography": 0.23877551020408164, + "Lampungic": 0.2653061224489796, + "Social science": 0.25542570951585974, + "Balinese": 0.25690021231422505, + "Makassarese": 0.26881720430107525, + "Banjarese": 0.2847222222222222, + "Chemistry": 0.21897810218978103, + "Biology": 0.20946745562130178, + "Science": 0.26109391124871, + "Christian religion": 0.24875621890547264, + "Art": 0.23627287853577372, + "Islam religion": 0.24324324324324326, + "Hindu religion": 0.18666666666666668, + "Madurese": 0.2542372881355932, + "Sport": 0.22972972972972974, + "Indonesian language": 0.2636986301369863, + "Physics": 0.18585858585858586, + "Minangkabau culture": 0.23618090452261306, + "Dayak language": 0.23853211009174313, + "Sociology": 0.18346774193548387, + "Economy": 0.2069672131147541, + "Sundanese": 0.24719101123595505, + "Javanese": 0.2439516129032258, + "Civic education": 0.24892703862660945 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.22898886961073495 - }, - "prompt_2": { - "bleu_score": 0.22131682108538805 - }, - "prompt_3": { - "bleu_score": 0.21876755119211821 - }, - "prompt_4": { - "bleu_score": 0.22093180904148585 - }, - "prompt_5": { - "bleu_score": 0.21802364406548647 + "bleu_score": 0.3287443508742123 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.18682046862305973 - }, - "prompt_2": { - "bleu_score": 0.17941321052949818 - }, - "prompt_3": { - "bleu_score": 0.16587480317190326 - }, - "prompt_4": { - "bleu_score": 0.18606680016936544 - }, - "prompt_5": { - "bleu_score": 0.18811132568607536 + "bleu_score": 0.2659669890064635 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.12167567389233164 - }, - "prompt_2": { - "bleu_score": 0.11634949386099766 - }, - "prompt_3": { - "bleu_score": 0.12113333625588714 - }, - "prompt_4": { - "bleu_score": 0.11659217841852082 - }, - "prompt_5": { - "bleu_score": 0.1197048824865453 + "bleu_score": 0.17170080951799402 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.21630817675207034 - }, - "prompt_2": { - "bleu_score": 0.21343494752169703 - }, - "prompt_3": { - "bleu_score": 0.21809019634391713 - }, - "prompt_4": { - "bleu_score": 0.2105236988675236 - }, - "prompt_5": { - "bleu_score": 0.21218896556966144 + "bleu_score": 0.32481248192705475 } }, "mmlu": { "prompt_1": { - "accuracy": 0.26721120186697783 - }, - "prompt_2": { - "accuracy": 0.27537922987164526 - }, - "prompt_3": { - "accuracy": 0.2847141190198366 - }, - "prompt_4": { - "accuracy": 0.2660443407234539 - }, - "prompt_5": { - "accuracy": 0.26954492415402564 + "accuracy": 0.2718786464410735 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.26485520200214513, + "accuracy": 0.2702895959957097, "category_acc": { - "high_school_european_history": 0.2682926829268293, - "business_ethics": 0.25252525252525254, - "clinical_knowledge": 0.24621212121212122, - "medical_genetics": 0.1919191919191919, - "high_school_us_history": 0.22167487684729065, - "high_school_physics": 0.24666666666666667, - "high_school_world_history": 0.2754237288135593, - "virology": 0.26666666666666666, - "high_school_microeconomics": 0.22784810126582278, - "econometrics": 0.20353982300884957, - "college_computer_science": 0.1919191919191919, - "high_school_biology": 0.2783171521035599, - "abstract_algebra": 0.29292929292929293, + "high_school_european_history": 0.2621951219512195, + "business_ethics": 0.29292929292929293, + "clinical_knowledge": 0.3522727272727273, + "medical_genetics": 0.35353535353535354, + "high_school_us_history": 0.270935960591133, + "high_school_physics": 0.34, + "high_school_world_history": 0.2033898305084746, + "virology": 0.2545454545454545, + "high_school_microeconomics": 0.29957805907172996, + "econometrics": 0.22123893805309736, + "college_computer_science": 0.25252525252525254, + "high_school_biology": 0.35275080906148865, + "abstract_algebra": 0.23232323232323232, "professional_accounting": 0.2313167259786477, "philosophy": 0.2709677419354839, - "professional_medicine": 0.24354243542435425, - "nutrition": 0.2819672131147541, - "global_facts": 0.31313131313131315, - "machine_learning": 0.2072072072072072, - "security_studies": 0.29508196721311475, - "public_relations": 0.21100917431192662, - "professional_psychology": 0.28805237315875615, - "prehistory": 0.2724458204334365, - "anatomy": 0.29850746268656714, - "human_sexuality": 0.26153846153846155, - "college_medicine": 0.25, - "high_school_government_and_politics": 0.296875, - "college_chemistry": 0.1919191919191919, - "logical_fallacies": 0.29012345679012347, - "high_school_geography": 0.3197969543147208, - "elementary_mathematics": 0.2493368700265252, - "human_aging": 0.22522522522522523, - "college_mathematics": 0.1717171717171717, - "high_school_psychology": 0.3014705882352941, - "formal_logic": 0.264, - "high_school_statistics": 0.2651162790697674, - "international_law": 0.3, - "high_school_mathematics": 0.2527881040892193, - "high_school_computer_science": 0.25252525252525254, - "conceptual_physics": 0.23504273504273504, - "miscellaneous": 0.29923273657289, - "high_school_chemistry": 0.26732673267326734, - "marketing": 0.2317596566523605, - "professional_law": 0.27332028701891714, - "management": 0.27450980392156865, - "college_physics": 0.24752475247524752, - "jurisprudence": 0.29906542056074764, - "world_religions": 0.25882352941176473, - "sociology": 0.25, - "us_foreign_policy": 0.31313131313131315, - "high_school_macroeconomics": 0.2467866323907455, - "computer_security": 0.29292929292929293, - "moral_scenarios": 0.232662192393736, - "moral_disputes": 0.3072463768115942, - "electrical_engineering": 0.2986111111111111, - "astronomy": 0.2913907284768212, - "college_biology": 0.2517482517482518 - } - }, - "prompt_2": { - "accuracy": 0.2643546657132642, - "category_acc": { - "high_school_european_history": 0.27439024390243905, - "business_ethics": 0.16161616161616163, - "clinical_knowledge": 0.2840909090909091, - "medical_genetics": 0.3333333333333333, - "high_school_us_history": 0.2561576354679803, - "high_school_physics": 0.36, - "high_school_world_history": 0.2711864406779661, - "virology": 0.21212121212121213, - "high_school_microeconomics": 0.21940928270042195, - "econometrics": 0.21238938053097345, - "college_computer_science": 0.21212121212121213, - "high_school_biology": 0.3106796116504854, - "abstract_algebra": 0.31313131313131315, - "professional_accounting": 0.24555160142348753, - "philosophy": 0.2903225806451613, - "professional_medicine": 0.28413284132841327, - "nutrition": 0.3081967213114754, - "global_facts": 0.24242424242424243, - "machine_learning": 0.2072072072072072, - "security_studies": 0.26229508196721313, - "public_relations": 0.22018348623853212, - "professional_psychology": 0.22749590834697217, - "prehistory": 0.22910216718266255, - "anatomy": 0.26865671641791045, - "human_sexuality": 0.2230769230769231, - "college_medicine": 0.2441860465116279, - "high_school_government_and_politics": 0.3125, - "college_chemistry": 0.2727272727272727, - "logical_fallacies": 0.25308641975308643, - "high_school_geography": 0.3299492385786802, - "elementary_mathematics": 0.246684350132626, - "human_aging": 0.18468468468468469, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.33088235294117646, - "formal_logic": 0.2, - "high_school_statistics": 0.26976744186046514, - "international_law": 0.25, - "high_school_mathematics": 0.23048327137546468, - "high_school_computer_science": 0.3434343434343434, - "conceptual_physics": 0.2863247863247863, - "miscellaneous": 0.2813299232736573, - "high_school_chemistry": 0.3118811881188119, - "marketing": 0.18025751072961374, - "professional_law": 0.26157860404435745, - "management": 0.3235294117647059, - "college_physics": 0.2376237623762376, - "jurisprudence": 0.2616822429906542, - "world_religions": 0.27647058823529413, - "sociology": 0.215, - "us_foreign_policy": 0.26262626262626265, - "high_school_macroeconomics": 0.26735218508997427, - "computer_security": 0.2222222222222222, - "moral_scenarios": 0.2595078299776286, - "moral_disputes": 0.26666666666666666, - "electrical_engineering": 0.2777777777777778, - "astronomy": 0.31125827814569534, - "college_biology": 0.3006993006993007 - } - }, - "prompt_3": { - "accuracy": 0.26528423310690025, - "category_acc": { - "high_school_european_history": 0.31097560975609756, - "business_ethics": 0.25252525252525254, - "clinical_knowledge": 0.2916666666666667, - "medical_genetics": 0.26262626262626265, - "high_school_us_history": 0.2660098522167488, - "high_school_physics": 0.3333333333333333, - "high_school_world_history": 0.2288135593220339, - "virology": 0.24242424242424243, - "high_school_microeconomics": 0.25316455696202533, - "econometrics": 0.19469026548672566, - "college_computer_science": 0.23232323232323232, - "high_school_biology": 0.32362459546925565, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.2846975088967972, - "philosophy": 0.27741935483870966, - "professional_medicine": 0.28044280442804426, - "nutrition": 0.26557377049180325, - "global_facts": 0.29292929292929293, - "machine_learning": 0.21621621621621623, - "security_studies": 0.2336065573770492, + "professional_medicine": 0.3874538745387454, + "nutrition": 0.26885245901639343, + "global_facts": 0.20202020202020202, + "machine_learning": 0.18018018018018017, + "security_studies": 0.3442622950819672, "public_relations": 0.23853211009174313, - "professional_psychology": 0.23567921440261866, - "prehistory": 0.24458204334365324, - "anatomy": 0.26865671641791045, - "human_sexuality": 0.26153846153846155, - "college_medicine": 0.26744186046511625, - "high_school_government_and_politics": 0.3229166666666667, - "college_chemistry": 0.23232323232323232, - "logical_fallacies": 0.30864197530864196, - "high_school_geography": 0.3248730964467005, - "elementary_mathematics": 0.27851458885941643, - "human_aging": 0.11711711711711711, - "college_mathematics": 0.2222222222222222, - "high_school_psychology": 0.3161764705882353, - "formal_logic": 0.24, - "high_school_statistics": 0.30697674418604654, - "international_law": 0.2833333333333333, - "high_school_mathematics": 0.2527881040892193, - "high_school_computer_science": 0.25252525252525254, - "conceptual_physics": 0.2264957264957265, - "miscellaneous": 0.26854219948849106, - "high_school_chemistry": 0.26732673267326734, - "marketing": 0.2017167381974249, - "professional_law": 0.26157860404435745, - "management": 0.3137254901960784, - "college_physics": 0.27722772277227725, - "jurisprudence": 0.24299065420560748, - "world_religions": 0.25882352941176473, - "sociology": 0.23, - "us_foreign_policy": 0.2727272727272727, - "high_school_macroeconomics": 0.2853470437017995, - "computer_security": 0.2222222222222222, - "moral_scenarios": 0.24496644295302014, - "moral_disputes": 0.28695652173913044, - "electrical_engineering": 0.3125, - "astronomy": 0.33112582781456956, - "college_biology": 0.27972027972027974 - } - }, - "prompt_4": { - "accuracy": 0.2606363961387201, - "category_acc": { - "high_school_european_history": 0.31097560975609756, - "business_ethics": 0.2727272727272727, - "clinical_knowledge": 0.30303030303030304, - "medical_genetics": 0.25252525252525254, - "high_school_us_history": 0.21182266009852216, - "high_school_physics": 0.30666666666666664, - "high_school_world_history": 0.2669491525423729, - "virology": 0.20606060606060606, - "high_school_microeconomics": 0.25738396624472576, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.24242424242424243, - "high_school_biology": 0.26537216828478966, - "abstract_algebra": 0.25252525252525254, - "professional_accounting": 0.21352313167259787, - "philosophy": 0.2645161290322581, - "professional_medicine": 0.34317343173431736, - "nutrition": 0.26229508196721313, - "global_facts": 0.32323232323232326, - "machine_learning": 0.24324324324324326, - "security_studies": 0.2827868852459016, - "public_relations": 0.22935779816513763, - "professional_psychology": 0.22913256955810146, - "prehistory": 0.24458204334365324, - "anatomy": 0.23134328358208955, - "human_sexuality": 0.24615384615384617, - "college_medicine": 0.23837209302325582, - "high_school_government_and_politics": 0.2708333333333333, - "college_chemistry": 0.25252525252525254, - "logical_fallacies": 0.24074074074074073, - "high_school_geography": 0.3350253807106599, - "elementary_mathematics": 0.21220159151193635, - "human_aging": 0.16216216216216217, - "college_mathematics": 0.16161616161616163, - "high_school_psychology": 0.3382352941176471, - "formal_logic": 0.208, - "high_school_statistics": 0.24186046511627907, - "international_law": 0.23333333333333334, - "high_school_mathematics": 0.30855018587360594, - "high_school_computer_science": 0.25252525252525254, - "conceptual_physics": 0.23931623931623933, - "miscellaneous": 0.2659846547314578, - "high_school_chemistry": 0.2623762376237624, - "marketing": 0.21030042918454936, - "professional_law": 0.2530984996738421, - "management": 0.35294117647058826, - "college_physics": 0.27722772277227725, - "jurisprudence": 0.2803738317757009, - "world_religions": 0.2235294117647059, - "sociology": 0.245, - "us_foreign_policy": 0.25252525252525254, - "high_school_macroeconomics": 0.29048843187660667, - "computer_security": 0.1919191919191919, - "moral_scenarios": 0.2606263982102908, - "moral_disputes": 0.2898550724637681, - "electrical_engineering": 0.2777777777777778, - "astronomy": 0.33112582781456956, - "college_biology": 0.25874125874125875 - } - }, - "prompt_5": { - "accuracy": 0.2591347872720772, - "category_acc": { - "high_school_european_history": 0.3170731707317073, - "business_ethics": 0.21212121212121213, - "clinical_knowledge": 0.25757575757575757, - "medical_genetics": 0.25252525252525254, - "high_school_us_history": 0.24630541871921183, - "high_school_physics": 0.3, - "high_school_world_history": 0.2838983050847458, - "virology": 0.23636363636363636, - "high_school_microeconomics": 0.2109704641350211, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.23232323232323232, - "high_school_biology": 0.2783171521035599, - "abstract_algebra": 0.31313131313131315, - "professional_accounting": 0.24199288256227758, - "philosophy": 0.3, - "professional_medicine": 0.22878228782287824, - "nutrition": 0.2819672131147541, - "global_facts": 0.26262626262626265, - "machine_learning": 0.21621621621621623, - "security_studies": 0.2581967213114754, - "public_relations": 0.22935779816513763, "professional_psychology": 0.25859247135842883, - "prehistory": 0.25077399380804954, - "anatomy": 0.31343283582089554, - "human_sexuality": 0.2, - "college_medicine": 0.22093023255813954, - "high_school_government_and_politics": 0.3020833333333333, - "college_chemistry": 0.21212121212121213, - "logical_fallacies": 0.2839506172839506, - "high_school_geography": 0.3299492385786802, - "elementary_mathematics": 0.2519893899204244, - "human_aging": 0.13963963963963963, - "college_mathematics": 0.2222222222222222, - "high_school_psychology": 0.31066176470588236, - "formal_logic": 0.152, - "high_school_statistics": 0.24186046511627907, - "international_law": 0.30833333333333335, - "high_school_mathematics": 0.26394052044609667, - "high_school_computer_science": 0.29292929292929293, - "conceptual_physics": 0.27350427350427353, - "miscellaneous": 0.2557544757033248, - "high_school_chemistry": 0.26732673267326734, + "prehistory": 0.25696594427244585, + "anatomy": 0.23134328358208955, + "human_sexuality": 0.26153846153846155, + "college_medicine": 0.313953488372093, + "high_school_government_and_politics": 0.3333333333333333, + "college_chemistry": 0.36363636363636365, + "logical_fallacies": 0.2037037037037037, + "high_school_geography": 0.3553299492385787, + "elementary_mathematics": 0.246684350132626, + "human_aging": 0.17567567567567569, + "college_mathematics": 0.26262626262626265, + "high_school_psychology": 0.31985294117647056, + "formal_logic": 0.28, + "high_school_statistics": 0.3674418604651163, + "international_law": 0.16666666666666666, + "high_school_mathematics": 0.2788104089219331, + "high_school_computer_science": 0.2222222222222222, + "conceptual_physics": 0.2606837606837607, + "miscellaneous": 0.21994884910485935, + "high_school_chemistry": 0.23267326732673269, "marketing": 0.22317596566523606, - "professional_law": 0.2583170254403131, - "management": 0.3235294117647059, - "college_physics": 0.21782178217821782, - "jurisprudence": 0.22429906542056074, - "world_religions": 0.2411764705882353, + "professional_law": 0.2648401826484018, + "management": 0.29411764705882354, + "college_physics": 0.31683168316831684, + "jurisprudence": 0.29906542056074764, + "world_religions": 0.20588235294117646, "sociology": 0.245, - "us_foreign_policy": 0.25252525252525254, - "high_school_macroeconomics": 0.2622107969151671, - "computer_security": 0.20202020202020202, - "moral_scenarios": 0.24049217002237136, - "moral_disputes": 0.2927536231884058, - "electrical_engineering": 0.2916666666666667, - "astronomy": 0.31125827814569534, - "college_biology": 0.3006993006993007 + "us_foreign_policy": 0.32323232323232326, + "high_school_macroeconomics": 0.3161953727506427, + "computer_security": 0.25252525252525254, + "moral_scenarios": 0.2595078299776286, + "moral_disputes": 0.2318840579710145, + "electrical_engineering": 0.2569444444444444, + "astronomy": 0.2847682119205298, + "college_biology": 0.21678321678321677 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.25185735512630014 - }, - "prompt_2": { - "accuracy": 0.262258543833581 - }, - "prompt_3": { - "accuracy": 0.2674591381872214 - }, - "prompt_4": { - "accuracy": 0.25334323922734026 - }, - "prompt_5": { - "accuracy": 0.25705794947994054 + "accuracy": 0.2563150074294205 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.2727272727272727, + "accuracy": 0.273972602739726, "category_acc": { - "computer_network": 0.25, - "operating_system": 0.2916666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.23809523809523808, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.38095238095238093, + "computer_network": 0.20833333333333334, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.3076923076923077, + "college_programming": 0.19047619047619047, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.17391304347826086, + "discrete_mathematics": 0.2857142857142857, "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.3793103448275862, + "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.30434782608695654, "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.4583333333333333, - "high_school_biology": 0.125, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.2692307692307692, + "high_school_chemistry": 0.375, + "high_school_biology": 0.25, + "middle_school_mathematics": 0.4166666666666667, + "middle_school_biology": 0.34615384615384615, "middle_school_physics": 0.25, "middle_school_chemistry": 0.28, "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.31666666666666665, - "business_administration": 0.3157894736842105, - "marxism": 0.20833333333333334, - "mao_zedong_thought": 0.20689655172413793, - "education_science": 0.20588235294117646, - "teacher_qualification": 0.30612244897959184, - "high_school_politics": 0.08333333333333333, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.4444444444444444, - "law": 0.1724137931034483, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.14705882352941177, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.25, - "high_school_history": 0.28, - "middle_school_history": 0.2222222222222222, - "civil_servant": 0.21153846153846154, - "sports_science": 0.2916666666666667, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.5, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.2222222222222222, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.2037037037037037, - "physician": 0.35185185185185186 - } - }, - "prompt_2": { - "accuracy": 0.2808219178082192, - "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.15384615384615385, - "college_programming": 0.2619047619047619, - "college_physics": 0.25, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.21428571428571427, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.4583333333333333, - "high_school_biology": 0.16666666666666666, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.16666666666666666, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.25, - "college_economics": 0.25, - "business_administration": 0.21052631578947367, + "college_economics": 0.2833333333333333, + "business_administration": 0.23684210526315788, "marxism": 0.2916666666666667, "mao_zedong_thought": 0.2413793103448276, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.2857142857142857, + "education_science": 0.4117647058823529, + "teacher_qualification": 0.20408163265306123, "high_school_politics": 0.16666666666666666, "high_school_geography": 0.20833333333333334, "middle_school_politics": 0.19230769230769232, "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.32142857142857145, + "modern_chinese_history": 0.5, "ideological_and_moral_cultivation": 0.25, "logic": 0.37037037037037035, "law": 0.1724137931034483, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.21052631578947367, - "professional_tour_guide": 0.20588235294117646, - "legal_professional": 0.25, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.32, - "middle_school_history": 0.4074074074074074, - "civil_servant": 0.25, - "sports_science": 0.16666666666666666, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.6666666666666666, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.17647058823529413, - "accountant": 0.24074074074074073, - "fire_engineer": 0.3611111111111111, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.15789473684210525, + "professional_tour_guide": 0.23529411764705882, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.25, + "high_school_history": 0.28, + "middle_school_history": 0.2962962962962963, + "civil_servant": 0.19230769230769232, + "sports_science": 0.25, + "plant_protection": 0.2962962962962963, + "basic_medicine": 0.4583333333333333, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.27450980392156865, + "accountant": 0.25925925925925924, + "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.3611111111111111, - "tax_accountant": 0.3148148148148148, - "physician": 0.3148148148148148 + "tax_accountant": 0.25925925925925924, + "physician": 0.35185185185185186 } - }, - "prompt_3": { - "accuracy": 0.2683686176836862, + } + }, + "cmmlu": { + "prompt_1": { + "accuracy": 0.25448028673835127 + } + }, + "cmmlu_full": { + "prompt_1": { + "accuracy": 0.26144016577447765, "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.21428571428571427, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.4482758620689655, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.19047619047619047, - "metrology_engineer": 0.27586206896551724, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.25, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.08333333333333333, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.08333333333333333, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.18333333333333332, - "business_administration": 0.23684210526315788, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.2413793103448276, - "education_science": 0.20588235294117646, - "teacher_qualification": 0.3877551020408163, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.5294117647058824, - "modern_chinese_history": 0.32142857142857145, - "ideological_and_moral_cultivation": 0.125, - "logic": 0.2962962962962963, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.23529411764705882, - "legal_professional": 0.14285714285714285, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.4, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.28846153846153844, - "sports_science": 0.20833333333333334, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.21568627450980393, - "accountant": 0.24074074074074073, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.2222222222222222, - "tax_accountant": 0.3148148148148148, - "physician": 0.2962962962962963 - } - }, - "prompt_4": { - "accuracy": 0.2590286425902864, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.25, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.2857142857142857, - "college_physics": 0.375, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.4166666666666667, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.3076923076923077, - "middle_school_physics": 0.3333333333333333, - "middle_school_chemistry": 0.24, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.21666666666666667, - "business_administration": 0.3157894736842105, - "marxism": 0.20833333333333334, - "mao_zedong_thought": 0.13793103448275862, - "education_science": 0.29411764705882354, - "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.0, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.19230769230769232, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.25, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.3333333333333333, - "law": 0.1724137931034483, - "chinese_language_and_literature": 0.25, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.23529411764705882, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.25, - "high_school_history": 0.36, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.21153846153846154, - "sports_science": 0.25, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.5833333333333334, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.17647058823529413, - "accountant": 0.24074074074074073, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.2777777777777778, - "physician": 0.2777777777777778 - } - }, - "prompt_5": { - "accuracy": 0.27085927770859275, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.30952380952380953, - "college_physics": 0.08333333333333333, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.375, - "high_school_chemistry": 0.5, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.16666666666666666, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.21666666666666667, - "business_administration": 0.39473684210526316, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.20689655172413793, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.4074074074074074, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.21052631578947367, - "professional_tour_guide": 0.17647058823529413, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.32, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.19230769230769232, - "sports_science": 0.25, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.2037037037037037, - "fire_engineer": 0.4166666666666667, - "environmental_impact_assessment_engineer": 0.16666666666666666, - "tax_accountant": 0.3148148148148148, - "physician": 0.18518518518518517 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.21863799283154123 - }, - "prompt_2": { - "accuracy": 0.2867383512544803 - }, - "prompt_3": { - "accuracy": 0.2078853046594982 - }, - "prompt_4": { - "accuracy": 0.24372759856630824 - }, - "prompt_5": { - "accuracy": 0.2616487455197133 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.2451217406320152, - "category_acc": { - "agronomy": 0.22485207100591717, - "anatomy": 0.25, - "ancient_chinese": 0.25609756097560976, - "arts": 0.25625, - "astronomy": 0.2606060606060606, - "business_ethics": 0.24401913875598086, - "chinese_civil_service_exam": 0.25, - "chinese_driving_rule": 0.2366412213740458, - "chinese_food_culture": 0.23529411764705882, - "chinese_foreign_policy": 0.18691588785046728, - "chinese_history": 0.25077399380804954, - "chinese_literature": 0.27941176470588236, - "chinese_teacher_qualification": 0.2122905027932961, - "clinical_knowledge": 0.2320675105485232, - "college_actuarial_science": 0.2169811320754717, - "college_education": 0.205607476635514, - "college_engineering_hydrology": 0.3490566037735849, - "college_law": 0.32407407407407407, - "college_mathematics": 0.3142857142857143, - "college_medical_statistics": 0.20754716981132076, + "agronomy": 0.2781065088757396, + "anatomy": 0.30405405405405406, + "ancient_chinese": 0.2621951219512195, + "arts": 0.26875, + "astronomy": 0.24848484848484848, + "business_ethics": 0.27751196172248804, + "chinese_civil_service_exam": 0.24375, + "chinese_driving_rule": 0.29770992366412213, + "chinese_food_culture": 0.22794117647058823, + "chinese_foreign_policy": 0.24299065420560748, + "chinese_history": 0.21981424148606812, + "chinese_literature": 0.2696078431372549, + "chinese_teacher_qualification": 0.2737430167597765, + "clinical_knowledge": 0.24472573839662448, + "college_actuarial_science": 0.3584905660377358, + "college_education": 0.17757009345794392, + "college_engineering_hydrology": 0.3113207547169811, + "college_law": 0.3055555555555556, + "college_mathematics": 0.3047619047619048, + "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.2454212454212454, - "computer_science": 0.25980392156862747, + "computer_science": 0.2647058823529412, "computer_security": 0.2573099415204678, - "conceptual_physics": 0.24489795918367346, - "construction_project_management": 0.20863309352517986, - "economics": 0.20125786163522014, + "conceptual_physics": 0.21768707482993196, + "construction_project_management": 0.302158273381295, + "economics": 0.24528301886792453, "education": 0.25766871165644173, - "electrical_engineering": 0.2616279069767442, + "electrical_engineering": 0.22674418604651161, "elementary_chinese": 0.26587301587301587, - "elementary_commonsense": 0.21212121212121213, - "elementary_information_and_technology": 0.2184873949579832, - "elementary_mathematics": 0.2217391304347826, - "ethnology": 0.23703703703703705, - "food_science": 0.2517482517482518, - "genetics": 0.2556818181818182, - "global_facts": 0.2953020134228188, - "high_school_biology": 0.25443786982248523, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.2796610169491525, - "high_school_mathematics": 0.21341463414634146, - "high_school_physics": 0.23636363636363636, - "high_school_politics": 0.2867132867132867, - "human_sexuality": 0.23015873015873015, + "elementary_commonsense": 0.2878787878787879, + "elementary_information_and_technology": 0.2647058823529412, + "elementary_mathematics": 0.21304347826086956, + "ethnology": 0.2740740740740741, + "food_science": 0.2097902097902098, + "genetics": 0.26136363636363635, + "global_facts": 0.2214765100671141, + "high_school_biology": 0.2958579881656805, + "high_school_chemistry": 0.24242424242424243, + "high_school_geography": 0.23728813559322035, + "high_school_mathematics": 0.2621951219512195, + "high_school_physics": 0.2, + "high_school_politics": 0.3006993006993007, + "human_sexuality": 0.2857142857142857, "international_law": 0.25405405405405407, - "journalism": 0.2441860465116279, - "jurisprudence": 0.22141119221411193, - "legal_and_moral_basis": 0.21495327102803738, - "logical": 0.21138211382113822, - "machine_learning": 0.21311475409836064, - "management": 0.2571428571428571, - "marketing": 0.28888888888888886, - "marxist_theory": 0.291005291005291, - "modern_chinese": 0.22413793103448276, - "nutrition": 0.2482758620689655, - "philosophy": 0.2857142857142857, - "professional_accounting": 0.24, - "professional_law": 0.2985781990521327, - "professional_medicine": 0.24202127659574468, - "professional_psychology": 0.21120689655172414, - "public_relations": 0.22988505747126436, - "security_study": 0.2074074074074074, - "sociology": 0.19469026548672566, - "sports_science": 0.2909090909090909, - "traditional_chinese_medicine": 0.23783783783783785, - "virology": 0.27218934911242604, - "world_history": 0.2608695652173913, - "world_religions": 0.2375 - } - }, - "prompt_2": { - "accuracy": 0.25133828354342946, - "category_acc": { - "agronomy": 0.26627218934911245, - "anatomy": 0.24324324324324326, - "ancient_chinese": 0.27439024390243905, - "arts": 0.2875, - "astronomy": 0.2, - "business_ethics": 0.22966507177033493, - "chinese_civil_service_exam": 0.26875, - "chinese_driving_rule": 0.20610687022900764, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.2523364485981308, - "chinese_history": 0.24458204334365324, - "chinese_literature": 0.2549019607843137, - "chinese_teacher_qualification": 0.24022346368715083, - "clinical_knowledge": 0.23628691983122363, - "college_actuarial_science": 0.20754716981132076, - "college_education": 0.205607476635514, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.3425925925925926, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.16981132075471697, - "college_medicine": 0.24175824175824176, - "computer_science": 0.2647058823529412, - "computer_security": 0.22807017543859648, - "conceptual_physics": 0.2653061224489796, - "construction_project_management": 0.22302158273381295, - "economics": 0.18867924528301888, - "education": 0.24539877300613497, - "electrical_engineering": 0.18604651162790697, - "elementary_chinese": 0.21031746031746032, - "elementary_commonsense": 0.21717171717171718, - "elementary_information_and_technology": 0.24369747899159663, - "elementary_mathematics": 0.2565217391304348, - "ethnology": 0.3111111111111111, - "food_science": 0.23776223776223776, - "genetics": 0.2727272727272727, - "global_facts": 0.24161073825503357, - "high_school_biology": 0.21301775147928995, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.3389830508474576, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.23636363636363636, - "high_school_politics": 0.32167832167832167, - "human_sexuality": 0.2619047619047619, - "international_law": 0.21081081081081082, "journalism": 0.2616279069767442, - "jurisprudence": 0.2384428223844282, - "legal_and_moral_basis": 0.2616822429906542, + "jurisprudence": 0.25790754257907544, + "legal_and_moral_basis": 0.29906542056074764, "logical": 0.24390243902439024, - "machine_learning": 0.22950819672131148, - "management": 0.2571428571428571, - "marketing": 0.2611111111111111, - "marxist_theory": 0.2698412698412698, - "modern_chinese": 0.29310344827586204, - "nutrition": 0.2827586206896552, - "philosophy": 0.29523809523809524, - "professional_accounting": 0.2742857142857143, - "professional_law": 0.26066350710900477, - "professional_medicine": 0.2553191489361702, - "professional_psychology": 0.2672413793103448, - "public_relations": 0.21264367816091953, - "security_study": 0.3037037037037037, - "sociology": 0.23008849557522124, - "sports_science": 0.2606060606060606, - "traditional_chinese_medicine": 0.2810810810810811, - "virology": 0.27218934911242604, - "world_history": 0.2857142857142857, - "world_religions": 0.28125 - } - }, - "prompt_3": { - "accuracy": 0.24909342082541874, - "category_acc": { - "agronomy": 0.23076923076923078, - "anatomy": 0.28378378378378377, - "ancient_chinese": 0.2682926829268293, - "arts": 0.3, - "astronomy": 0.21212121212121213, - "business_ethics": 0.24401913875598086, - "chinese_civil_service_exam": 0.2375, - "chinese_driving_rule": 0.2366412213740458, - "chinese_food_culture": 0.23529411764705882, - "chinese_foreign_policy": 0.27102803738317754, - "chinese_history": 0.26006191950464397, - "chinese_literature": 0.2107843137254902, - "chinese_teacher_qualification": 0.26256983240223464, - "clinical_knowledge": 0.22362869198312235, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.21495327102803738, - "college_engineering_hydrology": 0.330188679245283, - "college_law": 0.25925925925925924, - "college_mathematics": 0.3142857142857143, - "college_medical_statistics": 0.24528301886792453, - "college_medicine": 0.27472527472527475, - "computer_science": 0.25, - "computer_security": 0.27485380116959063, - "conceptual_physics": 0.21768707482993196, - "construction_project_management": 0.2158273381294964, - "economics": 0.20125786163522014, - "education": 0.2147239263803681, - "electrical_engineering": 0.19186046511627908, - "elementary_chinese": 0.25793650793650796, - "elementary_commonsense": 0.2474747474747475, - "elementary_information_and_technology": 0.21008403361344538, - "elementary_mathematics": 0.2608695652173913, - "ethnology": 0.2, - "food_science": 0.21678321678321677, - "genetics": 0.24431818181818182, - "global_facts": 0.2751677852348993, - "high_school_biology": 0.2603550295857988, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.2627118644067797, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.3146853146853147, - "human_sexuality": 0.25396825396825395, - "international_law": 0.2648648648648649, - "journalism": 0.20930232558139536, - "jurisprudence": 0.22384428223844283, - "legal_and_moral_basis": 0.21962616822429906, - "logical": 0.2601626016260163, "machine_learning": 0.2459016393442623, "management": 0.2571428571428571, - "marketing": 0.2388888888888889, - "marxist_theory": 0.26455026455026454, - "modern_chinese": 0.25862068965517243, - "nutrition": 0.30344827586206896, + "marketing": 0.2833333333333333, + "marxist_theory": 0.2857142857142857, + "modern_chinese": 0.23275862068965517, + "nutrition": 0.25517241379310346, "philosophy": 0.29523809523809524, "professional_accounting": 0.28, - "professional_law": 0.26540284360189575, - "professional_medicine": 0.26063829787234044, - "professional_psychology": 0.2543103448275862, - "public_relations": 0.21264367816091953, - "security_study": 0.2222222222222222, - "sociology": 0.23008849557522124, - "sports_science": 0.30303030303030304, - "traditional_chinese_medicine": 0.23783783783783785, - "virology": 0.24260355029585798, - "world_history": 0.2484472049689441, - "world_religions": 0.25 - } - }, - "prompt_4": { - "accuracy": 0.2566914177171473, - "category_acc": { - "agronomy": 0.30177514792899407, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.20121951219512196, - "arts": 0.275, - "astronomy": 0.22424242424242424, - "business_ethics": 0.22488038277511962, - "chinese_civil_service_exam": 0.225, - "chinese_driving_rule": 0.22900763358778625, - "chinese_food_culture": 0.27941176470588236, - "chinese_foreign_policy": 0.21495327102803738, - "chinese_history": 0.21981424148606812, - "chinese_literature": 0.27450980392156865, - "chinese_teacher_qualification": 0.2737430167597765, - "clinical_knowledge": 0.2489451476793249, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.2523364485981308, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.2777777777777778, - "college_mathematics": 0.3238095238095238, - "college_medical_statistics": 0.16037735849056603, - "college_medicine": 0.31135531135531136, - "computer_science": 0.24509803921568626, - "computer_security": 0.21637426900584794, - "conceptual_physics": 0.272108843537415, - "construction_project_management": 0.2517985611510791, - "economics": 0.2893081761006289, - "education": 0.27607361963190186, - "electrical_engineering": 0.20348837209302326, - "elementary_chinese": 0.2619047619047619, - "elementary_commonsense": 0.2474747474747475, - "elementary_information_and_technology": 0.2815126050420168, - "elementary_mathematics": 0.24782608695652175, - "ethnology": 0.2518518518518518, - "food_science": 0.23776223776223776, - "genetics": 0.2159090909090909, - "global_facts": 0.3221476510067114, - "high_school_biology": 0.26627218934911245, - "high_school_chemistry": 0.2878787878787879, - "high_school_geography": 0.3135593220338983, - "high_school_mathematics": 0.2073170731707317, - "high_school_physics": 0.3, - "high_school_politics": 0.27972027972027974, - "human_sexuality": 0.2857142857142857, - "international_law": 0.2864864864864865, - "journalism": 0.2441860465116279, - "jurisprudence": 0.24817518248175183, - "legal_and_moral_basis": 0.26635514018691586, - "logical": 0.2601626016260163, - "machine_learning": 0.29508196721311475, - "management": 0.24761904761904763, - "marketing": 0.26666666666666666, - "marxist_theory": 0.2857142857142857, - "modern_chinese": 0.1810344827586207, - "nutrition": 0.2896551724137931, - "philosophy": 0.26666666666666666, - "professional_accounting": 0.2742857142857143, - "professional_law": 0.23222748815165878, - "professional_medicine": 0.25, - "professional_psychology": 0.23275862068965517, - "public_relations": 0.2471264367816092, - "security_study": 0.28888888888888886, - "sociology": 0.23008849557522124, - "sports_science": 0.3151515151515151, - "traditional_chinese_medicine": 0.20540540540540542, - "virology": 0.2485207100591716, - "world_history": 0.2919254658385093, - "world_religions": 0.25 - } - }, - "prompt_5": { - "accuracy": 0.2548782593679848, - "category_acc": { - "agronomy": 0.27218934911242604, - "anatomy": 0.25, - "ancient_chinese": 0.3231707317073171, - "arts": 0.25, - "astronomy": 0.2545454545454545, - "business_ethics": 0.24880382775119617, - "chinese_civil_service_exam": 0.225, - "chinese_driving_rule": 0.2366412213740458, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.2336448598130841, - "chinese_history": 0.26625386996904027, - "chinese_literature": 0.27450980392156865, - "chinese_teacher_qualification": 0.2346368715083799, - "clinical_knowledge": 0.24050632911392406, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.2336448598130841, - "college_engineering_hydrology": 0.32075471698113206, - "college_law": 0.3055555555555556, - "college_mathematics": 0.20952380952380953, - "college_medical_statistics": 0.2358490566037736, - "college_medicine": 0.2271062271062271, - "computer_science": 0.2696078431372549, - "computer_security": 0.25146198830409355, - "conceptual_physics": 0.2789115646258503, - "construction_project_management": 0.23741007194244604, - "economics": 0.27044025157232704, - "education": 0.2822085889570552, - "electrical_engineering": 0.2441860465116279, - "elementary_chinese": 0.25, - "elementary_commonsense": 0.22727272727272727, - "elementary_information_and_technology": 0.21428571428571427, - "elementary_mathematics": 0.2217391304347826, - "ethnology": 0.24444444444444444, - "food_science": 0.25874125874125875, - "genetics": 0.23295454545454544, - "global_facts": 0.2953020134228188, - "high_school_biology": 0.2958579881656805, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.3220338983050847, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.23636363636363636, - "high_school_politics": 0.32167832167832167, - "human_sexuality": 0.30158730158730157, - "international_law": 0.23783783783783785, - "journalism": 0.2441860465116279, - "jurisprudence": 0.25304136253041365, - "legal_and_moral_basis": 0.2336448598130841, - "logical": 0.2845528455284553, - "machine_learning": 0.2459016393442623, - "management": 0.2571428571428571, - "marketing": 0.28888888888888886, - "marxist_theory": 0.24338624338624337, - "modern_chinese": 0.28448275862068967, - "nutrition": 0.2689655172413793, - "philosophy": 0.22857142857142856, - "professional_accounting": 0.2571428571428571, - "professional_law": 0.22274881516587677, - "professional_medicine": 0.25, - "professional_psychology": 0.22844827586206898, - "public_relations": 0.2413793103448276, + "professional_law": 0.26066350710900477, + "professional_medicine": 0.2526595744680851, + "professional_psychology": 0.27155172413793105, + "public_relations": 0.27011494252873564, "security_study": 0.2740740740740741, - "sociology": 0.22123893805309736, - "sports_science": 0.296969696969697, - "traditional_chinese_medicine": 0.2594594594594595, + "sociology": 0.3141592920353982, + "sports_science": 0.23030303030303031, + "traditional_chinese_medicine": 0.2918918918918919, "virology": 0.26627218934911245, - "world_history": 0.2484472049689441, - "world_religions": 0.28125 + "world_history": 0.2236024844720497, + "world_religions": 0.2625 } } }, "zbench": { "prompt_1": { "accuracy": 0.2727272727272727 - }, - "prompt_2": { - "accuracy": 0.36363636363636365 - }, - "prompt_3": { - "accuracy": 0.3333333333333333 - }, - "prompt_4": { - "accuracy": 0.3333333333333333 - }, - "prompt_5": { - "accuracy": 0.3333333333333333 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.225 - }, - "prompt_2": { - "accuracy": 0.2431818181818182 - }, - "prompt_3": { - "accuracy": 0.19545454545454546 - }, - "prompt_4": { - "accuracy": 0.17727272727272728 - }, - "prompt_5": { - "accuracy": 0.20681818181818182 + "accuracy": 0.18409090909090908 } }, "ocnli": { "prompt_1": { - "accuracy": 0.31966101694915255 - }, - "prompt_2": { - "accuracy": 0.3288135593220339 - }, - "prompt_3": { - "accuracy": 0.31966101694915255 - }, - "prompt_4": { - "accuracy": 0.3474576271186441 - }, - "prompt_5": { - "accuracy": 0.33932203389830506 + "accuracy": 0.3403389830508475 } }, "c3": { "prompt_1": { - "accuracy": 0.3029169783096485 - }, - "prompt_2": { - "accuracy": 0.3175018698578908 - }, - "prompt_3": { - "accuracy": 0.29842931937172773 - }, - "prompt_4": { - "accuracy": 0.3156320119670905 - }, - "prompt_5": { - "accuracy": 0.3145100972326103 + "accuracy": 0.28085265519820496 } }, "dream": { "prompt_1": { - "accuracy": 0.3679568838804508 - }, - "prompt_2": { - "accuracy": 0.38167564919157276 - }, - "prompt_3": { - "accuracy": 0.38755512003919645 - }, - "prompt_4": { - "accuracy": 0.3890249877511024 - }, - "prompt_5": { - "accuracy": 0.3718765311121999 + "accuracy": 0.3375796178343949 } }, "samsum": { "prompt_1": { - "rouge1": 0.28812226142725134, - "rouge2": 0.08436806841909454, - "rougeL": 0.2115083107671798, - "avg_rouge": 0.1946662135378419 - }, - "prompt_2": { - "rouge1": 0.3068992575059755, - "rouge2": 0.09142739580714761, - "rougeL": 0.22489507487194027, - "avg_rouge": 0.2077405760616878 - }, - "prompt_3": { - "rouge1": 0.28735900695962824, - "rouge2": 0.07936712339626861, - "rougeL": 0.20734731095583275, - "avg_rouge": 0.19135781377057656 - }, - "prompt_4": { - "rouge1": 0.2760646430336083, - "rouge2": 0.08177856097524426, - "rougeL": 0.2021823343091616, - "avg_rouge": 0.18667517943933806 - }, - "prompt_5": { - "rouge1": 0.3049031035421055, - "rouge2": 0.0934024704575301, - "rougeL": 0.22324872011486405, - "avg_rouge": 0.20718476470483319 + "rouge1": 0.3764256748130376, + "rouge2": 0.14284717779230788, + "rougeL": 0.30317993386561815, + "avg_rouge": 0.27415092882365455 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.2233175380553051, - "rouge2": 0.06003164899136425, - "rougeL": 0.1642759870024971, - "avg_rouge": 0.14920839134972216 - }, - "prompt_2": { - "rouge1": 0.21979203142912254, - "rouge2": 0.06045225399272618, - "rougeL": 0.1601884699414455, - "avg_rouge": 0.1468109184544314 - }, - "prompt_3": { - "rouge1": 0.217400854552657, - "rouge2": 0.05884551072689542, - "rougeL": 0.1601510539919537, - "avg_rouge": 0.14546580642383536 - }, - "prompt_4": { - "rouge1": 0.2196951985217058, - "rouge2": 0.05918298031835628, - "rougeL": 0.15950981348850893, - "avg_rouge": 0.1461293307761903 - }, - "prompt_5": { - "rouge1": 0.21095871875809746, - "rouge2": 0.0582916078071698, - "rougeL": 0.15623374594719722, - "avg_rouge": 0.1418280241708215 + "rouge1": 0.37555740903295204, + "rouge2": 0.12309897604398988, + "rougeL": 0.3049472937088673, + "avg_rouge": 0.2678678929286031 } }, "sst2": { "prompt_1": { - "accuracy": 0.6880733944954128 - }, - "prompt_2": { - "accuracy": 0.5688073394495413 - }, - "prompt_3": { - "accuracy": 0.5791284403669725 - }, - "prompt_4": { - "accuracy": 0.5951834862385321 - }, - "prompt_5": { - "accuracy": 0.6823394495412844 + "accuracy": 0.5424311926605505 } }, "cola": { "prompt_1": { - "accuracy": 0.5196548418024928 - }, - "prompt_2": { - "accuracy": 0.5292425695110259 - }, - "prompt_3": { - "accuracy": 0.5560882070949185 - }, - "prompt_4": { - "accuracy": 0.6653883029721956 - }, - "prompt_5": { - "accuracy": 0.5311601150527325 + "accuracy": 0.49185043144774687 } }, "qqp": { "prompt_1": { - "accuracy": 0.497 - }, - "prompt_2": { "accuracy": 0.4985 - }, - "prompt_3": { - "accuracy": 0.495 - }, - "prompt_4": { - "accuracy": 0.4955 - }, - "prompt_5": { - "accuracy": 0.4955 } }, "mnli": { "prompt_1": { - "accuracy": 0.3605 - }, - "prompt_2": { - "accuracy": 0.363 - }, - "prompt_3": { - "accuracy": 0.3625 - }, - "prompt_4": { - "accuracy": 0.365 - }, - "prompt_5": { - "accuracy": 0.357 + "accuracy": 0.3 } }, "qnli": { "prompt_1": { - "accuracy": 0.5205 - }, - "prompt_2": { - "accuracy": 0.506 - }, - "prompt_3": { - "accuracy": 0.5015 - }, - "prompt_4": { - "accuracy": 0.5075 - }, - "prompt_5": { - "accuracy": 0.519 + "accuracy": 0.7 } }, "wnli": { "prompt_1": { - "accuracy": 0.4788732394366197 - }, - "prompt_2": { - "accuracy": 0.4225352112676056 - }, - "prompt_3": { - "accuracy": 0.43661971830985913 - }, - "prompt_4": { - "accuracy": 0.4507042253521127 - }, - "prompt_5": { - "accuracy": 0.4225352112676056 + "accuracy": 0.2 } }, "rte": { "prompt_1": { - "accuracy": 0.51985559566787 - }, - "prompt_2": { - "accuracy": 0.5342960288808665 - }, - "prompt_3": { - "accuracy": 0.5054151624548736 - }, - "prompt_4": { - "accuracy": 0.5090252707581228 - }, - "prompt_5": { - "accuracy": 0.516245487364621 + "accuracy": 0.3 } }, "mrpc": { "prompt_1": { - "accuracy": 0.5343137254901961 - }, - "prompt_2": { - "accuracy": 0.5343137254901961 - }, - "prompt_3": { - "accuracy": 0.5318627450980392 - }, - "prompt_4": { - "accuracy": 0.5343137254901961 - }, - "prompt_5": { - "accuracy": 0.5343137254901961 + "accuracy": 0.5 } } + } + }, + "sailor_4b": { + "model_size": "4B", + "model_link": "https://huggingface.co/sail/Sailor-4B", + "zero_shot": { + "cross_xquad": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cross_mmlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cross_logiqa": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "sg_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cn_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "us_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ph_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "sing2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "indommlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_ind2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_vie2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_zho2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_zsm2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mmlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mmlu_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c_eval_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cmmlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cmmlu_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "zbench": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ind_emotion": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ocnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c3": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "dream": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "samsum": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "dialogsum": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "sst2": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cola": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "qqp": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "qnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "wnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "rte": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mrpc": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + } }, "five_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.2642857142857143, + "overall_acc": 0.4957983193277311, "language_acc": { - "English": 0.27058823529411763, - "Chinese": 0.2722689075630252, - "Spanish": 0.25882352941176473, - "Vietnamese": 0.25546218487394956 - }, - "consistency_score_2": 0.4816526610644258, - "consistency_score_3": 0.28445378151260503, - "consistency_score_4": 0.19747899159663865, + "Spanish": 0.46554621848739497, + "English": 0.5092436974789916, + "Chinese": 0.5016806722689076, + "Vietnamese": 0.5067226890756302 + }, + "consistency_score_2": 0.6123249299719888, + "consistency_score_3": 0.4619747899159664, + "consistency_score_4": 0.37815126050420167, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.49411764705882355, - "English,Spanish": 0.5210084033613446, - "English,Vietnamese": 0.49159663865546216, - "Chinese,Spanish": 0.4579831932773109, - "Chinese,Vietnamese": 0.446218487394958, - "Spanish,Vietnamese": 0.4789915966386555 + "Spanish,English": 0.607563025210084, + "Spanish,Chinese": 0.5873949579831933, + "Spanish,Vietnamese": 0.6310924369747899, + "English,Chinese": 0.6327731092436975, + "English,Vietnamese": 0.6235294117647059, + "Chinese,Vietnamese": 0.5915966386554622 }, "3_combine": { - "English,Chinese,Spanish": 0.29747899159663865, - "English,Chinese,Vietnamese": 0.27983193277310925, - "English,Spanish,Vietnamese": 0.3042016806722689, - "Chinese,Spanish,Vietnamese": 0.25630252100840334 + "Spanish,English,Chinese": 0.45966386554621846, + "Spanish,English,Vietnamese": 0.4722689075630252, + "Spanish,Chinese,Vietnamese": 0.44789915966386556, + "English,Chinese,Vietnamese": 0.4680672268907563 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.19747899159663865 + "Spanish,English,Chinese,Vietnamese": 0.37815126050420167 } }, - "AC3_2": 0.3412987500213022, - "AC3_3": 0.2739991248680286, - "AC3_4": 0.22604965548204875 + "AC3_2": 0.5479348462854469, + "AC3_3": 0.47828931962616705, + "AC3_4": 0.42905623783067337 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.2561904761904762, + "overall_acc": 0.37523809523809526, "language_acc": { - "Spanish": 0.26666666666666666, - "English": 0.32666666666666666, - "Chinese": 0.22666666666666666, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.23333333333333334, - "Indonesian": 0.20666666666666667, - "Malay": 0.24666666666666667 - }, - "consistency_score_2": 0.4041269841269841, - "consistency_score_3": 0.18457142857142858, - "consistency_score_4": 0.08952380952380949, - "consistency_score_5": 0.04507936507936508, - "consistency_score_6": 0.023809523809523808, - "consistency_score_7": 0.013333333333333334, + "Filipino": 0.3333333333333333, + "Vietnamese": 0.36666666666666664, + "Chinese": 0.3933333333333333, + "Spanish": 0.36666666666666664, + "Malay": 0.38666666666666666, + "Indonesian": 0.35333333333333333, + "English": 0.4266666666666667 + }, + "consistency_score_2": 0.5292063492063492, + "consistency_score_3": 0.34228571428571425, + "consistency_score_4": 0.24476190476190482, + "consistency_score_5": 0.18571428571428572, + "consistency_score_6": 0.14666666666666667, + "consistency_score_7": 0.12, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.4, - "Spanish,Chinese": 0.36666666666666664, - "Spanish,Vietnamese": 0.36, - "Spanish,Filipino": 0.4, - "Spanish,Indonesian": 0.36666666666666664, - "Spanish,Malay": 0.4, - "English,Chinese": 0.35333333333333333, - "English,Vietnamese": 0.47333333333333333, - "English,Filipino": 0.43333333333333335, - "English,Indonesian": 0.43333333333333335, - "English,Malay": 0.4666666666666667, - "Chinese,Vietnamese": 0.37333333333333335, - "Chinese,Filipino": 0.35333333333333333, - "Chinese,Indonesian": 0.36, - "Chinese,Malay": 0.4066666666666667, - "Vietnamese,Filipino": 0.43333333333333335, - "Vietnamese,Indonesian": 0.38, - "Vietnamese,Malay": 0.4066666666666667, - "Filipino,Indonesian": 0.4066666666666667, - "Filipino,Malay": 0.46, - "Indonesian,Malay": 0.4533333333333333 + "Filipino,Vietnamese": 0.58, + "Filipino,Chinese": 0.5133333333333333, + "Filipino,Spanish": 0.5266666666666666, + "Filipino,Malay": 0.4866666666666667, + "Filipino,Indonesian": 0.5066666666666667, + "Filipino,English": 0.4666666666666667, + "Vietnamese,Chinese": 0.6, + "Vietnamese,Spanish": 0.5466666666666666, + "Vietnamese,Malay": 0.5333333333333333, + "Vietnamese,Indonesian": 0.5466666666666666, + "Vietnamese,English": 0.56, + "Chinese,Spanish": 0.58, + "Chinese,Malay": 0.52, + "Chinese,Indonesian": 0.54, + "Chinese,English": 0.6066666666666667, + "Spanish,Malay": 0.47333333333333333, + "Spanish,Indonesian": 0.5066666666666667, + "Spanish,English": 0.49333333333333335, + "Malay,Indonesian": 0.56, + "Malay,English": 0.47333333333333333, + "Indonesian,English": 0.49333333333333335 }, "3_combine": { - "Spanish,English,Chinese": 0.14, - "Spanish,English,Vietnamese": 0.18666666666666668, - "Spanish,English,Filipino": 0.19333333333333333, - "Spanish,English,Indonesian": 0.17333333333333334, - "Spanish,English,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese": 0.12, - "Spanish,Chinese,Filipino": 0.14666666666666667, - "Spanish,Chinese,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,Vietnamese,Indonesian": 0.14, - "Spanish,Vietnamese,Malay": 0.16666666666666666, - "Spanish,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Filipino,Malay": 0.2, - "Spanish,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese": 0.17333333333333334, - "English,Chinese,Filipino": 0.16, - "English,Chinese,Indonesian": 0.15333333333333332, - "English,Chinese,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino": 0.22666666666666666, - "English,Vietnamese,Indonesian": 0.23333333333333334, - "English,Vietnamese,Malay": 0.23333333333333334, - "English,Filipino,Indonesian": 0.19333333333333333, - "English,Filipino,Malay": 0.24666666666666667, - "English,Indonesian,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Filipino": 0.18, - "Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "Chinese,Vietnamese,Malay": 0.16666666666666666, - "Chinese,Filipino,Indonesian": 0.18, - "Chinese,Filipino,Malay": 0.20666666666666667, - "Chinese,Indonesian,Malay": 0.19333333333333333, - "Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Vietnamese,Filipino,Malay": 0.22, - "Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Filipino,Indonesian,Malay": 0.24 + "Filipino,Vietnamese,Chinese": 0.37333333333333335, + "Filipino,Vietnamese,Spanish": 0.36, + "Filipino,Vietnamese,Malay": 0.37333333333333335, + "Filipino,Vietnamese,Indonesian": 0.36, + "Filipino,Vietnamese,English": 0.34, + "Filipino,Chinese,Spanish": 0.36, + "Filipino,Chinese,Malay": 0.30666666666666664, + "Filipino,Chinese,Indonesian": 0.32, + "Filipino,Chinese,English": 0.34, + "Filipino,Spanish,Malay": 0.29333333333333333, + "Filipino,Spanish,Indonesian": 0.31333333333333335, + "Filipino,Spanish,English": 0.31333333333333335, + "Filipino,Malay,Indonesian": 0.32666666666666666, + "Filipino,Malay,English": 0.28, + "Filipino,Indonesian,English": 0.28, + "Vietnamese,Chinese,Spanish": 0.3933333333333333, + "Vietnamese,Chinese,Malay": 0.37333333333333335, + "Vietnamese,Chinese,Indonesian": 0.38, + "Vietnamese,Chinese,English": 0.42, + "Vietnamese,Spanish,Malay": 0.3333333333333333, + "Vietnamese,Spanish,Indonesian": 0.3466666666666667, + "Vietnamese,Spanish,English": 0.3466666666666667, + "Vietnamese,Malay,Indonesian": 0.36666666666666664, + "Vietnamese,Malay,English": 0.3333333333333333, + "Vietnamese,Indonesian,English": 0.35333333333333333, + "Chinese,Spanish,Malay": 0.34, + "Chinese,Spanish,Indonesian": 0.36, + "Chinese,Spanish,English": 0.36666666666666664, + "Chinese,Malay,Indonesian": 0.36, + "Chinese,Malay,English": 0.3466666666666667, + "Chinese,Indonesian,English": 0.37333333333333335, + "Spanish,Malay,Indonesian": 0.34, + "Spanish,Malay,English": 0.29333333333333333, + "Spanish,Indonesian,English": 0.30666666666666664, + "Malay,Indonesian,English": 0.30666666666666664 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.04666666666666667, - "Spanish,English,Chinese,Filipino": 0.07333333333333333, - "Spanish,English,Chinese,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Malay": 0.08666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.08666666666666667, - "Spanish,English,Vietnamese,Malay": 0.10666666666666667, - "Spanish,English,Filipino,Indonesian": 0.08666666666666667, - "Spanish,English,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Indonesian,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Filipino": 0.06666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.05333333333333334, - "Spanish,Chinese,Vietnamese,Malay": 0.07333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.08, - "Spanish,Chinese,Filipino,Malay": 0.10666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.08666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.08, - "Spanish,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.07333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.08, - "English,Chinese,Vietnamese,Malay": 0.07333333333333333, - "English,Chinese,Filipino,Indonesian": 0.06666666666666667, - "English,Chinese,Filipino,Malay": 0.09333333333333334, - "English,Chinese,Indonesian,Malay": 0.08666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "English,Vietnamese,Filipino,Malay": 0.11333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "English,Filipino,Indonesian,Malay": 0.11333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Chinese,Vietnamese,Filipino,Malay": 0.1, - "Chinese,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 + "Filipino,Vietnamese,Chinese,Spanish": 0.26, + "Filipino,Vietnamese,Chinese,Malay": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,English": 0.26666666666666666, + "Filipino,Vietnamese,Spanish,Malay": 0.24666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, + "Filipino,Vietnamese,Malay,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Malay,English": 0.22666666666666666, + "Filipino,Vietnamese,Indonesian,English": 0.22, + "Filipino,Chinese,Spanish,Malay": 0.22666666666666666, + "Filipino,Chinese,Spanish,Indonesian": 0.23333333333333334, + "Filipino,Chinese,Spanish,English": 0.25333333333333335, + "Filipino,Chinese,Malay,Indonesian": 0.24, + "Filipino,Chinese,Malay,English": 0.22, + "Filipino,Chinese,Indonesian,English": 0.24, + "Filipino,Spanish,Malay,Indonesian": 0.23333333333333334, + "Filipino,Spanish,Malay,English": 0.2, + "Filipino,Spanish,Indonesian,English": 0.20666666666666667, + "Filipino,Malay,Indonesian,English": 0.21333333333333335, + "Vietnamese,Chinese,Spanish,Malay": 0.26, + "Vietnamese,Chinese,Spanish,Indonesian": 0.2733333333333333, + "Vietnamese,Chinese,Spanish,English": 0.28, + "Vietnamese,Chinese,Malay,Indonesian": 0.2866666666666667, + "Vietnamese,Chinese,Malay,English": 0.26666666666666666, + "Vietnamese,Chinese,Indonesian,English": 0.28, + "Vietnamese,Spanish,Malay,Indonesian": 0.24, + "Vietnamese,Spanish,Malay,English": 0.22, + "Vietnamese,Spanish,Indonesian,English": 0.23333333333333334, + "Vietnamese,Malay,Indonesian,English": 0.24666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.26, + "Chinese,Spanish,Malay,English": 0.24, + "Chinese,Spanish,Indonesian,English": 0.25333333333333335, + "Chinese,Malay,Indonesian,English": 0.26666666666666666, + "Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.03333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.02666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.03333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.05333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.04666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.04666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.06666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.05333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.06666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.04, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.04666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.04666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.06, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.02, - "English,Chinese,Vietnamese,Filipino,Malay": 0.03333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.04666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.04, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.04666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.18, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Spanish,Malay,English": 0.16, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.18, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, + "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Indonesian,English": 0.18, + "Filipino,Chinese,Malay,Indonesian,English": 0.19333333333333333, + "Filipino,Spanish,Malay,Indonesian,English": 0.16666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2, + "Vietnamese,Chinese,Spanish,Malay,English": 0.18666666666666668, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.2, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.22, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.16666666666666666, + "Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.02, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.02666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.02666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.03333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.03333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.16, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.13333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 } }, - "AC3_2": 0.313586996289506, - "AC3_3": 0.21456229159226345, - "AC3_4": 0.1326826708261097, - "AC3_5": 0.07666817198518942, - "AC3_6": 0.04356980886037067, - "AC3_7": 0.025347467599547174 + "AC3_2": 0.4391168162111224, + "AC3_3": 0.35800523336813417, + "AC3_4": 0.29627093843513447, + "AC3_5": 0.2484598592814412, + "AC3_6": 0.21090024326859275, + "AC3_7": 0.18184615380943492 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.27922077922077926, + "overall_acc": 0.3092532467532468, "language_acc": { - "English": 0.2897727272727273, - "Filipino": 0.26704545454545453, - "Spanish": 0.2897727272727273, - "Malay": 0.29545454545454547, - "Chinese": 0.30113636363636365, - "Indonesian": 0.30113636363636365, - "Vietnamese": 0.21022727272727273 + "Indonesian": 0.3068181818181818, + "English": 0.2840909090909091, + "Filipino": 0.2727272727272727, + "Spanish": 0.3125, + "Chinese": 0.32386363636363635, + "Malay": 0.3522727272727273, + "Vietnamese": 0.3125 }, - "consistency_score_2": 0.3866341991341991, - "consistency_score_3": 0.15876623376623375, - "consistency_score_4": 0.06542207792207792, - "consistency_score_5": 0.024891774891774885, - "consistency_score_6": 0.007305194805194806, - "consistency_score_7": 0.0, + "consistency_score_2": 0.4945887445887447, + "consistency_score_3": 0.2849025974025974, + "consistency_score_4": 0.18116883116883112, + "consistency_score_5": 0.125, + "consistency_score_6": 0.09253246753246754, + "consistency_score_7": 0.07386363636363637, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.3181818181818182, - "English,Malay": 0.4715909090909091, - "English,Chinese": 0.375, - "English,Indonesian": 0.42045454545454547, - "English,Vietnamese": 0.3465909090909091, - "Filipino,Spanish": 0.38636363636363635, - "Filipino,Malay": 0.4034090909090909, - "Filipino,Chinese": 0.3068181818181818, - "Filipino,Indonesian": 0.4318181818181818, - "Filipino,Vietnamese": 0.36363636363636365, - "Spanish,Malay": 0.35795454545454547, - "Spanish,Chinese": 0.4090909090909091, - "Spanish,Indonesian": 0.39204545454545453, - "Spanish,Vietnamese": 0.4147727272727273, - "Malay,Chinese": 0.4147727272727273, - "Malay,Indonesian": 0.48295454545454547, - "Malay,Vietnamese": 0.3806818181818182, - "Chinese,Indonesian": 0.32954545454545453, - "Chinese,Vietnamese": 0.4431818181818182, - "Indonesian,Vietnamese": 0.3352272727272727 + "Indonesian,English": 0.4943181818181818, + "Indonesian,Filipino": 0.4318181818181818, + "Indonesian,Spanish": 0.4772727272727273, + "Indonesian,Chinese": 0.5, + "Indonesian,Malay": 0.5965909090909091, + "Indonesian,Vietnamese": 0.5852272727272727, + "English,Filipino": 0.4375, + "English,Spanish": 0.5454545454545454, + "English,Chinese": 0.5738636363636364, + "English,Malay": 0.4772727272727273, + "English,Vietnamese": 0.5, + "Filipino,Spanish": 0.36363636363636365, + "Filipino,Chinese": 0.3522727272727273, + "Filipino,Malay": 0.4772727272727273, + "Filipino,Vietnamese": 0.3977272727272727, + "Spanish,Chinese": 0.5284090909090909, + "Spanish,Malay": 0.4943181818181818, + "Spanish,Vietnamese": 0.5568181818181818, + "Chinese,Malay": 0.4431818181818182, + "Chinese,Vietnamese": 0.5681818181818182, + "Malay,Vietnamese": 0.5852272727272727 }, "3_combine": { - "English,Filipino,Spanish": 0.09659090909090909, - "English,Filipino,Malay": 0.19318181818181818, - "English,Filipino,Chinese": 0.10795454545454546, - "English,Filipino,Indonesian": 0.1534090909090909, - "English,Filipino,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay": 0.14204545454545456, - "English,Spanish,Chinese": 0.14204545454545456, - "English,Spanish,Indonesian": 0.14772727272727273, - "English,Spanish,Vietnamese": 0.125, - "English,Malay,Chinese": 0.20454545454545456, - "English,Malay,Indonesian": 0.26136363636363635, - "English,Malay,Vietnamese": 0.17613636363636365, - "English,Chinese,Indonesian": 0.14772727272727273, - "English,Chinese,Vietnamese": 0.1590909090909091, - "English,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay": 0.1590909090909091, - "Filipino,Spanish,Chinese": 0.13636363636363635, - "Filipino,Spanish,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese": 0.1534090909090909, - "Filipino,Malay,Indonesian": 0.2159090909090909, - "Filipino,Malay,Vietnamese": 0.13636363636363635, - "Filipino,Chinese,Indonesian": 0.11931818181818182, - "Filipino,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese": 0.17613636363636365, - "Spanish,Malay,Indonesian": 0.17045454545454544, - "Spanish,Malay,Vietnamese": 0.1534090909090909, - "Spanish,Chinese,Indonesian": 0.14204545454545456, - "Spanish,Chinese,Vietnamese": 0.20454545454545456, - "Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "Malay,Chinese,Indonesian": 0.19318181818181818, - "Malay,Chinese,Vietnamese": 0.18181818181818182, - "Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Chinese,Indonesian,Vietnamese": 0.125 + "Indonesian,English,Filipino": 0.23863636363636365, + "Indonesian,English,Spanish": 0.29545454545454547, + "Indonesian,English,Chinese": 0.3068181818181818, + "Indonesian,English,Malay": 0.32386363636363635, + "Indonesian,English,Vietnamese": 0.32386363636363635, + "Indonesian,Filipino,Spanish": 0.21022727272727273, + "Indonesian,Filipino,Chinese": 0.19886363636363635, + "Indonesian,Filipino,Malay": 0.3068181818181818, + "Indonesian,Filipino,Vietnamese": 0.25, + "Indonesian,Spanish,Chinese": 0.2897727272727273, + "Indonesian,Spanish,Malay": 0.32954545454545453, + "Indonesian,Spanish,Vietnamese": 0.32386363636363635, + "Indonesian,Chinese,Malay": 0.3068181818181818, + "Indonesian,Chinese,Vietnamese": 0.3522727272727273, + "Indonesian,Malay,Vietnamese": 0.4147727272727273, + "English,Filipino,Spanish": 0.23863636363636365, + "English,Filipino,Chinese": 0.24431818181818182, + "English,Filipino,Malay": 0.26704545454545453, + "English,Filipino,Vietnamese": 0.22727272727272727, + "English,Spanish,Chinese": 0.3522727272727273, + "English,Spanish,Malay": 0.2840909090909091, + "English,Spanish,Vietnamese": 0.32954545454545453, + "English,Chinese,Malay": 0.26704545454545453, + "English,Chinese,Vietnamese": 0.3465909090909091, + "English,Malay,Vietnamese": 0.3125, + "Filipino,Spanish,Chinese": 0.19318181818181818, + "Filipino,Spanish,Malay": 0.22727272727272727, + "Filipino,Spanish,Vietnamese": 0.20454545454545456, + "Filipino,Chinese,Malay": 0.2215909090909091, + "Filipino,Chinese,Vietnamese": 0.20454545454545456, + "Filipino,Malay,Vietnamese": 0.2897727272727273, + "Spanish,Chinese,Malay": 0.26704545454545453, + "Spanish,Chinese,Vietnamese": 0.3465909090909091, + "Spanish,Malay,Vietnamese": 0.3409090909090909, + "Chinese,Malay,Vietnamese": 0.3352272727272727 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.056818181818181816, - "English,Filipino,Spanish,Chinese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Vietnamese": 0.045454545454545456, - "English,Filipino,Malay,Chinese": 0.06818181818181818, - "English,Filipino,Malay,Indonesian": 0.10227272727272728, - "English,Filipino,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Chinese,Vietnamese": 0.05113636363636364, - "English,Filipino,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese": 0.07386363636363637, - "English,Spanish,Malay,Indonesian": 0.07954545454545454, - "English,Spanish,Malay,Vietnamese": 0.0625, - "English,Spanish,Chinese,Indonesian": 0.0625, - "English,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "English,Spanish,Indonesian,Vietnamese": 0.056818181818181816, - "English,Malay,Chinese,Indonesian": 0.10227272727272728, - "English,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese": 0.06818181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.08522727272727272, - "Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Chinese,Indonesian": 0.05113636363636364, - "Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Chinese,Indonesian": 0.0625, - "Filipino,Malay,Chinese,Vietnamese": 0.0625, - "Filipino,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Spanish,Malay,Indonesian,Vietnamese": 0.05113636363636364, - "Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Malay,Chinese,Indonesian,Vietnamese": 0.0625 + "Indonesian,English,Filipino,Spanish": 0.1590909090909091, + "Indonesian,English,Filipino,Chinese": 0.14772727272727273, + "Indonesian,English,Filipino,Malay": 0.18181818181818182, + "Indonesian,English,Filipino,Vietnamese": 0.1590909090909091, + "Indonesian,English,Spanish,Chinese": 0.20454545454545456, + "Indonesian,English,Spanish,Malay": 0.22727272727272727, + "Indonesian,English,Spanish,Vietnamese": 0.20454545454545456, + "Indonesian,English,Chinese,Malay": 0.19318181818181818, + "Indonesian,English,Chinese,Vietnamese": 0.2215909090909091, + "Indonesian,English,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Spanish,Chinese": 0.13068181818181818, + "Indonesian,Filipino,Spanish,Malay": 0.17613636363636365, + "Indonesian,Filipino,Spanish,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Chinese,Malay": 0.14772727272727273, + "Indonesian,Filipino,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Malay,Vietnamese": 0.20454545454545456, + "Indonesian,Spanish,Chinese,Malay": 0.19886363636363635, + "Indonesian,Spanish,Chinese,Vietnamese": 0.2215909090909091, + "Indonesian,Spanish,Malay,Vietnamese": 0.25, + "Indonesian,Chinese,Malay,Vietnamese": 0.24431818181818182, + "English,Filipino,Spanish,Chinese": 0.1534090909090909, + "English,Filipino,Spanish,Malay": 0.1590909090909091, + "English,Filipino,Spanish,Vietnamese": 0.1534090909090909, + "English,Filipino,Chinese,Malay": 0.14204545454545456, + "English,Filipino,Chinese,Vietnamese": 0.1534090909090909, + "English,Filipino,Malay,Vietnamese": 0.17613636363636365, + "English,Spanish,Chinese,Malay": 0.19318181818181818, + "English,Spanish,Chinese,Vietnamese": 0.24431818181818182, + "English,Spanish,Malay,Vietnamese": 0.20454545454545456, + "English,Chinese,Malay,Vietnamese": 0.21022727272727273, + "Filipino,Spanish,Chinese,Malay": 0.13068181818181818, + "Filipino,Spanish,Chinese,Vietnamese": 0.125, + "Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, + "Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, + "Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Indonesian": 0.03409090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.022727272727272728, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 + "Indonesian,English,Filipino,Spanish,Chinese": 0.11363636363636363, + "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10795454545454546, + "Indonesian,English,Filipino,Chinese,Malay": 0.10795454545454546, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.10795454545454546, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese,Malay": 0.1534090909090909, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1590909090909091, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.125, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727, + "English,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, + "English,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, + "English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, + "English,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.125, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 } }, - "AC3_2": 0.3242637086115966, - "AC3_3": 0.20242989030064762, - "AC3_4": 0.10600656994189732, - "AC3_5": 0.04570873964388737, - "AC3_6": 0.014237886754164077, - "AC3_7": 0.0 + "AC3_2": 0.38055532480283305, + "AC3_3": 0.2965789333118043, + "AC3_4": 0.22848502039473678, + "AC3_5": 0.17803738313657158, + "AC3_6": 0.1424439196811943, + "AC3_7": 0.11924595528474537 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.21359223300970873 + "accuracy": 0.33980582524271846 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.20952380952380953 + "accuracy": 0.2571428571428571 } }, "us_eval": { "prompt_1": { - "accuracy": 0.24299065420560748 + "accuracy": 0.3925233644859813 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.21, + "accuracy": 0.32, "category_acc": { - "brand": 0.2, + "brand": 0.3, "demographics": 0.2, - "biology": 0.1, - "history": 0.26666666666666666, - "literature": 0.1, - "politics": 0.3, - "culture": 0.0, - "film": 0.3, - "law": 0.2, + "biology": 0.3, + "history": 0.4, + "literature": 0.2, + "politics": 0.4, + "culture": 0.4, + "film": 0.2, + "law": 0.3, "geography": 0.4 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.13004777584638155 + "bleu_score": 0.2859232180639829 } }, "indommlu": { - "prompt_1": -1 + "prompt_1": { + "accuracy": 0.33994258628746915, + "category_acc": { + "History": 0.29116465863453816, + "Geography": 0.26326530612244897, + "Lampungic": 0.32653061224489793, + "Social science": 0.5158597662771286, + "Balinese": 0.2781316348195329, + "Makassarese": 0.27956989247311825, + "Banjarese": 0.2361111111111111, + "Chemistry": 0.23503649635036497, + "Biology": 0.3029585798816568, + "Science": 0.3735810113519092, + "Christian religion": 0.3880597014925373, + "Art": 0.39767054908485855, + "Islam religion": 0.4039829302987198, + "Hindu religion": 0.35333333333333333, + "Madurese": 0.27796610169491526, + "Sport": 0.34459459459459457, + "Indonesian language": 0.3788916562889166, + "Physics": 0.2828282828282828, + "Minangkabau culture": 0.32663316582914576, + "Dayak language": 0.22935779816513763, + "Sociology": 0.33669354838709675, + "Economy": 0.28688524590163933, + "Sundanese": 0.3025064822817632, + "Javanese": 0.2842741935483871, + "Civic education": 0.4177396280400572 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.14232500222341316 + "bleu_score": 0.3824192602359019 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.11010354975069649 + "bleu_score": 0.3128439608609861 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.06534027304610379 + "bleu_score": 0.2169986161156318 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.13823983749981356 + "bleu_score": 0.38545186546036936 } }, "mmlu": { "prompt_1": { - "accuracy": 0.23453908984830804 + "accuracy": 0.3547257876312719 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.26077940650697173, + "accuracy": 0.3407937075437969, "category_acc": { - "high_school_european_history": 0.24390243902439024, - "business_ethics": 0.18181818181818182, - "clinical_knowledge": 0.26136363636363635, - "medical_genetics": 0.29292929292929293, - "high_school_us_history": 0.2660098522167488, - "high_school_physics": 0.2866666666666667, - "high_school_world_history": 0.2288135593220339, - "virology": 0.2545454545454545, - "high_school_microeconomics": 0.24050632911392406, - "econometrics": 0.23893805309734514, - "college_computer_science": 0.29292929292929293, - "high_school_biology": 0.27184466019417475, - "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.2277580071174377, - "philosophy": 0.2645161290322581, + "high_school_european_history": 0.4146341463414634, + "business_ethics": 0.32323232323232326, + "clinical_knowledge": 0.4053030303030303, + "medical_genetics": 0.3838383838383838, + "high_school_us_history": 0.4433497536945813, + "high_school_physics": 0.38, + "high_school_world_history": 0.3601694915254237, + "virology": 0.2727272727272727, + "high_school_microeconomics": 0.25316455696202533, + "econometrics": 0.25663716814159293, + "college_computer_science": 0.40404040404040403, + "high_school_biology": 0.35275080906148865, + "abstract_algebra": 0.20202020202020202, + "professional_accounting": 0.28825622775800713, + "philosophy": 0.36129032258064514, "professional_medicine": 0.35793357933579334, - "nutrition": 0.2, - "global_facts": 0.1919191919191919, - "machine_learning": 0.18018018018018017, - "security_studies": 0.3073770491803279, - "public_relations": 0.23853211009174313, - "professional_psychology": 0.21112929623567922, - "prehistory": 0.2786377708978328, - "anatomy": 0.26865671641791045, - "human_sexuality": 0.23076923076923078, - "college_medicine": 0.25, - "high_school_government_and_politics": 0.3177083333333333, - "college_chemistry": 0.30303030303030304, - "logical_fallacies": 0.24074074074074073, - "high_school_geography": 0.29441624365482233, - "elementary_mathematics": 0.246684350132626, - "human_aging": 0.26576576576576577, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.2867647058823529, - "formal_logic": 0.288, - "high_school_statistics": 0.32558139534883723, - "international_law": 0.24166666666666667, - "high_school_mathematics": 0.26765799256505574, - "high_school_computer_science": 0.26262626262626265, - "conceptual_physics": 0.27350427350427353, - "miscellaneous": 0.2391304347826087, - "high_school_chemistry": 0.24257425742574257, - "marketing": 0.2446351931330472, - "professional_law": 0.2628832354859752, - "management": 0.28431372549019607, - "college_physics": 0.32673267326732675, - "jurisprudence": 0.32710280373831774, - "world_religions": 0.2235294117647059, - "sociology": 0.27, - "us_foreign_policy": 0.26262626262626265, - "high_school_macroeconomics": 0.2699228791773779, - "computer_security": 0.2828282828282828, - "moral_scenarios": 0.27293064876957496, - "moral_disputes": 0.20869565217391303, - "electrical_engineering": 0.2569444444444444, - "astronomy": 0.31125827814569534, - "college_biology": 0.2727272727272727 + "nutrition": 0.38688524590163936, + "global_facts": 0.23232323232323232, + "machine_learning": 0.25225225225225223, + "security_studies": 0.3483606557377049, + "public_relations": 0.28440366972477066, + "professional_psychology": 0.353518821603928, + "prehistory": 0.3653250773993808, + "anatomy": 0.3582089552238806, + "human_sexuality": 0.43846153846153846, + "college_medicine": 0.3430232558139535, + "high_school_government_and_politics": 0.4635416666666667, + "college_chemistry": 0.3434343434343434, + "logical_fallacies": 0.35802469135802467, + "high_school_geography": 0.4720812182741117, + "elementary_mathematics": 0.2864721485411141, + "human_aging": 0.3108108108108108, + "college_mathematics": 0.29292929292929293, + "high_school_psychology": 0.375, + "formal_logic": 0.264, + "high_school_statistics": 0.3116279069767442, + "international_law": 0.45, + "high_school_mathematics": 0.23048327137546468, + "high_school_computer_science": 0.3939393939393939, + "conceptual_physics": 0.32905982905982906, + "miscellaneous": 0.4143222506393862, + "high_school_chemistry": 0.3118811881188119, + "marketing": 0.4034334763948498, + "professional_law": 0.2759295499021526, + "management": 0.39215686274509803, + "college_physics": 0.31683168316831684, + "jurisprudence": 0.3177570093457944, + "world_religions": 0.4823529411764706, + "sociology": 0.43, + "us_foreign_policy": 0.5151515151515151, + "high_school_macroeconomics": 0.29562982005141386, + "computer_security": 0.3838383838383838, + "moral_scenarios": 0.24496644295302014, + "moral_disputes": 0.37971014492753624, + "electrical_engineering": 0.3333333333333333, + "astronomy": 0.40397350993377484, + "college_biology": 0.3916083916083916 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.2726597325408618 + "accuracy": 0.3150074294205052 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.25404732254047324, + "accuracy": 0.30884184308841844, "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.25, + "computer_network": 0.5833333333333334, + "operating_system": 0.08333333333333333, "computer_architecture": 0.19230769230769232, - "college_programming": 0.21428571428571427, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.27586206896551724, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.19230769230769232, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.10714285714285714, - "college_economics": 0.16666666666666666, - "business_administration": 0.21052631578947367, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.2413793103448276, - "education_science": 0.058823529411764705, - "teacher_qualification": 0.2857142857142857, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.08333333333333333, - "middle_school_politics": 0.34615384615384615, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.25, - "logic": 0.14814814814814814, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.28, + "college_programming": 0.19047619047619047, + "college_physics": 0.4166666666666667, + "college_chemistry": 0.3793103448275862, + "advanced_mathematics": 0.125, + "probability_and_statistics": 0.34782608695652173, + "discrete_mathematics": 0.14285714285714285, + "electrical_engineer": 0.23809523809523808, + "metrology_engineer": 0.2413793103448276, + "high_school_mathematics": 0.17391304347826086, + "high_school_physics": 0.25, + "high_school_chemistry": 0.25, + "high_school_biology": 0.125, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.46153846153846156, + "middle_school_physics": 0.5833333333333334, + "middle_school_chemistry": 0.4, + "veterinary_medicine": 0.39285714285714285, + "college_economics": 0.23333333333333334, + "business_administration": 0.2894736842105263, + "marxism": 0.3333333333333333, + "mao_zedong_thought": 0.3793103448275862, + "education_science": 0.2647058823529412, + "teacher_qualification": 0.30612244897959184, + "high_school_politics": 0.5416666666666666, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.38461538461538464, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.5, + "ideological_and_moral_cultivation": 0.375, + "logic": 0.25925925925925924, + "law": 0.27586206896551724, + "chinese_language_and_literature": 0.25, + "art_studies": 0.2894736842105263, + "professional_tour_guide": 0.3235294117647059, + "legal_professional": 0.5357142857142857, + "high_school_chinese": 0.4583333333333333, + "high_school_history": 0.36, "middle_school_history": 0.3333333333333333, - "civil_servant": 0.3269230769230769, - "sports_science": 0.25, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.20833333333333334, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.2962962962962963, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.2777777777777778, + "civil_servant": 0.38461538461538464, + "sports_science": 0.375, + "plant_protection": 0.25925925925925924, + "basic_medicine": 0.3333333333333333, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.21568627450980393, + "accountant": 0.4074074074074074, + "fire_engineer": 0.19444444444444445, + "environmental_impact_assessment_engineer": 0.3055555555555556, + "tax_accountant": 0.2037037037037037, "physician": 0.3148148148148148 } } }, "cmmlu": { "prompt_1": { - "accuracy": 0.26881720430107525 + "accuracy": 0.2616487455197133 } }, "cmmlu_full": { "prompt_1": { - "accuracy": 0.2571231220859955, + "accuracy": 0.28708340528406145, "category_acc": { - "agronomy": 0.30177514792899407, - "anatomy": 0.31756756756756754, - "ancient_chinese": 0.25609756097560976, - "arts": 0.21875, - "astronomy": 0.24848484848484848, - "business_ethics": 0.27751196172248804, - "chinese_civil_service_exam": 0.23125, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.18691588785046728, - "chinese_history": 0.2786377708978328, - "chinese_literature": 0.25980392156862747, - "chinese_teacher_qualification": 0.2737430167597765, - "clinical_knowledge": 0.22362869198312235, - "college_actuarial_science": 0.22641509433962265, - "college_education": 0.24299065420560748, - "college_engineering_hydrology": 0.3584905660377358, - "college_law": 0.2222222222222222, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.16981132075471697, - "college_medicine": 0.28205128205128205, - "computer_science": 0.27450980392156865, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.2585034013605442, - "construction_project_management": 0.2805755395683453, - "economics": 0.3270440251572327, - "education": 0.20245398773006135, - "electrical_engineering": 0.19186046511627908, - "elementary_chinese": 0.24603174603174602, - "elementary_commonsense": 0.23737373737373738, - "elementary_information_and_technology": 0.29411764705882354, - "elementary_mathematics": 0.26521739130434785, - "ethnology": 0.25925925925925924, - "food_science": 0.25874125874125875, - "genetics": 0.26704545454545453, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.22485207100591717, - "high_school_chemistry": 0.3181818181818182, - "high_school_geography": 0.2711864406779661, + "agronomy": 0.27218934911242604, + "anatomy": 0.25, + "ancient_chinese": 0.29878048780487804, + "arts": 0.3375, + "astronomy": 0.28484848484848485, + "business_ethics": 0.23444976076555024, + "chinese_civil_service_exam": 0.25, + "chinese_driving_rule": 0.3282442748091603, + "chinese_food_culture": 0.3161764705882353, + "chinese_foreign_policy": 0.29906542056074764, + "chinese_history": 0.29102167182662536, + "chinese_literature": 0.29411764705882354, + "chinese_teacher_qualification": 0.30726256983240224, + "clinical_knowledge": 0.25738396624472576, + "college_actuarial_science": 0.27358490566037735, + "college_education": 0.32710280373831774, + "college_engineering_hydrology": 0.22641509433962265, + "college_law": 0.23148148148148148, + "college_mathematics": 0.19047619047619047, + "college_medical_statistics": 0.3490566037735849, + "college_medicine": 0.2783882783882784, + "computer_science": 0.25980392156862747, + "computer_security": 0.3157894736842105, + "conceptual_physics": 0.30612244897959184, + "construction_project_management": 0.2949640287769784, + "economics": 0.2893081761006289, + "education": 0.31901840490797545, + "electrical_engineering": 0.32558139534883723, + "elementary_chinese": 0.2896825396825397, + "elementary_commonsense": 0.24242424242424243, + "elementary_information_and_technology": 0.33613445378151263, + "elementary_mathematics": 0.2391304347826087, + "ethnology": 0.3333333333333333, + "food_science": 0.2937062937062937, + "genetics": 0.2556818181818182, + "global_facts": 0.35570469798657717, + "high_school_biology": 0.28994082840236685, + "high_school_chemistry": 0.24242424242424243, + "high_school_geography": 0.2627118644067797, "high_school_mathematics": 0.24390243902439024, - "high_school_physics": 0.3, - "high_school_politics": 0.23076923076923078, - "human_sexuality": 0.2777777777777778, - "international_law": 0.2864864864864865, - "journalism": 0.16279069767441862, - "jurisprudence": 0.26034063260340634, - "legal_and_moral_basis": 0.2523364485981308, - "logical": 0.2845528455284553, - "machine_learning": 0.2459016393442623, - "management": 0.2857142857142857, - "marketing": 0.24444444444444444, - "marxist_theory": 0.24338624338624337, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.20689655172413793, - "philosophy": 0.2571428571428571, + "high_school_physics": 0.20909090909090908, + "high_school_politics": 0.3146853146853147, + "human_sexuality": 0.29365079365079366, + "international_law": 0.2594594594594595, + "journalism": 0.29651162790697677, + "jurisprudence": 0.2846715328467153, + "legal_and_moral_basis": 0.40654205607476634, + "logical": 0.2682926829268293, + "machine_learning": 0.27049180327868855, + "management": 0.29523809523809524, + "marketing": 0.29444444444444445, + "marxist_theory": 0.32275132275132273, + "modern_chinese": 0.25862068965517243, + "nutrition": 0.2620689655172414, + "philosophy": 0.3142857142857143, "professional_accounting": 0.2742857142857143, - "professional_law": 0.3033175355450237, - "professional_medicine": 0.23404255319148937, - "professional_psychology": 0.24568965517241378, - "public_relations": 0.28160919540229884, - "security_study": 0.23703703703703705, - "sociology": 0.2345132743362832, - "sports_science": 0.2545454545454545, - "traditional_chinese_medicine": 0.2756756756756757, - "virology": 0.23668639053254437, - "world_history": 0.2670807453416149, - "world_religions": 0.30625 + "professional_law": 0.2890995260663507, + "professional_medicine": 0.24202127659574468, + "professional_psychology": 0.31896551724137934, + "public_relations": 0.2471264367816092, + "security_study": 0.37037037037037035, + "sociology": 0.2610619469026549, + "sports_science": 0.3333333333333333, + "traditional_chinese_medicine": 0.25405405405405407, + "virology": 0.3136094674556213, + "world_history": 0.30434782608695654, + "world_religions": 0.3 } } }, "zbench": { "prompt_1": { - "accuracy": 0.18181818181818182 + "accuracy": 0.2727272727272727 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.20454545454545456 + "accuracy": 0.18636363636363637 } }, "ocnli": { "prompt_1": { - "accuracy": 0.3494915254237288 + "accuracy": 0.3288135593220339 } }, "c3": { "prompt_1": { - "accuracy": 0.2868362004487659 + "accuracy": 0.4143605086013463 } }, "dream": { "prompt_1": { - "accuracy": 0.3184713375796178 + "accuracy": 0.5227829495345419 } }, "samsum": { "prompt_1": { - "rouge1": 0.252576810027494, - "rouge2": 0.07129142729784035, - "rougeL": 0.19596293864871944, - "avg_rouge": 0.1732770586580179 + "rouge1": 0.41687717824020265, + "rouge2": 0.17505033429566638, + "rougeL": 0.33363971351255944, + "avg_rouge": 0.3085224086828095 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.2790104857102043, - "rouge2": 0.07314162231498228, - "rougeL": 0.21665939847446175, - "avg_rouge": 0.1896038354998828 + "rouge1": 0.39976991490542363, + "rouge2": 0.15292232665909905, + "rougeL": 0.3215121347124994, + "avg_rouge": 0.29140145875900736 } }, "sst2": { "prompt_1": { - "accuracy": 0.5011467889908257 + "accuracy": 0.5630733944954128 } }, "cola": { "prompt_1": { - "accuracy": 0.4947267497603068 + "accuracy": 0.5321188878235859 } }, "qqp": { "prompt_1": { - "accuracy": 0.494 + "accuracy": 0.4955 } }, "mnli": { "prompt_1": { - "accuracy": 0.324 + "accuracy": 0.4 } }, "qnli": { "prompt_1": { - "accuracy": 0.496 + "accuracy": 0.7 } }, "wnli": { "prompt_1": { - "accuracy": 0.4788732394366197 + "accuracy": 0.4 } }, "rte": { "prompt_1": { - "accuracy": 0.516245487364621 + "accuracy": 0.4 } }, "mrpc": { "prompt_1": { - "accuracy": 0.5343137254901961 + "accuracy": 0.6 } } } }, - "sailor_1_8b_chat": { - "model_size": "1.8B", - "model_link": "https://huggingface.co/sail/Sailor-1.8B-Chat", + "sailor_7b": { + "model_size": "7B", + "model_link": "https://huggingface.co/sail/Sailor-7B", "zero_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.6336134453781512, - "language_acc": { - "English": 0.6705882352941176, - "Chinese": 0.6302521008403361, - "Spanish": 0.619327731092437, - "Vietnamese": 0.6142857142857143 - }, - "consistency_score_2": 0.6260504201680672, - "consistency_score_3": 0.47773109243697476, - "consistency_score_4": 0.3873949579831933, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.6159663865546219, - "English,Spanish": 0.6529411764705882, - "English,Vietnamese": 0.646218487394958, - "Chinese,Spanish": 0.6092436974789915, - "Chinese,Vietnamese": 0.5882352941176471, - "Spanish,Vietnamese": 0.6436974789915967 - }, - "3_combine": { - "English,Chinese,Spanish": 0.4789915966386555, - "English,Chinese,Vietnamese": 0.46638655462184875, - "English,Spanish,Vietnamese": 0.5042016806722689, - "Chinese,Spanish,Vietnamese": 0.4613445378151261 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3873949579831933 - } - }, - "AC3_2": 0.6298092285113396, - "AC3_3": 0.5447398769982109, - "AC3_4": 0.4808161288736118 - }, - "prompt_2": { - "overall_acc": 0.638235294117647, - "language_acc": { - "English": 0.6579831932773109, - "Chinese": 0.6394957983193277, - "Spanish": 0.6411764705882353, - "Vietnamese": 0.6142857142857143 - }, - "consistency_score_2": 0.6133053221288516, - "consistency_score_3": 0.4632352941176471, - "consistency_score_4": 0.37478991596638656, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.5966386554621849, - "English,Spanish": 0.6554621848739496, - "English,Vietnamese": 0.6319327731092437, - "Chinese,Spanish": 0.5941176470588235, - "Chinese,Vietnamese": 0.5831932773109244, - "Spanish,Vietnamese": 0.6184873949579832 - }, - "3_combine": { - "English,Chinese,Spanish": 0.4647058823529412, - "English,Chinese,Vietnamese": 0.4512605042016807, - "English,Spanish,Vietnamese": 0.49327731092436977, - "Chinese,Spanish,Vietnamese": 0.4436974789915966 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.37478991596638656 - } - }, - "AC3_2": 0.6255220126942401, - "AC3_3": 0.5368334249100306, - "AC3_4": 0.47225705706905236 - }, - "prompt_3": { - "overall_acc": 0.6640756302521008, - "language_acc": { - "English": 0.7025210084033613, - "Chinese": 0.6386554621848739, - "Spanish": 0.6789915966386555, - "Vietnamese": 0.6361344537815126 - }, - "consistency_score_2": 0.6452380952380953, - "consistency_score_3": 0.5052521008403361, - "consistency_score_4": 0.42100840336134454, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.6092436974789915, - "English,Spanish": 0.7134453781512605, - "English,Vietnamese": 0.6571428571428571, - "Chinese,Spanish": 0.6226890756302521, - "Chinese,Vietnamese": 0.5907563025210084, - "Spanish,Vietnamese": 0.6781512605042017 - }, - "3_combine": { - "English,Chinese,Spanish": 0.5084033613445378, - "English,Chinese,Vietnamese": 0.4764705882352941, - "English,Spanish,Vietnamese": 0.5521008403361345, - "Chinese,Spanish,Vietnamese": 0.48403361344537815 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.42100840336134454 - } - }, - "AC3_2": 0.6545213517329549, - "AC3_3": 0.57387779037719, - "AC3_4": 0.5153175461384567 - }, - "prompt_4": { - "overall_acc": 0.6373949579831933, - "language_acc": { - "English": 0.6857142857142857, - "Chinese": 0.6235294117647059, - "Spanish": 0.6420168067226891, - "Vietnamese": 0.5983193277310924 - }, - "consistency_score_2": 0.6103641456582632, - "consistency_score_3": 0.4569327731092437, - "consistency_score_4": 0.3663865546218487, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.5966386554621849, - "English,Spanish": 0.6655462184873949, - "English,Vietnamese": 0.619327731092437, - "Chinese,Spanish": 0.6042016806722689, - "Chinese,Vietnamese": 0.5605042016806723, - "Spanish,Vietnamese": 0.6159663865546219 - }, - "3_combine": { - "English,Chinese,Spanish": 0.4689075630252101, - "English,Chinese,Vietnamese": 0.4369747899159664, - "English,Spanish,Vietnamese": 0.4857142857142857, - "Chinese,Spanish,Vietnamese": 0.4361344537815126 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3663865546218487 - } - }, - "AC3_2": 0.6235867609536728, - "AC3_3": 0.5322841364894458, - "AC3_4": 0.46530632340391215 - }, - "prompt_5": { - "overall_acc": 0.6600840336134454, - "language_acc": { - "English": 0.6957983193277311, - "Chinese": 0.6521008403361345, - "Spanish": 0.6680672268907563, - "Vietnamese": 0.6243697478991597 - }, - "consistency_score_2": 0.6682072829131652, - "consistency_score_3": 0.5323529411764706, - "consistency_score_4": 0.4495798319327731, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.6411764705882353, - "English,Spanish": 0.7235294117647059, - "English,Vietnamese": 0.692436974789916, - "Chinese,Spanish": 0.6386554621848739, - "Chinese,Vietnamese": 0.6168067226890757, - "Spanish,Vietnamese": 0.6966386554621848 - }, - "3_combine": { - "English,Chinese,Spanish": 0.5294117647058824, - "English,Chinese,Vietnamese": 0.5067226890756302, - "English,Spanish,Vietnamese": 0.5806722689075631, - "Chinese,Spanish,Vietnamese": 0.5126050420168067 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.4495798319327731 - } - }, - "AC3_2": 0.664120819091661, - "AC3_3": 0.5893773576589848, - "AC3_4": 0.5348655175333112 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.37047619047619046, - "language_acc": { - "Spanish": 0.3933333333333333, - "English": 0.44666666666666666, - "Chinese": 0.35333333333333333, - "Vietnamese": 0.3466666666666667, - "Filipino": 0.22666666666666666, - "Indonesian": 0.42, - "Malay": 0.4066666666666667 - }, - "consistency_score_2": 0.4653968253968255, - "consistency_score_3": 0.2773333333333334, - "consistency_score_4": 0.18819047619047619, - "consistency_score_5": 0.1384126984126984, - "consistency_score_6": 0.10571428571428572, - "consistency_score_7": 0.08, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.49333333333333335, - "Spanish,Chinese": 0.4533333333333333, - "Spanish,Vietnamese": 0.5, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.49333333333333335, - "Spanish,Malay": 0.44, - "English,Chinese": 0.44666666666666666, - "English,Vietnamese": 0.49333333333333335, - "English,Filipino": 0.38666666666666666, - "English,Indonesian": 0.5466666666666666, - "English,Malay": 0.5533333333333333, - "Chinese,Vietnamese": 0.4533333333333333, - "Chinese,Filipino": 0.4, - "Chinese,Indonesian": 0.44666666666666666, - "Chinese,Malay": 0.4066666666666667, - "Vietnamese,Filipino": 0.4666666666666667, - "Vietnamese,Indonesian": 0.5, - "Vietnamese,Malay": 0.43333333333333335, - "Filipino,Indonesian": 0.41333333333333333, - "Filipino,Malay": 0.44666666666666666, - "Indonesian,Malay": 0.56 - }, - "3_combine": { - "Spanish,English,Chinese": 0.26, - "Spanish,English,Vietnamese": 0.31333333333333335, - "Spanish,English,Filipino": 0.25333333333333335, - "Spanish,English,Indonesian": 0.32, - "Spanish,English,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese": 0.28, - "Spanish,Chinese,Filipino": 0.23333333333333334, - "Spanish,Chinese,Indonesian": 0.2733333333333333, - "Spanish,Chinese,Malay": 0.26, - "Spanish,Vietnamese,Filipino": 0.28, - "Spanish,Vietnamese,Indonesian": 0.32, - "Spanish,Vietnamese,Malay": 0.2866666666666667, - "Spanish,Filipino,Indonesian": 0.24666666666666667, - "Spanish,Filipino,Malay": 0.28, - "Spanish,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese": 0.2733333333333333, - "English,Chinese,Filipino": 0.2, - "English,Chinese,Indonesian": 0.28, - "English,Chinese,Malay": 0.28, - "English,Vietnamese,Filipino": 0.25333333333333335, - "English,Vietnamese,Indonesian": 0.34, - "English,Vietnamese,Malay": 0.3333333333333333, - "English,Filipino,Indonesian": 0.24666666666666667, - "English,Filipino,Malay": 0.26, - "English,Indonesian,Malay": 0.38666666666666666, - "Chinese,Vietnamese,Filipino": 0.24, - "Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Malay": 0.24, - "Chinese,Filipino,Indonesian": 0.22666666666666666, - "Chinese,Filipino,Malay": 0.23333333333333334, - "Chinese,Indonesian,Malay": 0.26666666666666666, - "Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Vietnamese,Filipino,Malay": 0.26, - "Vietnamese,Indonesian,Malay": 0.32, - "Filipino,Indonesian,Malay": 0.2866666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.18, - "Spanish,English,Chinese,Filipino": 0.13333333333333333, - "Spanish,English,Chinese,Indonesian": 0.2, - "Spanish,English,Chinese,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.18, - "Spanish,English,Vietnamese,Indonesian": 0.24666666666666667, - "Spanish,English,Vietnamese,Malay": 0.22666666666666666, - "Spanish,English,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Filipino,Malay": 0.18666666666666668, - "Spanish,English,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Filipino,Malay": 0.16666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Malay": 0.2, - "Spanish,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.14, - "English,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "English,Chinese,Vietnamese,Malay": 0.19333333333333333, - "English,Chinese,Filipino,Indonesian": 0.15333333333333332, - "English,Chinese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Indonesian,Malay": 0.20666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.2 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.16, - "Spanish,English,Chinese,Vietnamese,Malay": 0.14, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "English,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - } - }, - "AC3_2": 0.41254697696401, - "AC3_3": 0.3172086640694302, - "AC3_4": 0.24959459674391923, - "AC3_5": 0.2015316519224926, - "AC3_6": 0.16449142853688536, - "AC3_7": 0.1315856236494366 - }, - "prompt_2": { - "overall_acc": 0.32095238095238093, - "language_acc": { - "Spanish": 0.29333333333333333, - "English": 0.35333333333333333, - "Chinese": 0.32, - "Vietnamese": 0.32666666666666666, - "Filipino": 0.24666666666666667, - "Indonesian": 0.36, - "Malay": 0.3466666666666667 - }, - "consistency_score_2": 0.4187301587301586, - "consistency_score_3": 0.23180952380952385, - "consistency_score_4": 0.15466666666666667, - "consistency_score_5": 0.11650793650793652, - "consistency_score_6": 0.09238095238095238, - "consistency_score_7": 0.07333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.3933333333333333, - "Spanish,Chinese": 0.44666666666666666, - "Spanish,Vietnamese": 0.3466666666666667, - "Spanish,Filipino": 0.3933333333333333, - "Spanish,Indonesian": 0.38666666666666666, - "Spanish,Malay": 0.38, - "English,Chinese": 0.49333333333333335, - "English,Vietnamese": 0.44666666666666666, - "English,Filipino": 0.37333333333333335, - "English,Indonesian": 0.4533333333333333, - "English,Malay": 0.49333333333333335, - "Chinese,Vietnamese": 0.41333333333333333, - "Chinese,Filipino": 0.4, - "Chinese,Indonesian": 0.37333333333333335, - "Chinese,Malay": 0.46, - "Vietnamese,Filipino": 0.4066666666666667, - "Vietnamese,Indonesian": 0.4666666666666667, - "Vietnamese,Malay": 0.48, - "Filipino,Indonesian": 0.3466666666666667, - "Filipino,Malay": 0.4, - "Indonesian,Malay": 0.44 - }, - "3_combine": { - "Spanish,English,Chinese": 0.24666666666666667, - "Spanish,English,Vietnamese": 0.2, - "Spanish,English,Filipino": 0.20666666666666667, - "Spanish,English,Indonesian": 0.24666666666666667, - "Spanish,English,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese": 0.22, - "Spanish,Chinese,Filipino": 0.23333333333333334, - "Spanish,Chinese,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino": 0.2, - "Spanish,Vietnamese,Indonesian": 0.22, - "Spanish,Vietnamese,Malay": 0.20666666666666667, - "Spanish,Filipino,Indonesian": 0.2, - "Spanish,Filipino,Malay": 0.22, - "Spanish,Indonesian,Malay": 0.22666666666666666, - "English,Chinese,Vietnamese": 0.2733333333333333, - "English,Chinese,Filipino": 0.22666666666666666, - "English,Chinese,Indonesian": 0.24, - "English,Chinese,Malay": 0.3, - "English,Vietnamese,Filipino": 0.22, - "English,Vietnamese,Indonesian": 0.26666666666666666, - "English,Vietnamese,Malay": 0.29333333333333333, - "English,Filipino,Indonesian": 0.21333333333333335, - "English,Filipino,Malay": 0.22666666666666666, - "English,Indonesian,Malay": 0.28, - "Chinese,Vietnamese,Filipino": 0.2, - "Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "Chinese,Vietnamese,Malay": 0.24666666666666667, - "Chinese,Filipino,Indonesian": 0.18666666666666668, - "Chinese,Filipino,Malay": 0.24, - "Chinese,Indonesian,Malay": 0.24, - "Vietnamese,Filipino,Indonesian": 0.24, - "Vietnamese,Filipino,Malay": 0.23333333333333334, - "Vietnamese,Indonesian,Malay": 0.28, - "Filipino,Indonesian,Malay": 0.21333333333333335 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.16, - "Spanish,English,Chinese,Filipino": 0.14666666666666667, - "Spanish,English,Chinese,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.17333333333333334, - "Spanish,English,Vietnamese,Malay": 0.14666666666666667, - "Spanish,English,Filipino,Indonesian": 0.14, - "Spanish,English,Filipino,Malay": 0.14, - "Spanish,English,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian": 0.12, - "Spanish,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.14, - "Spanish,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.14, - "Spanish,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "English,Chinese,Vietnamese,Malay": 0.18666666666666668, - "English,Chinese,Filipino,Indonesian": 0.15333333333333332, - "English,Chinese,Filipino,Malay": 0.17333333333333334, - "English,Chinese,Indonesian,Malay": 0.18, - "English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "English,Vietnamese,Filipino,Malay": 0.16666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "English,Filipino,Indonesian,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Chinese,Filipino,Indonesian,Malay": 0.14, - "Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12, - "Spanish,English,Chinese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.12, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino,Malay": 0.12, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.12, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 - } - }, - "AC3_2": 0.3633787042222818, - "AC3_3": 0.26919300313465505, - "AC3_4": 0.20874115601004306, - "AC3_5": 0.1709572188426677, - "AC3_6": 0.14346719329423466, - "AC3_7": 0.11938808370563024 - }, - "prompt_3": { - "overall_acc": 0.35619047619047617, - "language_acc": { - "Spanish": 0.36, - "English": 0.4666666666666667, - "Chinese": 0.35333333333333333, - "Vietnamese": 0.3466666666666667, - "Filipino": 0.24, - "Indonesian": 0.36666666666666664, - "Malay": 0.36 - }, - "consistency_score_2": 0.5155555555555555, - "consistency_score_3": 0.33904761904761904, - "consistency_score_4": 0.2537142857142856, - "consistency_score_5": 0.20571428571428574, - "consistency_score_6": 0.17523809523809522, - "consistency_score_7": 0.15333333333333332, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5266666666666666, - "Spanish,Chinese": 0.47333333333333333, - "Spanish,Vietnamese": 0.5133333333333333, - "Spanish,Filipino": 0.4866666666666667, - "Spanish,Indonesian": 0.54, - "Spanish,Malay": 0.48, - "English,Chinese": 0.5266666666666666, - "English,Vietnamese": 0.58, - "English,Filipino": 0.46, - "English,Indonesian": 0.5666666666666667, - "English,Malay": 0.6, - "Chinese,Vietnamese": 0.5066666666666667, - "Chinese,Filipino": 0.49333333333333335, - "Chinese,Indonesian": 0.44, - "Chinese,Malay": 0.5266666666666666, - "Vietnamese,Filipino": 0.49333333333333335, - "Vietnamese,Indonesian": 0.5066666666666667, - "Vietnamese,Malay": 0.5466666666666666, - "Filipino,Indonesian": 0.44666666666666666, - "Filipino,Malay": 0.52, - "Indonesian,Malay": 0.5933333333333334 - }, - "3_combine": { - "Spanish,English,Chinese": 0.32666666666666666, - "Spanish,English,Vietnamese": 0.36666666666666664, - "Spanish,English,Filipino": 0.32666666666666666, - "Spanish,English,Indonesian": 0.36, - "Spanish,English,Malay": 0.35333333333333333, - "Spanish,Chinese,Vietnamese": 0.32, - "Spanish,Chinese,Filipino": 0.31333333333333335, - "Spanish,Chinese,Indonesian": 0.3, - "Spanish,Chinese,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino": 0.31333333333333335, - "Spanish,Vietnamese,Indonesian": 0.36, - "Spanish,Vietnamese,Malay": 0.35333333333333333, - "Spanish,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Filipino,Malay": 0.34, - "Spanish,Indonesian,Malay": 0.3933333333333333, - "English,Chinese,Vietnamese": 0.36666666666666664, - "English,Chinese,Filipino": 0.3, - "English,Chinese,Indonesian": 0.3333333333333333, - "English,Chinese,Malay": 0.36666666666666664, - "English,Vietnamese,Filipino": 0.32, - "English,Vietnamese,Indonesian": 0.4, - "English,Vietnamese,Malay": 0.41333333333333333, - "English,Filipino,Indonesian": 0.31333333333333335, - "English,Filipino,Malay": 0.3333333333333333, - "English,Indonesian,Malay": 0.43333333333333335, - "Chinese,Vietnamese,Filipino": 0.3, - "Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Malay": 0.36, - "Chinese,Filipino,Indonesian": 0.26, - "Chinese,Filipino,Malay": 0.32, - "Chinese,Indonesian,Malay": 0.32666666666666666, - "Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Vietnamese,Filipino,Malay": 0.34, - "Vietnamese,Indonesian,Malay": 0.38, - "Filipino,Indonesian,Malay": 0.3466666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.25333333333333335, - "Spanish,English,Chinese,Filipino": 0.24, - "Spanish,English,Chinese,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.29333333333333333, - "Spanish,English,Vietnamese,Malay": 0.30666666666666664, - "Spanish,English,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Filipino,Malay": 0.26, - "Spanish,English,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Malay": 0.24666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.23333333333333334, - "Spanish,Chinese,Indonesian,Malay": 0.26, - "Spanish,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,Vietnamese,Filipino,Malay": 0.26, - "Spanish,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "English,Chinese,Vietnamese,Indonesian": 0.26, - "English,Chinese,Vietnamese,Malay": 0.28, - "English,Chinese,Filipino,Indonesian": 0.22, - "English,Chinese,Filipino,Malay": 0.22666666666666666, - "English,Chinese,Indonesian,Malay": 0.28, - "English,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "English,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "English,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.20666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Chinese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.2, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.18, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - } - }, - "AC3_2": 0.42130614221921164, - "AC3_3": 0.34740769727553594, - "AC3_4": 0.29634335678196383, - "AC3_5": 0.2608038740455954, - "AC3_6": 0.2349069806692294, - "AC3_7": 0.21438006226322134 - }, - "prompt_4": { - "overall_acc": 0.3552380952380953, + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cross_logiqa": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "sg_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cn_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "us_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ph_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "sing2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "indommlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_ind2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_vie2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_zho2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_zsm2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mmlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mmlu_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c_eval_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cmmlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cmmlu_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "zbench": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ind_emotion": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ocnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c3": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "dream": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "samsum": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "dialogsum": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "sst2": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cola": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "qqp": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "qnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "wnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "rte": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mrpc": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + } + }, + "five_shot": { + "cross_xquad": { + "prompt_1": { + "overall_acc": 0.7960084033613446, "language_acc": { - "Spanish": 0.3333333333333333, - "English": 0.44, - "Chinese": 0.35333333333333333, - "Vietnamese": 0.36, - "Filipino": 0.23333333333333334, - "Indonesian": 0.37333333333333335, - "Malay": 0.3933333333333333 + "Spanish": 0.7798319327731092, + "English": 0.8134453781512605, + "Chinese": 0.7974789915966387, + "Vietnamese": 0.7932773109243697 }, - "consistency_score_2": 0.4704761904761905, - "consistency_score_3": 0.28971428571428576, - "consistency_score_4": 0.2081904761904762, - "consistency_score_5": 0.16317460317460317, - "consistency_score_6": 0.13428571428571429, - "consistency_score_7": 0.11333333333333333, + "consistency_score_2": 0.8275910364145659, + "consistency_score_3": 0.751890756302521, + "consistency_score_4": 0.704201680672269, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5, - "Spanish,Chinese": 0.47333333333333333, - "Spanish,Vietnamese": 0.44666666666666666, - "Spanish,Filipino": 0.5, - "Spanish,Indonesian": 0.43333333333333335, - "Spanish,Malay": 0.44, - "English,Chinese": 0.5666666666666667, - "English,Vietnamese": 0.47333333333333333, - "English,Filipino": 0.4, - "English,Indonesian": 0.5133333333333333, - "English,Malay": 0.5533333333333333, - "Chinese,Vietnamese": 0.46, - "Chinese,Filipino": 0.41333333333333333, - "Chinese,Indonesian": 0.41333333333333333, - "Chinese,Malay": 0.47333333333333333, - "Vietnamese,Filipino": 0.48, - "Vietnamese,Indonesian": 0.48, - "Vietnamese,Malay": 0.48, - "Filipino,Indonesian": 0.42, - "Filipino,Malay": 0.4066666666666667, - "Indonesian,Malay": 0.5533333333333333 + "Spanish,English": 0.8344537815126051, + "Spanish,Chinese": 0.8016806722689076, + "Spanish,Vietnamese": 0.8252100840336134, + "English,Chinese": 0.8336134453781513, + "English,Vietnamese": 0.8529411764705882, + "Chinese,Vietnamese": 0.8176470588235294 }, "3_combine": { - "Spanish,English,Chinese": 0.35333333333333333, - "Spanish,English,Vietnamese": 0.30666666666666664, - "Spanish,English,Filipino": 0.2866666666666667, - "Spanish,English,Indonesian": 0.30666666666666664, - "Spanish,English,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese": 0.28, - "Spanish,Chinese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Malay": 0.2866666666666667, - "Spanish,Vietnamese,Filipino": 0.29333333333333333, - "Spanish,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Malay": 0.2733333333333333, - "Spanish,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Filipino,Malay": 0.26, - "Spanish,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese": 0.32, - "English,Chinese,Filipino": 0.2733333333333333, - "English,Chinese,Indonesian": 0.32, - "English,Chinese,Malay": 0.35333333333333333, - "English,Vietnamese,Filipino": 0.24666666666666667, - "English,Vietnamese,Indonesian": 0.3, - "English,Vietnamese,Malay": 0.31333333333333335, - "English,Filipino,Indonesian": 0.26666666666666666, - "English,Filipino,Malay": 0.25333333333333335, - "English,Indonesian,Malay": 0.36666666666666664, - "Chinese,Vietnamese,Filipino": 0.2733333333333333, - "Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Malay": 0.29333333333333333, - "Chinese,Filipino,Indonesian": 0.24, - "Chinese,Filipino,Malay": 0.24, - "Chinese,Indonesian,Malay": 0.30666666666666664, - "Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Vietnamese,Filipino,Malay": 0.2733333333333333, - "Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Filipino,Indonesian,Malay": 0.2733333333333333 + "Spanish,English,Chinese": 0.746218487394958, + "Spanish,English,Vietnamese": 0.7647058823529411, + "Spanish,Chinese,Vietnamese": 0.7336134453781512, + "English,Chinese,Vietnamese": 0.7630252100840336 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.22666666666666666, - "Spanish,English,Chinese,Filipino": 0.22, - "Spanish,English,Chinese,Indonesian": 0.21333333333333335, - "Spanish,English,Chinese,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian": 0.2, - "Spanish,English,Filipino,Malay": 0.18, - "Spanish,English,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.2, - "Spanish,Chinese,Filipino,Indonesian": 0.18, - "Spanish,Chinese,Filipino,Malay": 0.18, - "Spanish,Chinese,Indonesian,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "English,Chinese,Vietnamese,Malay": 0.23333333333333334, - "English,Chinese,Filipino,Indonesian": 0.18, - "English,Chinese,Filipino,Malay": 0.18666666666666668, - "English,Chinese,Indonesian,Malay": 0.26, - "English,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.24, - "Chinese,Filipino,Indonesian,Malay": 0.18, - "Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,English,Chinese,Vietnamese,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Malay": 0.14, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,English,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "English,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 + "Spanish,English,Chinese,Vietnamese": 0.704201680672269 } }, - "AC3_2": 0.40481573016462796, - "AC3_3": 0.31914775119513683, - "AC3_4": 0.2625255159069136, - "AC3_5": 0.22362814564257139, - "AC3_6": 0.194897165051904, - "AC3_7": 0.1718428183915105 - }, - "prompt_5": { - "overall_acc": 0.3428571428571429, + "AC3_2": 0.8114925434845776, + "AC3_3": 0.7733208673849625, + "AC3_4": 0.747295943954452 + } + }, + "cross_mmlu": { + "prompt_1": { + "overall_acc": 0.5219047619047619, "language_acc": { - "Spanish": 0.35333333333333333, - "English": 0.4066666666666667, - "Chinese": 0.36, - "Vietnamese": 0.34, - "Filipino": 0.24, - "Indonesian": 0.37333333333333335, - "Malay": 0.32666666666666666 + "Filipino": 0.36, + "Vietnamese": 0.5266666666666666, + "Chinese": 0.5466666666666666, + "Spanish": 0.48, + "Malay": 0.5133333333333333, + "Indonesian": 0.5933333333333334, + "English": 0.6333333333333333 }, - "consistency_score_2": 0.5393650793650793, - "consistency_score_3": 0.37466666666666665, - "consistency_score_4": 0.2908571428571428, - "consistency_score_5": 0.23873015873015874, - "consistency_score_6": 0.2019047619047619, - "consistency_score_7": 0.17333333333333334, + "consistency_score_2": 0.6107936507936507, + "consistency_score_3": 0.4615238095238095, + "consistency_score_4": 0.3739047619047619, + "consistency_score_5": 0.313015873015873, + "consistency_score_6": 0.2657142857142857, + "consistency_score_7": 0.22666666666666666, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5533333333333333, - "Spanish,Chinese": 0.5266666666666666, - "Spanish,Vietnamese": 0.5533333333333333, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.5066666666666667, - "Spanish,Malay": 0.47333333333333333, - "English,Chinese": 0.6266666666666667, - "English,Vietnamese": 0.6, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.56, - "English,Malay": 0.5466666666666666, - "Chinese,Vietnamese": 0.5266666666666666, - "Chinese,Filipino": 0.49333333333333335, - "Chinese,Indonesian": 0.5466666666666666, - "Chinese,Malay": 0.5866666666666667, - "Vietnamese,Filipino": 0.4866666666666667, - "Vietnamese,Indonesian": 0.58, - "Vietnamese,Malay": 0.5933333333333334, - "Filipino,Indonesian": 0.52, - "Filipino,Malay": 0.5066666666666667, - "Indonesian,Malay": 0.6333333333333333 + "Filipino,Vietnamese": 0.4866666666666667, + "Filipino,Chinese": 0.4866666666666667, + "Filipino,Spanish": 0.4666666666666667, + "Filipino,Malay": 0.4866666666666667, + "Filipino,Indonesian": 0.5066666666666667, + "Filipino,English": 0.4666666666666667, + "Vietnamese,Chinese": 0.6333333333333333, + "Vietnamese,Spanish": 0.5933333333333334, + "Vietnamese,Malay": 0.6533333333333333, + "Vietnamese,Indonesian": 0.6333333333333333, + "Vietnamese,English": 0.6866666666666666, + "Chinese,Spanish": 0.6466666666666666, + "Chinese,Malay": 0.6733333333333333, + "Chinese,Indonesian": 0.6866666666666666, + "Chinese,English": 0.7333333333333333, + "Spanish,Malay": 0.6133333333333333, + "Spanish,Indonesian": 0.6333333333333333, + "Spanish,English": 0.6333333333333333, + "Malay,Indonesian": 0.6666666666666666, + "Malay,English": 0.7133333333333334, + "Indonesian,English": 0.7266666666666667 }, "3_combine": { - "Spanish,English,Chinese": 0.4066666666666667, - "Spanish,English,Vietnamese": 0.4266666666666667, - "Spanish,English,Filipino": 0.30666666666666664, - "Spanish,English,Indonesian": 0.36666666666666664, - "Spanish,English,Malay": 0.38, - "Spanish,Chinese,Vietnamese": 0.38666666666666666, - "Spanish,Chinese,Filipino": 0.3, - "Spanish,Chinese,Indonesian": 0.35333333333333333, - "Spanish,Chinese,Malay": 0.38, - "Spanish,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,Vietnamese,Indonesian": 0.4, - "Spanish,Vietnamese,Malay": 0.3933333333333333, - "Spanish,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Filipino,Malay": 0.31333333333333335, - "Spanish,Indonesian,Malay": 0.38666666666666666, - "English,Chinese,Vietnamese": 0.4266666666666667, - "English,Chinese,Filipino": 0.35333333333333333, - "English,Chinese,Indonesian": 0.42, - "English,Chinese,Malay": 0.44, - "English,Vietnamese,Filipino": 0.34, - "English,Vietnamese,Indonesian": 0.44, - "English,Vietnamese,Malay": 0.4533333333333333, - "English,Filipino,Indonesian": 0.31333333333333335, - "English,Filipino,Malay": 0.31333333333333335, - "English,Indonesian,Malay": 0.42, - "Chinese,Vietnamese,Filipino": 0.31333333333333335, - "Chinese,Vietnamese,Indonesian": 0.3933333333333333, - "Chinese,Vietnamese,Malay": 0.42, - "Chinese,Filipino,Indonesian": 0.3333333333333333, - "Chinese,Filipino,Malay": 0.34, - "Chinese,Indonesian,Malay": 0.43333333333333335, - "Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "Vietnamese,Filipino,Malay": 0.36, - "Vietnamese,Indonesian,Malay": 0.4666666666666667, - "Filipino,Indonesian,Malay": 0.3933333333333333 + "Filipino,Vietnamese,Chinese": 0.34, + "Filipino,Vietnamese,Spanish": 0.34, + "Filipino,Vietnamese,Malay": 0.36666666666666664, + "Filipino,Vietnamese,Indonesian": 0.36, + "Filipino,Vietnamese,English": 0.36666666666666664, + "Filipino,Chinese,Spanish": 0.36666666666666664, + "Filipino,Chinese,Malay": 0.38, + "Filipino,Chinese,Indonesian": 0.3933333333333333, + "Filipino,Chinese,English": 0.38666666666666666, + "Filipino,Spanish,Malay": 0.35333333333333333, + "Filipino,Spanish,Indonesian": 0.38, + "Filipino,Spanish,English": 0.36, + "Filipino,Malay,Indonesian": 0.3933333333333333, + "Filipino,Malay,English": 0.38666666666666666, + "Filipino,Indonesian,English": 0.3933333333333333, + "Vietnamese,Chinese,Spanish": 0.4866666666666667, + "Vietnamese,Chinese,Malay": 0.5133333333333333, + "Vietnamese,Chinese,Indonesian": 0.5, + "Vietnamese,Chinese,English": 0.5533333333333333, + "Vietnamese,Spanish,Malay": 0.4866666666666667, + "Vietnamese,Spanish,Indonesian": 0.48, + "Vietnamese,Spanish,English": 0.5, + "Vietnamese,Malay,Indonesian": 0.5066666666666667, + "Vietnamese,Malay,English": 0.56, + "Vietnamese,Indonesian,English": 0.5466666666666666, + "Chinese,Spanish,Malay": 0.5133333333333333, + "Chinese,Spanish,Indonesian": 0.5133333333333333, + "Chinese,Spanish,English": 0.5533333333333333, + "Chinese,Malay,Indonesian": 0.54, + "Chinese,Malay,English": 0.6, + "Chinese,Indonesian,English": 0.6066666666666667, + "Spanish,Malay,Indonesian": 0.4866666666666667, + "Spanish,Malay,English": 0.5266666666666666, + "Spanish,Indonesian,English": 0.5333333333333333, + "Malay,Indonesian,English": 0.58 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.3333333333333333, - "Spanish,English,Chinese,Filipino": 0.25333333333333335, - "Spanish,English,Chinese,Indonesian": 0.3, - "Spanish,English,Chinese,Malay": 0.3333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.26, - "Spanish,English,Vietnamese,Indonesian": 0.32666666666666666, - "Spanish,English,Vietnamese,Malay": 0.34, - "Spanish,English,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Indonesian,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.32666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.26, - "Spanish,Chinese,Indonesian,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.3466666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.26, - "English,Chinese,Vietnamese,Filipino": 0.26, - "English,Chinese,Vietnamese,Indonesian": 0.3333333333333333, - "English,Chinese,Vietnamese,Malay": 0.36666666666666664, - "English,Chinese,Filipino,Indonesian": 0.26, - "English,Chinese,Filipino,Malay": 0.26666666666666666, - "English,Chinese,Indonesian,Malay": 0.3466666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "English,Vietnamese,Filipino,Malay": 0.28, - "English,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "English,Filipino,Indonesian,Malay": 0.26, - "Chinese,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.2733333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.3, - "Spanish,English,Chinese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.28, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Filipino,Indonesian,Malay": 0.22, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 + "Filipino,Vietnamese,Chinese,Spanish": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Malay": 0.3, + "Filipino,Vietnamese,Chinese,Indonesian": 0.3, + "Filipino,Vietnamese,Chinese,English": 0.30666666666666664, + "Filipino,Vietnamese,Spanish,Malay": 0.2866666666666667, + "Filipino,Vietnamese,Spanish,Indonesian": 0.29333333333333333, + "Filipino,Vietnamese,Spanish,English": 0.3, + "Filipino,Vietnamese,Malay,Indonesian": 0.3, + "Filipino,Vietnamese,Malay,English": 0.31333333333333335, + "Filipino,Vietnamese,Indonesian,English": 0.31333333333333335, + "Filipino,Chinese,Spanish,Malay": 0.31333333333333335, + "Filipino,Chinese,Spanish,Indonesian": 0.32666666666666666, + "Filipino,Chinese,Spanish,English": 0.3333333333333333, + "Filipino,Chinese,Malay,Indonesian": 0.3466666666666667, + "Filipino,Chinese,Malay,English": 0.3466666666666667, + "Filipino,Chinese,Indonesian,English": 0.35333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.30666666666666664, + "Filipino,Spanish,Malay,English": 0.30666666666666664, + "Filipino,Spanish,Indonesian,English": 0.32666666666666666, + "Filipino,Malay,Indonesian,English": 0.3333333333333333, + "Vietnamese,Chinese,Spanish,Malay": 0.43333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian": 0.41333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.4533333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.43333333333333335, + "Vietnamese,Chinese,Malay,English": 0.4866666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.46, + "Vietnamese,Spanish,Malay,Indonesian": 0.4066666666666667, + "Vietnamese,Spanish,Malay,English": 0.44666666666666666, + "Vietnamese,Spanish,Indonesian,English": 0.44, + "Vietnamese,Malay,Indonesian,English": 0.47333333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.43333333333333335, + "Chinese,Spanish,Malay,English": 0.4866666666666667, + "Chinese,Spanish,Indonesian,English": 0.48, + "Chinese,Malay,Indonesian,English": 0.5, + "Spanish,Malay,Indonesian,English": 0.44 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.28, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.28, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.28, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.26, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2733333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.2733333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2866666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.29333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.30666666666666664, + "Filipino,Chinese,Malay,Indonesian,English": 0.31333333333333335, + "Filipino,Spanish,Malay,Indonesian,English": 0.2733333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.37333333333333335, + "Vietnamese,Chinese,Spanish,Malay,English": 0.42, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.3933333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.41333333333333333, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.38666666666666666, + "Chinese,Spanish,Malay,Indonesian,English": 0.41333333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.26, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.23333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.36 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666 } }, - "AC3_2": 0.4192258263031721, - "AC3_3": 0.3580568090820335, - "AC3_4": 0.3147236892457122, - "AC3_5": 0.28147223950244493, - "AC3_6": 0.2541458540992015, - "AC3_7": 0.23025830253841859 + "AC3_2": 0.5628614135740533, + "AC3_3": 0.48986063837561483, + "AC3_4": 0.4356789485454739, + "AC3_5": 0.39132934994407675, + "AC3_6": 0.35214372080517775, + "AC3_7": 0.31606446136575045 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.3206168831168831, + "overall_acc": 0.42045454545454547, "language_acc": { - "English": 0.3068181818181818, - "Filipino": 0.2784090909090909, - "Spanish": 0.3352272727272727, - "Malay": 0.3409090909090909, - "Chinese": 0.35795454545454547, - "Indonesian": 0.32954545454545453, - "Vietnamese": 0.29545454545454547 + "Indonesian": 0.48295454545454547, + "English": 0.45454545454545453, + "Filipino": 0.3068181818181818, + "Spanish": 0.42613636363636365, + "Chinese": 0.42045454545454547, + "Malay": 0.4375, + "Vietnamese": 0.4147727272727273 }, - "consistency_score_2": 0.5008116883116882, - "consistency_score_3": 0.3173701298701299, - "consistency_score_4": 0.21850649350649354, - "consistency_score_5": 0.1561147186147186, - "consistency_score_6": 0.11444805194805195, - "consistency_score_7": 0.08522727272727272, + "consistency_score_2": 0.49918831168831174, + "consistency_score_3": 0.3222402597402598, + "consistency_score_4": 0.23490259740259742, + "consistency_score_5": 0.18290043290043292, + "consistency_score_6": 0.14853896103896105, + "consistency_score_7": 0.125, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.3181818181818182, - "English,Spanish": 0.5625, - "English,Malay": 0.5568181818181818, - "English,Chinese": 0.5625, - "English,Indonesian": 0.6079545454545454, - "English,Vietnamese": 0.6136363636363636, - "Filipino,Spanish": 0.4602272727272727, - "Filipino,Malay": 0.4318181818181818, - "Filipino,Chinese": 0.25, - "Filipino,Indonesian": 0.3352272727272727, - "Filipino,Vietnamese": 0.3465909090909091, - "Spanish,Malay": 0.6193181818181818, - "Spanish,Chinese": 0.48295454545454547, - "Spanish,Indonesian": 0.5227272727272727, - "Spanish,Vietnamese": 0.5340909090909091, - "Malay,Chinese": 0.5227272727272727, - "Malay,Indonesian": 0.6079545454545454, - "Malay,Vietnamese": 0.5511363636363636, - "Chinese,Indonesian": 0.5170454545454546, - "Chinese,Vietnamese": 0.5170454545454546, - "Indonesian,Vietnamese": 0.5965909090909091 + "Indonesian,English": 0.5681818181818182, + "Indonesian,Filipino": 0.3977272727272727, + "Indonesian,Spanish": 0.5056818181818182, + "Indonesian,Chinese": 0.4715909090909091, + "Indonesian,Malay": 0.6534090909090909, + "Indonesian,Vietnamese": 0.5625, + "English,Filipino": 0.4659090909090909, + "English,Spanish": 0.5170454545454546, + "English,Chinese": 0.5909090909090909, + "English,Malay": 0.5738636363636364, + "English,Vietnamese": 0.5909090909090909, + "Filipino,Spanish": 0.3352272727272727, + "Filipino,Chinese": 0.48863636363636365, + "Filipino,Malay": 0.38636363636363635, + "Filipino,Vietnamese": 0.3806818181818182, + "Spanish,Chinese": 0.4431818181818182, + "Spanish,Malay": 0.48295454545454547, + "Spanish,Vietnamese": 0.5625, + "Chinese,Malay": 0.4715909090909091, + "Chinese,Vietnamese": 0.5056818181818182, + "Malay,Vietnamese": 0.5284090909090909 }, "3_combine": { - "English,Filipino,Spanish": 0.22727272727272727, - "English,Filipino,Malay": 0.2215909090909091, - "English,Filipino,Chinese": 0.17613636363636365, - "English,Filipino,Indonesian": 0.2159090909090909, - "English,Filipino,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay": 0.4090909090909091, - "English,Spanish,Chinese": 0.3465909090909091, - "English,Spanish,Indonesian": 0.39204545454545453, - "English,Spanish,Vietnamese": 0.4034090909090909, - "English,Malay,Chinese": 0.3806818181818182, - "English,Malay,Indonesian": 0.4431818181818182, - "English,Malay,Vietnamese": 0.4090909090909091, - "English,Chinese,Indonesian": 0.3977272727272727, - "English,Chinese,Vietnamese": 0.4034090909090909, - "English,Indonesian,Vietnamese": 0.4659090909090909, - "Filipino,Spanish,Malay": 0.3352272727272727, - "Filipino,Spanish,Chinese": 0.1875, - "Filipino,Spanish,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Vietnamese": 0.23295454545454544, - "Filipino,Malay,Chinese": 0.19886363636363635, - "Filipino,Malay,Indonesian": 0.25, + "Indonesian,English,Filipino": 0.2897727272727273, + "Indonesian,English,Spanish": 0.3693181818181818, + "Indonesian,English,Chinese": 0.3693181818181818, + "Indonesian,English,Malay": 0.4375, + "Indonesian,English,Vietnamese": 0.42045454545454547, + "Indonesian,Filipino,Spanish": 0.22727272727272727, + "Indonesian,Filipino,Chinese": 0.26136363636363635, + "Indonesian,Filipino,Malay": 0.29545454545454547, + "Indonesian,Filipino,Vietnamese": 0.26704545454545453, + "Indonesian,Spanish,Chinese": 0.2897727272727273, + "Indonesian,Spanish,Malay": 0.3693181818181818, + "Indonesian,Spanish,Vietnamese": 0.3693181818181818, + "Indonesian,Chinese,Malay": 0.3806818181818182, + "Indonesian,Chinese,Vietnamese": 0.3352272727272727, + "Indonesian,Malay,Vietnamese": 0.42045454545454547, + "English,Filipino,Spanish": 0.2727272727272727, + "English,Filipino,Chinese": 0.3522727272727273, + "English,Filipino,Malay": 0.2840909090909091, + "English,Filipino,Vietnamese": 0.2897727272727273, + "English,Spanish,Chinese": 0.3352272727272727, + "English,Spanish,Malay": 0.36363636363636365, + "English,Spanish,Vietnamese": 0.3806818181818182, + "English,Chinese,Malay": 0.3806818181818182, + "English,Chinese,Vietnamese": 0.3977272727272727, + "English,Malay,Vietnamese": 0.4034090909090909, + "Filipino,Spanish,Chinese": 0.23295454545454544, + "Filipino,Spanish,Malay": 0.21022727272727273, + "Filipino,Spanish,Vietnamese": 0.2215909090909091, + "Filipino,Chinese,Malay": 0.26704545454545453, + "Filipino,Chinese,Vietnamese": 0.2727272727272727, "Filipino,Malay,Vietnamese": 0.23863636363636365, - "Filipino,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Indonesian,Vietnamese": 0.21022727272727273, - "Spanish,Malay,Chinese": 0.35795454545454547, - "Spanish,Malay,Indonesian": 0.4318181818181818, - "Spanish,Malay,Vietnamese": 0.39204545454545453, - "Spanish,Chinese,Indonesian": 0.3125, - "Spanish,Chinese,Vietnamese": 0.3465909090909091, - "Spanish,Indonesian,Vietnamese": 0.38636363636363635, - "Malay,Chinese,Indonesian": 0.3977272727272727, - "Malay,Chinese,Vietnamese": 0.375, - "Malay,Indonesian,Vietnamese": 0.42045454545454547, - "Chinese,Indonesian,Vietnamese": 0.38636363636363635 + "Spanish,Chinese,Malay": 0.2897727272727273, + "Spanish,Chinese,Vietnamese": 0.3068181818181818, + "Spanish,Malay,Vietnamese": 0.3409090909090909, + "Chinese,Malay,Vietnamese": 0.3352272727272727 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.1875, - "English,Filipino,Spanish,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese": 0.14204545454545456, - "English,Filipino,Malay,Indonesian": 0.16477272727272727, - "English,Filipino,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Indonesian,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay,Chinese": 0.2840909090909091, - "English,Spanish,Malay,Indonesian": 0.3352272727272727, - "English,Spanish,Malay,Vietnamese": 0.3181818181818182, - "English,Spanish,Chinese,Indonesian": 0.26704545454545453, - "English,Spanish,Chinese,Vietnamese": 0.2897727272727273, - "English,Spanish,Indonesian,Vietnamese": 0.3181818181818182, - "English,Malay,Chinese,Indonesian": 0.3181818181818182, - "English,Malay,Chinese,Vietnamese": 0.3125, - "English,Malay,Indonesian,Vietnamese": 0.3522727272727273, - "English,Chinese,Indonesian,Vietnamese": 0.3352272727272727, - "Filipino,Spanish,Malay,Chinese": 0.16477272727272727, - "Filipino,Spanish,Malay,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Chinese,Vietnamese": 0.125, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Spanish,Malay,Chinese,Indonesian": 0.2727272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.3181818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.30113636363636365 + "Indonesian,English,Filipino,Spanish": 0.20454545454545456, + "Indonesian,English,Filipino,Chinese": 0.23863636363636365, + "Indonesian,English,Filipino,Malay": 0.23295454545454544, + "Indonesian,English,Filipino,Vietnamese": 0.22727272727272727, + "Indonesian,English,Spanish,Chinese": 0.2556818181818182, + "Indonesian,English,Spanish,Malay": 0.3068181818181818, + "Indonesian,English,Spanish,Vietnamese": 0.29545454545454547, + "Indonesian,English,Chinese,Malay": 0.3125, + "Indonesian,English,Chinese,Vietnamese": 0.29545454545454547, + "Indonesian,English,Malay,Vietnamese": 0.3352272727272727, + "Indonesian,Filipino,Spanish,Chinese": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Malay": 0.18181818181818182, + "Indonesian,Filipino,Spanish,Vietnamese": 0.18181818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.2159090909090909, + "Indonesian,Filipino,Chinese,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Malay,Vietnamese": 0.20454545454545456, + "Indonesian,Spanish,Chinese,Malay": 0.25, + "Indonesian,Spanish,Chinese,Vietnamese": 0.23295454545454544, + "Indonesian,Spanish,Malay,Vietnamese": 0.2897727272727273, + "Indonesian,Chinese,Malay,Vietnamese": 0.2784090909090909, + "English,Filipino,Spanish,Chinese": 0.2159090909090909, + "English,Filipino,Spanish,Malay": 0.19318181818181818, + "English,Filipino,Spanish,Vietnamese": 0.19318181818181818, + "English,Filipino,Chinese,Malay": 0.23863636363636365, + "English,Filipino,Chinese,Vietnamese": 0.23863636363636365, + "English,Filipino,Malay,Vietnamese": 0.20454545454545456, + "English,Spanish,Chinese,Malay": 0.26136363636363635, + "English,Spanish,Chinese,Vietnamese": 0.26704545454545453, + "English,Spanish,Malay,Vietnamese": 0.2840909090909091, + "English,Chinese,Malay,Vietnamese": 0.29545454545454547, + "Filipino,Spanish,Chinese,Malay": 0.17045454545454544, + "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, + "Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, + "Filipino,Chinese,Malay,Vietnamese": 0.1875, + "Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Indonesian": 0.23863636363636365, - "English,Spanish,Malay,Chinese,Vietnamese": 0.25, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2727272727272727, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727 + "Indonesian,English,Filipino,Spanish,Chinese": 0.17045454545454544, + "Indonesian,English,Filipino,Spanish,Malay": 0.17045454545454544, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.17045454545454544, + "Indonesian,English,Filipino,Chinese,Malay": 0.19886363636363635, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1875, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,English,Spanish,Chinese,Malay": 0.23295454545454544, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2159090909090909, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.25, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.25, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, + "English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, + "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, + "English,Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, + "English,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 } }, - "AC3_2": 0.3909498356897912, - "AC3_3": 0.31898524498486464, - "AC3_4": 0.2598917943964336, - "AC3_5": 0.2099840425875556, - "AC3_6": 0.16868276309361813, - "AC3_7": 0.1346590908759109 - }, - "prompt_2": { - "overall_acc": 0.32142857142857145, - "language_acc": { - "English": 0.39204545454545453, - "Filipino": 0.2727272727272727, - "Spanish": 0.29545454545454547, - "Malay": 0.3181818181818182, - "Chinese": 0.3409090909090909, - "Indonesian": 0.3125, - "Vietnamese": 0.3181818181818182 - }, - "consistency_score_2": 0.44751082251082247, - "consistency_score_3": 0.25373376623376626, - "consistency_score_4": 0.15925324675324673, - "consistency_score_5": 0.1055194805194805, - "consistency_score_6": 0.07061688311688312, - "consistency_score_7": 0.045454545454545456, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.2897727272727273, - "English,Spanish": 0.48863636363636365, - "English,Malay": 0.5113636363636364, - "English,Chinese": 0.48863636363636365, - "English,Indonesian": 0.5625, - "English,Vietnamese": 0.5738636363636364, - "Filipino,Spanish": 0.42613636363636365, - "Filipino,Malay": 0.2897727272727273, - "Filipino,Chinese": 0.32954545454545453, - "Filipino,Indonesian": 0.2840909090909091, - "Filipino,Vietnamese": 0.29545454545454547, - "Spanish,Malay": 0.4659090909090909, - "Spanish,Chinese": 0.4375, - "Spanish,Indonesian": 0.4715909090909091, - "Spanish,Vietnamese": 0.5113636363636364, - "Malay,Chinese": 0.45454545454545453, - "Malay,Indonesian": 0.5454545454545454, - "Malay,Vietnamese": 0.5056818181818182, - "Chinese,Indonesian": 0.4431818181818182, - "Chinese,Vietnamese": 0.4772727272727273, - "Indonesian,Vietnamese": 0.5454545454545454 - }, - "3_combine": { - "English,Filipino,Spanish": 0.18181818181818182, - "English,Filipino,Malay": 0.14772727272727273, - "English,Filipino,Chinese": 0.16477272727272727, - "English,Filipino,Indonesian": 0.1590909090909091, - "English,Filipino,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay": 0.30113636363636365, - "English,Spanish,Chinese": 0.2727272727272727, - "English,Spanish,Indonesian": 0.32954545454545453, - "English,Spanish,Vietnamese": 0.3522727272727273, - "English,Malay,Chinese": 0.30113636363636365, - "English,Malay,Indonesian": 0.39204545454545453, - "English,Malay,Vietnamese": 0.3522727272727273, - "English,Chinese,Indonesian": 0.3125, - "English,Chinese,Vietnamese": 0.3409090909090909, - "English,Indonesian,Vietnamese": 0.4034090909090909, - "Filipino,Spanish,Malay": 0.20454545454545456, - "Filipino,Spanish,Chinese": 0.20454545454545456, - "Filipino,Spanish,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese": 0.14204545454545456, - "Filipino,Malay,Indonesian": 0.1534090909090909, - "Filipino,Malay,Vietnamese": 0.13636363636363635, - "Filipino,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Indonesian,Vietnamese": 0.1590909090909091, - "Spanish,Malay,Chinese": 0.2556818181818182, - "Spanish,Malay,Indonesian": 0.3068181818181818, - "Spanish,Malay,Vietnamese": 0.3068181818181818, - "Spanish,Chinese,Indonesian": 0.2556818181818182, - "Spanish,Chinese,Vietnamese": 0.2727272727272727, - "Spanish,Indonesian,Vietnamese": 0.3409090909090909, - "Malay,Chinese,Indonesian": 0.29545454545454547, - "Malay,Chinese,Vietnamese": 0.2897727272727273, - "Malay,Indonesian,Vietnamese": 0.38636363636363635, - "Chinese,Indonesian,Vietnamese": 0.3068181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.125, - "English,Filipino,Spanish,Chinese": 0.10795454545454546, - "English,Filipino,Spanish,Indonesian": 0.125, - "English,Filipino,Spanish,Vietnamese": 0.125, - "English,Filipino,Malay,Chinese": 0.09659090909090909, - "English,Filipino,Malay,Indonesian": 0.10795454545454546, - "English,Filipino,Malay,Vietnamese": 0.09659090909090909, - "English,Filipino,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese": 0.20454545454545456, - "English,Spanish,Malay,Indonesian": 0.24431818181818182, - "English,Spanish,Malay,Vietnamese": 0.22727272727272727, - "English,Spanish,Chinese,Indonesian": 0.21022727272727273, - "English,Spanish,Chinese,Vietnamese": 0.2215909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.26136363636363635, - "English,Malay,Chinese,Indonesian": 0.23295454545454544, - "English,Malay,Chinese,Vietnamese": 0.23295454545454544, - "English,Malay,Indonesian,Vietnamese": 0.29545454545454547, - "English,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Spanish,Malay,Chinese": 0.09659090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.125, - "Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Filipino,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "Spanish,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Spanish,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Indonesian": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Indonesian": 0.17045454545454544, - "English,Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - } - }, - "AC3_2": 0.37413290434462865, - "AC3_3": 0.28359743553796735, - "AC3_4": 0.21298306552234364, - "AC3_5": 0.15888104287424795, - "AC3_6": 0.1157941437149184, - "AC3_7": 0.07964601767740621 - }, - "prompt_3": { - "overall_acc": 0.336038961038961, - "language_acc": { - "English": 0.38636363636363635, - "Filipino": 0.30113636363636365, - "Spanish": 0.29545454545454547, - "Malay": 0.3352272727272727, - "Chinese": 0.39204545454545453, - "Indonesian": 0.3352272727272727, - "Vietnamese": 0.3068181818181818 - }, - "consistency_score_2": 0.491883116883117, - "consistency_score_3": 0.3029220779220779, - "consistency_score_4": 0.20162337662337665, - "consistency_score_5": 0.137987012987013, - "consistency_score_6": 0.09415584415584417, - "consistency_score_7": 0.0625, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3181818181818182, - "English,Spanish": 0.5454545454545454, - "English,Malay": 0.5511363636363636, - "English,Chinese": 0.4943181818181818, - "English,Indonesian": 0.5909090909090909, - "English,Vietnamese": 0.5625, - "Filipino,Spanish": 0.4715909090909091, - "Filipino,Malay": 0.42613636363636365, - "Filipino,Chinese": 0.35795454545454547, - "Filipino,Indonesian": 0.3465909090909091, - "Filipino,Vietnamese": 0.3522727272727273, - "Spanish,Malay": 0.5681818181818182, - "Spanish,Chinese": 0.4318181818181818, - "Spanish,Indonesian": 0.5397727272727273, - "Spanish,Vietnamese": 0.5909090909090909, - "Malay,Chinese": 0.4715909090909091, - "Malay,Indonesian": 0.6136363636363636, - "Malay,Vietnamese": 0.5738636363636364, - "Chinese,Indonesian": 0.4375, - "Chinese,Vietnamese": 0.5, - "Indonesian,Vietnamese": 0.5852272727272727 - }, - "3_combine": { - "English,Filipino,Spanish": 0.23295454545454544, - "English,Filipino,Malay": 0.2215909090909091, - "English,Filipino,Chinese": 0.18181818181818182, - "English,Filipino,Indonesian": 0.21022727272727273, - "English,Filipino,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay": 0.3806818181818182, - "English,Spanish,Chinese": 0.29545454545454547, - "English,Spanish,Indonesian": 0.3806818181818182, - "English,Spanish,Vietnamese": 0.4147727272727273, - "English,Malay,Chinese": 0.32386363636363635, - "English,Malay,Indonesian": 0.4318181818181818, - "English,Malay,Vietnamese": 0.42045454545454547, - "English,Chinese,Indonesian": 0.3125, - "English,Chinese,Vietnamese": 0.3352272727272727, - "English,Indonesian,Vietnamese": 0.4090909090909091, - "Filipino,Spanish,Malay": 0.3068181818181818, - "Filipino,Spanish,Chinese": 0.2159090909090909, - "Filipino,Spanish,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Vietnamese": 0.26704545454545453, - "Filipino,Malay,Chinese": 0.2159090909090909, - "Filipino,Malay,Indonesian": 0.25, - "Filipino,Malay,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese": 0.3068181818181818, - "Spanish,Malay,Indonesian": 0.3977272727272727, - "Spanish,Malay,Vietnamese": 0.42045454545454547, - "Spanish,Chinese,Indonesian": 0.2556818181818182, - "Spanish,Chinese,Vietnamese": 0.3181818181818182, - "Spanish,Indonesian,Vietnamese": 0.39204545454545453, - "Malay,Chinese,Indonesian": 0.3181818181818182, - "Malay,Chinese,Vietnamese": 0.3352272727272727, - "Malay,Indonesian,Vietnamese": 0.4431818181818182, - "Chinese,Indonesian,Vietnamese": 0.32386363636363635 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.18181818181818182, - "English,Filipino,Spanish,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Malay,Indonesian": 0.16477272727272727, - "English,Filipino,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese": 0.2215909090909091, - "English,Spanish,Malay,Indonesian": 0.2897727272727273, - "English,Spanish,Malay,Vietnamese": 0.3352272727272727, - "English,Spanish,Chinese,Indonesian": 0.20454545454545456, - "English,Spanish,Chinese,Vietnamese": 0.24431818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.3068181818181818, - "English,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Malay,Chinese,Vietnamese": 0.25, - "English,Malay,Indonesian,Vietnamese": 0.3409090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Malay,Chinese": 0.1590909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese,Indonesian": 0.125, - "Filipino,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian": 0.20454545454545456, - "Spanish,Malay,Chinese,Vietnamese": 0.25, - "Spanish,Malay,Indonesian,Vietnamese": 0.32386363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.25 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.26136363636363635, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.18181818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - } - }, - "AC3_2": 0.39929335365689006, - "AC3_3": 0.3186223075206126, - "AC3_4": 0.2520292207323458, - "AC3_5": 0.195639121111548, - "AC3_6": 0.14709629989229567, - "AC3_7": 0.10539714864972519 - }, - "prompt_4": { - "overall_acc": 0.3246753246753246, - "language_acc": { - "English": 0.3181818181818182, - "Filipino": 0.2840909090909091, - "Spanish": 0.32386363636363635, - "Malay": 0.3465909090909091, - "Chinese": 0.39204545454545453, - "Indonesian": 0.3181818181818182, - "Vietnamese": 0.2897727272727273 - }, - "consistency_score_2": 0.4691558441558442, - "consistency_score_3": 0.27775974025974026, - "consistency_score_4": 0.17922077922077922, - "consistency_score_5": 0.12067099567099565, - "consistency_score_6": 0.08279220779220779, - "consistency_score_7": 0.056818181818181816, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3181818181818182, - "English,Spanish": 0.5056818181818182, - "English,Malay": 0.5454545454545454, - "English,Chinese": 0.5170454545454546, - "English,Indonesian": 0.5397727272727273, - "English,Vietnamese": 0.5625, - "Filipino,Spanish": 0.4659090909090909, - "Filipino,Malay": 0.39204545454545453, - "Filipino,Chinese": 0.2897727272727273, - "Filipino,Indonesian": 0.29545454545454547, - "Filipino,Vietnamese": 0.3693181818181818, - "Spanish,Malay": 0.5511363636363636, - "Spanish,Chinese": 0.3977272727272727, - "Spanish,Indonesian": 0.5113636363636364, - "Spanish,Vietnamese": 0.5397727272727273, - "Malay,Chinese": 0.4659090909090909, - "Malay,Indonesian": 0.5681818181818182, - "Malay,Vietnamese": 0.5738636363636364, - "Chinese,Indonesian": 0.42613636363636365, - "Chinese,Vietnamese": 0.45454545454545453, - "Indonesian,Vietnamese": 0.5625 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2215909090909091, - "English,Filipino,Malay": 0.20454545454545456, - "English,Filipino,Chinese": 0.1875, - "English,Filipino,Indonesian": 0.17045454545454544, - "English,Filipino,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay": 0.35795454545454547, - "English,Spanish,Chinese": 0.26136363636363635, - "English,Spanish,Indonesian": 0.3352272727272727, - "English,Spanish,Vietnamese": 0.36363636363636365, - "English,Malay,Chinese": 0.32386363636363635, - "English,Malay,Indonesian": 0.3806818181818182, - "English,Malay,Vietnamese": 0.4090909090909091, - "English,Chinese,Indonesian": 0.29545454545454547, - "English,Chinese,Vietnamese": 0.3352272727272727, - "English,Indonesian,Vietnamese": 0.3806818181818182, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.1875, - "Filipino,Spanish,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Vietnamese": 0.25, - "Filipino,Malay,Chinese": 0.17613636363636365, - "Filipino,Malay,Indonesian": 0.21022727272727273, - "Filipino,Malay,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian": 0.125, - "Filipino,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Indonesian,Vietnamese": 0.19886363636363635, - "Spanish,Malay,Chinese": 0.2727272727272727, - "Spanish,Malay,Indonesian": 0.3693181818181818, - "Spanish,Malay,Vietnamese": 0.3806818181818182, - "Spanish,Chinese,Indonesian": 0.25, - "Spanish,Chinese,Vietnamese": 0.26704545454545453, - "Spanish,Indonesian,Vietnamese": 0.35795454545454547, - "Malay,Chinese,Indonesian": 0.29545454545454547, - "Malay,Chinese,Vietnamese": 0.32954545454545453, - "Malay,Indonesian,Vietnamese": 0.4034090909090909, - "Chinese,Indonesian,Vietnamese": 0.29545454545454547 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.16477272727272727, - "English,Filipino,Spanish,Chinese": 0.11931818181818182, - "English,Filipino,Spanish,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, - "English,Filipino,Malay,Chinese": 0.125, - "English,Filipino,Malay,Indonesian": 0.125, - "English,Filipino,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese": 0.19886363636363635, - "English,Spanish,Malay,Indonesian": 0.2727272727272727, - "English,Spanish,Malay,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese,Indonesian": 0.1875, - "English,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "English,Spanish,Indonesian,Vietnamese": 0.26704545454545453, - "English,Malay,Chinese,Indonesian": 0.22727272727272727, - "English,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Malay,Indonesian,Vietnamese": 0.3068181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Malay,Chinese": 0.14204545454545456, - "Filipino,Spanish,Malay,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Spanish,Chinese,Vietnamese": 0.125, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Chinese,Indonesian": 0.20454545454545456, - "Spanish,Malay,Chinese,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0625, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 - } - }, - "AC3_2": 0.3837675616333284, - "AC3_3": 0.2993907197128904, - "AC3_4": 0.2309546123520776, - "AC3_5": 0.17594798878818904, - "AC3_6": 0.13193977334165588, - "AC3_7": 0.09671179881410756 - }, - "prompt_5": { - "overall_acc": 0.32467532467532473, - "language_acc": { - "English": 0.3181818181818182, - "Filipino": 0.2897727272727273, - "Spanish": 0.30113636363636365, - "Malay": 0.36363636363636365, - "Chinese": 0.32386363636363635, - "Indonesian": 0.3522727272727273, - "Vietnamese": 0.32386363636363635 - }, - "consistency_score_2": 0.48701298701298706, - "consistency_score_3": 0.29951298701298706, - "consistency_score_4": 0.20649350649350648, - "consistency_score_5": 0.15151515151515152, - "consistency_score_6": 0.1176948051948052, - "consistency_score_7": 0.09659090909090909, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.5511363636363636, - "English,Malay": 0.5397727272727273, - "English,Chinese": 0.5227272727272727, - "English,Indonesian": 0.5625, - "English,Vietnamese": 0.5284090909090909, - "Filipino,Spanish": 0.45454545454545453, - "Filipino,Malay": 0.35795454545454547, - "Filipino,Chinese": 0.3522727272727273, - "Filipino,Indonesian": 0.3409090909090909, - "Filipino,Vietnamese": 0.3977272727272727, - "Spanish,Malay": 0.5852272727272727, - "Spanish,Chinese": 0.48295454545454547, - "Spanish,Indonesian": 0.5170454545454546, - "Spanish,Vietnamese": 0.5795454545454546, - "Malay,Chinese": 0.48863636363636365, - "Malay,Indonesian": 0.6136363636363636, - "Malay,Vietnamese": 0.5681818181818182, - "Chinese,Indonesian": 0.4375, - "Chinese,Vietnamese": 0.45454545454545453, - "Indonesian,Vietnamese": 0.5568181818181818 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2556818181818182, - "English,Filipino,Malay": 0.2215909090909091, - "English,Filipino,Chinese": 0.20454545454545456, - "English,Filipino,Indonesian": 0.2159090909090909, - "English,Filipino,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay": 0.39204545454545453, - "English,Spanish,Chinese": 0.32386363636363635, - "English,Spanish,Indonesian": 0.375, - "English,Spanish,Vietnamese": 0.375, - "English,Malay,Chinese": 0.3409090909090909, - "English,Malay,Indonesian": 0.4147727272727273, - "English,Malay,Vietnamese": 0.375, - "English,Chinese,Indonesian": 0.3125, - "English,Chinese,Vietnamese": 0.32386363636363635, - "English,Indonesian,Vietnamese": 0.3693181818181818, - "Filipino,Spanish,Malay": 0.26136363636363635, - "Filipino,Spanish,Chinese": 0.24431818181818182, - "Filipino,Spanish,Indonesian": 0.23295454545454544, - "Filipino,Spanish,Vietnamese": 0.2784090909090909, - "Filipino,Malay,Chinese": 0.1875, - "Filipino,Malay,Indonesian": 0.2159090909090909, - "Filipino,Malay,Vietnamese": 0.23295454545454544, - "Filipino,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Chinese,Vietnamese": 0.21022727272727273, - "Filipino,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese": 0.32386363636363635, - "Spanish,Malay,Indonesian": 0.3977272727272727, - "Spanish,Malay,Vietnamese": 0.4034090909090909, - "Spanish,Chinese,Indonesian": 0.30113636363636365, - "Spanish,Chinese,Vietnamese": 0.3352272727272727, - "Spanish,Indonesian,Vietnamese": 0.36363636363636365, - "Malay,Chinese,Indonesian": 0.3409090909090909, - "Malay,Chinese,Vietnamese": 0.32954545454545453, - "Malay,Indonesian,Vietnamese": 0.42613636363636365, - "Chinese,Indonesian,Vietnamese": 0.2897727272727273 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.19886363636363635, - "English,Filipino,Spanish,Chinese": 0.17045454545454544, - "English,Filipino,Spanish,Indonesian": 0.18181818181818182, - "English,Filipino,Spanish,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese": 0.26136363636363635, - "English,Spanish,Malay,Indonesian": 0.3068181818181818, - "English,Spanish,Malay,Vietnamese": 0.2897727272727273, - "English,Spanish,Chinese,Indonesian": 0.23295454545454544, - "English,Spanish,Chinese,Vietnamese": 0.25, - "English,Spanish,Indonesian,Vietnamese": 0.2727272727272727, - "English,Malay,Chinese,Indonesian": 0.2727272727272727, - "English,Malay,Chinese,Vietnamese": 0.25, - "English,Malay,Indonesian,Vietnamese": 0.3068181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Malay,Chinese": 0.1534090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.23863636363636365, - "Spanish,Malay,Chinese,Vietnamese": 0.25, - "Spanish,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Indonesian": 0.125, - "English,Filipino,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian": 0.21022727272727273, - "English,Spanish,Malay,Chinese,Vietnamese": 0.20454545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1875 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - } - }, - "AC3_2": 0.38961038956238964, - "AC3_3": 0.3115869825383643, - "AC3_4": 0.25243704945553935, - "AC3_5": 0.20661157020454546, - "AC3_6": 0.17276301676066014, - "AC3_7": 0.14888772110671347 + "AC3_2": 0.45645109519228255, + "AC3_3": 0.3648534525092451, + "AC3_4": 0.301410813564475, + "AC3_5": 0.25491235217952013, + "AC3_6": 0.21952405650402038, + "AC3_7": 0.1927083332980035 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.3592233009708738 - }, - "prompt_2": { - "accuracy": 0.42718446601941745 - }, - "prompt_3": { - "accuracy": 0.3786407766990291 - }, - "prompt_4": { - "accuracy": 0.39805825242718446 - }, - "prompt_5": { - "accuracy": 0.33980582524271846 + "accuracy": 0.5631067961165048 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.2857142857142857 - }, - "prompt_2": { - "accuracy": 0.3904761904761905 - }, - "prompt_3": { - "accuracy": 0.3523809523809524 - }, - "prompt_4": { - "accuracy": 0.3047619047619048 - }, - "prompt_5": { - "accuracy": 0.37142857142857144 + "accuracy": 0.6 } }, "us_eval": { "prompt_1": { - "accuracy": 0.3925233644859813 - }, - "prompt_2": { - "accuracy": 0.35514018691588783 - }, - "prompt_3": { - "accuracy": 0.3925233644859813 - }, - "prompt_4": { - "accuracy": 0.3925233644859813 - }, - "prompt_5": { - "accuracy": 0.4205607476635514 + "accuracy": 0.5887850467289719 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.25, - "category_acc": { - "brand": 0.2, - "demographics": 0.0, - "biology": 0.3, - "history": 0.3333333333333333, - "literature": 0.4, - "politics": 0.1, - "culture": 0.1, - "film": 0.3, - "law": 0.1, - "geography": 0.5 - } - }, - "prompt_2": { - "accuracy": 0.31, - "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.5, - "history": 0.3333333333333333, - "literature": 0.5, - "politics": 0.0, - "culture": 0.2, - "film": 0.3, - "law": 0.3, - "geography": 0.6 - } - }, - "prompt_3": { - "accuracy": 0.25, - "category_acc": { - "brand": 0.3, - "demographics": 0.0, - "biology": 0.6, - "history": 0.13333333333333333, - "literature": 0.4, - "politics": 0.1, - "culture": 0.1, - "film": 0.2, - "law": 0.1, - "geography": 0.5 - } - }, - "prompt_4": { - "accuracy": 0.27, + "accuracy": 0.37, "category_acc": { "brand": 0.3, - "demographics": 0.0, + "demographics": 0.2, "biology": 0.4, "history": 0.26666666666666666, - "literature": 0.4, - "politics": 0.2, - "culture": 0.2, - "film": 0.2, - "law": 0.2, - "geography": 0.4 - } - }, - "prompt_5": { - "accuracy": 0.24, - "category_acc": { - "brand": 0.2, - "demographics": 0.0, - "biology": 0.6, - "history": 0.06666666666666667, - "literature": 0.4, - "politics": 0.2, + "literature": 0.3, + "politics": 0.5, "culture": 0.3, "film": 0.2, - "law": 0.0, - "geography": 0.4 + "law": 0.7, + "geography": 0.5 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.06305082715215274 - }, - "prompt_2": { - "bleu_score": 0.06463430550758704 - }, - "prompt_3": { - "bleu_score": 0.06356003237143983 - }, - "prompt_4": { - "bleu_score": 0.06539930841182513 - }, - "prompt_5": { - "bleu_score": 0.06031168781615755 + "bleu_score": 0.32178565305954265 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "prompt_1": { + "accuracy": 0.490219640830496, + "category_acc": { + "History": 0.5321285140562249, + "Geography": 0.44285714285714284, + "Lampungic": 0.3741496598639456, + "Social science": 0.7378964941569283, + "Balinese": 0.3248407643312102, + "Makassarese": 0.34946236559139787, + "Banjarese": 0.4166666666666667, + "Chemistry": 0.28175182481751826, + "Biology": 0.4437869822485207, + "Science": 0.6140350877192983, + "Christian religion": 0.6218905472636815, + "Art": 0.5973377703826955, + "Islam religion": 0.6244665718349929, + "Hindu religion": 0.48, + "Madurese": 0.26440677966101694, + "Sport": 0.5675675675675675, + "Indonesian language": 0.547011207970112, + "Physics": 0.3696969696969697, + "Minangkabau culture": 0.3869346733668342, + "Dayak language": 0.3302752293577982, + "Sociology": 0.45161290322580644, + "Economy": 0.4487704918032787, + "Sundanese": 0.3975799481417459, + "Javanese": 0.3719758064516129, + "Civic education": 0.630901287553648 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.07908898359419463 - }, - "prompt_2": { - "bleu_score": 0.07897628396673814 - }, - "prompt_3": { - "bleu_score": 0.07899232385079807 - }, - "prompt_4": { - "bleu_score": 0.07891920764416933 - }, - "prompt_5": { - "bleu_score": 0.07468075077628368 + "bleu_score": 0.4081953488837232 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.06631314380326508 - }, - "prompt_2": { - "bleu_score": 0.06715912321244885 - }, - "prompt_3": { - "bleu_score": 0.0669243196705807 - }, - "prompt_4": { - "bleu_score": 0.06705363387875442 - }, - "prompt_5": { - "bleu_score": 0.06463296313543722 + "bleu_score": 0.33776063614449015 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.04429631011508256 - }, - "prompt_2": { - "bleu_score": 0.04422337129052579 - }, - "prompt_3": { - "bleu_score": 0.044872517066552334 - }, - "prompt_4": { - "bleu_score": 0.04557580925559713 - }, - "prompt_5": { - "bleu_score": 0.04355200215183761 + "bleu_score": 0.2477760479226549 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.07958207120800556 - }, - "prompt_2": { - "bleu_score": 0.07948625371017644 - }, - "prompt_3": { - "bleu_score": 0.07995669494424623 - }, - "prompt_4": { - "bleu_score": 0.07923605117741923 - }, - "prompt_5": { - "bleu_score": 0.07715463343756887 + "bleu_score": 0.4184670269252103 } }, "mmlu": { "prompt_1": { - "accuracy": 0.3605600933488915 - }, - "prompt_2": { - "accuracy": 0.3628938156359393 - }, - "prompt_3": { - "accuracy": 0.3547257876312719 - }, - "prompt_4": { - "accuracy": 0.3500583430571762 - }, - "prompt_5": { - "accuracy": 0.3547257876312719 + "accuracy": 0.4830805134189031 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.3662495530925992, + "accuracy": 0.5003217733285663, "category_acc": { - "high_school_european_history": 0.43902439024390244, - "business_ethics": 0.3939393939393939, - "clinical_knowledge": 0.4015151515151515, - "medical_genetics": 0.3838383838383838, - "high_school_us_history": 0.46798029556650245, - "high_school_physics": 0.26, - "high_school_world_history": 0.4957627118644068, - "virology": 0.30303030303030304, - "high_school_microeconomics": 0.33755274261603374, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.29292929292929293, - "high_school_biology": 0.36245954692556637, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.3202846975088968, - "philosophy": 0.3741935483870968, - "professional_medicine": 0.3210332103321033, - "nutrition": 0.4163934426229508, + "high_school_european_history": 0.6219512195121951, + "business_ethics": 0.494949494949495, + "clinical_knowledge": 0.5795454545454546, + "medical_genetics": 0.5555555555555556, + "high_school_us_history": 0.6699507389162561, + "high_school_physics": 0.32, + "high_school_world_history": 0.635593220338983, + "virology": 0.43636363636363634, + "high_school_microeconomics": 0.48945147679324896, + "econometrics": 0.3274336283185841, + "college_computer_science": 0.41414141414141414, + "high_school_biology": 0.6245954692556634, + "abstract_algebra": 0.30303030303030304, + "professional_accounting": 0.35587188612099646, + "philosophy": 0.5774193548387097, + "professional_medicine": 0.4907749077490775, + "nutrition": 0.5770491803278689, "global_facts": 0.3333333333333333, - "machine_learning": 0.23423423423423423, - "security_studies": 0.29098360655737704, - "public_relations": 0.46788990825688076, - "professional_psychology": 0.3617021276595745, - "prehistory": 0.4086687306501548, - "anatomy": 0.43283582089552236, - "human_sexuality": 0.4307692307692308, - "college_medicine": 0.4011627906976744, - "high_school_government_and_politics": 0.4739583333333333, - "college_chemistry": 0.30303030303030304, - "logical_fallacies": 0.4012345679012346, - "high_school_geography": 0.467005076142132, - "elementary_mathematics": 0.2917771883289125, - "human_aging": 0.35585585585585583, + "machine_learning": 0.44144144144144143, + "security_studies": 0.569672131147541, + "public_relations": 0.5504587155963303, + "professional_psychology": 0.4746317512274959, + "prehistory": 0.5882352941176471, + "anatomy": 0.5522388059701493, + "human_sexuality": 0.5923076923076923, + "college_medicine": 0.46511627906976744, + "high_school_government_and_politics": 0.7135416666666666, + "college_chemistry": 0.46464646464646464, + "logical_fallacies": 0.5864197530864198, + "high_school_geography": 0.6649746192893401, + "elementary_mathematics": 0.3209549071618037, + "human_aging": 0.5990990990990991, "college_mathematics": 0.36363636363636365, - "high_school_psychology": 0.4485294117647059, - "formal_logic": 0.344, - "high_school_statistics": 0.26046511627906976, - "international_law": 0.5333333333333333, - "high_school_mathematics": 0.2342007434944238, - "high_school_computer_science": 0.3333333333333333, - "conceptual_physics": 0.34615384615384615, - "miscellaneous": 0.4948849104859335, - "high_school_chemistry": 0.2871287128712871, - "marketing": 0.5665236051502146, - "professional_law": 0.30071754729288974, - "management": 0.46078431372549017, - "college_physics": 0.2376237623762376, - "jurisprudence": 0.35514018691588783, - "world_religions": 0.4764705882352941, - "sociology": 0.475, - "us_foreign_policy": 0.5656565656565656, - "high_school_macroeconomics": 0.3444730077120823, - "computer_security": 0.42424242424242425, + "high_school_psychology": 0.6709558823529411, + "formal_logic": 0.336, + "high_school_statistics": 0.4, + "international_law": 0.6583333333333333, + "high_school_mathematics": 0.25650557620817843, + "high_school_computer_science": 0.5151515151515151, + "conceptual_physics": 0.49145299145299143, + "miscellaneous": 0.7161125319693095, + "high_school_chemistry": 0.4306930693069307, + "marketing": 0.776824034334764, + "professional_law": 0.3457273320287019, + "management": 0.6078431372549019, + "college_physics": 0.27722772277227725, + "jurisprudence": 0.6074766355140186, + "world_religions": 0.6764705882352942, + "sociology": 0.655, + "us_foreign_policy": 0.797979797979798, + "high_school_macroeconomics": 0.4832904884318766, + "computer_security": 0.6666666666666666, "moral_scenarios": 0.24049217002237136, - "moral_disputes": 0.37681159420289856, - "electrical_engineering": 0.3680555555555556, - "astronomy": 0.423841059602649, - "college_biology": 0.3146853146853147 + "moral_disputes": 0.5478260869565217, + "electrical_engineering": 0.4722222222222222, + "astronomy": 0.5695364238410596, + "college_biology": 0.5524475524475524 } - }, - "prompt_2": { - "accuracy": 0.3667500893814802, + } + }, + "c_eval": { + "prompt_1": { + "accuracy": 0.512630014858841 + } + }, + "c_eval_full": { + "prompt_1": { + "accuracy": 0.5336239103362391, "category_acc": { - "high_school_european_history": 0.49390243902439024, - "business_ethics": 0.43434343434343436, - "clinical_knowledge": 0.39015151515151514, - "medical_genetics": 0.29292929292929293, - "high_school_us_history": 0.43349753694581283, - "high_school_physics": 0.26666666666666666, - "high_school_world_history": 0.5423728813559322, - "virology": 0.3212121212121212, - "high_school_microeconomics": 0.3755274261603376, - "econometrics": 0.22123893805309736, - "college_computer_science": 0.31313131313131315, - "high_school_biology": 0.3656957928802589, - "abstract_algebra": 0.2727272727272727, - "professional_accounting": 0.3274021352313167, - "philosophy": 0.4064516129032258, - "professional_medicine": 0.3505535055350554, - "nutrition": 0.4262295081967213, - "global_facts": 0.2222222222222222, - "machine_learning": 0.2882882882882883, - "security_studies": 0.30327868852459017, - "public_relations": 0.45871559633027525, - "professional_psychology": 0.3486088379705401, - "prehistory": 0.3931888544891641, - "anatomy": 0.4253731343283582, - "human_sexuality": 0.4307692307692308, - "college_medicine": 0.3546511627906977, - "high_school_government_and_politics": 0.3697916666666667, - "college_chemistry": 0.31313131313131315, - "logical_fallacies": 0.3950617283950617, - "high_school_geography": 0.4619289340101523, - "elementary_mathematics": 0.3129973474801061, - "human_aging": 0.38738738738738737, - "college_mathematics": 0.32323232323232326, - "high_school_psychology": 0.4209558823529412, - "formal_logic": 0.368, - "high_school_statistics": 0.2558139534883721, - "international_law": 0.5416666666666666, - "high_school_mathematics": 0.21933085501858737, - "high_school_computer_science": 0.3838383838383838, - "conceptual_physics": 0.3076923076923077, - "miscellaneous": 0.46930946291560105, - "high_school_chemistry": 0.2871287128712871, - "marketing": 0.5708154506437768, - "professional_law": 0.32550554468362686, - "management": 0.4803921568627451, - "college_physics": 0.21782178217821782, - "jurisprudence": 0.3177570093457944, - "world_religions": 0.4, - "sociology": 0.5, - "us_foreign_policy": 0.5858585858585859, - "high_school_macroeconomics": 0.33676092544987146, - "computer_security": 0.5252525252525253, - "moral_scenarios": 0.2483221476510067, - "moral_disputes": 0.3681159420289855, - "electrical_engineering": 0.4027777777777778, - "astronomy": 0.4105960264900662, - "college_biology": 0.32867132867132864 - } - }, - "prompt_3": { - "accuracy": 0.3599570968895245, - "category_acc": { - "high_school_european_history": 0.4817073170731707, - "business_ethics": 0.42424242424242425, - "clinical_knowledge": 0.39015151515151514, - "medical_genetics": 0.3333333333333333, - "high_school_us_history": 0.4433497536945813, - "high_school_physics": 0.25333333333333335, - "high_school_world_history": 0.5508474576271186, - "virology": 0.3212121212121212, - "high_school_microeconomics": 0.3080168776371308, - "econometrics": 0.24778761061946902, - "college_computer_science": 0.31313131313131315, - "high_school_biology": 0.3592233009708738, - "abstract_algebra": 0.2727272727272727, - "professional_accounting": 0.302491103202847, - "philosophy": 0.4064516129032258, - "professional_medicine": 0.3062730627306273, - "nutrition": 0.4262295081967213, - "global_facts": 0.2727272727272727, - "machine_learning": 0.2972972972972973, - "security_studies": 0.2786885245901639, - "public_relations": 0.44036697247706424, - "professional_psychology": 0.3567921440261866, - "prehistory": 0.4024767801857585, - "anatomy": 0.40298507462686567, - "human_sexuality": 0.4307692307692308, - "college_medicine": 0.36627906976744184, - "high_school_government_and_politics": 0.3645833333333333, - "college_chemistry": 0.2828282828282828, - "logical_fallacies": 0.35185185185185186, - "high_school_geography": 0.4720812182741117, - "elementary_mathematics": 0.2838196286472148, - "human_aging": 0.4144144144144144, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.4338235294117647, - "formal_logic": 0.352, - "high_school_statistics": 0.25116279069767444, - "international_law": 0.525, - "high_school_mathematics": 0.2342007434944238, - "high_school_computer_science": 0.30303030303030304, - "conceptual_physics": 0.2948717948717949, - "miscellaneous": 0.4859335038363171, - "high_school_chemistry": 0.2722772277227723, - "marketing": 0.47639484978540775, - "professional_law": 0.3059360730593607, - "management": 0.49019607843137253, - "college_physics": 0.24752475247524752, - "jurisprudence": 0.32710280373831774, - "world_religions": 0.45294117647058824, - "sociology": 0.47, - "us_foreign_policy": 0.5252525252525253, - "high_school_macroeconomics": 0.31876606683804626, - "computer_security": 0.42424242424242425, - "moral_scenarios": 0.2348993288590604, - "moral_disputes": 0.391304347826087, - "electrical_engineering": 0.4027777777777778, - "astronomy": 0.46357615894039733, - "college_biology": 0.36363636363636365 - } - }, - "prompt_4": { - "accuracy": 0.3488737933500179, - "category_acc": { - "high_school_european_history": 0.47560975609756095, - "business_ethics": 0.45454545454545453, - "clinical_knowledge": 0.375, - "medical_genetics": 0.35353535353535354, - "high_school_us_history": 0.4187192118226601, - "high_school_physics": 0.2733333333333333, - "high_school_world_history": 0.4957627118644068, - "virology": 0.2787878787878788, - "high_school_microeconomics": 0.3333333333333333, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.3333333333333333, - "high_school_biology": 0.3268608414239482, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.3167259786476868, - "philosophy": 0.3419354838709677, - "professional_medicine": 0.31365313653136534, - "nutrition": 0.4065573770491803, - "global_facts": 0.29292929292929293, - "machine_learning": 0.2972972972972973, - "security_studies": 0.26229508196721313, - "public_relations": 0.4036697247706422, - "professional_psychology": 0.32896890343698854, - "prehistory": 0.38699690402476783, - "anatomy": 0.40298507462686567, - "human_sexuality": 0.4230769230769231, - "college_medicine": 0.3953488372093023, - "high_school_government_and_politics": 0.4375, - "college_chemistry": 0.3434343434343434, - "logical_fallacies": 0.35802469135802467, - "high_school_geography": 0.4416243654822335, - "elementary_mathematics": 0.3050397877984085, - "human_aging": 0.34234234234234234, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.41544117647058826, - "formal_logic": 0.352, - "high_school_statistics": 0.2744186046511628, - "international_law": 0.45, - "high_school_mathematics": 0.22304832713754646, - "high_school_computer_science": 0.36363636363636365, - "conceptual_physics": 0.29914529914529914, - "miscellaneous": 0.42327365728900257, - "high_school_chemistry": 0.2722772277227723, - "marketing": 0.49356223175965663, - "professional_law": 0.3020221787345075, - "management": 0.43137254901960786, - "college_physics": 0.22772277227722773, - "jurisprudence": 0.2897196261682243, - "world_religions": 0.4294117647058823, - "sociology": 0.435, - "us_foreign_policy": 0.5353535353535354, - "high_school_macroeconomics": 0.3341902313624679, - "computer_security": 0.42424242424242425, - "moral_scenarios": 0.2348993288590604, - "moral_disputes": 0.3739130434782609, - "electrical_engineering": 0.375, - "astronomy": 0.41721854304635764, - "college_biology": 0.34265734265734266 + "computer_network": 0.2916666666666667, + "operating_system": 0.5833333333333334, + "computer_architecture": 0.4230769230769231, + "college_programming": 0.6190476190476191, + "college_physics": 0.4583333333333333, + "college_chemistry": 0.4827586206896552, + "advanced_mathematics": 0.375, + "probability_and_statistics": 0.30434782608695654, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.38095238095238093, + "metrology_engineer": 0.5862068965517241, + "high_school_mathematics": 0.13043478260869565, + "high_school_physics": 0.5, + "high_school_chemistry": 0.625, + "high_school_biology": 0.4166666666666667, + "middle_school_mathematics": 0.4166666666666667, + "middle_school_biology": 0.8461538461538461, + "middle_school_physics": 0.75, + "middle_school_chemistry": 0.96, + "veterinary_medicine": 0.5, + "college_economics": 0.43333333333333335, + "business_administration": 0.4473684210526316, + "marxism": 0.6666666666666666, + "mao_zedong_thought": 0.8275862068965517, + "education_science": 0.6470588235294118, + "teacher_qualification": 0.7551020408163265, + "high_school_politics": 0.7916666666666666, + "high_school_geography": 0.7083333333333334, + "middle_school_politics": 0.7307692307692307, + "middle_school_geography": 0.7058823529411765, + "modern_chinese_history": 0.6428571428571429, + "ideological_and_moral_cultivation": 0.7916666666666666, + "logic": 0.4444444444444444, + "law": 0.27586206896551724, + "chinese_language_and_literature": 0.5714285714285714, + "art_studies": 0.5526315789473685, + "professional_tour_guide": 0.5588235294117647, + "legal_professional": 0.42857142857142855, + "high_school_chinese": 0.5, + "high_school_history": 0.8, + "middle_school_history": 0.6296296296296297, + "civil_servant": 0.34615384615384615, + "sports_science": 0.4583333333333333, + "plant_protection": 0.5925925925925926, + "basic_medicine": 0.5, + "clinical_medicine": 0.37037037037037035, + "urban_and_rural_planner": 0.5882352941176471, + "accountant": 0.4444444444444444, + "fire_engineer": 0.4166666666666667, + "environmental_impact_assessment_engineer": 0.5, + "tax_accountant": 0.46296296296296297, + "physician": 0.5555555555555556 } - }, - "prompt_5": { - "accuracy": 0.3618877368609224, + } + }, + "cmmlu": { + "prompt_1": { + "accuracy": 0.5770609318996416 + } + }, + "cmmlu_full": { + "prompt_1": { + "accuracy": 0.5230530132964946, "category_acc": { - "high_school_european_history": 0.42073170731707316, - "business_ethics": 0.3939393939393939, - "clinical_knowledge": 0.3977272727272727, - "medical_genetics": 0.41414141414141414, - "high_school_us_history": 0.4827586206896552, - "high_school_physics": 0.24666666666666667, - "high_school_world_history": 0.4788135593220339, - "virology": 0.3333333333333333, - "high_school_microeconomics": 0.350210970464135, - "econometrics": 0.23008849557522124, - "college_computer_science": 0.2727272727272727, - "high_school_biology": 0.3656957928802589, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.33451957295373663, - "philosophy": 0.38387096774193546, - "professional_medicine": 0.31365313653136534, - "nutrition": 0.43278688524590164, - "global_facts": 0.2828282828282828, - "machine_learning": 0.2972972972972973, - "security_studies": 0.29918032786885246, - "public_relations": 0.4036697247706422, - "professional_psychology": 0.34206219312602293, - "prehistory": 0.39628482972136225, - "anatomy": 0.44029850746268656, - "human_sexuality": 0.4307692307692308, - "college_medicine": 0.4127906976744186, - "high_school_government_and_politics": 0.4270833333333333, - "college_chemistry": 0.26262626262626265, - "logical_fallacies": 0.3950617283950617, - "high_school_geography": 0.49746192893401014, - "elementary_mathematics": 0.3129973474801061, - "human_aging": 0.3783783783783784, - "college_mathematics": 0.25252525252525254, - "high_school_psychology": 0.42830882352941174, - "formal_logic": 0.256, - "high_school_statistics": 0.2558139534883721, - "international_law": 0.49166666666666664, - "high_school_mathematics": 0.22676579925650558, - "high_school_computer_science": 0.37373737373737376, - "conceptual_physics": 0.2863247863247863, - "miscellaneous": 0.4884910485933504, - "high_school_chemistry": 0.2722772277227723, - "marketing": 0.5278969957081545, - "professional_law": 0.2974559686888454, - "management": 0.5098039215686274, - "college_physics": 0.2079207920792079, - "jurisprudence": 0.37383177570093457, - "world_religions": 0.4117647058823529, - "sociology": 0.515, - "us_foreign_policy": 0.5757575757575758, - "high_school_macroeconomics": 0.3341902313624679, - "computer_security": 0.45454545454545453, - "moral_scenarios": 0.24161073825503357, - "moral_disputes": 0.391304347826087, - "electrical_engineering": 0.3819444444444444, - "astronomy": 0.40397350993377484, - "college_biology": 0.3706293706293706 + "agronomy": 0.47928994082840237, + "anatomy": 0.33783783783783783, + "ancient_chinese": 0.36585365853658536, + "arts": 0.76875, + "astronomy": 0.3333333333333333, + "business_ethics": 0.507177033492823, + "chinese_civil_service_exam": 0.4625, + "chinese_driving_rule": 0.6412213740458015, + "chinese_food_culture": 0.5294117647058824, + "chinese_foreign_policy": 0.6261682242990654, + "chinese_history": 0.6222910216718266, + "chinese_literature": 0.4019607843137255, + "chinese_teacher_qualification": 0.664804469273743, + "clinical_knowledge": 0.4430379746835443, + "college_actuarial_science": 0.2358490566037736, + "college_education": 0.7102803738317757, + "college_engineering_hydrology": 0.4339622641509434, + "college_law": 0.46296296296296297, + "college_mathematics": 0.26666666666666666, + "college_medical_statistics": 0.37735849056603776, + "college_medicine": 0.4835164835164835, + "computer_science": 0.5343137254901961, + "computer_security": 0.6257309941520468, + "conceptual_physics": 0.5578231292517006, + "construction_project_management": 0.4172661870503597, + "economics": 0.5660377358490566, + "education": 0.6196319018404908, + "electrical_engineering": 0.5, + "elementary_chinese": 0.5158730158730159, + "elementary_commonsense": 0.601010101010101, + "elementary_information_and_technology": 0.7184873949579832, + "elementary_mathematics": 0.3217391304347826, + "ethnology": 0.5111111111111111, + "food_science": 0.5384615384615384, + "genetics": 0.4147727272727273, + "global_facts": 0.5838926174496645, + "high_school_biology": 0.5384615384615384, + "high_school_chemistry": 0.4696969696969697, + "high_school_geography": 0.6271186440677966, + "high_school_mathematics": 0.31097560975609756, + "high_school_physics": 0.4090909090909091, + "high_school_politics": 0.5594405594405595, + "human_sexuality": 0.5476190476190477, + "international_law": 0.4540540540540541, + "journalism": 0.5697674418604651, + "jurisprudence": 0.5523114355231143, + "legal_and_moral_basis": 0.8738317757009346, + "logical": 0.4146341463414634, + "machine_learning": 0.4098360655737705, + "management": 0.6190476190476191, + "marketing": 0.5444444444444444, + "marxist_theory": 0.7195767195767195, + "modern_chinese": 0.4051724137931034, + "nutrition": 0.5655172413793104, + "philosophy": 0.6, + "professional_accounting": 0.5828571428571429, + "professional_law": 0.44075829383886256, + "professional_medicine": 0.39361702127659576, + "professional_psychology": 0.5775862068965517, + "public_relations": 0.5459770114942529, + "security_study": 0.5851851851851851, + "sociology": 0.5221238938053098, + "sports_science": 0.5212121212121212, + "traditional_chinese_medicine": 0.4540540540540541, + "virology": 0.5325443786982249, + "world_history": 0.639751552795031, + "world_religions": 0.575 } } }, - "c_eval": { + "zbench": { "prompt_1": { - "accuracy": 0.31426448736998513 + "accuracy": 0.3939393939393939 + } + }, + "ind_emotion": { + "prompt_1": { + "accuracy": 0.26590909090909093 + } + }, + "ocnli": { + "prompt_1": { + "accuracy": 0.32372881355932204 + } + }, + "c3": { + "prompt_1": { + "accuracy": 0.7550486163051608 + } + }, + "dream": { + "prompt_1": { + "accuracy": 0.7437530622243999 + } + }, + "samsum": { + "prompt_1": { + "rouge1": 0.45036232390257586, + "rouge2": 0.20575253780038455, + "rougeL": 0.3643735012321646, + "avg_rouge": 0.34016278764504165 + } + }, + "dialogsum": { + "prompt_1": { + "rouge1": 0.4231691817172813, + "rouge2": 0.17154835835598875, + "rougeL": 0.35000262687479355, + "avg_rouge": 0.3149067223160212 + } + }, + "sst2": { + "prompt_1": { + "accuracy": 0.7821100917431193 + } + }, + "cola": { + "prompt_1": { + "accuracy": 0.5944391179290508 + } + }, + "qqp": { + "prompt_1": { + "accuracy": 0.509 + } + }, + "mnli": { + "prompt_1": { + "accuracy": 0.2 + } + }, + "qnli": { + "prompt_1": { + "accuracy": 0.6 + } + }, + "wnli": { + "prompt_1": { + "accuracy": 0.6 + } + }, + "rte": { + "prompt_1": { + "accuracy": 0.5 + } + }, + "mrpc": { + "prompt_1": { + "accuracy": 0.5 + } + } + } + }, + "sailor_0_5b_chat": { + "model_size": "0.5B", + "model_link": "https://huggingface.co/sail/Sailor-0.5B-Chat", + "zero_shot": { + "cross_xquad": { + "prompt_1": { + "overall_acc": 0.3468487394957983, + "language_acc": { + "Spanish": 0.33361344537815124, + "English": 0.380672268907563, + "Chinese": 0.32857142857142857, + "Vietnamese": 0.3445378151260504 + }, + "consistency_score_2": 0.3995798319327731, + "consistency_score_3": 0.1953781512605042, + "consistency_score_4": 0.10336134453781512, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.4504201680672269, + "Spanish,Chinese": 0.36554621848739494, + "Spanish,Vietnamese": 0.47058823529411764, + "English,Chinese": 0.37815126050420167, + "English,Vietnamese": 0.4008403361344538, + "Chinese,Vietnamese": 0.3319327731092437 + }, + "3_combine": { + "Spanish,English,Chinese": 0.1915966386554622, + "Spanish,English,Vietnamese": 0.2453781512605042, + "Spanish,Chinese,Vietnamese": 0.18067226890756302, + "English,Chinese,Vietnamese": 0.1638655462184874 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.10336134453781512 + } + }, + "AC3_2": 0.3713517041557144, + "AC3_3": 0.249956859877259, + "AC3_4": 0.1592623236528496 }, "prompt_2": { - "accuracy": 0.31946508172362553 + "overall_acc": 0.3340336134453782, + "language_acc": { + "Spanish": 0.33865546218487397, + "English": 0.34789915966386553, + "Chinese": 0.3100840336134454, + "Vietnamese": 0.33949579831932775 + }, + "consistency_score_2": 0.47142857142857136, + "consistency_score_3": 0.2819327731092437, + "consistency_score_4": 0.19327731092436976, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.5042016806722689, + "Spanish,Chinese": 0.4764705882352941, + "Spanish,Vietnamese": 0.4823529411764706, + "English,Chinese": 0.5134453781512605, + "English,Vietnamese": 0.46050420168067224, + "Chinese,Vietnamese": 0.3915966386554622 + }, + "3_combine": { + "Spanish,English,Chinese": 0.31596638655462184, + "Spanish,English,Vietnamese": 0.292436974789916, + "Spanish,Chinese,Vietnamese": 0.25630252100840334, + "English,Chinese,Vietnamese": 0.2630252100840336 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.19327731092436976 + } + }, + "AC3_2": 0.391012743076874, + "AC3_3": 0.3057797470470136, + "AC3_4": 0.24486926239957693 }, "prompt_3": { - "accuracy": 0.35438335809806837 + "overall_acc": 0.3401260504201681, + "language_acc": { + "Spanish": 0.3445378151260504, + "English": 0.3487394957983193, + "Chinese": 0.32941176470588235, + "Vietnamese": 0.3378151260504202 + }, + "consistency_score_2": 0.4343137254901961, + "consistency_score_3": 0.2346638655462185, + "consistency_score_4": 0.13865546218487396, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.47478991596638653, + "Spanish,Chinese": 0.40252100840336136, + "Spanish,Vietnamese": 0.4857142857142857, + "English,Chinese": 0.45630252100840335, + "English,Vietnamese": 0.43781512605042017, + "Chinese,Vietnamese": 0.3487394957983193 + }, + "3_combine": { + "Spanish,English,Chinese": 0.24873949579831933, + "Spanish,English,Vietnamese": 0.27563025210084036, + "Spanish,Chinese,Vietnamese": 0.20840336134453782, + "English,Chinese,Vietnamese": 0.20588235294117646 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.13865546218487396 + } + }, + "AC3_2": 0.38149231656270854, + "AC3_3": 0.2777198817640574, + "AC3_4": 0.19700148594157488 }, "prompt_4": { - "accuracy": 0.34323922734026746 + "overall_acc": 0.3191176470588235, + "language_acc": { + "Spanish": 0.3226890756302521, + "English": 0.3176470588235294, + "Chinese": 0.32521008403361346, + "Vietnamese": 0.31092436974789917 + }, + "consistency_score_2": 0.4189075630252101, + "consistency_score_3": 0.2130252100840336, + "consistency_score_4": 0.11932773109243698, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.4588235294117647, + "Spanish,Chinese": 0.4142857142857143, + "Spanish,Vietnamese": 0.44285714285714284, + "English,Chinese": 0.46218487394957986, + "English,Vietnamese": 0.3739495798319328, + "Chinese,Vietnamese": 0.36134453781512604 + }, + "3_combine": { + "Spanish,English,Chinese": 0.25126050420168067, + "Spanish,English,Vietnamese": 0.21428571428571427, + "Spanish,Chinese,Vietnamese": 0.19663865546218487, + "English,Chinese,Vietnamese": 0.1899159663865546 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.11932773109243698 + } + }, + "AC3_2": 0.3622662044685879, + "AC3_3": 0.2554956921096242, + "AC3_4": 0.1737027537020321 }, "prompt_5": { - "accuracy": 0.32838038632986627 + "overall_acc": 0.32941176470588235, + "language_acc": { + "Spanish": 0.334453781512605, + "English": 0.32016806722689073, + "Chinese": 0.3411764705882353, + "Vietnamese": 0.32184873949579834 + }, + "consistency_score_2": 0.42198879551820734, + "consistency_score_3": 0.2161764705882353, + "consistency_score_4": 0.126890756302521, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.4226890756302521, + "Spanish,Chinese": 0.44873949579831934, + "Spanish,Vietnamese": 0.4647058823529412, + "English,Chinese": 0.43025210084033616, + "English,Vietnamese": 0.38823529411764707, + "Chinese,Vietnamese": 0.3773109243697479 + }, + "3_combine": { + "Spanish,English,Chinese": 0.22605042016806723, + "Spanish,English,Vietnamese": 0.22521008403361345, + "Spanish,Chinese,Vietnamese": 0.2235294117647059, + "English,Chinese,Vietnamese": 0.1899159663865546 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.126890756302521 + } + }, + "AC3_2": 0.36999725887296725, + "AC3_3": 0.2610432851907775, + "AC3_4": 0.18320875307435475 } }, - "c_eval_full": { + "cross_mmlu": { "prompt_1": { - "accuracy": 0.3424657534246575, - "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.16666666666666666, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.16666666666666666, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.4482758620689655, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.375, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.48, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.36666666666666664, - "business_administration": 0.2894736842105263, - "marxism": 0.375, - "mao_zedong_thought": 0.41379310344827586, - "education_science": 0.4117647058823529, - "teacher_qualification": 0.40816326530612246, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.34615384615384615, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.25, - "logic": 0.4074074074074074, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.5, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.56, - "middle_school_history": 0.48148148148148145, - "civil_servant": 0.3269230769230769, - "sports_science": 0.20833333333333334, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.4117647058823529, - "accountant": 0.3333333333333333, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.3611111111111111, - "tax_accountant": 0.35185185185185186, - "physician": 0.2962962962962963 - } - }, - "prompt_2": { - "accuracy": 0.32938978829389787, - "category_acc": { - "computer_network": 0.16666666666666666, - "operating_system": 0.25, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.30952380952380953, - "college_physics": 0.25, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.4782608695652174, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.4166666666666667, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.125, - "middle_school_biology": 0.46153846153846156, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.43333333333333335, - "business_administration": 0.3157894736842105, - "marxism": 0.375, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.4411764705882353, - "teacher_qualification": 0.46938775510204084, - "high_school_politics": 0.25, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.25, - "ideological_and_moral_cultivation": 0.4166666666666667, - "logic": 0.4074074074074074, - "law": 0.1724137931034483, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.48, - "middle_school_history": 0.4444444444444444, - "civil_servant": 0.2692307692307692, - "sports_science": 0.25, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.5, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.39215686274509803, - "accountant": 0.2777777777777778, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.4074074074074074, - "physician": 0.37037037037037035 - } - }, - "prompt_3": { - "accuracy": 0.3318804483188045, - "category_acc": { - "computer_network": 0.375, - "operating_system": 0.16666666666666666, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.21428571428571427, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.4782608695652174, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.375, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.3076923076923077, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.4, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.4166666666666667, - "business_administration": 0.4473684210526316, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.5294117647058824, - "teacher_qualification": 0.42857142857142855, - "high_school_politics": 0.4166666666666667, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.25, - "ideological_and_moral_cultivation": 0.375, - "logic": 0.48148148148148145, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.25, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.4, - "middle_school_history": 0.4074074074074074, - "civil_servant": 0.23076923076923078, - "sports_science": 0.375, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.5, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.37254901960784315, - "accountant": 0.24074074074074073, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.35185185185185186, - "physician": 0.3148148148148148 - } - }, - "prompt_4": { - "accuracy": 0.32938978829389787, - "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.25, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.16666666666666666, - "college_physics": 0.125, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.4827586206896552, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.375, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.44, - "veterinary_medicine": 0.42857142857142855, - "college_economics": 0.35, - "business_administration": 0.3684210526315789, - "marxism": 0.375, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.42857142857142855, - "high_school_politics": 0.4583333333333333, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.4166666666666667, - "logic": 0.4444444444444444, - "law": 0.4482758620689655, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.6, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.25, - "sports_science": 0.3333333333333333, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.18518518518518517, - "urban_and_rural_planner": 0.37254901960784315, - "accountant": 0.2962962962962963, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.3148148148148148, - "physician": 0.4074074074074074 - } - }, - "prompt_5": { - "accuracy": 0.3125778331257783, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.2916666666666667, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.19047619047619047, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.375, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.38333333333333336, - "business_administration": 0.23684210526315788, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.27586206896551724, - "education_science": 0.38235294117647056, - "teacher_qualification": 0.32653061224489793, - "high_school_politics": 0.4166666666666667, - "high_school_geography": 0.25, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.37037037037037035, - "law": 0.1724137931034483, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.25, - "high_school_chinese": 0.375, - "high_school_history": 0.4, - "middle_school_history": 0.48148148148148145, - "civil_servant": 0.34615384615384615, - "sports_science": 0.20833333333333334, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.375, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.39215686274509803, - "accountant": 0.3333333333333333, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.37037037037037035, - "physician": 0.37037037037037035 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.36200716845878134 + "overall_acc": 0.25047619047619046, + "language_acc": { + "Filipino": 0.26666666666666666, + "Vietnamese": 0.25333333333333335, + "Chinese": 0.26666666666666666, + "Spanish": 0.23333333333333334, + "Malay": 0.26, + "Indonesian": 0.21333333333333335, + "English": 0.26 + }, + "consistency_score_2": 0.5441269841269841, + "consistency_score_3": 0.3451428571428572, + "consistency_score_4": 0.23638095238095239, + "consistency_score_5": 0.17079365079365078, + "consistency_score_6": 0.1285714285714286, + "consistency_score_7": 0.1, + "detailed_consistency_score": { + "2_combine": { + "Filipino,Vietnamese": 0.5933333333333334, + "Filipino,Chinese": 0.5466666666666666, + "Filipino,Spanish": 0.58, + "Filipino,Malay": 0.68, + "Filipino,Indonesian": 0.6733333333333333, + "Filipino,English": 0.5066666666666667, + "Vietnamese,Chinese": 0.5, + "Vietnamese,Spanish": 0.5133333333333333, + "Vietnamese,Malay": 0.54, + "Vietnamese,Indonesian": 0.5866666666666667, + "Vietnamese,English": 0.43333333333333335, + "Chinese,Spanish": 0.47333333333333333, + "Chinese,Malay": 0.5533333333333333, + "Chinese,Indonesian": 0.5266666666666666, + "Chinese,English": 0.5, + "Spanish,Malay": 0.5, + "Spanish,Indonesian": 0.5733333333333334, + "Spanish,English": 0.52, + "Malay,Indonesian": 0.76, + "Malay,English": 0.44, + "Indonesian,English": 0.4266666666666667 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.35333333333333333, + "Filipino,Vietnamese,Spanish": 0.36666666666666664, + "Filipino,Vietnamese,Malay": 0.42, + "Filipino,Vietnamese,Indonesian": 0.44, + "Filipino,Vietnamese,English": 0.3, + "Filipino,Chinese,Spanish": 0.3333333333333333, + "Filipino,Chinese,Malay": 0.4066666666666667, + "Filipino,Chinese,Indonesian": 0.38666666666666666, + "Filipino,Chinese,English": 0.32, + "Filipino,Spanish,Malay": 0.3933333333333333, + "Filipino,Spanish,Indonesian": 0.42, + "Filipino,Spanish,English": 0.3333333333333333, + "Filipino,Malay,Indonesian": 0.5666666666666667, + "Filipino,Malay,English": 0.3333333333333333, + "Filipino,Indonesian,English": 0.32666666666666666, + "Vietnamese,Chinese,Spanish": 0.28, + "Vietnamese,Chinese,Malay": 0.3466666666666667, + "Vietnamese,Chinese,Indonesian": 0.35333333333333333, + "Vietnamese,Chinese,English": 0.24666666666666667, + "Vietnamese,Spanish,Malay": 0.30666666666666664, + "Vietnamese,Spanish,Indonesian": 0.36, + "Vietnamese,Spanish,English": 0.2733333333333333, + "Vietnamese,Malay,Indonesian": 0.4666666666666667, + "Vietnamese,Malay,English": 0.24, + "Vietnamese,Indonesian,English": 0.26, + "Chinese,Spanish,Malay": 0.30666666666666664, + "Chinese,Spanish,Indonesian": 0.32, + "Chinese,Spanish,English": 0.2866666666666667, + "Chinese,Malay,Indonesian": 0.43333333333333335, + "Chinese,Malay,English": 0.29333333333333333, + "Chinese,Indonesian,English": 0.2733333333333333, + "Spanish,Malay,Indonesian": 0.4266666666666667, + "Spanish,Malay,English": 0.2733333333333333, + "Spanish,Indonesian,English": 0.2866666666666667, + "Malay,Indonesian,English": 0.3466666666666667 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Malay": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.28, + "Filipino,Vietnamese,Chinese,English": 0.19333333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.26666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,Spanish,English": 0.21333333333333335, + "Filipino,Vietnamese,Malay,Indonesian": 0.36666666666666664, + "Filipino,Vietnamese,Malay,English": 0.20666666666666667, + "Filipino,Vietnamese,Indonesian,English": 0.21333333333333335, + "Filipino,Chinese,Spanish,Malay": 0.26666666666666666, + "Filipino,Chinese,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Chinese,Spanish,English": 0.20666666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.3333333333333333, + "Filipino,Chinese,Malay,English": 0.22, + "Filipino,Chinese,Indonesian,English": 0.21333333333333335, + "Filipino,Spanish,Malay,Indonesian": 0.3466666666666667, + "Filipino,Spanish,Malay,English": 0.21333333333333335, + "Filipino,Spanish,Indonesian,English": 0.22666666666666666, + "Filipino,Malay,Indonesian,English": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,Malay": 0.22666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian": 0.22666666666666666, + "Vietnamese,Chinese,Spanish,English": 0.16666666666666666, + "Vietnamese,Chinese,Malay,Indonesian": 0.3, + "Vietnamese,Chinese,Malay,English": 0.16, + "Vietnamese,Chinese,Indonesian,English": 0.16, + "Vietnamese,Spanish,Malay,Indonesian": 0.28, + "Vietnamese,Spanish,Malay,English": 0.16666666666666666, + "Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, + "Vietnamese,Malay,Indonesian,English": 0.20666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.26, + "Chinese,Spanish,Malay,English": 0.18666666666666668, + "Chinese,Spanish,Indonesian,English": 0.18, + "Chinese,Malay,Indonesian,English": 0.22666666666666666, + "Spanish,Malay,Indonesian,English": 0.22 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.2, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.14, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Malay,English": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.14, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.24, + "Filipino,Vietnamese,Spanish,Malay,English": 0.14666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, + "Filipino,Chinese,Spanish,Malay,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Malay,Indonesian,English": 0.17333333333333334, + "Filipino,Spanish,Malay,Indonesian,English": 0.18, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2, + "Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.12666666666666668, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.13333333333333333, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.14666666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 + } + }, + "AC3_2": 0.3430413027743999, + "AC3_3": 0.2902864451977339, + "AC3_4": 0.24322453232950383, + "AC3_5": 0.2030990059445401, + "AC3_6": 0.16992103369530082, + "AC3_7": 0.14293478256791262 }, "prompt_2": { - "accuracy": 0.3727598566308244 + "overall_acc": 0.25428571428571434, + "language_acc": { + "Filipino": 0.2733333333333333, + "Vietnamese": 0.24666666666666667, + "Chinese": 0.22666666666666666, + "Spanish": 0.26, + "Malay": 0.23333333333333334, + "Indonesian": 0.26, + "English": 0.28 + }, + "consistency_score_2": 0.5292063492063492, + "consistency_score_3": 0.3257142857142856, + "consistency_score_4": 0.20933333333333332, + "consistency_score_5": 0.13555555555555554, + "consistency_score_6": 0.08666666666666667, + "consistency_score_7": 0.05333333333333334, + "detailed_consistency_score": { + "2_combine": { + "Filipino,Vietnamese": 0.58, + "Filipino,Chinese": 0.4666666666666667, + "Filipino,Spanish": 0.5933333333333334, + "Filipino,Malay": 0.6266666666666667, + "Filipino,Indonesian": 0.6133333333333333, + "Filipino,English": 0.44, + "Vietnamese,Chinese": 0.44, + "Vietnamese,Spanish": 0.5066666666666667, + "Vietnamese,Malay": 0.5866666666666667, + "Vietnamese,Indonesian": 0.5933333333333334, + "Vietnamese,English": 0.4666666666666667, + "Chinese,Spanish": 0.5, + "Chinese,Malay": 0.43333333333333335, + "Chinese,Indonesian": 0.4666666666666667, + "Chinese,English": 0.54, + "Spanish,Malay": 0.5933333333333334, + "Spanish,Indonesian": 0.6066666666666667, + "Spanish,English": 0.5533333333333333, + "Malay,Indonesian": 0.6066666666666667, + "Malay,English": 0.41333333333333333, + "Indonesian,English": 0.4866666666666667 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.28, + "Filipino,Vietnamese,Spanish": 0.3466666666666667, + "Filipino,Vietnamese,Malay": 0.41333333333333333, + "Filipino,Vietnamese,Indonesian": 0.4, + "Filipino,Vietnamese,English": 0.26666666666666666, + "Filipino,Chinese,Spanish": 0.31333333333333335, + "Filipino,Chinese,Malay": 0.3, + "Filipino,Chinese,Indonesian": 0.3, + "Filipino,Chinese,English": 0.26, + "Filipino,Spanish,Malay": 0.4266666666666667, + "Filipino,Spanish,Indonesian": 0.41333333333333333, + "Filipino,Spanish,English": 0.32, + "Filipino,Malay,Indonesian": 0.4533333333333333, + "Filipino,Malay,English": 0.28, + "Filipino,Indonesian,English": 0.3, + "Vietnamese,Chinese,Spanish": 0.26666666666666666, + "Vietnamese,Chinese,Malay": 0.26666666666666666, + "Vietnamese,Chinese,Indonesian": 0.29333333333333333, + "Vietnamese,Chinese,English": 0.2733333333333333, + "Vietnamese,Spanish,Malay": 0.37333333333333335, + "Vietnamese,Spanish,Indonesian": 0.36666666666666664, + "Vietnamese,Spanish,English": 0.3, + "Vietnamese,Malay,Indonesian": 0.4266666666666667, + "Vietnamese,Malay,English": 0.2733333333333333, + "Vietnamese,Indonesian,English": 0.30666666666666664, + "Chinese,Spanish,Malay": 0.3, + "Chinese,Spanish,Indonesian": 0.32, + "Chinese,Spanish,English": 0.34, + "Chinese,Malay,Indonesian": 0.28, + "Chinese,Malay,English": 0.24, + "Chinese,Indonesian,English": 0.29333333333333333, + "Spanish,Malay,Indonesian": 0.4266666666666667, + "Spanish,Malay,English": 0.31333333333333335, + "Spanish,Indonesian,English": 0.36, + "Malay,Indonesian,English": 0.30666666666666664 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Malay": 0.2, + "Filipino,Vietnamese,Chinese,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,English": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Malay": 0.2733333333333333, + "Filipino,Vietnamese,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Spanish,English": 0.18666666666666668, + "Filipino,Vietnamese,Malay,Indonesian": 0.31333333333333335, + "Filipino,Vietnamese,Malay,English": 0.18666666666666668, + "Filipino,Vietnamese,Indonesian,English": 0.19333333333333333, + "Filipino,Chinese,Spanish,Malay": 0.2, + "Filipino,Chinese,Spanish,Indonesian": 0.19333333333333333, + "Filipino,Chinese,Spanish,English": 0.2, + "Filipino,Chinese,Malay,Indonesian": 0.20666666666666667, + "Filipino,Chinese,Malay,English": 0.16, + "Filipino,Chinese,Indonesian,English": 0.16666666666666666, + "Filipino,Spanish,Malay,Indonesian": 0.32666666666666666, + "Filipino,Spanish,Malay,English": 0.21333333333333335, + "Filipino,Spanish,Indonesian,English": 0.22666666666666666, + "Filipino,Malay,Indonesian,English": 0.21333333333333335, + "Vietnamese,Chinese,Spanish,Malay": 0.19333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.20666666666666667, + "Vietnamese,Chinese,Spanish,English": 0.18, + "Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, + "Vietnamese,Chinese,Malay,English": 0.14666666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.18, + "Vietnamese,Spanish,Malay,Indonesian": 0.29333333333333333, + "Vietnamese,Spanish,Malay,English": 0.2, + "Vietnamese,Spanish,Indonesian,English": 0.21333333333333335, + "Vietnamese,Malay,Indonesian,English": 0.22, + "Chinese,Spanish,Malay,Indonesian": 0.2, + "Chinese,Spanish,Malay,English": 0.19333333333333333, + "Chinese,Spanish,Indonesian,English": 0.23333333333333334, + "Chinese,Malay,Indonesian,English": 0.15333333333333332, + "Spanish,Malay,Indonesian,English": 0.24666666666666667 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.14, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.1, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.10666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22, + "Filipino,Vietnamese,Spanish,Malay,English": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.14666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, + "Filipino,Chinese,Spanish,Malay,English": 0.12, + "Filipino,Chinese,Spanish,Indonesian,English": 0.12666666666666668, + "Filipino,Chinese,Malay,Indonesian,English": 0.1, + "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.12, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.10666666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.16666666666666666, + "Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.06666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.11333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.08, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334 + } + }, + "AC3_2": 0.34351238708280596, + "AC3_3": 0.28560168890220095, + "AC3_4": 0.22963023824134926, + "AC3_5": 0.17684039083411668, + "AC3_6": 0.12927374297884428, + "AC3_7": 0.08817337458433994 }, "prompt_3": { - "accuracy": 0.3548387096774194 + "overall_acc": 0.2571428571428572, + "language_acc": { + "Filipino": 0.23333333333333334, + "Vietnamese": 0.22666666666666666, + "Chinese": 0.26666666666666666, + "Spanish": 0.22666666666666666, + "Malay": 0.24, + "Indonesian": 0.28, + "English": 0.32666666666666666 + }, + "consistency_score_2": 0.5425396825396825, + "consistency_score_3": 0.34609523809523807, + "consistency_score_4": 0.2375238095238095, + "consistency_score_5": 0.17111111111111116, + "consistency_score_6": 0.1285714285714286, + "consistency_score_7": 0.1, + "detailed_consistency_score": { + "2_combine": { + "Filipino,Vietnamese": 0.5666666666666667, + "Filipino,Chinese": 0.4866666666666667, + "Filipino,Spanish": 0.6333333333333333, + "Filipino,Malay": 0.6266666666666667, + "Filipino,Indonesian": 0.64, + "Filipino,English": 0.47333333333333333, + "Vietnamese,Chinese": 0.42, + "Vietnamese,Spanish": 0.56, + "Vietnamese,Malay": 0.5333333333333333, + "Vietnamese,Indonesian": 0.5466666666666666, + "Vietnamese,English": 0.4666666666666667, + "Chinese,Spanish": 0.49333333333333335, + "Chinese,Malay": 0.5133333333333333, + "Chinese,Indonesian": 0.43333333333333335, + "Chinese,English": 0.56, + "Spanish,Malay": 0.6066666666666667, + "Spanish,Indonesian": 0.6466666666666666, + "Spanish,English": 0.54, + "Malay,Indonesian": 0.6466666666666666, + "Malay,English": 0.4866666666666667, + "Indonesian,English": 0.5133333333333333 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.26666666666666666, + "Filipino,Vietnamese,Spanish": 0.38666666666666666, + "Filipino,Vietnamese,Malay": 0.38, + "Filipino,Vietnamese,Indonesian": 0.4, + "Filipino,Vietnamese,English": 0.29333333333333333, + "Filipino,Chinese,Spanish": 0.32, + "Filipino,Chinese,Malay": 0.36, + "Filipino,Chinese,Indonesian": 0.31333333333333335, + "Filipino,Chinese,English": 0.3, + "Filipino,Spanish,Malay": 0.44666666666666666, + "Filipino,Spanish,Indonesian": 0.4866666666666667, + "Filipino,Spanish,English": 0.35333333333333333, + "Filipino,Malay,Indonesian": 0.47333333333333333, + "Filipino,Malay,English": 0.34, + "Filipino,Indonesian,English": 0.34, + "Vietnamese,Chinese,Spanish": 0.26666666666666666, + "Vietnamese,Chinese,Malay": 0.28, + "Vietnamese,Chinese,Indonesian": 0.24666666666666667, + "Vietnamese,Chinese,English": 0.2733333333333333, + "Vietnamese,Spanish,Malay": 0.36666666666666664, + "Vietnamese,Spanish,Indonesian": 0.4, + "Vietnamese,Spanish,English": 0.32666666666666666, + "Vietnamese,Malay,Indonesian": 0.38666666666666666, + "Vietnamese,Malay,English": 0.29333333333333333, + "Vietnamese,Indonesian,English": 0.30666666666666664, + "Chinese,Spanish,Malay": 0.34, + "Chinese,Spanish,Indonesian": 0.32, + "Chinese,Spanish,English": 0.32666666666666666, + "Chinese,Malay,Indonesian": 0.34, + "Chinese,Malay,English": 0.32666666666666666, + "Chinese,Indonesian,English": 0.2866666666666667, + "Spanish,Malay,Indonesian": 0.47333333333333333, + "Spanish,Malay,English": 0.36, + "Spanish,Indonesian,English": 0.37333333333333335, + "Malay,Indonesian,English": 0.36 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Malay": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.2, + "Filipino,Vietnamese,Chinese,English": 0.18, + "Filipino,Vietnamese,Spanish,Malay": 0.29333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian": 0.32, + "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, + "Filipino,Vietnamese,Malay,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Malay,English": 0.22, + "Filipino,Vietnamese,Indonesian,English": 0.21333333333333335, + "Filipino,Chinese,Spanish,Malay": 0.25333333333333335, + "Filipino,Chinese,Spanish,Indonesian": 0.24, + "Filipino,Chinese,Spanish,English": 0.20666666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.25333333333333335, + "Filipino,Chinese,Malay,English": 0.22666666666666666, + "Filipino,Chinese,Indonesian,English": 0.21333333333333335, + "Filipino,Spanish,Malay,Indonesian": 0.36666666666666664, + "Filipino,Spanish,Malay,English": 0.2733333333333333, + "Filipino,Spanish,Indonesian,English": 0.28, + "Filipino,Malay,Indonesian,English": 0.26, + "Vietnamese,Chinese,Spanish,Malay": 0.2, + "Vietnamese,Chinese,Spanish,Indonesian": 0.2, + "Vietnamese,Chinese,Spanish,English": 0.19333333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.21333333333333335, + "Vietnamese,Chinese,Malay,English": 0.19333333333333333, + "Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, + "Vietnamese,Spanish,Malay,Indonesian": 0.3, + "Vietnamese,Spanish,Malay,English": 0.22666666666666666, + "Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, + "Vietnamese,Malay,Indonesian,English": 0.21333333333333335, + "Chinese,Spanish,Malay,Indonesian": 0.26, + "Chinese,Spanish,Malay,English": 0.24, + "Chinese,Spanish,Indonesian,English": 0.21333333333333335, + "Chinese,Malay,Indonesian,English": 0.23333333333333334, + "Spanish,Malay,Indonesian,English": 0.28 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.16, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,Malay,English": 0.14, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay,English": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2, + "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Indonesian,English": 0.16, + "Filipino,Chinese,Malay,Indonesian,English": 0.17333333333333334, + "Filipino,Spanish,Malay,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Malay,English": 0.14666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.14, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.18, + "Chinese,Spanish,Malay,Indonesian,English": 0.18 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.11333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.14, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 + } + }, + "AC3_2": 0.34891396808431036, + "AC3_3": 0.2950606702564686, + "AC3_4": 0.24694427631291485, + "AC3_5": 0.20548554480005346, + "AC3_6": 0.17142857138412698, + "AC3_7": 0.14399999995968 }, "prompt_4": { - "accuracy": 0.36200716845878134 + "overall_acc": 0.25047619047619046, + "language_acc": { + "Filipino": 0.26666666666666666, + "Vietnamese": 0.26, + "Chinese": 0.24666666666666667, + "Spanish": 0.26666666666666666, + "Malay": 0.2, + "Indonesian": 0.23333333333333334, + "English": 0.28 + }, + "consistency_score_2": 0.5377777777777778, + "consistency_score_3": 0.3401904761904762, + "consistency_score_4": 0.24019047619047618, + "consistency_score_5": 0.18603174603174605, + "consistency_score_6": 0.1542857142857143, + "consistency_score_7": 0.13333333333333333, + "detailed_consistency_score": { + "2_combine": { + "Filipino,Vietnamese": 0.5266666666666666, + "Filipino,Chinese": 0.4533333333333333, + "Filipino,Spanish": 0.6066666666666667, + "Filipino,Malay": 0.64, + "Filipino,Indonesian": 0.6933333333333334, + "Filipino,English": 0.5133333333333333, + "Vietnamese,Chinese": 0.5733333333333334, + "Vietnamese,Spanish": 0.5066666666666667, + "Vietnamese,Malay": 0.52, + "Vietnamese,Indonesian": 0.5533333333333333, + "Vietnamese,English": 0.42, + "Chinese,Spanish": 0.5133333333333333, + "Chinese,Malay": 0.52, + "Chinese,Indonesian": 0.5333333333333333, + "Chinese,English": 0.47333333333333333, + "Spanish,Malay": 0.5733333333333334, + "Spanish,Indonesian": 0.6, + "Spanish,English": 0.5266666666666666, + "Malay,Indonesian": 0.66, + "Malay,English": 0.44, + "Indonesian,English": 0.44666666666666666 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.32, + "Filipino,Vietnamese,Spanish": 0.3466666666666667, + "Filipino,Vietnamese,Malay": 0.35333333333333333, + "Filipino,Vietnamese,Indonesian": 0.4, + "Filipino,Vietnamese,English": 0.28, + "Filipino,Chinese,Spanish": 0.31333333333333335, + "Filipino,Chinese,Malay": 0.3333333333333333, + "Filipino,Chinese,Indonesian": 0.37333333333333335, + "Filipino,Chinese,English": 0.26666666666666666, + "Filipino,Spanish,Malay": 0.43333333333333335, + "Filipino,Spanish,Indonesian": 0.46, + "Filipino,Spanish,English": 0.35333333333333333, + "Filipino,Malay,Indonesian": 0.5133333333333333, + "Filipino,Malay,English": 0.34, + "Filipino,Indonesian,English": 0.36, + "Vietnamese,Chinese,Spanish": 0.34, + "Vietnamese,Chinese,Malay": 0.3466666666666667, + "Vietnamese,Chinese,Indonesian": 0.36666666666666664, + "Vietnamese,Chinese,English": 0.2866666666666667, + "Vietnamese,Spanish,Malay": 0.3333333333333333, + "Vietnamese,Spanish,Indonesian": 0.36, + "Vietnamese,Spanish,English": 0.28, + "Vietnamese,Malay,Indonesian": 0.37333333333333335, + "Vietnamese,Malay,English": 0.24, + "Vietnamese,Indonesian,English": 0.24666666666666667, + "Chinese,Spanish,Malay": 0.3333333333333333, + "Chinese,Spanish,Indonesian": 0.35333333333333333, + "Chinese,Spanish,English": 0.30666666666666664, + "Chinese,Malay,Indonesian": 0.3933333333333333, + "Chinese,Malay,English": 0.25333333333333335, + "Chinese,Indonesian,English": 0.2733333333333333, + "Spanish,Malay,Indonesian": 0.44, + "Spanish,Malay,English": 0.3, + "Spanish,Indonesian,English": 0.31333333333333335, + "Malay,Indonesian,English": 0.32 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Malay": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.28, + "Filipino,Vietnamese,Chinese,English": 0.18, + "Filipino,Vietnamese,Spanish,Malay": 0.26666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.28, + "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, + "Filipino,Vietnamese,Malay,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,Malay,English": 0.19333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.2, + "Filipino,Chinese,Spanish,Malay": 0.25333333333333335, + "Filipino,Chinese,Spanish,Indonesian": 0.26, + "Filipino,Chinese,Spanish,English": 0.22666666666666666, + "Filipino,Chinese,Malay,Indonesian": 0.31333333333333335, + "Filipino,Chinese,Malay,English": 0.21333333333333335, + "Filipino,Chinese,Indonesian,English": 0.22, + "Filipino,Spanish,Malay,Indonesian": 0.36, + "Filipino,Spanish,Malay,English": 0.24, + "Filipino,Spanish,Indonesian,English": 0.25333333333333335, + "Filipino,Malay,Indonesian,English": 0.28, + "Vietnamese,Chinese,Spanish,Malay": 0.25333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian": 0.26, + "Vietnamese,Chinese,Spanish,English": 0.2, + "Vietnamese,Chinese,Malay,Indonesian": 0.28, + "Vietnamese,Chinese,Malay,English": 0.18, + "Vietnamese,Chinese,Indonesian,English": 0.18, + "Vietnamese,Spanish,Malay,Indonesian": 0.2733333333333333, + "Vietnamese,Spanish,Malay,English": 0.19333333333333333, + "Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, + "Vietnamese,Malay,Indonesian,English": 0.18, + "Chinese,Spanish,Malay,Indonesian": 0.28, + "Chinese,Spanish,Malay,English": 0.21333333333333335, + "Chinese,Spanish,Indonesian,English": 0.23333333333333334, + "Chinese,Malay,Indonesian,English": 0.22, + "Spanish,Malay,Indonesian,English": 0.23333333333333334 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.17333333333333334, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, + "Filipino,Chinese,Spanish,Malay,English": 0.18666666666666668, + "Filipino,Chinese,Spanish,Indonesian,English": 0.18666666666666668, + "Filipino,Chinese,Malay,Indonesian,English": 0.2, + "Filipino,Spanish,Malay,Indonesian,English": 0.2, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22, + "Vietnamese,Chinese,Spanish,Malay,English": 0.16, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16666666666666666, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.15333333333333332, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.16, + "Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 + } + }, + "AC3_2": 0.34176936497208055, + "AC3_3": 0.2885201393839017, + "AC3_4": 0.2452255249426277, + "AC3_5": 0.21349679648788636, + "AC3_6": 0.19095126045702548, + "AC3_7": 0.17402812236987686 }, "prompt_5": { - "accuracy": 0.3225806451612903 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.31540321188050424, - "category_acc": { - "agronomy": 0.35502958579881655, - "anatomy": 0.32432432432432434, - "ancient_chinese": 0.3048780487804878, - "arts": 0.4, - "astronomy": 0.28484848484848485, - "business_ethics": 0.3397129186602871, - "chinese_civil_service_exam": 0.3, - "chinese_driving_rule": 0.3816793893129771, - "chinese_food_culture": 0.25735294117647056, - "chinese_foreign_policy": 0.3644859813084112, - "chinese_history": 0.3219814241486068, - "chinese_literature": 0.29411764705882354, - "chinese_teacher_qualification": 0.37988826815642457, - "clinical_knowledge": 0.26582278481012656, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.3177570093457944, - "college_engineering_hydrology": 0.3490566037735849, - "college_law": 0.23148148148148148, - "college_mathematics": 0.29523809523809524, - "college_medical_statistics": 0.2358490566037736, - "college_medicine": 0.31135531135531136, - "computer_science": 0.27941176470588236, - "computer_security": 0.3216374269005848, - "conceptual_physics": 0.29931972789115646, - "construction_project_management": 0.2302158273381295, - "economics": 0.34591194968553457, - "education": 0.3987730061349693, - "electrical_engineering": 0.2558139534883721, - "elementary_chinese": 0.3055555555555556, - "elementary_commonsense": 0.29797979797979796, - "elementary_information_and_technology": 0.3907563025210084, - "elementary_mathematics": 0.2826086956521739, - "ethnology": 0.26666666666666666, - "food_science": 0.3356643356643357, - "genetics": 0.2840909090909091, - "global_facts": 0.35570469798657717, - "high_school_biology": 0.2485207100591716, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.2033898305084746, - "high_school_mathematics": 0.21341463414634146, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.3986013986013986, - "human_sexuality": 0.3333333333333333, - "international_law": 0.2810810810810811, - "journalism": 0.3313953488372093, - "jurisprudence": 0.3260340632603406, - "legal_and_moral_basis": 0.5373831775700935, - "logical": 0.3008130081300813, - "machine_learning": 0.2540983606557377, - "management": 0.3047619047619048, - "marketing": 0.35555555555555557, - "marxist_theory": 0.37566137566137564, - "modern_chinese": 0.29310344827586204, - "nutrition": 0.31724137931034485, - "philosophy": 0.3619047619047619, - "professional_accounting": 0.3314285714285714, - "professional_law": 0.27488151658767773, - "professional_medicine": 0.2553191489361702, - "professional_psychology": 0.3448275862068966, - "public_relations": 0.3793103448275862, - "security_study": 0.35555555555555557, - "sociology": 0.30973451327433627, - "sports_science": 0.2727272727272727, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.3668639053254438, - "world_history": 0.32298136645962733, - "world_religions": 0.35625 - } - }, - "prompt_2": { - "accuracy": 0.32420998100500775, - "category_acc": { - "agronomy": 0.38461538461538464, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.2804878048780488, - "arts": 0.48125, - "astronomy": 0.2606060606060606, - "business_ethics": 0.39712918660287083, - "chinese_civil_service_exam": 0.3125, - "chinese_driving_rule": 0.4198473282442748, - "chinese_food_culture": 0.25735294117647056, - "chinese_foreign_policy": 0.3925233644859813, - "chinese_history": 0.3746130030959752, - "chinese_literature": 0.3137254901960784, - "chinese_teacher_qualification": 0.39106145251396646, - "clinical_knowledge": 0.3080168776371308, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.35514018691588783, - "college_engineering_hydrology": 0.3490566037735849, - "college_law": 0.2777777777777778, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.24528301886792453, - "college_medicine": 0.2857142857142857, - "computer_science": 0.29411764705882354, - "computer_security": 0.30409356725146197, - "conceptual_physics": 0.30612244897959184, - "construction_project_management": 0.31654676258992803, - "economics": 0.3836477987421384, - "education": 0.3803680981595092, - "electrical_engineering": 0.3546511627906977, - "elementary_chinese": 0.3333333333333333, - "elementary_commonsense": 0.2878787878787879, - "elementary_information_and_technology": 0.3445378151260504, - "elementary_mathematics": 0.23043478260869565, - "ethnology": 0.3037037037037037, - "food_science": 0.24475524475524477, - "genetics": 0.2727272727272727, - "global_facts": 0.3288590604026846, - "high_school_biology": 0.3076923076923077, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.3220338983050847, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.3, - "high_school_politics": 0.3916083916083916, - "human_sexuality": 0.3333333333333333, - "international_law": 0.3027027027027027, - "journalism": 0.37209302325581395, - "jurisprudence": 0.35279805352798055, - "legal_and_moral_basis": 0.4672897196261682, - "logical": 0.2764227642276423, - "machine_learning": 0.2540983606557377, - "management": 0.3047619047619048, - "marketing": 0.34444444444444444, - "marxist_theory": 0.35978835978835977, - "modern_chinese": 0.3103448275862069, - "nutrition": 0.3103448275862069, - "philosophy": 0.3619047619047619, - "professional_accounting": 0.32571428571428573, - "professional_law": 0.2796208530805687, - "professional_medicine": 0.23404255319148937, - "professional_psychology": 0.375, - "public_relations": 0.3390804597701149, - "security_study": 0.362962962962963, - "sociology": 0.3230088495575221, - "sports_science": 0.2727272727272727, - "traditional_chinese_medicine": 0.2972972972972973, - "virology": 0.3136094674556213, - "world_history": 0.4409937888198758, - "world_religions": 0.31875 - } - }, - "prompt_3": { - "accuracy": 0.33595233983767914, - "category_acc": { - "agronomy": 0.3609467455621302, - "anatomy": 0.27702702702702703, - "ancient_chinese": 0.25, - "arts": 0.41875, - "astronomy": 0.24848484848484848, - "business_ethics": 0.35406698564593303, - "chinese_civil_service_exam": 0.36875, - "chinese_driving_rule": 0.3969465648854962, - "chinese_food_culture": 0.3235294117647059, - "chinese_foreign_policy": 0.34579439252336447, - "chinese_history": 0.38390092879256965, - "chinese_literature": 0.28921568627450983, - "chinese_teacher_qualification": 0.39664804469273746, - "clinical_knowledge": 0.34177215189873417, - "college_actuarial_science": 0.3018867924528302, - "college_education": 0.38317757009345793, - "college_engineering_hydrology": 0.33962264150943394, - "college_law": 0.3148148148148148, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.19811320754716982, - "college_medicine": 0.3443223443223443, - "computer_science": 0.29411764705882354, - "computer_security": 0.3157894736842105, - "conceptual_physics": 0.29931972789115646, - "construction_project_management": 0.302158273381295, - "economics": 0.389937106918239, - "education": 0.4233128834355828, - "electrical_engineering": 0.3023255813953488, - "elementary_chinese": 0.3253968253968254, - "elementary_commonsense": 0.30808080808080807, - "elementary_information_and_technology": 0.4369747899159664, - "elementary_mathematics": 0.24347826086956523, - "ethnology": 0.28888888888888886, - "food_science": 0.26573426573426573, - "genetics": 0.26704545454545453, - "global_facts": 0.3825503355704698, - "high_school_biology": 0.2958579881656805, - "high_school_chemistry": 0.29545454545454547, - "high_school_geography": 0.3898305084745763, - "high_school_mathematics": 0.25, - "high_school_physics": 0.3, - "high_school_politics": 0.38461538461538464, - "human_sexuality": 0.31746031746031744, - "international_law": 0.3567567567567568, - "journalism": 0.4127906976744186, - "jurisprudence": 0.35036496350364965, - "legal_and_moral_basis": 0.5420560747663551, - "logical": 0.34146341463414637, - "machine_learning": 0.3524590163934426, - "management": 0.32857142857142857, - "marketing": 0.3888888888888889, - "marxist_theory": 0.4126984126984127, - "modern_chinese": 0.2672413793103448, - "nutrition": 0.31724137931034485, - "philosophy": 0.38095238095238093, - "professional_accounting": 0.3314285714285714, - "professional_law": 0.26540284360189575, - "professional_medicine": 0.26063829787234044, - "professional_psychology": 0.34051724137931033, - "public_relations": 0.3850574712643678, - "security_study": 0.37037037037037035, - "sociology": 0.33185840707964603, - "sports_science": 0.3090909090909091, - "traditional_chinese_medicine": 0.2756756756756757, - "virology": 0.3076923076923077, - "world_history": 0.4161490683229814, - "world_religions": 0.4125 - } - }, - "prompt_4": { - "accuracy": 0.3312035917803488, - "category_acc": { - "agronomy": 0.33727810650887574, - "anatomy": 0.3108108108108108, - "ancient_chinese": 0.22560975609756098, - "arts": 0.3875, - "astronomy": 0.23636363636363636, - "business_ethics": 0.3444976076555024, - "chinese_civil_service_exam": 0.3, - "chinese_driving_rule": 0.3816793893129771, - "chinese_food_culture": 0.2867647058823529, - "chinese_foreign_policy": 0.3644859813084112, - "chinese_history": 0.3560371517027864, - "chinese_literature": 0.3088235294117647, - "chinese_teacher_qualification": 0.37988826815642457, - "clinical_knowledge": 0.31645569620253167, - "college_actuarial_science": 0.3113207547169811, - "college_education": 0.3177570093457944, - "college_engineering_hydrology": 0.4056603773584906, - "college_law": 0.25, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.2830188679245283, - "college_medicine": 0.32234432234432236, - "computer_science": 0.29901960784313725, - "computer_security": 0.30409356725146197, - "conceptual_physics": 0.2925170068027211, - "construction_project_management": 0.37410071942446044, - "economics": 0.36477987421383645, - "education": 0.37423312883435583, - "electrical_engineering": 0.3313953488372093, - "elementary_chinese": 0.31746031746031744, - "elementary_commonsense": 0.29797979797979796, - "elementary_information_and_technology": 0.40336134453781514, - "elementary_mathematics": 0.2565217391304348, - "ethnology": 0.3037037037037037, - "food_science": 0.2937062937062937, - "genetics": 0.29545454545454547, - "global_facts": 0.3221476510067114, - "high_school_biology": 0.25443786982248523, - "high_school_chemistry": 0.30303030303030304, - "high_school_geography": 0.2542372881355932, - "high_school_mathematics": 0.2682926829268293, - "high_school_physics": 0.23636363636363636, - "high_school_politics": 0.3776223776223776, - "human_sexuality": 0.3412698412698413, - "international_law": 0.33513513513513515, - "journalism": 0.38953488372093026, - "jurisprudence": 0.31386861313868614, - "legal_and_moral_basis": 0.616822429906542, - "logical": 0.34959349593495936, - "machine_learning": 0.28688524590163933, - "management": 0.35714285714285715, - "marketing": 0.4222222222222222, - "marxist_theory": 0.3439153439153439, - "modern_chinese": 0.31896551724137934, - "nutrition": 0.3931034482758621, - "philosophy": 0.34285714285714286, - "professional_accounting": 0.3314285714285714, - "professional_law": 0.2843601895734597, - "professional_medicine": 0.2526595744680851, - "professional_psychology": 0.3793103448275862, - "public_relations": 0.3505747126436782, - "security_study": 0.3851851851851852, - "sociology": 0.38495575221238937, - "sports_science": 0.3151515151515151, - "traditional_chinese_medicine": 0.2702702702702703, - "virology": 0.3431952662721893, - "world_history": 0.38509316770186336, - "world_religions": 0.40625 - } - }, - "prompt_5": { - "accuracy": 0.3054740113969953, - "category_acc": { - "agronomy": 0.30177514792899407, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.29878048780487804, - "arts": 0.3625, - "astronomy": 0.24242424242424243, - "business_ethics": 0.31100478468899523, - "chinese_civil_service_exam": 0.39375, - "chinese_driving_rule": 0.42748091603053434, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.3364485981308411, - "chinese_history": 0.35294117647058826, - "chinese_literature": 0.28431372549019607, - "chinese_teacher_qualification": 0.3240223463687151, - "clinical_knowledge": 0.2489451476793249, - "college_actuarial_science": 0.3584905660377358, - "college_education": 0.3177570093457944, - "college_engineering_hydrology": 0.3018867924528302, - "college_law": 0.28703703703703703, - "college_mathematics": 0.3142857142857143, - "college_medical_statistics": 0.2169811320754717, - "college_medicine": 0.30036630036630035, - "computer_science": 0.28431372549019607, - "computer_security": 0.3391812865497076, - "conceptual_physics": 0.3129251700680272, - "construction_project_management": 0.2949640287769784, - "economics": 0.3270440251572327, - "education": 0.3803680981595092, - "electrical_engineering": 0.32558139534883723, - "elementary_chinese": 0.25396825396825395, - "elementary_commonsense": 0.30303030303030304, - "elementary_information_and_technology": 0.33613445378151263, - "elementary_mathematics": 0.26521739130434785, - "ethnology": 0.22962962962962963, - "food_science": 0.3006993006993007, - "genetics": 0.2727272727272727, - "global_facts": 0.3825503355704698, - "high_school_biology": 0.22485207100591717, - "high_school_chemistry": 0.1590909090909091, - "high_school_geography": 0.22033898305084745, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.3, - "high_school_politics": 0.36363636363636365, - "human_sexuality": 0.2777777777777778, - "international_law": 0.2810810810810811, - "journalism": 0.3488372093023256, - "jurisprudence": 0.29683698296836986, - "legal_and_moral_basis": 0.5, - "logical": 0.2764227642276423, - "machine_learning": 0.20491803278688525, - "management": 0.34285714285714286, - "marketing": 0.37222222222222223, - "marxist_theory": 0.37566137566137564, - "modern_chinese": 0.31896551724137934, - "nutrition": 0.32413793103448274, - "philosophy": 0.3047619047619048, - "professional_accounting": 0.28, - "professional_law": 0.26540284360189575, - "professional_medicine": 0.24468085106382978, - "professional_psychology": 0.2974137931034483, - "public_relations": 0.3275862068965517, - "security_study": 0.362962962962963, - "sociology": 0.27876106194690264, - "sports_science": 0.23636363636363636, - "traditional_chinese_medicine": 0.25405405405405407, - "virology": 0.3136094674556213, - "world_history": 0.40993788819875776, - "world_religions": 0.3375 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.42424242424242425 - }, - "prompt_2": { - "accuracy": 0.42424242424242425 - }, - "prompt_3": { - "accuracy": 0.3939393939393939 - }, - "prompt_4": { - "accuracy": 0.36363636363636365 - }, - "prompt_5": { - "accuracy": 0.42424242424242425 + "overall_acc": 0.23333333333333336, + "language_acc": { + "Filipino": 0.22666666666666666, + "Vietnamese": 0.28, + "Chinese": 0.18, + "Spanish": 0.20666666666666667, + "Malay": 0.26666666666666666, + "Indonesian": 0.23333333333333334, + "English": 0.24 + }, + "consistency_score_2": 0.5333333333333333, + "consistency_score_3": 0.338095238095238, + "consistency_score_4": 0.23161904761904764, + "consistency_score_5": 0.16730158730158734, + "consistency_score_6": 0.12476190476190477, + "consistency_score_7": 0.09333333333333334, + "detailed_consistency_score": { + "2_combine": { + "Filipino,Vietnamese": 0.4666666666666667, + "Filipino,Chinese": 0.5266666666666666, + "Filipino,Spanish": 0.62, + "Filipino,Malay": 0.5866666666666667, + "Filipino,Indonesian": 0.6266666666666667, + "Filipino,English": 0.5066666666666667, + "Vietnamese,Chinese": 0.5466666666666666, + "Vietnamese,Spanish": 0.52, + "Vietnamese,Malay": 0.5333333333333333, + "Vietnamese,Indonesian": 0.5666666666666667, + "Vietnamese,English": 0.56, + "Chinese,Spanish": 0.5066666666666667, + "Chinese,Malay": 0.48, + "Chinese,Indonesian": 0.5266666666666666, + "Chinese,English": 0.5333333333333333, + "Spanish,Malay": 0.5066666666666667, + "Spanish,Indonesian": 0.5333333333333333, + "Spanish,English": 0.52, + "Malay,Indonesian": 0.62, + "Malay,English": 0.43333333333333335, + "Indonesian,English": 0.48 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.31333333333333335, + "Filipino,Vietnamese,Spanish": 0.3333333333333333, + "Filipino,Vietnamese,Malay": 0.32666666666666666, + "Filipino,Vietnamese,Indonesian": 0.36, + "Filipino,Vietnamese,English": 0.30666666666666664, + "Filipino,Chinese,Spanish": 0.3466666666666667, + "Filipino,Chinese,Malay": 0.32666666666666666, + "Filipino,Chinese,Indonesian": 0.38, + "Filipino,Chinese,English": 0.32, + "Filipino,Spanish,Malay": 0.37333333333333335, + "Filipino,Spanish,Indonesian": 0.41333333333333333, + "Filipino,Spanish,English": 0.3466666666666667, + "Filipino,Malay,Indonesian": 0.44, + "Filipino,Malay,English": 0.30666666666666664, + "Filipino,Indonesian,English": 0.3466666666666667, + "Vietnamese,Chinese,Spanish": 0.3333333333333333, + "Vietnamese,Chinese,Malay": 0.31333333333333335, + "Vietnamese,Chinese,Indonesian": 0.37333333333333335, + "Vietnamese,Chinese,English": 0.38, + "Vietnamese,Spanish,Malay": 0.32, + "Vietnamese,Spanish,Indonesian": 0.35333333333333333, + "Vietnamese,Spanish,English": 0.34, + "Vietnamese,Malay,Indonesian": 0.4, + "Vietnamese,Malay,English": 0.32, + "Vietnamese,Indonesian,English": 0.3466666666666667, + "Chinese,Spanish,Malay": 0.26666666666666666, + "Chinese,Spanish,Indonesian": 0.32666666666666666, + "Chinese,Spanish,English": 0.31333333333333335, + "Chinese,Malay,Indonesian": 0.36, + "Chinese,Malay,English": 0.2733333333333333, + "Chinese,Indonesian,English": 0.31333333333333335, + "Spanish,Malay,Indonesian": 0.36, + "Spanish,Malay,English": 0.26, + "Spanish,Indonesian,English": 0.32, + "Malay,Indonesian,English": 0.32 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Malay": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,English": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Malay": 0.24666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian": 0.28, + "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.26, + "Filipino,Vietnamese,Malay,English": 0.21333333333333335, + "Filipino,Vietnamese,Indonesian,English": 0.25333333333333335, + "Filipino,Chinese,Spanish,Malay": 0.2, + "Filipino,Chinese,Spanish,Indonesian": 0.24666666666666667, + "Filipino,Chinese,Spanish,English": 0.21333333333333335, + "Filipino,Chinese,Malay,Indonesian": 0.25333333333333335, + "Filipino,Chinese,Malay,English": 0.20666666666666667, + "Filipino,Chinese,Indonesian,English": 0.22666666666666666, + "Filipino,Spanish,Malay,Indonesian": 0.2866666666666667, + "Filipino,Spanish,Malay,English": 0.20666666666666667, + "Filipino,Spanish,Indonesian,English": 0.24, + "Filipino,Malay,Indonesian,English": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.2, + "Vietnamese,Chinese,Spanish,Indonesian": 0.25333333333333335, + "Vietnamese,Chinese,Spanish,English": 0.23333333333333334, + "Vietnamese,Chinese,Malay,Indonesian": 0.26666666666666666, + "Vietnamese,Chinese,Malay,English": 0.21333333333333335, + "Vietnamese,Chinese,Indonesian,English": 0.24666666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.26, + "Vietnamese,Spanish,Malay,English": 0.2, + "Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, + "Vietnamese,Malay,Indonesian,English": 0.25333333333333335, + "Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, + "Chinese,Spanish,Malay,English": 0.16666666666666666, + "Chinese,Spanish,Indonesian,English": 0.21333333333333335, + "Chinese,Malay,Indonesian,English": 0.20666666666666667, + "Spanish,Malay,Indonesian,English": 0.22 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.2, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.18, + "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.18666666666666668, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.16, + "Filipino,Chinese,Spanish,Malay,English": 0.13333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Malay,Indonesian,English": 0.15333333333333332, + "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18, + "Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.17333333333333334, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.18, + "Chinese,Spanish,Malay,Indonesian,English": 0.14 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 + } + }, + "AC3_2": 0.32463768111707625, + "AC3_3": 0.2761111110627917, + "AC3_4": 0.23247303012888915, + "AC3_5": 0.19487585837713747, + "AC3_6": 0.16258865243686577, + "AC3_7": 0.133333333292517 } }, - "ind_emotion": { + "cross_logiqa": { "prompt_1": { - "accuracy": 0.5045454545454545 + "overall_acc": 0.26461038961038963, + "language_acc": { + "Indonesian": 0.26136363636363635, + "English": 0.26136363636363635, + "Filipino": 0.26704545454545453, + "Spanish": 0.2897727272727273, + "Chinese": 0.22727272727272727, + "Malay": 0.2897727272727273, + "Vietnamese": 0.2556818181818182 + }, + "consistency_score_2": 0.38609307359307365, + "consistency_score_3": 0.18944805194805198, + "consistency_score_4": 0.10487012987012986, + "consistency_score_5": 0.06114718614718615, + "consistency_score_6": 0.036525974025974024, + "consistency_score_7": 0.022727272727272728, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.375, + "Indonesian,Filipino": 0.48295454545454547, + "Indonesian,Spanish": 0.4772727272727273, + "Indonesian,Chinese": 0.2215909090909091, + "Indonesian,Malay": 0.5227272727272727, + "Indonesian,Vietnamese": 0.5284090909090909, + "English,Filipino": 0.36363636363636365, + "English,Spanish": 0.42613636363636365, + "English,Chinese": 0.3465909090909091, + "English,Malay": 0.3068181818181818, + "English,Vietnamese": 0.3465909090909091, + "Filipino,Spanish": 0.4943181818181818, + "Filipino,Chinese": 0.2159090909090909, + "Filipino,Malay": 0.44886363636363635, + "Filipino,Vietnamese": 0.5227272727272727, + "Spanish,Chinese": 0.25, + "Spanish,Malay": 0.4318181818181818, + "Spanish,Vietnamese": 0.44886363636363635, + "Chinese,Malay": 0.19318181818181818, + "Chinese,Vietnamese": 0.26136363636363635, + "Malay,Vietnamese": 0.4431818181818182 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.2215909090909091, + "Indonesian,English,Spanish": 0.23863636363636365, + "Indonesian,English,Chinese": 0.10227272727272728, + "Indonesian,English,Malay": 0.20454545454545456, + "Indonesian,English,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Spanish": 0.2897727272727273, + "Indonesian,Filipino,Chinese": 0.14772727272727273, + "Indonesian,Filipino,Malay": 0.26704545454545453, + "Indonesian,Filipino,Vietnamese": 0.32386363636363635, + "Indonesian,Spanish,Chinese": 0.125, + "Indonesian,Spanish,Malay": 0.26704545454545453, + "Indonesian,Spanish,Vietnamese": 0.29545454545454547, + "Indonesian,Chinese,Malay": 0.13068181818181818, + "Indonesian,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,Malay,Vietnamese": 0.3068181818181818, + "English,Filipino,Spanish": 0.2215909090909091, + "English,Filipino,Chinese": 0.10795454545454546, + "English,Filipino,Malay": 0.1875, + "English,Filipino,Vietnamese": 0.2215909090909091, + "English,Spanish,Chinese": 0.13068181818181818, + "English,Spanish,Malay": 0.1875, + "English,Spanish,Vietnamese": 0.19318181818181818, + "English,Chinese,Malay": 0.08522727272727272, + "English,Chinese,Vietnamese": 0.10227272727272728, + "English,Malay,Vietnamese": 0.1875, + "Filipino,Spanish,Chinese": 0.11363636363636363, + "Filipino,Spanish,Malay": 0.26136363636363635, + "Filipino,Spanish,Vietnamese": 0.3068181818181818, + "Filipino,Chinese,Malay": 0.11931818181818182, + "Filipino,Chinese,Vietnamese": 0.13068181818181818, + "Filipino,Malay,Vietnamese": 0.2784090909090909, + "Spanish,Chinese,Malay": 0.09090909090909091, + "Spanish,Chinese,Vietnamese": 0.10227272727272728, + "Spanish,Malay,Vietnamese": 0.23295454545454544, + "Chinese,Malay,Vietnamese": 0.10795454545454546 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.14204545454545456, + "Indonesian,English,Filipino,Chinese": 0.06818181818181818, + "Indonesian,English,Filipino,Malay": 0.125, + "Indonesian,English,Filipino,Vietnamese": 0.14772727272727273, + "Indonesian,English,Spanish,Chinese": 0.06818181818181818, + "Indonesian,English,Spanish,Malay": 0.13636363636363635, + "Indonesian,English,Spanish,Vietnamese": 0.13636363636363635, + "Indonesian,English,Chinese,Malay": 0.06818181818181818, + "Indonesian,English,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,English,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Spanish,Chinese": 0.09659090909090909, + "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Vietnamese": 0.2159090909090909, + "Indonesian,Filipino,Chinese,Malay": 0.10227272727272728, + "Indonesian,Filipino,Chinese,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Malay,Vietnamese": 0.20454545454545456, + "Indonesian,Spanish,Chinese,Malay": 0.07386363636363637, + "Indonesian,Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,Spanish,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,Chinese,Malay,Vietnamese": 0.09090909090909091, + "English,Filipino,Spanish,Chinese": 0.0625, + "English,Filipino,Spanish,Malay": 0.13068181818181818, + "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, + "English,Filipino,Chinese,Malay": 0.0625, + "English,Filipino,Chinese,Vietnamese": 0.06818181818181818, + "English,Filipino,Malay,Vietnamese": 0.13068181818181818, + "English,Spanish,Chinese,Malay": 0.05113636363636364, + "English,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "English,Spanish,Malay,Vietnamese": 0.11363636363636363, + "English,Chinese,Malay,Vietnamese": 0.05113636363636364, + "Filipino,Spanish,Chinese,Malay": 0.0625, + "Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, + "Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, + "Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.045454545454545456, + "Indonesian,English,Filipino,Spanish,Malay": 0.09090909090909091, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, + "Indonesian,English,Filipino,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,English,Spanish,Chinese,Malay": 0.045454545454545456, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0625, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, + "English,Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, + "English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 + }, + "6_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 + }, + "7_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 + } + }, + "AC3_2": 0.3140116639634068, + "AC3_3": 0.22080824069168092, + "AC3_4": 0.15020941269072685, + "AC3_5": 0.09933878410895389, + "AC3_6": 0.06419119961462726, + "AC3_7": 0.041859270658262025 }, "prompt_2": { - "accuracy": 0.4409090909090909 + "overall_acc": 0.26623376623376627, + "language_acc": { + "Indonesian": 0.25, + "English": 0.2897727272727273, + "Filipino": 0.24431818181818182, + "Spanish": 0.2784090909090909, + "Chinese": 0.25, + "Malay": 0.26704545454545453, + "Vietnamese": 0.2840909090909091 + }, + "consistency_score_2": 0.3663419913419913, + "consistency_score_3": 0.15616883116883118, + "consistency_score_4": 0.07175324675324674, + "consistency_score_5": 0.03219696969696971, + "consistency_score_6": 0.011363636363636364, + "consistency_score_7": 0.0, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.3465909090909091, + "Indonesian,Filipino": 0.5340909090909091, + "Indonesian,Spanish": 0.3977272727272727, + "Indonesian,Chinese": 0.1590909090909091, + "Indonesian,Malay": 0.5056818181818182, + "Indonesian,Vietnamese": 0.42045454545454547, + "English,Filipino": 0.24431818181818182, + "English,Spanish": 0.4715909090909091, + "English,Chinese": 0.4943181818181818, + "English,Malay": 0.29545454545454547, + "English,Vietnamese": 0.36363636363636365, + "Filipino,Spanish": 0.35795454545454547, + "Filipino,Chinese": 0.08522727272727272, + "Filipino,Malay": 0.5, + "Filipino,Vietnamese": 0.4715909090909091, + "Spanish,Chinese": 0.42045454545454547, + "Spanish,Malay": 0.3977272727272727, + "Spanish,Vietnamese": 0.42613636363636365, + "Chinese,Malay": 0.14772727272727273, + "Chinese,Vietnamese": 0.22727272727272727, + "Malay,Vietnamese": 0.42613636363636365 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.1534090909090909, + "Indonesian,English,Spanish": 0.20454545454545456, + "Indonesian,English,Chinese": 0.09659090909090909, + "Indonesian,English,Malay": 0.18181818181818182, + "Indonesian,English,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Spanish": 0.23295454545454544, + "Indonesian,Filipino,Chinese": 0.03977272727272727, + "Indonesian,Filipino,Malay": 0.3125, + "Indonesian,Filipino,Vietnamese": 0.26704545454545453, + "Indonesian,Spanish,Chinese": 0.10227272727272728, + "Indonesian,Spanish,Malay": 0.23863636363636365, + "Indonesian,Spanish,Vietnamese": 0.22727272727272727, + "Indonesian,Chinese,Malay": 0.07386363636363637, + "Indonesian,Chinese,Vietnamese": 0.07954545454545454, + "Indonesian,Malay,Vietnamese": 0.23295454545454544, + "English,Filipino,Spanish": 0.13068181818181818, + "English,Filipino,Chinese": 0.028409090909090908, + "English,Filipino,Malay": 0.13636363636363635, + "English,Filipino,Vietnamese": 0.1534090909090909, + "English,Spanish,Chinese": 0.2556818181818182, + "English,Spanish,Malay": 0.1875, + "English,Spanish,Vietnamese": 0.19318181818181818, + "English,Chinese,Malay": 0.09090909090909091, + "English,Chinese,Vietnamese": 0.13636363636363635, + "English,Malay,Vietnamese": 0.14772727272727273, + "Filipino,Spanish,Chinese": 0.03977272727272727, + "Filipino,Spanish,Malay": 0.23295454545454544, + "Filipino,Spanish,Vietnamese": 0.24431818181818182, + "Filipino,Chinese,Malay": 0.022727272727272728, + "Filipino,Chinese,Vietnamese": 0.0625, + "Filipino,Malay,Vietnamese": 0.2897727272727273, + "Spanish,Chinese,Malay": 0.10227272727272728, + "Spanish,Chinese,Vietnamese": 0.125, + "Spanish,Malay,Vietnamese": 0.2159090909090909, + "Chinese,Malay,Vietnamese": 0.05113636363636364 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.09659090909090909, + "Indonesian,English,Filipino,Chinese": 0.005681818181818182, + "Indonesian,English,Filipino,Malay": 0.09659090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, + "Indonesian,English,Spanish,Chinese": 0.0625, + "Indonesian,English,Spanish,Malay": 0.125, + "Indonesian,English,Spanish,Vietnamese": 0.10795454545454546, + "Indonesian,English,Chinese,Malay": 0.05113636363636364, + "Indonesian,English,Chinese,Vietnamese": 0.03977272727272727, + "Indonesian,English,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,Filipino,Spanish,Chinese": 0.028409090909090908, + "Indonesian,Filipino,Spanish,Malay": 0.1590909090909091, + "Indonesian,Filipino,Spanish,Vietnamese": 0.1534090909090909, + "Indonesian,Filipino,Chinese,Malay": 0.011363636363636364, + "Indonesian,Filipino,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,Filipino,Malay,Vietnamese": 0.1875, + "Indonesian,Spanish,Chinese,Malay": 0.056818181818181816, + "Indonesian,Spanish,Chinese,Vietnamese": 0.0625, + "Indonesian,Spanish,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Chinese": 0.017045454545454544, + "English,Filipino,Spanish,Malay": 0.08522727272727272, + "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, + "English,Filipino,Chinese,Malay": 0.005681818181818182, + "English,Filipino,Chinese,Vietnamese": 0.022727272727272728, + "English,Filipino,Malay,Vietnamese": 0.10795454545454546, + "English,Spanish,Chinese,Malay": 0.07386363636363637, + "English,Spanish,Chinese,Vietnamese": 0.07954545454545454, + "English,Spanish,Malay,Vietnamese": 0.09090909090909091, + "English,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Filipino,Spanish,Chinese,Malay": 0.017045454545454544, + "Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, + "Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.005681818181818182, + "Indonesian,English,Filipino,Spanish,Malay": 0.0625, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.06818181818181818, + "Indonesian,English,Filipino,Chinese,Malay": 0.0, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Spanish,Chinese,Malay": 0.03977272727272727, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.0625, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Chinese,Malay": 0.005681818181818182, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, + "English,Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, + "English,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 + }, + "6_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 + }, + "7_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0 + } + }, + "AC3_2": 0.3083665692245794, + "AC3_3": 0.19686155500096064, + "AC3_4": 0.11304065767580032, + "AC3_5": 0.05744663313001279, + "AC3_6": 0.02179691652590665, + "AC3_7": 0.0 }, "prompt_3": { - "accuracy": 0.42272727272727273 + "overall_acc": 0.26461038961038963, + "language_acc": { + "Indonesian": 0.26136363636363635, + "English": 0.2784090909090909, + "Filipino": 0.24431818181818182, + "Spanish": 0.2840909090909091, + "Chinese": 0.2840909090909091, + "Malay": 0.25, + "Vietnamese": 0.25 + }, + "consistency_score_2": 0.40854978354978355, + "consistency_score_3": 0.21217532467532468, + "consistency_score_4": 0.12045454545454548, + "consistency_score_5": 0.06737012987012987, + "consistency_score_6": 0.03327922077922078, + "consistency_score_7": 0.011363636363636364, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.36363636363636365, + "Indonesian,Filipino": 0.6022727272727273, + "Indonesian,Spanish": 0.5227272727272727, + "Indonesian,Chinese": 0.26704545454545453, + "Indonesian,Malay": 0.5568181818181818, + "Indonesian,Vietnamese": 0.48863636363636365, + "English,Filipino": 0.29545454545454547, + "English,Spanish": 0.45454545454545453, + "English,Chinese": 0.4090909090909091, + "English,Malay": 0.29545454545454547, + "English,Vietnamese": 0.375, + "Filipino,Spanish": 0.4715909090909091, + "Filipino,Chinese": 0.25, + "Filipino,Malay": 0.5113636363636364, + "Filipino,Vietnamese": 0.48863636363636365, + "Spanish,Chinese": 0.3465909090909091, + "Spanish,Malay": 0.44886363636363635, + "Spanish,Vietnamese": 0.5, + "Chinese,Malay": 0.23863636363636365, + "Chinese,Vietnamese": 0.2727272727272727, + "Malay,Vietnamese": 0.42045454545454547 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.2159090909090909, + "Indonesian,English,Spanish": 0.2727272727272727, + "Indonesian,English,Chinese": 0.125, + "Indonesian,English,Malay": 0.23863636363636365, + "Indonesian,English,Vietnamese": 0.2215909090909091, + "Indonesian,Filipino,Spanish": 0.35795454545454547, + "Indonesian,Filipino,Chinese": 0.17045454545454544, + "Indonesian,Filipino,Malay": 0.38636363636363635, + "Indonesian,Filipino,Vietnamese": 0.3352272727272727, + "Indonesian,Spanish,Chinese": 0.17045454545454544, + "Indonesian,Spanish,Malay": 0.3352272727272727, + "Indonesian,Spanish,Vietnamese": 0.3352272727272727, + "Indonesian,Chinese,Malay": 0.1590909090909091, + "Indonesian,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,Malay,Vietnamese": 0.2840909090909091, + "English,Filipino,Spanish": 0.20454545454545456, + "English,Filipino,Chinese": 0.09090909090909091, + "English,Filipino,Malay": 0.17613636363636365, + "English,Filipino,Vietnamese": 0.19318181818181818, + "English,Spanish,Chinese": 0.18181818181818182, + "English,Spanish,Malay": 0.2159090909090909, + "English,Spanish,Vietnamese": 0.2556818181818182, + "English,Chinese,Malay": 0.09659090909090909, + "English,Chinese,Vietnamese": 0.13068181818181818, + "English,Malay,Vietnamese": 0.17045454545454544, + "Filipino,Spanish,Chinese": 0.1534090909090909, + "Filipino,Spanish,Malay": 0.2784090909090909, + "Filipino,Spanish,Vietnamese": 0.2897727272727273, + "Filipino,Chinese,Malay": 0.1590909090909091, + "Filipino,Chinese,Vietnamese": 0.14204545454545456, + "Filipino,Malay,Vietnamese": 0.26136363636363635, + "Spanish,Chinese,Malay": 0.14772727272727273, + "Spanish,Chinese,Vietnamese": 0.1534090909090909, + "Spanish,Malay,Vietnamese": 0.26136363636363635, + "Chinese,Malay,Vietnamese": 0.11931818181818182 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.1590909090909091, + "Indonesian,English,Filipino,Chinese": 0.0625, + "Indonesian,English,Filipino,Malay": 0.1534090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.13636363636363635, + "Indonesian,English,Spanish,Chinese": 0.09659090909090909, + "Indonesian,English,Spanish,Malay": 0.17613636363636365, + "Indonesian,English,Spanish,Vietnamese": 0.18181818181818182, + "Indonesian,English,Chinese,Malay": 0.07954545454545454, + "Indonesian,English,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,English,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Spanish,Chinese": 0.11363636363636363, + "Indonesian,Filipino,Spanish,Malay": 0.23863636363636365, + "Indonesian,Filipino,Spanish,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Chinese,Malay": 0.125, + "Indonesian,Filipino,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,Filipino,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Spanish,Chinese,Malay": 0.11363636363636363, + "Indonesian,Spanish,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,Spanish,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Chinese,Malay,Vietnamese": 0.08522727272727272, + "English,Filipino,Spanish,Chinese": 0.06818181818181818, + "English,Filipino,Spanish,Malay": 0.13636363636363635, + "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, + "English,Filipino,Chinese,Malay": 0.05113636363636364, + "English,Filipino,Chinese,Vietnamese": 0.0625, + "English,Filipino,Malay,Vietnamese": 0.10795454545454546, + "English,Spanish,Chinese,Malay": 0.06818181818181818, + "English,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "English,Spanish,Malay,Vietnamese": 0.13636363636363635, + "English,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Filipino,Spanish,Chinese,Malay": 0.10795454545454546, + "Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, + "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, + "Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, + "Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.045454545454545456, + "Indonesian,English,Filipino,Spanish,Malay": 0.11363636363636363, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, + "Indonesian,English,Filipino,Chinese,Malay": 0.045454545454545456, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,English,Spanish,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, + "English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, + "English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816 + }, + "6_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.03409090909090909, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 + }, + "7_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 + } + }, + "AC3_2": 0.32119106772695516, + "AC3_3": 0.23550955338661372, + "AC3_4": 0.16554882718374375, + "AC3_5": 0.10739688181440923, + "AC3_6": 0.05912275733179299, + "AC3_7": 0.021791443842371187 }, "prompt_4": { - "accuracy": 0.42045454545454547 - }, - "prompt_5": { - "accuracy": 0.3931818181818182 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.31898305084745765 - }, - "prompt_2": { - "accuracy": 0.32 - }, - "prompt_3": { - "accuracy": 0.32372881355932204 - }, - "prompt_4": { - "accuracy": 0.3254237288135593 + "overall_acc": 0.26461038961038963, + "language_acc": { + "Indonesian": 0.25, + "English": 0.2897727272727273, + "Filipino": 0.23863636363636365, + "Spanish": 0.29545454545454547, + "Chinese": 0.26704545454545453, + "Malay": 0.26136363636363635, + "Vietnamese": 0.25 + }, + "consistency_score_2": 0.4323593073593074, + "consistency_score_3": 0.2308441558441558, + "consistency_score_4": 0.13636363636363638, + "consistency_score_5": 0.08306277056277056, + "consistency_score_6": 0.04951298701298701, + "consistency_score_7": 0.028409090909090908, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.35795454545454547, + "Indonesian,Filipino": 0.6136363636363636, + "Indonesian,Spanish": 0.5568181818181818, + "Indonesian,Chinese": 0.21022727272727273, + "Indonesian,Malay": 0.5795454545454546, + "Indonesian,Vietnamese": 0.5340909090909091, + "English,Filipino": 0.32386363636363635, + "English,Spanish": 0.48863636363636365, + "English,Chinese": 0.4772727272727273, + "English,Malay": 0.3125, + "English,Vietnamese": 0.39204545454545453, + "Filipino,Spanish": 0.5625, + "Filipino,Chinese": 0.20454545454545456, + "Filipino,Malay": 0.5965909090909091, + "Filipino,Vietnamese": 0.5454545454545454, + "Spanish,Chinese": 0.36363636363636365, + "Spanish,Malay": 0.48295454545454547, + "Spanish,Vietnamese": 0.5227272727272727, + "Chinese,Malay": 0.17613636363636365, + "Chinese,Vietnamese": 0.2556818181818182, + "Malay,Vietnamese": 0.5227272727272727 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.2556818181818182, + "Indonesian,English,Spanish": 0.2784090909090909, + "Indonesian,English,Chinese": 0.10795454545454546, + "Indonesian,English,Malay": 0.22727272727272727, + "Indonesian,English,Vietnamese": 0.24431818181818182, + "Indonesian,Filipino,Spanish": 0.4147727272727273, + "Indonesian,Filipino,Chinese": 0.1590909090909091, + "Indonesian,Filipino,Malay": 0.4090909090909091, + "Indonesian,Filipino,Vietnamese": 0.3806818181818182, + "Indonesian,Spanish,Chinese": 0.13068181818181818, + "Indonesian,Spanish,Malay": 0.36363636363636365, + "Indonesian,Spanish,Vietnamese": 0.35795454545454547, + "Indonesian,Chinese,Malay": 0.125, + "Indonesian,Chinese,Vietnamese": 0.11931818181818182, + "Indonesian,Malay,Vietnamese": 0.3522727272727273, + "English,Filipino,Spanish": 0.25, + "English,Filipino,Chinese": 0.09659090909090909, + "English,Filipino,Malay": 0.2159090909090909, + "English,Filipino,Vietnamese": 0.22727272727272727, + "English,Spanish,Chinese": 0.21022727272727273, + "English,Spanish,Malay": 0.23295454545454544, + "English,Spanish,Vietnamese": 0.26704545454545453, + "English,Chinese,Malay": 0.08522727272727272, + "English,Chinese,Vietnamese": 0.14772727272727273, + "English,Malay,Vietnamese": 0.2159090909090909, + "Filipino,Spanish,Chinese": 0.1534090909090909, + "Filipino,Spanish,Malay": 0.375, + "Filipino,Spanish,Vietnamese": 0.36363636363636365, + "Filipino,Chinese,Malay": 0.13068181818181818, + "Filipino,Chinese,Vietnamese": 0.13068181818181818, + "Filipino,Malay,Vietnamese": 0.36363636363636365, + "Spanish,Chinese,Malay": 0.11931818181818182, + "Spanish,Chinese,Vietnamese": 0.14204545454545456, + "Spanish,Malay,Vietnamese": 0.3181818181818182, + "Chinese,Malay,Vietnamese": 0.10795454545454546 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.20454545454545456, + "Indonesian,English,Filipino,Chinese": 0.07954545454545454, + "Indonesian,English,Filipino,Malay": 0.17613636363636365, + "Indonesian,English,Filipino,Vietnamese": 0.18181818181818182, + "Indonesian,English,Spanish,Chinese": 0.06818181818181818, + "Indonesian,English,Spanish,Malay": 0.1875, + "Indonesian,English,Spanish,Vietnamese": 0.19318181818181818, + "Indonesian,English,Chinese,Malay": 0.0625, + "Indonesian,English,Chinese,Vietnamese": 0.0625, + "Indonesian,English,Malay,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Chinese": 0.11931818181818182, + "Indonesian,Filipino,Spanish,Malay": 0.29545454545454547, + "Indonesian,Filipino,Spanish,Vietnamese": 0.2727272727272727, + "Indonesian,Filipino,Chinese,Malay": 0.11363636363636363, + "Indonesian,Filipino,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,Spanish,Malay,Vietnamese": 0.25, + "Indonesian,Chinese,Malay,Vietnamese": 0.08522727272727272, + "English,Filipino,Spanish,Chinese": 0.07386363636363637, + "English,Filipino,Spanish,Malay": 0.17045454545454544, + "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, + "English,Filipino,Chinese,Malay": 0.0625, + "English,Filipino,Chinese,Vietnamese": 0.0625, + "English,Filipino,Malay,Vietnamese": 0.1590909090909091, + "English,Spanish,Chinese,Malay": 0.0625, + "English,Spanish,Chinese,Vietnamese": 0.06818181818181818, + "English,Spanish,Malay,Vietnamese": 0.18181818181818182, + "English,Chinese,Malay,Vietnamese": 0.0625, + "Filipino,Spanish,Chinese,Malay": 0.09659090909090909, + "Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, + "Filipino,Spanish,Malay,Vietnamese": 0.26136363636363635, + "Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, + "Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.056818181818181816, + "Indonesian,English,Filipino,Spanish,Malay": 0.14772727272727273, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "Indonesian,English,Filipino,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese,Malay": 0.03977272727272727, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0625, + "English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, + "English,Filipino,Chinese,Malay,Vietnamese": 0.045454545454545456, + "English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 + }, + "6_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 + }, + "7_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 + } + }, + "AC3_2": 0.32829767277696675, + "AC3_3": 0.24657665396463196, + "AC3_4": 0.17997791677513691, + "AC3_5": 0.12643640402649675, + "AC3_6": 0.08341722873951041, + "AC3_7": 0.05130949381275885 }, "prompt_5": { - "accuracy": 0.3240677966101695 + "overall_acc": 0.2621753246753247, + "language_acc": { + "Indonesian": 0.2215909090909091, + "English": 0.2727272727272727, + "Filipino": 0.2727272727272727, + "Spanish": 0.29545454545454547, + "Chinese": 0.2727272727272727, + "Malay": 0.25, + "Vietnamese": 0.25 + }, + "consistency_score_2": 0.4388528138528139, + "consistency_score_3": 0.24464285714285705, + "consistency_score_4": 0.1547077922077922, + "consistency_score_5": 0.10551948051948054, + "consistency_score_6": 0.07386363636363637, + "consistency_score_7": 0.05113636363636364, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.3409090909090909, + "Indonesian,Filipino": 0.6022727272727273, + "Indonesian,Spanish": 0.5397727272727273, + "Indonesian,Chinese": 0.26704545454545453, + "Indonesian,Malay": 0.5965909090909091, + "Indonesian,Vietnamese": 0.5284090909090909, + "English,Filipino": 0.3693181818181818, + "English,Spanish": 0.48295454545454547, + "English,Chinese": 0.4715909090909091, + "English,Malay": 0.30113636363636365, + "English,Vietnamese": 0.42045454545454547, + "Filipino,Spanish": 0.5681818181818182, + "Filipino,Chinese": 0.32386363636363635, + "Filipino,Malay": 0.5113636363636364, + "Filipino,Vietnamese": 0.5113636363636364, + "Spanish,Chinese": 0.35795454545454547, + "Spanish,Malay": 0.4034090909090909, + "Spanish,Vietnamese": 0.5738636363636364, + "Chinese,Malay": 0.23863636363636365, + "Chinese,Vietnamese": 0.3181818181818182, + "Malay,Vietnamese": 0.48863636363636365 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.23295454545454544, + "Indonesian,English,Spanish": 0.26704545454545453, + "Indonesian,English,Chinese": 0.1590909090909091, + "Indonesian,English,Malay": 0.21022727272727273, + "Indonesian,English,Vietnamese": 0.24431818181818182, + "Indonesian,Filipino,Spanish": 0.38636363636363635, + "Indonesian,Filipino,Chinese": 0.2159090909090909, + "Indonesian,Filipino,Malay": 0.375, + "Indonesian,Filipino,Vietnamese": 0.3522727272727273, + "Indonesian,Spanish,Chinese": 0.1875, + "Indonesian,Spanish,Malay": 0.32386363636363635, + "Indonesian,Spanish,Vietnamese": 0.35795454545454547, + "Indonesian,Chinese,Malay": 0.16477272727272727, + "Indonesian,Chinese,Vietnamese": 0.17045454545454544, + "Indonesian,Malay,Vietnamese": 0.36363636363636365, + "English,Filipino,Spanish": 0.2784090909090909, + "English,Filipino,Chinese": 0.17613636363636365, + "English,Filipino,Malay": 0.20454545454545456, + "English,Filipino,Vietnamese": 0.25, + "English,Spanish,Chinese": 0.2159090909090909, + "English,Spanish,Malay": 0.2159090909090909, + "English,Spanish,Vietnamese": 0.3181818181818182, + "English,Chinese,Malay": 0.14204545454545456, + "English,Chinese,Vietnamese": 0.20454545454545456, + "English,Malay,Vietnamese": 0.20454545454545456, + "Filipino,Spanish,Chinese": 0.21022727272727273, + "Filipino,Spanish,Malay": 0.2897727272727273, + "Filipino,Spanish,Vietnamese": 0.3693181818181818, + "Filipino,Chinese,Malay": 0.18181818181818182, + "Filipino,Chinese,Vietnamese": 0.19318181818181818, + "Filipino,Malay,Vietnamese": 0.30113636363636365, + "Spanish,Chinese,Malay": 0.14204545454545456, + "Spanish,Chinese,Vietnamese": 0.2215909090909091, + "Spanish,Malay,Vietnamese": 0.2840909090909091, + "Chinese,Malay,Vietnamese": 0.14772727272727273 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.19318181818181818, + "Indonesian,English,Filipino,Chinese": 0.11931818181818182, + "Indonesian,English,Filipino,Malay": 0.1534090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.18181818181818182, + "Indonesian,English,Spanish,Chinese": 0.11931818181818182, + "Indonesian,English,Spanish,Malay": 0.17045454545454544, + "Indonesian,English,Spanish,Vietnamese": 0.20454545454545456, + "Indonesian,English,Chinese,Malay": 0.10227272727272728, + "Indonesian,English,Chinese,Vietnamese": 0.11931818181818182, + "Indonesian,English,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Filipino,Spanish,Chinese": 0.14772727272727273, + "Indonesian,Filipino,Spanish,Malay": 0.23863636363636365, + "Indonesian,Filipino,Spanish,Vietnamese": 0.26704545454545453, + "Indonesian,Filipino,Chinese,Malay": 0.14204545454545456, + "Indonesian,Filipino,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Malay,Vietnamese": 0.25, + "Indonesian,Spanish,Chinese,Malay": 0.11363636363636363, + "Indonesian,Spanish,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,Spanish,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,Chinese,Malay,Vietnamese": 0.125, + "English,Filipino,Spanish,Chinese": 0.125, + "English,Filipino,Spanish,Malay": 0.1590909090909091, + "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, + "English,Filipino,Chinese,Malay": 0.10795454545454546, + "English,Filipino,Chinese,Vietnamese": 0.125, + "English,Filipino,Malay,Vietnamese": 0.1590909090909091, + "English,Spanish,Chinese,Malay": 0.09090909090909091, + "English,Spanish,Chinese,Vietnamese": 0.14204545454545456, + "English,Spanish,Malay,Vietnamese": 0.17613636363636365, + "English,Chinese,Malay,Vietnamese": 0.09659090909090909, + "Filipino,Spanish,Chinese,Malay": 0.11931818181818182, + "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, + "Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, + "Filipino,Chinese,Malay,Vietnamese": 0.125, + "Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.09090909090909091, + "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1590909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.08522727272727272, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese,Malay": 0.07386363636363637, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.09659090909090909, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, + "English,Filipino,Spanish,Chinese,Malay": 0.07386363636363637, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, + "English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, + "English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, + "English,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 + }, + "6_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0625, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 + }, + "7_combine": { + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364 + } + }, + "AC3_2": 0.3282503871576349, + "AC3_3": 0.25310583869811504, + "AC3_4": 0.1945896295604757, + "AC3_5": 0.15047590374623793, + "AC3_6": 0.11525581902586854, + "AC3_7": 0.08558054636929374 } }, - "c3": { + "sg_eval": { "prompt_1": { - "accuracy": 0.5478683620044876 + "accuracy": 0.3300970873786408 }, "prompt_2": { - "accuracy": 0.5366492146596858 + "accuracy": 0.2912621359223301 }, "prompt_3": { - "accuracy": 0.5142109199700823 + "accuracy": 0.2815533980582524 }, "prompt_4": { - "accuracy": 0.5243081525804039 + "accuracy": 0.22330097087378642 }, "prompt_5": { - "accuracy": 0.5224382946896036 + "accuracy": 0.2621359223300971 } }, - "dream": { + "cn_eval": { "prompt_1": { - "accuracy": 0.5928466438020578 + "accuracy": 0.22857142857142856 }, "prompt_2": { - "accuracy": 0.6457618814306713 + "accuracy": 0.2761904761904762 }, "prompt_3": { - "accuracy": 0.6266536011758942 + "accuracy": 0.24761904761904763 }, "prompt_4": { - "accuracy": 0.5678588926996571 + "accuracy": 0.2 }, "prompt_5": { - "accuracy": 0.6188143067123959 + "accuracy": 0.20952380952380953 } }, - "samsum": { + "us_eval": { "prompt_1": { - "rouge1": 0.19421979959087435, - "rouge2": 0.06983749460043516, - "rougeL": 0.168321567036734, - "avg_rouge": 0.14412628707601452 + "accuracy": 0.2897196261682243 }, "prompt_2": { - "rouge1": 0.19866330633085436, - "rouge2": 0.07228786591276251, - "rougeL": 0.17140262631047645, - "avg_rouge": 0.14745126618469775 + "accuracy": 0.27102803738317754 }, "prompt_3": { - "rouge1": 0.1823010767794734, - "rouge2": 0.062186205910091226, - "rougeL": 0.1637899495200591, - "avg_rouge": 0.13609241073654124 + "accuracy": 0.2803738317757009 }, "prompt_4": { - "rouge1": 0.19574582248288472, - "rouge2": 0.07082506606400779, - "rougeL": 0.16926419301155568, - "avg_rouge": 0.14527836051948273 + "accuracy": 0.2616822429906542 }, "prompt_5": { - "rouge1": 0.19145248800250056, - "rouge2": 0.06645109075593836, - "rougeL": 0.16682834576905436, - "avg_rouge": 0.1415773081758311 + "accuracy": 0.2336448598130841 } }, - "dialogsum": { + "ph_eval": { "prompt_1": { - "rouge1": 0.19354612820856404, - "rouge2": 0.05220115551483978, - "rougeL": 0.1475132846001137, - "avg_rouge": 0.13108685610783918 + "accuracy": 0.25, + "category_acc": { + "brand": 0.4, + "demographics": 0.4, + "biology": 0.3, + "history": 0.2, + "literature": 0.1, + "politics": 0.1, + "culture": 0.2, + "film": 0.4, + "law": 0.3, + "geography": 0.2 + } }, "prompt_2": { - "rouge1": 0.18930219636318057, - "rouge2": 0.0509957241945143, - "rougeL": 0.14481650495763934, - "avg_rouge": 0.12837147517177808 + "accuracy": 0.18, + "category_acc": { + "brand": 0.2, + "demographics": 0.2, + "biology": 0.2, + "history": 0.13333333333333333, + "literature": 0.4, + "politics": 0.0, + "culture": 0.1, + "film": 0.2, + "law": 0.2, + "geography": 0.2 + } }, "prompt_3": { - "rouge1": 0.18856305511927057, - "rouge2": 0.05047203018596512, - "rougeL": 0.14516669076855662, - "avg_rouge": 0.12806725869126412 + "accuracy": 0.22, + "category_acc": { + "brand": 0.3, + "demographics": 0.2, + "biology": 0.4, + "history": 0.2, + "literature": 0.3, + "politics": 0.0, + "culture": 0.1, + "film": 0.2, + "law": 0.2, + "geography": 0.3 + } }, "prompt_4": { - "rouge1": 0.19987887059378412, - "rouge2": 0.054264246884584, - "rougeL": 0.15137121232130749, - "avg_rouge": 0.13517144326655853 + "accuracy": 0.24, + "category_acc": { + "brand": 0.1, + "demographics": 0.4, + "biology": 0.3, + "history": 0.26666666666666666, + "literature": 0.3, + "politics": 0.2, + "culture": 0.3, + "film": 0.1, + "law": 0.2, + "geography": 0.3 + } }, "prompt_5": { - "rouge1": 0.1887689700181949, - "rouge2": 0.050424274072061896, - "rougeL": 0.14477589018893963, - "avg_rouge": 0.1279897114263988 + "accuracy": 0.23, + "category_acc": { + "brand": 0.2, + "demographics": 0.4, + "biology": 0.2, + "history": 0.13333333333333333, + "literature": 0.3, + "politics": 0.0, + "culture": 0.3, + "film": 0.4, + "law": 0.3, + "geography": 0.2 + } } }, - "sst2": { + "sing2eng": { "prompt_1": { - "accuracy": 0.805045871559633 + "bleu_score": 0.17364911315618559 }, "prompt_2": { - "accuracy": 0.7786697247706422 + "bleu_score": 0.1816094741862085 }, "prompt_3": { - "accuracy": 0.7442660550458715 + "bleu_score": 0.15745870540541765 }, "prompt_4": { - "accuracy": 0.7981651376146789 + "bleu_score": 0.1604700546938792 }, "prompt_5": { - "accuracy": 0.8027522935779816 + "bleu_score": 0.14500235127771283 } }, - "cola": { + "indommlu": { "prompt_1": { - "accuracy": 0.5704697986577181 + "accuracy": 0.2488817678082649, + "category_acc": { + "History": 0.20080321285140562, + "Geography": 0.24285714285714285, + "Lampungic": 0.24489795918367346, + "Social science": 0.3005008347245409, + "Balinese": 0.2611464968152866, + "Makassarese": 0.25806451612903225, + "Banjarese": 0.3680555555555556, + "Chemistry": 0.2072992700729927, + "Biology": 0.22130177514792898, + "Science": 0.24974200206398348, + "Christian religion": 0.27860696517412936, + "Art": 0.2628951747088186, + "Islam religion": 0.22617354196301565, + "Hindu religion": 0.23333333333333334, + "Madurese": 0.27796610169491526, + "Sport": 0.24324324324324326, + "Indonesian language": 0.26774595267745954, + "Physics": 0.2202020202020202, + "Minangkabau culture": 0.24623115577889448, + "Dayak language": 0.23853211009174313, + "Sociology": 0.19556451612903225, + "Economy": 0.19262295081967212, + "Sundanese": 0.26534140017286084, + "Javanese": 0.26310483870967744, + "Civic education": 0.24177396280400573 + } }, "prompt_2": { - "accuracy": 0.5100671140939598 + "accuracy": 0.2415381534147807, + "category_acc": { + "History": 0.1827309236947791, + "Geography": 0.23061224489795917, + "Lampungic": 0.2653061224489796, + "Social science": 0.3005008347245409, + "Balinese": 0.2632696390658174, + "Makassarese": 0.25806451612903225, + "Banjarese": 0.25, + "Chemistry": 0.22189781021897811, + "Biology": 0.20828402366863905, + "Science": 0.24251805985552116, + "Christian religion": 0.24378109452736318, + "Art": 0.24459234608985025, + "Islam religion": 0.22901849217638692, + "Hindu religion": 0.24666666666666667, + "Madurese": 0.2440677966101695, + "Sport": 0.2972972972972973, + "Indonesian language": 0.26058530510585304, + "Physics": 0.19595959595959597, + "Minangkabau culture": 0.24623115577889448, + "Dayak language": 0.22018348623853212, + "Sociology": 0.19153225806451613, + "Economy": 0.22336065573770492, + "Sundanese": 0.2808988764044944, + "Javanese": 0.22983870967741934, + "Civic education": 0.2145922746781116 + } }, "prompt_3": { - "accuracy": 0.5906040268456376 + "accuracy": 0.24667868349021965, + "category_acc": { + "History": 0.23493975903614459, + "Geography": 0.19795918367346937, + "Lampungic": 0.2108843537414966, + "Social science": 0.327212020033389, + "Balinese": 0.24416135881104034, + "Makassarese": 0.25806451612903225, + "Banjarese": 0.2777777777777778, + "Chemistry": 0.21897810218978103, + "Biology": 0.21420118343195266, + "Science": 0.25696594427244585, + "Christian religion": 0.22388059701492538, + "Art": 0.2778702163061564, + "Islam religion": 0.25889046941678523, + "Hindu religion": 0.25333333333333335, + "Madurese": 0.26440677966101694, + "Sport": 0.2635135135135135, + "Indonesian language": 0.2562266500622665, + "Physics": 0.21212121212121213, + "Minangkabau culture": 0.23618090452261306, + "Dayak language": 0.28440366972477066, + "Sociology": 0.22782258064516128, + "Economy": 0.19467213114754098, + "Sundanese": 0.2757130509939499, + "Javanese": 0.24193548387096775, + "Civic education": 0.21316165951359084 + } }, "prompt_4": { - "accuracy": 0.6049856184084372 + "accuracy": 0.25435609853795316, + "category_acc": { + "History": 0.2469879518072289, + "Geography": 0.23877551020408164, + "Lampungic": 0.2653061224489796, + "Social science": 0.29549248747913187, + "Balinese": 0.27388535031847133, + "Makassarese": 0.24731182795698925, + "Banjarese": 0.2916666666666667, + "Chemistry": 0.24379562043795622, + "Biology": 0.2106508875739645, + "Science": 0.2579979360165119, + "Christian religion": 0.21890547263681592, + "Art": 0.27121464226289516, + "Islam religion": 0.2802275960170697, + "Hindu religion": 0.24, + "Madurese": 0.24067796610169492, + "Sport": 0.24324324324324326, + "Indonesian language": 0.2714819427148194, + "Physics": 0.21616161616161617, + "Minangkabau culture": 0.2964824120603015, + "Dayak language": 0.24770642201834864, + "Sociology": 0.22983870967741934, + "Economy": 0.22131147540983606, + "Sundanese": 0.2783059636992221, + "Javanese": 0.22983870967741934, + "Civic education": 0.22603719599427755 + } }, "prompt_5": { - "accuracy": 0.6788111217641419 + "accuracy": 0.2538887776220041, + "category_acc": { + "History": 0.21887550200803213, + "Geography": 0.21836734693877552, + "Lampungic": 0.25170068027210885, + "Social science": 0.3005008347245409, + "Balinese": 0.2653927813163482, + "Makassarese": 0.2903225806451613, + "Banjarese": 0.24305555555555555, + "Chemistry": 0.22189781021897811, + "Biology": 0.22603550295857988, + "Science": 0.27657378740970073, + "Christian religion": 0.2835820895522388, + "Art": 0.27287853577371046, + "Islam religion": 0.2233285917496444, + "Hindu religion": 0.22, + "Madurese": 0.2677966101694915, + "Sport": 0.22297297297297297, + "Indonesian language": 0.27179327521793273, + "Physics": 0.19797979797979798, + "Minangkabau culture": 0.24120603015075376, + "Dayak language": 0.2018348623853211, + "Sociology": 0.21370967741935484, + "Economy": 0.2540983606557377, + "Sundanese": 0.30682800345721695, + "Javanese": 0.23286290322580644, + "Civic education": 0.23605150214592274 + } } }, - "qqp": { + "flores_ind2eng": { "prompt_1": { - "accuracy": 0.475 + "bleu_score": 0.22847830117086618 }, "prompt_2": { - "accuracy": 0.49 + "bleu_score": 0.2250685143300185 }, "prompt_3": { - "accuracy": 0.4995 + "bleu_score": 0.22062239532038108 }, "prompt_4": { - "accuracy": 0.4565 + "bleu_score": 0.22033637777813436 }, "prompt_5": { - "accuracy": 0.474 + "bleu_score": 0.21347066048139857 } }, - "mnli": { + "flores_vie2eng": { "prompt_1": { - "accuracy": 0.387 + "bleu_score": 0.18936731971081208 }, "prompt_2": { - "accuracy": 0.39 + "bleu_score": 0.18190614295595098 }, "prompt_3": { - "accuracy": 0.4035 + "bleu_score": 0.16718291578910777 }, "prompt_4": { - "accuracy": 0.3535 + "bleu_score": 0.1853209684094906 }, "prompt_5": { - "accuracy": 0.4045 + "bleu_score": 0.18652646523521288 } }, - "qnli": { + "flores_zho2eng": { "prompt_1": { - "accuracy": 0.577 + "bleu_score": 0.1249721609421984 }, "prompt_2": { - "accuracy": 0.6325 + "bleu_score": 0.11767975443160957 }, "prompt_3": { - "accuracy": 0.6435 + "bleu_score": 0.12164300828346879 }, "prompt_4": { - "accuracy": 0.564 + "bleu_score": 0.11784310604903017 }, "prompt_5": { - "accuracy": 0.5415 + "bleu_score": 0.11834728651092338 } }, - "wnli": { + "flores_zsm2eng": { "prompt_1": { - "accuracy": 0.5492957746478874 + "bleu_score": 0.21558251564062889 }, "prompt_2": { - "accuracy": 0.5070422535211268 + "bleu_score": 0.21464399589286035 }, "prompt_3": { - "accuracy": 0.43661971830985913 + "bleu_score": 0.21852482863861056 }, "prompt_4": { - "accuracy": 0.4225352112676056 + "bleu_score": 0.2121958104676107 }, "prompt_5": { - "accuracy": 0.4788732394366197 + "bleu_score": 0.21493092223269536 } }, - "rte": { + "mmlu": { "prompt_1": { - "accuracy": 0.5126353790613718 + "accuracy": 0.28004667444574094 }, "prompt_2": { - "accuracy": 0.5884476534296029 + "accuracy": 0.26837806301050177 }, "prompt_3": { - "accuracy": 0.5451263537906137 + "accuracy": 0.27887981330221706 }, "prompt_4": { - "accuracy": 0.5379061371841155 + "accuracy": 0.2660443407234539 }, "prompt_5": { - "accuracy": 0.516245487364621 + "accuracy": 0.25204200700116686 } }, - "mrpc": { + "mmlu_full": { "prompt_1": { - "accuracy": 0.678921568627451 + "accuracy": 0.26478369681801933, + "category_acc": { + "high_school_european_history": 0.34146341463414637, + "business_ethics": 0.24242424242424243, + "clinical_knowledge": 0.2727272727272727, + "medical_genetics": 0.24242424242424243, + "high_school_us_history": 0.2413793103448276, + "high_school_physics": 0.23333333333333334, + "high_school_world_history": 0.2711864406779661, + "virology": 0.23030303030303031, + "high_school_microeconomics": 0.24050632911392406, + "econometrics": 0.23893805309734514, + "college_computer_science": 0.20202020202020202, + "high_school_biology": 0.28802588996763756, + "abstract_algebra": 0.30303030303030304, + "professional_accounting": 0.28825622775800713, + "philosophy": 0.2645161290322581, + "professional_medicine": 0.23985239852398524, + "nutrition": 0.26885245901639343, + "global_facts": 0.2828282828282828, + "machine_learning": 0.25225225225225223, + "security_studies": 0.2336065573770492, + "public_relations": 0.28440366972477066, + "professional_psychology": 0.25040916530278234, + "prehistory": 0.25696594427244585, + "anatomy": 0.35074626865671643, + "human_sexuality": 0.24615384615384617, + "college_medicine": 0.2616279069767442, + "high_school_government_and_politics": 0.265625, + "college_chemistry": 0.2222222222222222, + "logical_fallacies": 0.2962962962962963, + "high_school_geography": 0.3299492385786802, + "elementary_mathematics": 0.26525198938992045, + "human_aging": 0.18018018018018017, + "college_mathematics": 0.2727272727272727, + "high_school_psychology": 0.26838235294117646, + "formal_logic": 0.184, + "high_school_statistics": 0.26976744186046514, + "international_law": 0.25833333333333336, + "high_school_mathematics": 0.26394052044609667, + "high_school_computer_science": 0.24242424242424243, + "conceptual_physics": 0.24786324786324787, + "miscellaneous": 0.29539641943734013, + "high_school_chemistry": 0.297029702970297, + "marketing": 0.2532188841201717, + "professional_law": 0.2622309197651663, + "management": 0.28431372549019607, + "college_physics": 0.25742574257425743, + "jurisprudence": 0.32710280373831774, + "world_religions": 0.31176470588235294, + "sociology": 0.275, + "us_foreign_policy": 0.21212121212121213, + "high_school_macroeconomics": 0.2416452442159383, + "computer_security": 0.23232323232323232, + "moral_scenarios": 0.2539149888143177, + "moral_disputes": 0.3159420289855073, + "electrical_engineering": 0.2638888888888889, + "astronomy": 0.2582781456953642, + "college_biology": 0.2727272727272727 + } }, "prompt_2": { - "accuracy": 0.6691176470588235 + "accuracy": 0.2619949946371112, + "category_acc": { + "high_school_european_history": 0.2804878048780488, + "business_ethics": 0.25252525252525254, + "clinical_knowledge": 0.3333333333333333, + "medical_genetics": 0.26262626262626265, + "high_school_us_history": 0.2413793103448276, + "high_school_physics": 0.26666666666666666, + "high_school_world_history": 0.2245762711864407, + "virology": 0.21212121212121213, + "high_school_microeconomics": 0.2109704641350211, + "econometrics": 0.23008849557522124, + "college_computer_science": 0.16161616161616163, + "high_school_biology": 0.27184466019417475, + "abstract_algebra": 0.31313131313131315, + "professional_accounting": 0.3096085409252669, + "philosophy": 0.267741935483871, + "professional_medicine": 0.2878228782287823, + "nutrition": 0.2754098360655738, + "global_facts": 0.31313131313131315, + "machine_learning": 0.1981981981981982, + "security_studies": 0.26229508196721313, + "public_relations": 0.26605504587155965, + "professional_psychology": 0.24386252045826515, + "prehistory": 0.23839009287925697, + "anatomy": 0.2835820895522388, + "human_sexuality": 0.2692307692307692, + "college_medicine": 0.25, + "high_school_government_and_politics": 0.328125, + "college_chemistry": 0.21212121212121213, + "logical_fallacies": 0.2654320987654321, + "high_school_geography": 0.3248730964467005, + "elementary_mathematics": 0.23607427055702918, + "human_aging": 0.16666666666666666, + "college_mathematics": 0.2222222222222222, + "high_school_psychology": 0.28308823529411764, + "formal_logic": 0.184, + "high_school_statistics": 0.3116279069767442, + "international_law": 0.275, + "high_school_mathematics": 0.24535315985130113, + "high_school_computer_science": 0.29292929292929293, + "conceptual_physics": 0.3076923076923077, + "miscellaneous": 0.25831202046035806, + "high_school_chemistry": 0.2524752475247525, + "marketing": 0.1888412017167382, + "professional_law": 0.26157860404435745, + "management": 0.3333333333333333, + "college_physics": 0.2376237623762376, + "jurisprudence": 0.2336448598130841, + "world_religions": 0.2529411764705882, + "sociology": 0.255, + "us_foreign_policy": 0.2828282828282828, + "high_school_macroeconomics": 0.2802056555269923, + "computer_security": 0.2828282828282828, + "moral_scenarios": 0.2505592841163311, + "moral_disputes": 0.2927536231884058, + "electrical_engineering": 0.2847222222222222, + "astronomy": 0.31788079470198677, + "college_biology": 0.26573426573426573 + } }, "prompt_3": { - "accuracy": 0.6838235294117647 - }, - "prompt_4": { - "accuracy": 0.6078431372549019 - }, - "prompt_5": { - "accuracy": 0.6666666666666666 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.5107142857142857, - "language_acc": { - "English": 0.5369747899159664, - "Chinese": 0.5226890756302521, - "Spanish": 0.46638655462184875, - "Vietnamese": 0.5168067226890757 - }, - "consistency_score_2": 0.5107843137254902, - "consistency_score_3": 0.338235294117647, - "consistency_score_4": 0.253781512605042, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.5327731092436975, - "English,Spanish": 0.5084033613445378, - "English,Vietnamese": 0.5352941176470588, - "Chinese,Spanish": 0.49159663865546216, - "Chinese,Vietnamese": 0.5117647058823529, - "Spanish,Vietnamese": 0.48487394957983193 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3369747899159664, - "English,Chinese,Vietnamese": 0.35714285714285715, - "English,Spanish,Vietnamese": 0.3352941176470588, - "Chinese,Spanish,Vietnamese": 0.3235294117647059 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.253781512605042 - } - }, - "AC3_2": 0.5107492972695311, - "AC3_3": 0.4069537242777258, - "AC3_4": 0.33907274364504414 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.34857142857142864, - "language_acc": { - "Spanish": 0.3333333333333333, - "English": 0.4, - "Chinese": 0.37333333333333335, - "Vietnamese": 0.36666666666666664, - "Filipino": 0.28, - "Indonesian": 0.3333333333333333, - "Malay": 0.35333333333333333 - }, - "consistency_score_2": 0.39174603174603173, - "consistency_score_3": 0.19657142857142865, - "consistency_score_4": 0.11447619047619045, - "consistency_score_5": 0.07206349206349207, - "consistency_score_6": 0.04761904761904762, - "consistency_score_7": 0.03333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.3466666666666667, - "Spanish,Chinese": 0.3333333333333333, - "Spanish,Vietnamese": 0.34, - "Spanish,Filipino": 0.32666666666666666, - "Spanish,Indonesian": 0.4066666666666667, - "Spanish,Malay": 0.3933333333333333, - "English,Chinese": 0.44666666666666666, - "English,Vietnamese": 0.42, - "English,Filipino": 0.44, - "English,Indonesian": 0.4066666666666667, - "English,Malay": 0.4666666666666667, - "Chinese,Vietnamese": 0.4066666666666667, - "Chinese,Filipino": 0.3, - "Chinese,Indonesian": 0.36666666666666664, - "Chinese,Malay": 0.4533333333333333, - "Vietnamese,Filipino": 0.36, - "Vietnamese,Indonesian": 0.31333333333333335, - "Vietnamese,Malay": 0.41333333333333333, - "Filipino,Indonesian": 0.3933333333333333, - "Filipino,Malay": 0.4, - "Indonesian,Malay": 0.49333333333333335 - }, - "3_combine": { - "Spanish,English,Chinese": 0.19333333333333333, - "Spanish,English,Vietnamese": 0.16, - "Spanish,English,Filipino": 0.16, - "Spanish,English,Indonesian": 0.19333333333333333, - "Spanish,English,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese": 0.16, - "Spanish,Chinese,Filipino": 0.14666666666666667, - "Spanish,Chinese,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,Vietnamese,Malay": 0.18666666666666668, - "Spanish,Filipino,Indonesian": 0.18, - "Spanish,Filipino,Malay": 0.18, - "Spanish,Indonesian,Malay": 0.22, - "English,Chinese,Vietnamese": 0.20666666666666667, - "English,Chinese,Filipino": 0.2, - "English,Chinese,Indonesian": 0.22666666666666666, - "English,Chinese,Malay": 0.2866666666666667, - "English,Vietnamese,Filipino": 0.19333333333333333, - "English,Vietnamese,Indonesian": 0.18, - "English,Vietnamese,Malay": 0.24, - "English,Filipino,Indonesian": 0.21333333333333335, - "English,Filipino,Malay": 0.24666666666666667, - "English,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino": 0.16, - "Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Chinese,Vietnamese,Malay": 0.24, - "Chinese,Filipino,Indonesian": 0.14666666666666667, - "Chinese,Filipino,Malay": 0.19333333333333333, - "Chinese,Indonesian,Malay": 0.24666666666666667, - "Vietnamese,Filipino,Indonesian": 0.14, - "Vietnamese,Filipino,Malay": 0.20666666666666667, - "Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Filipino,Indonesian,Malay": 0.24666666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.1, - "Spanish,English,Chinese,Filipino": 0.1, - "Spanish,English,Chinese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino": 0.06666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.08, - "Spanish,English,Vietnamese,Malay": 0.1, - "Spanish,English,Filipino,Indonesian": 0.12, - "Spanish,English,Filipino,Malay": 0.12, - "Spanish,English,Indonesian,Malay": 0.14, - "Spanish,Chinese,Vietnamese,Filipino": 0.08, - "Spanish,Chinese,Vietnamese,Indonesian": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.12666666666666668, - "Spanish,Chinese,Filipino,Indonesian": 0.1, - "Spanish,Chinese,Filipino,Malay": 0.10666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.1, - "Spanish,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.1, - "English,Chinese,Vietnamese,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian": 0.10666666666666667, - "English,Chinese,Filipino,Malay": 0.14666666666666667, - "English,Chinese,Indonesian,Malay": 0.16666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "English,Vietnamese,Filipino,Malay": 0.13333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.14, - "English,Filipino,Indonesian,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.05333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.08, - "Spanish,English,Chinese,Filipino,Indonesian": 0.08, - "Spanish,English,Chinese,Filipino,Malay": 0.08, - "Spanish,English,Chinese,Indonesian,Malay": 0.08666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.04666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.06, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.06, - "Spanish,English,Filipino,Indonesian,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.05333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.08, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.05333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.08, - "English,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.08, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.04, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.06666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.04, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.03333333333333333 - } - }, - "AC3_2": 0.3688997794159873, - "AC3_3": 0.25138065284397415, - "AC3_4": 0.17235000290101743, - "AC3_5": 0.11943504040287285, - "AC3_6": 0.08379120877005956, - "AC3_7": 0.06084788028331913 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.325487012987013, - "language_acc": { - "English": 0.3522727272727273, - "Filipino": 0.2727272727272727, - "Spanish": 0.3125, - "Malay": 0.3352272727272727, - "Chinese": 0.32386363636363635, - "Indonesian": 0.32386363636363635, - "Vietnamese": 0.35795454545454547 - }, - "consistency_score_2": 0.36688311688311687, - "consistency_score_3": 0.17077922077922075, - "consistency_score_4": 0.09220779220779218, - "consistency_score_5": 0.05438311688311689, - "consistency_score_6": 0.032467532467532464, - "consistency_score_7": 0.017045454545454544, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3125, - "English,Spanish": 0.44886363636363635, - "English,Malay": 0.3977272727272727, - "English,Chinese": 0.3409090909090909, - "English,Indonesian": 0.39204545454545453, - "English,Vietnamese": 0.4090909090909091, - "Filipino,Spanish": 0.3693181818181818, - "Filipino,Malay": 0.32954545454545453, - "Filipino,Chinese": 0.3068181818181818, - "Filipino,Indonesian": 0.3068181818181818, - "Filipino,Vietnamese": 0.3409090909090909, - "Spanish,Malay": 0.38636363636363635, - "Spanish,Chinese": 0.35795454545454547, - "Spanish,Indonesian": 0.3806818181818182, - "Spanish,Vietnamese": 0.4431818181818182, - "Malay,Chinese": 0.2727272727272727, - "Malay,Indonesian": 0.4431818181818182, - "Malay,Vietnamese": 0.42613636363636365, - "Chinese,Indonesian": 0.3068181818181818, - "Chinese,Vietnamese": 0.3352272727272727, - "Indonesian,Vietnamese": 0.3977272727272727 - }, - "3_combine": { - "English,Filipino,Spanish": 0.17613636363636365, - "English,Filipino,Malay": 0.13636363636363635, - "English,Filipino,Chinese": 0.11363636363636363, - "English,Filipino,Indonesian": 0.11931818181818182, - "English,Filipino,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay": 0.20454545454545456, - "English,Spanish,Chinese": 0.19886363636363635, - "English,Spanish,Indonesian": 0.21022727272727273, - "English,Spanish,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese": 0.14204545454545456, - "English,Malay,Indonesian": 0.2159090909090909, - "English,Malay,Vietnamese": 0.2159090909090909, - "English,Chinese,Indonesian": 0.14772727272727273, - "English,Chinese,Vietnamese": 0.16477272727272727, - "English,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Malay": 0.16477272727272727, - "Filipino,Spanish,Chinese": 0.14772727272727273, - "Filipino,Spanish,Indonesian": 0.1534090909090909, - "Filipino,Spanish,Vietnamese": 0.21022727272727273, - "Filipino,Malay,Chinese": 0.11931818181818182, - "Filipino,Malay,Indonesian": 0.17045454545454544, - "Filipino,Malay,Vietnamese": 0.17045454545454544, - "Filipino,Chinese,Indonesian": 0.125, - "Filipino,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese": 0.13068181818181818, - "Spanish,Malay,Indonesian": 0.2159090909090909, - "Spanish,Malay,Vietnamese": 0.2215909090909091, - "Spanish,Chinese,Indonesian": 0.1534090909090909, - "Spanish,Chinese,Vietnamese": 0.18181818181818182, - "Spanish,Indonesian,Vietnamese": 0.23295454545454544, - "Malay,Chinese,Indonesian": 0.14772727272727273, - "Malay,Chinese,Vietnamese": 0.14772727272727273, - "Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Chinese,Indonesian,Vietnamese": 0.13636363636363635 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.10227272727272728, - "English,Filipino,Spanish,Chinese": 0.08522727272727272, - "English,Filipino,Spanish,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese": 0.056818181818181816, - "English,Filipino,Malay,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.0625, - "English,Filipino,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian": 0.125, - "English,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian": 0.09659090909090909, - "English,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "English,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "English,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese": 0.07386363636363637, - "Filipino,Spanish,Malay,Indonesian": 0.10227272727272728, - "Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Malay,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.03977272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.05113636363636364, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544 - } - }, - "AC3_2": 0.3449475494698267, - "AC3_3": 0.22401853947879616, - "AC3_4": 0.14370510946938372, - "AC3_5": 0.09319499942046607, - "AC3_6": 0.05904526311040059, - "AC3_7": 0.032394442041382816 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.4174757281553398 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.3047619047619048 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.308411214953271 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.3, + "accuracy": 0.2679299249195567, "category_acc": { - "brand": 0.5, - "demographics": 0.2, - "biology": 0.2, - "history": 0.3333333333333333, - "literature": 0.5, - "politics": 0.3, - "culture": 0.3, - "film": 0.0, - "law": 0.2, - "geography": 0.4 + "high_school_european_history": 0.2804878048780488, + "business_ethics": 0.2727272727272727, + "clinical_knowledge": 0.2916666666666667, + "medical_genetics": 0.29292929292929293, + "high_school_us_history": 0.2660098522167488, + "high_school_physics": 0.34, + "high_school_world_history": 0.2669491525423729, + "virology": 0.20606060606060606, + "high_school_microeconomics": 0.25738396624472576, + "econometrics": 0.23893805309734514, + "college_computer_science": 0.23232323232323232, + "high_school_biology": 0.284789644012945, + "abstract_algebra": 0.32323232323232326, + "professional_accounting": 0.2846975088967972, + "philosophy": 0.2903225806451613, + "professional_medicine": 0.2952029520295203, + "nutrition": 0.2721311475409836, + "global_facts": 0.2828282828282828, + "machine_learning": 0.23423423423423423, + "security_studies": 0.22540983606557377, + "public_relations": 0.25688073394495414, + "professional_psychology": 0.2553191489361702, + "prehistory": 0.24458204334365324, + "anatomy": 0.3208955223880597, + "human_sexuality": 0.23846153846153847, + "college_medicine": 0.27325581395348836, + "high_school_government_and_politics": 0.3177083333333333, + "college_chemistry": 0.23232323232323232, + "logical_fallacies": 0.29012345679012347, + "high_school_geography": 0.3299492385786802, + "elementary_mathematics": 0.2519893899204244, + "human_aging": 0.16666666666666666, + "college_mathematics": 0.1919191919191919, + "high_school_psychology": 0.33639705882352944, + "formal_logic": 0.192, + "high_school_statistics": 0.2930232558139535, + "international_law": 0.21666666666666667, + "high_school_mathematics": 0.26765799256505574, + "high_school_computer_science": 0.24242424242424243, + "conceptual_physics": 0.2564102564102564, + "miscellaneous": 0.29028132992327366, + "high_school_chemistry": 0.2524752475247525, + "marketing": 0.21888412017167383, + "professional_law": 0.2576647097195042, + "management": 0.3431372549019608, + "college_physics": 0.24752475247524752, + "jurisprudence": 0.2523364485981308, + "world_religions": 0.21764705882352942, + "sociology": 0.23, + "us_foreign_policy": 0.29292929292929293, + "high_school_macroeconomics": 0.2879177377892031, + "computer_security": 0.23232323232323232, + "moral_scenarios": 0.24384787472035793, + "moral_disputes": 0.28405797101449276, + "electrical_engineering": 0.3541666666666667, + "astronomy": 0.33112582781456956, + "college_biology": 0.24475524475524477 } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.04899461984382978 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.05271608034037656 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.03996174387805273 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.02477939464744664 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.054636159464664505 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.3045507584597433 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.30818734358240973, + }, + "prompt_4": { + "accuracy": 0.2679299249195567, "category_acc": { - "high_school_european_history": 0.39634146341463417, - "business_ethics": 0.3939393939393939, - "clinical_knowledge": 0.3181818181818182, - "medical_genetics": 0.41414141414141414, - "high_school_us_history": 0.37438423645320196, - "high_school_physics": 0.3, - "high_school_world_history": 0.4152542372881356, - "virology": 0.2545454545454545, - "high_school_microeconomics": 0.33755274261603374, + "high_school_european_history": 0.3048780487804878, + "business_ethics": 0.2727272727272727, + "clinical_knowledge": 0.2689393939393939, + "medical_genetics": 0.29292929292929293, + "high_school_us_history": 0.2660098522167488, + "high_school_physics": 0.30666666666666664, + "high_school_world_history": 0.2669491525423729, + "virology": 0.20606060606060606, + "high_school_microeconomics": 0.2742616033755274, + "econometrics": 0.2743362831858407, + "college_computer_science": 0.20202020202020202, + "high_school_biology": 0.31715210355987056, + "abstract_algebra": 0.25252525252525254, + "professional_accounting": 0.2918149466192171, + "philosophy": 0.26129032258064516, + "professional_medicine": 0.2915129151291513, + "nutrition": 0.3114754098360656, + "global_facts": 0.25252525252525254, + "machine_learning": 0.23423423423423423, + "security_studies": 0.27049180327868855, + "public_relations": 0.23853211009174313, + "professional_psychology": 0.23240589198036007, + "prehistory": 0.26006191950464397, + "anatomy": 0.26865671641791045, + "human_sexuality": 0.3230769230769231, + "college_medicine": 0.28488372093023256, + "high_school_government_and_politics": 0.2864583333333333, + "college_chemistry": 0.32323232323232326, + "logical_fallacies": 0.2716049382716049, + "high_school_geography": 0.350253807106599, + "elementary_mathematics": 0.26525198938992045, + "human_aging": 0.14414414414414414, + "college_mathematics": 0.1717171717171717, + "high_school_psychology": 0.3235294117647059, + "formal_logic": 0.184, + "high_school_statistics": 0.26046511627906976, + "international_law": 0.25, + "high_school_mathematics": 0.31970260223048325, + "high_school_computer_science": 0.30303030303030304, + "conceptual_physics": 0.27350427350427353, + "miscellaneous": 0.2595907928388747, + "high_school_chemistry": 0.22772277227722773, + "marketing": 0.2017167381974249, + "professional_law": 0.2563600782778865, + "management": 0.3137254901960784, + "college_physics": 0.3465346534653465, + "jurisprudence": 0.2803738317757009, + "world_religions": 0.2, + "sociology": 0.285, + "us_foreign_policy": 0.25252525252525254, + "high_school_macroeconomics": 0.2647814910025707, + "computer_security": 0.21212121212121213, + "moral_scenarios": 0.25727069351230425, + "moral_disputes": 0.28405797101449276, + "electrical_engineering": 0.3125, + "astronomy": 0.3443708609271523, + "college_biology": 0.2517482517482518 + } + }, + "prompt_5": { + "accuracy": 0.2608509116910976, + "category_acc": { + "high_school_european_history": 0.3048780487804878, + "business_ethics": 0.25252525252525254, + "clinical_knowledge": 0.26515151515151514, + "medical_genetics": 0.25252525252525254, + "high_school_us_history": 0.2857142857142857, + "high_school_physics": 0.2733333333333333, + "high_school_world_history": 0.2584745762711864, + "virology": 0.23636363636363636, + "high_school_microeconomics": 0.25316455696202533, "econometrics": 0.23008849557522124, - "college_computer_science": 0.31313131313131315, - "high_school_biology": 0.2621359223300971, - "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.23487544483985764, - "philosophy": 0.32903225806451614, - "professional_medicine": 0.23985239852398524, - "nutrition": 0.32131147540983607, + "college_computer_science": 0.21212121212121213, + "high_school_biology": 0.284789644012945, + "abstract_algebra": 0.31313131313131315, + "professional_accounting": 0.29537366548042704, + "philosophy": 0.2709677419354839, + "professional_medicine": 0.26199261992619927, + "nutrition": 0.2459016393442623, "global_facts": 0.2727272727272727, - "machine_learning": 0.25225225225225223, - "security_studies": 0.2459016393442623, + "machine_learning": 0.21621621621621623, + "security_studies": 0.27049180327868855, "public_relations": 0.23853211009174313, - "professional_psychology": 0.2978723404255319, - "prehistory": 0.2724458204334365, - "anatomy": 0.35074626865671643, - "human_sexuality": 0.3076923076923077, - "college_medicine": 0.29651162790697677, - "high_school_government_and_politics": 0.359375, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.30246913580246915, - "high_school_geography": 0.4010152284263959, - "elementary_mathematics": 0.27851458885941643, - "human_aging": 0.2882882882882883, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.3161764705882353, - "formal_logic": 0.304, - "high_school_statistics": 0.2651162790697674, - "international_law": 0.4, - "high_school_mathematics": 0.24535315985130113, - "high_school_computer_science": 0.30303030303030304, - "conceptual_physics": 0.2606837606837607, - "miscellaneous": 0.36445012787723785, - "high_school_chemistry": 0.2623762376237624, - "marketing": 0.45493562231759654, - "professional_law": 0.29354207436399216, - "management": 0.3431372549019608, - "college_physics": 0.25742574257425743, - "jurisprudence": 0.308411214953271, - "world_religions": 0.3764705882352941, - "sociology": 0.38, - "us_foreign_policy": 0.5353535353535354, - "high_school_macroeconomics": 0.2776349614395887, - "computer_security": 0.3838383838383838, - "moral_scenarios": 0.2516778523489933, - "moral_disputes": 0.30144927536231886, - "electrical_engineering": 0.3333333333333333, - "astronomy": 0.31788079470198677, - "college_biology": 0.2867132867132867 + "professional_psychology": 0.25368248772504093, + "prehistory": 0.24148606811145512, + "anatomy": 0.3208955223880597, + "human_sexuality": 0.2230769230769231, + "college_medicine": 0.2558139534883721, + "high_school_government_and_politics": 0.2916666666666667, + "college_chemistry": 0.2222222222222222, + "logical_fallacies": 0.32098765432098764, + "high_school_geography": 0.29949238578680204, + "elementary_mathematics": 0.26525198938992045, + "human_aging": 0.16216216216216217, + "college_mathematics": 0.21212121212121213, + "high_school_psychology": 0.29044117647058826, + "formal_logic": 0.176, + "high_school_statistics": 0.26046511627906976, + "international_law": 0.275, + "high_school_mathematics": 0.25650557620817843, + "high_school_computer_science": 0.2727272727272727, + "conceptual_physics": 0.26495726495726496, + "miscellaneous": 0.2659846547314578, + "high_school_chemistry": 0.30198019801980197, + "marketing": 0.21888412017167383, + "professional_law": 0.253750815394651, + "management": 0.29411764705882354, + "college_physics": 0.2376237623762376, + "jurisprudence": 0.24299065420560748, + "world_religions": 0.2411764705882353, + "sociology": 0.255, + "us_foreign_policy": 0.25252525252525254, + "high_school_macroeconomics": 0.2647814910025707, + "computer_security": 0.2727272727272727, + "moral_scenarios": 0.24720357941834453, + "moral_disputes": 0.2782608695652174, + "electrical_engineering": 0.2847222222222222, + "astronomy": 0.304635761589404, + "college_biology": 0.24475524475524477 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.324665676077266 + "accuracy": 0.25705794947994054 + }, + "prompt_2": { + "accuracy": 0.2540861812778603 + }, + "prompt_3": { + "accuracy": 0.2600297176820208 + }, + "prompt_4": { + "accuracy": 0.25482912332838037 + }, + "prompt_5": { + "accuracy": 0.2451708766716196 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.3206724782067248, + "accuracy": 0.27023661270236615, "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.2916666666666667, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.2857142857142857, - "college_physics": 0.375, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.20833333333333334, + "computer_network": 0.2916666666666667, + "operating_system": 0.375, + "computer_architecture": 0.15384615384615385, + "college_programming": 0.2619047619047619, + "college_physics": 0.2916666666666667, + "college_chemistry": 0.4827586206896552, + "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.41379310344827586, - "high_school_mathematics": 0.21739130434782608, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.27586206896551724, + "high_school_mathematics": 0.43478260869565216, + "high_school_physics": 0.16666666666666666, + "high_school_chemistry": 0.4583333333333333, + "high_school_biology": 0.041666666666666664, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.46153846153846156, + "middle_school_physics": 0.25, + "middle_school_chemistry": 0.36, + "veterinary_medicine": 0.35714285714285715, + "college_economics": 0.18333333333333332, + "business_administration": 0.21052631578947367, + "marxism": 0.25, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.20588235294117646, + "teacher_qualification": 0.32653061224489793, + "high_school_politics": 0.08333333333333333, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.23076923076923078, + "middle_school_geography": 0.5882352941176471, + "modern_chinese_history": 0.32142857142857145, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.3333333333333333, + "law": 0.13793103448275862, + "chinese_language_and_literature": 0.35714285714285715, + "art_studies": 0.2894736842105263, + "professional_tour_guide": 0.23529411764705882, + "legal_professional": 0.17857142857142858, + "high_school_chinese": 0.16666666666666666, + "high_school_history": 0.32, + "middle_school_history": 0.2962962962962963, + "civil_servant": 0.21153846153846154, + "sports_science": 0.3333333333333333, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.5833333333333334, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.29411764705882354, + "accountant": 0.2222222222222222, + "fire_engineer": 0.2222222222222222, + "environmental_impact_assessment_engineer": 0.1388888888888889, + "tax_accountant": 0.2962962962962963, + "physician": 0.2777777777777778 + } + }, + "prompt_2": { + "accuracy": 0.28393524283935245, + "category_acc": { + "computer_network": 0.2916666666666667, + "operating_system": 0.375, + "computer_architecture": 0.15384615384615385, + "college_programming": 0.16666666666666666, + "college_physics": 0.25, + "college_chemistry": 0.3448275862068966, + "advanced_mathematics": 0.375, + "probability_and_statistics": 0.13043478260869565, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.3103448275862069, + "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.4166666666666667, - "middle_school_mathematics": 0.20833333333333334, + "high_school_chemistry": 0.4583333333333333, + "high_school_biology": 0.16666666666666666, + "middle_school_mathematics": 0.375, + "middle_school_biology": 0.19230769230769232, + "middle_school_physics": 0.25, + "middle_school_chemistry": 0.4, + "veterinary_medicine": 0.25, + "college_economics": 0.23333333333333334, + "business_administration": 0.23684210526315788, + "marxism": 0.375, + "mao_zedong_thought": 0.27586206896551724, + "education_science": 0.2647058823529412, + "teacher_qualification": 0.3469387755102041, + "high_school_politics": 0.08333333333333333, + "high_school_geography": 0.2916666666666667, + "middle_school_politics": 0.3076923076923077, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.25, + "ideological_and_moral_cultivation": 0.20833333333333334, + "logic": 0.4444444444444444, + "law": 0.27586206896551724, + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.21052631578947367, + "professional_tour_guide": 0.20588235294117646, + "legal_professional": 0.25, + "high_school_chinese": 0.3333333333333333, + "high_school_history": 0.28, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.19230769230769232, + "sports_science": 0.16666666666666666, + "plant_protection": 0.25925925925925924, + "basic_medicine": 0.5, + "clinical_medicine": 0.2222222222222222, + "urban_and_rural_planner": 0.3137254901960784, + "accountant": 0.2777777777777778, + "fire_engineer": 0.3611111111111111, + "environmental_impact_assessment_engineer": 0.3611111111111111, + "tax_accountant": 0.25925925925925924, + "physician": 0.2777777777777778 + } + }, + "prompt_3": { + "accuracy": 0.2714819427148194, + "category_acc": { + "computer_network": 0.25, + "operating_system": 0.3333333333333333, + "computer_architecture": 0.19230769230769232, + "college_programming": 0.19047619047619047, + "college_physics": 0.25, + "college_chemistry": 0.3103448275862069, + "advanced_mathematics": 0.3333333333333333, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.23809523809523808, + "metrology_engineer": 0.20689655172413793, + "high_school_mathematics": 0.5217391304347826, + "high_school_physics": 0.25, + "high_school_chemistry": 0.375, + "high_school_biology": 0.125, + "middle_school_mathematics": 0.3333333333333333, "middle_school_biology": 0.3076923076923077, - "middle_school_physics": 0.375, + "middle_school_physics": 0.16666666666666666, "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.26666666666666666, - "business_administration": 0.3157894736842105, - "marxism": 0.4583333333333333, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.3235294117647059, - "teacher_qualification": 0.3877551020408163, - "high_school_politics": 0.4166666666666667, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.34615384615384615, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.5, - "logic": 0.18518518518518517, - "law": 0.10344827586206896, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.4117647058823529, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.44, - "middle_school_history": 0.4444444444444444, - "civil_servant": 0.3076923076923077, - "sports_science": 0.25, + "veterinary_medicine": 0.25, + "college_economics": 0.2, + "business_administration": 0.2894736842105263, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.27586206896551724, + "education_science": 0.17647058823529413, + "teacher_qualification": 0.42857142857142855, + "high_school_politics": 0.125, + "high_school_geography": 0.25, + "middle_school_politics": 0.15384615384615385, + "middle_school_geography": 0.5294117647058824, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.08333333333333333, + "logic": 0.3333333333333333, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.42105263157894735, + "professional_tour_guide": 0.23529411764705882, + "legal_professional": 0.14285714285714285, + "high_school_chinese": 0.125, + "high_school_history": 0.4, + "middle_school_history": 0.25925925925925924, + "civil_servant": 0.19230769230769232, + "sports_science": 0.2916666666666667, "plant_protection": 0.25925925925925924, "basic_medicine": 0.4166666666666667, "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.4117647058823529, + "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.2962962962962963, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.4074074074074074, - "physician": 0.3888888888888889 + "fire_engineer": 0.19444444444444445, + "environmental_impact_assessment_engineer": 0.2222222222222222, + "tax_accountant": 0.2962962962962963, + "physician": 0.25925925925925924 } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.3548387096774194 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.3274045933344846, + }, + "prompt_4": { + "accuracy": 0.26899128268991285, "category_acc": { - "agronomy": 0.28994082840236685, - "anatomy": 0.31756756756756754, - "ancient_chinese": 0.2682926829268293, - "arts": 0.3, - "astronomy": 0.3151515151515151, - "business_ethics": 0.35406698564593303, - "chinese_civil_service_exam": 0.225, - "chinese_driving_rule": 0.5114503816793893, - "chinese_food_culture": 0.29411764705882354, - "chinese_foreign_policy": 0.2897196261682243, - "chinese_history": 0.3591331269349845, - "chinese_literature": 0.28921568627450983, - "chinese_teacher_qualification": 0.45251396648044695, - "clinical_knowledge": 0.3206751054852321, - "college_actuarial_science": 0.22641509433962265, - "college_education": 0.3925233644859813, - "college_engineering_hydrology": 0.33962264150943394, - "college_law": 0.26851851851851855, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.330188679245283, - "college_medicine": 0.2967032967032967, - "computer_science": 0.3235294117647059, - "computer_security": 0.3391812865497076, - "conceptual_physics": 0.3197278911564626, - "construction_project_management": 0.34532374100719426, - "economics": 0.3584905660377358, - "education": 0.38650306748466257, - "electrical_engineering": 0.3023255813953488, - "elementary_chinese": 0.28174603174603174, - "elementary_commonsense": 0.3434343434343434, - "elementary_information_and_technology": 0.38235294117647056, - "elementary_mathematics": 0.2608695652173913, - "ethnology": 0.3037037037037037, - "food_science": 0.3356643356643357, - "genetics": 0.30113636363636365, - "global_facts": 0.3825503355704698, - "high_school_biology": 0.28994082840236685, - "high_school_chemistry": 0.29545454545454547, - "high_school_geography": 0.2711864406779661, - "high_school_mathematics": 0.25, - "high_school_physics": 0.3, - "high_school_politics": 0.3146853146853147, - "human_sexuality": 0.2777777777777778, - "international_law": 0.31891891891891894, - "journalism": 0.436046511627907, - "jurisprudence": 0.2846715328467153, - "legal_and_moral_basis": 0.5514018691588785, - "logical": 0.2926829268292683, - "machine_learning": 0.26229508196721313, - "management": 0.3142857142857143, - "marketing": 0.37777777777777777, - "marxist_theory": 0.3968253968253968, - "modern_chinese": 0.25862068965517243, - "nutrition": 0.3310344827586207, - "philosophy": 0.3333333333333333, - "professional_accounting": 0.36, - "professional_law": 0.27488151658767773, - "professional_medicine": 0.27393617021276595, - "professional_psychology": 0.3448275862068966, - "public_relations": 0.39080459770114945, - "security_study": 0.37037037037037035, - "sociology": 0.3938053097345133, - "sports_science": 0.32727272727272727, - "traditional_chinese_medicine": 0.32432432432432434, - "virology": 0.3254437869822485, - "world_history": 0.32298136645962733, - "world_religions": 0.33125 + "computer_network": 0.2916666666666667, + "operating_system": 0.375, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.2857142857142857, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.41379310344827586, + "advanced_mathematics": 0.125, + "probability_and_statistics": 0.21739130434782608, + "discrete_mathematics": 0.47619047619047616, + "electrical_engineer": 0.23809523809523808, + "metrology_engineer": 0.27586206896551724, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.20833333333333334, + "high_school_chemistry": 0.4583333333333333, + "high_school_biology": 0.25, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.38461538461538464, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.16, + "veterinary_medicine": 0.32142857142857145, + "college_economics": 0.23333333333333334, + "business_administration": 0.39473684210526316, + "marxism": 0.25, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.20408163265306123, + "high_school_politics": 0.20833333333333334, + "high_school_geography": 0.25, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.32142857142857145, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.3333333333333333, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.25, + "art_studies": 0.21052631578947367, + "professional_tour_guide": 0.29411764705882354, + "legal_professional": 0.14285714285714285, + "high_school_chinese": 0.3333333333333333, + "high_school_history": 0.36, + "middle_school_history": 0.2222222222222222, + "civil_servant": 0.25, + "sports_science": 0.375, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.4583333333333333, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.19607843137254902, + "accountant": 0.2037037037037037, + "fire_engineer": 0.16666666666666666, + "environmental_impact_assessment_engineer": 0.3333333333333333, + "tax_accountant": 0.25925925925925924, + "physician": 0.3333333333333333 + } + }, + "prompt_5": { + "accuracy": 0.2627646326276463, + "category_acc": { + "computer_network": 0.16666666666666666, + "operating_system": 0.3333333333333333, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.19047619047619047, + "college_physics": 0.16666666666666666, + "college_chemistry": 0.3103448275862069, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.3793103448275862, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.4166666666666667, + "high_school_biology": 0.20833333333333334, + "middle_school_mathematics": 0.375, + "middle_school_biology": 0.3076923076923077, + "middle_school_physics": 0.20833333333333334, + "middle_school_chemistry": 0.24, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.25, + "business_administration": 0.42105263157894735, + "marxism": 0.25, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.20588235294117646, + "teacher_qualification": 0.30612244897959184, + "high_school_politics": 0.125, + "high_school_geography": 0.25, + "middle_school_politics": 0.3076923076923077, + "middle_school_geography": 0.23529411764705882, + "modern_chinese_history": 0.25, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.25925925925925924, + "law": 0.2413793103448276, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.34210526315789475, + "professional_tour_guide": 0.17647058823529413, + "legal_professional": 0.14285714285714285, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.36, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.2692307692307692, + "sports_science": 0.16666666666666666, + "plant_protection": 0.2222222222222222, + "basic_medicine": 0.4583333333333333, + "clinical_medicine": 0.14814814814814814, + "urban_and_rural_planner": 0.2549019607843137, + "accountant": 0.2037037037037037, + "fire_engineer": 0.2777777777777778, + "environmental_impact_assessment_engineer": 0.2222222222222222, + "tax_accountant": 0.2777777777777778, + "physician": 0.24074074074074073 } } }, - "zbench": { - "prompt_1": { - "accuracy": 0.24242424242424243 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.17727272727272728 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3111864406779661 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.5100972326103216 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.47280744732974034 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.14578848605052028, - "rouge2": 0.045012180901180816, - "rougeL": 0.133012512087138, - "avg_rouge": 0.1079377263462797 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.1535418416166175, - "rouge2": 0.044680654790048846, - "rougeL": 0.14081557717090992, - "avg_rouge": 0.11301269119252542 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.5997706422018348 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.5004793863854267 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.511 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.323 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.5035 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.5211267605633803 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.5126353790613718 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.5367647058823529 - } - } - } - }, - "sailor_4b_chat": { - "model_size": "4B", - "model_link": "https://huggingface.co/sail/Sailor-4B-Chat", - "zero_shot": { - "cross_xquad": { + "cmmlu": { "prompt_1": { - "overall_acc": 0.8611344537815127, - "language_acc": { - "English": 0.8823529411764706, - "Chinese": 0.8487394957983193, - "Spanish": 0.8579831932773109, - "Vietnamese": 0.8554621848739495 - }, - "consistency_score_2": 0.8491596638655462, - "consistency_score_3": 0.782983193277311, - "consistency_score_4": 0.7378151260504202, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8596638655462185, - "English,Spanish": 0.8815126050420168, - "English,Vietnamese": 0.846218487394958, - "Chinese,Spanish": 0.8411764705882353, - "Chinese,Vietnamese": 0.8168067226890756, - "Spanish,Vietnamese": 0.8495798319327731 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8, - "English,Chinese,Vietnamese": 0.7714285714285715, - "English,Spanish,Vietnamese": 0.7974789915966387, - "Chinese,Spanish,Vietnamese": 0.7630252100840336 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7378151260504202 - } - }, - "AC3_2": 0.8551051374478218, - "AC3_3": 0.8202014078835905, - "AC3_4": 0.7947192751258781 + "accuracy": 0.2222222222222222 }, "prompt_2": { - "overall_acc": 0.8758403361344538, - "language_acc": { - "English": 0.8966386554621849, - "Chinese": 0.8680672268907563, - "Spanish": 0.8680672268907563, - "Vietnamese": 0.8705882352941177 - }, - "consistency_score_2": 0.8596638655462184, - "consistency_score_3": 0.7978991596638656, - "consistency_score_4": 0.7554621848739496, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8705882352941177, - "English,Spanish": 0.8873949579831932, - "English,Vietnamese": 0.8605042016806723, - "Chinese,Spanish": 0.8512605042016806, - "Chinese,Vietnamese": 0.838655462184874, - "Spanish,Vietnamese": 0.8495798319327731 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8117647058823529, - "English,Chinese,Vietnamese": 0.7949579831932773, - "English,Spanish,Vietnamese": 0.8058823529411765, - "Chinese,Spanish,Vietnamese": 0.7789915966386555 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7554621848739496 - } - }, - "AC3_2": 0.8676767111138307, - "AC3_3": 0.8350550009890579, - "AC3_4": 0.811209748498758 + "accuracy": 0.2867383512544803 }, "prompt_3": { - "overall_acc": 0.8747899159663866, - "language_acc": { - "English": 0.8949579831932774, - "Chinese": 0.8689075630252101, - "Spanish": 0.8697478991596639, - "Vietnamese": 0.865546218487395 - }, - "consistency_score_2": 0.8592436974789915, - "consistency_score_3": 0.7974789915966387, - "consistency_score_4": 0.7579831932773109, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.865546218487395, - "English,Spanish": 0.888235294117647, - "English,Vietnamese": 0.8613445378151261, - "Chinese,Spanish": 0.8487394957983193, - "Chinese,Vietnamese": 0.8378151260504202, - "Spanish,Vietnamese": 0.853781512605042 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8084033613445378, - "English,Chinese,Vietnamese": 0.7924369747899159, - "English,Spanish,Vietnamese": 0.8092436974789916, - "Chinese,Spanish,Vietnamese": 0.7798319327731092 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7579831932773109 - } - }, - "AC3_2": 0.8669471180266991, - "AC3_3": 0.8343473670376454, - "AC3_4": 0.8122084448310388 + "accuracy": 0.24372759856630824 }, "prompt_4": { - "overall_acc": 0.8638655462184874, - "language_acc": { - "English": 0.888235294117647, - "Chinese": 0.846218487394958, - "Spanish": 0.8563025210084033, - "Vietnamese": 0.8647058823529412 - }, - "consistency_score_2": 0.8467787114845938, - "consistency_score_3": 0.7792016806722689, - "consistency_score_4": 0.7319327731092437, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.853781512605042, - "English,Spanish": 0.8815126050420168, - "English,Vietnamese": 0.8436974789915966, - "Chinese,Spanish": 0.8361344537815126, - "Chinese,Vietnamese": 0.8159663865546218, - "Spanish,Vietnamese": 0.8495798319327731 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7932773109243697, - "English,Chinese,Vietnamese": 0.7689075630252101, - "English,Spanish,Vietnamese": 0.7941176470588235, - "Chinese,Spanish,Vietnamese": 0.7605042016806722 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7319327731092437 - } - }, - "AC3_2": 0.8552367925547167, - "AC3_3": 0.8193523362049262, - "AC3_4": 0.7924453825267529 + "accuracy": 0.23655913978494625 }, "prompt_5": { - "overall_acc": 0.8594537815126051, - "language_acc": { - "English": 0.8831932773109243, - "Chinese": 0.8495798319327731, - "Spanish": 0.8428571428571429, - "Vietnamese": 0.8621848739495799 - }, - "consistency_score_2": 0.8582633053221288, - "consistency_score_3": 0.7955882352941177, - "consistency_score_4": 0.7529411764705882, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8638655462184874, - "English,Spanish": 0.8831932773109243, - "English,Vietnamese": 0.8613445378151261, - "Chinese,Spanish": 0.8504201680672269, - "Chinese,Vietnamese": 0.83109243697479, - "Spanish,Vietnamese": 0.8596638655462185 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8050420168067227, - "English,Chinese,Vietnamese": 0.788235294117647, - "English,Spanish,Vietnamese": 0.8092436974789916, - "Chinese,Spanish,Vietnamese": 0.7798319327731092 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7529411764705882 - } - }, - "AC3_2": 0.8588581308333207, - "AC3_3": 0.8262887713611523, - "AC3_4": 0.8026794404560622 + "accuracy": 0.25806451612903225 } }, - "cross_mmlu": { + "cmmlu_full": { "prompt_1": { - "overall_acc": 0.4714285714285715, - "language_acc": { - "Spanish": 0.4533333333333333, - "English": 0.5533333333333333, - "Chinese": 0.4866666666666667, - "Vietnamese": 0.4533333333333333, - "Filipino": 0.36, - "Indonesian": 0.4866666666666667, - "Malay": 0.5066666666666667 - }, - "consistency_score_2": 0.632063492063492, - "consistency_score_3": 0.48914285714285716, - "consistency_score_4": 0.41333333333333333, - "consistency_score_5": 0.3666666666666667, - "consistency_score_6": 0.33333333333333337, - "consistency_score_7": 0.30666666666666664, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6733333333333333, - "Spanish,Chinese": 0.5933333333333334, - "Spanish,Vietnamese": 0.64, - "Spanish,Filipino": 0.5466666666666666, - "Spanish,Indonesian": 0.6533333333333333, - "Spanish,Malay": 0.6266666666666667, - "English,Chinese": 0.68, - "English,Vietnamese": 0.6666666666666666, - "English,Filipino": 0.56, - "English,Indonesian": 0.7066666666666667, - "English,Malay": 0.7666666666666667, - "Chinese,Vietnamese": 0.62, - "Chinese,Filipino": 0.5066666666666667, - "Chinese,Indonesian": 0.6666666666666666, - "Chinese,Malay": 0.66, - "Vietnamese,Filipino": 0.54, - "Vietnamese,Indonesian": 0.6733333333333333, - "Vietnamese,Malay": 0.68, - "Filipino,Indonesian": 0.52, - "Filipino,Malay": 0.58, - "Indonesian,Malay": 0.7133333333333334 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5, - "Spanish,English,Vietnamese": 0.5266666666666666, - "Spanish,English,Filipino": 0.44666666666666666, - "Spanish,English,Indonesian": 0.54, - "Spanish,English,Malay": 0.56, - "Spanish,Chinese,Vietnamese": 0.46, - "Spanish,Chinese,Filipino": 0.3933333333333333, - "Spanish,Chinese,Indonesian": 0.49333333333333335, - "Spanish,Chinese,Malay": 0.48, - "Spanish,Vietnamese,Filipino": 0.42, - "Spanish,Vietnamese,Indonesian": 0.52, - "Spanish,Vietnamese,Malay": 0.52, - "Spanish,Filipino,Indonesian": 0.41333333333333333, - "Spanish,Filipino,Malay": 0.4533333333333333, - "Spanish,Indonesian,Malay": 0.5133333333333333, - "English,Chinese,Vietnamese": 0.5133333333333333, - "English,Chinese,Filipino": 0.41333333333333333, - "English,Chinese,Indonesian": 0.5533333333333333, - "English,Chinese,Malay": 0.58, - "English,Vietnamese,Filipino": 0.44, - "English,Vietnamese,Indonesian": 0.5466666666666666, - "English,Vietnamese,Malay": 0.5933333333333334, - "English,Filipino,Indonesian": 0.44, - "English,Filipino,Malay": 0.4866666666666667, - "English,Indonesian,Malay": 0.6066666666666667, - "Chinese,Vietnamese,Filipino": 0.4066666666666667, - "Chinese,Vietnamese,Indonesian": 0.5133333333333333, - "Chinese,Vietnamese,Malay": 0.5333333333333333, - "Chinese,Filipino,Indonesian": 0.3933333333333333, - "Chinese,Filipino,Malay": 0.42, - "Chinese,Indonesian,Malay": 0.5466666666666666, - "Vietnamese,Filipino,Indonesian": 0.42, - "Vietnamese,Filipino,Malay": 0.44, - "Vietnamese,Indonesian,Malay": 0.5733333333333334, - "Filipino,Indonesian,Malay": 0.46 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.43333333333333335, - "Spanish,English,Chinese,Filipino": 0.35333333333333333, - "Spanish,English,Chinese,Indonesian": 0.44666666666666666, - "Spanish,English,Chinese,Malay": 0.44666666666666666, - "Spanish,English,Vietnamese,Filipino": 0.37333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.4533333333333333, - "Spanish,English,Vietnamese,Malay": 0.48, - "Spanish,English,Filipino,Indonesian": 0.38, - "Spanish,English,Filipino,Malay": 0.42, - "Spanish,English,Indonesian,Malay": 0.48, - "Spanish,Chinese,Vietnamese,Filipino": 0.34, - "Spanish,Chinese,Vietnamese,Indonesian": 0.4266666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.43333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.3466666666666667, - "Spanish,Chinese,Filipino,Malay": 0.36, - "Spanish,Chinese,Indonesian,Malay": 0.42, - "Spanish,Vietnamese,Filipino,Indonesian": 0.37333333333333335, - "Spanish,Vietnamese,Filipino,Malay": 0.38666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.4666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.38666666666666666, - "English,Chinese,Vietnamese,Filipino": 0.36666666666666664, - "English,Chinese,Vietnamese,Indonesian": 0.4666666666666667, - "English,Chinese,Vietnamese,Malay": 0.4866666666666667, - "English,Chinese,Filipino,Indonesian": 0.36, - "English,Chinese,Filipino,Malay": 0.38666666666666666, - "English,Chinese,Indonesian,Malay": 0.5, - "English,Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "English,Vietnamese,Filipino,Malay": 0.41333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.5133333333333333, - "English,Filipino,Indonesian,Malay": 0.41333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.37333333333333335, - "Chinese,Vietnamese,Indonesian,Malay": 0.47333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.37333333333333335, - "Vietnamese,Filipino,Indonesian,Malay": 0.3933333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.32666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.4066666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.41333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.34, - "Spanish,English,Chinese,Indonesian,Malay": 0.4066666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.36666666666666664, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.43333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.36666666666666664, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.3333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.4, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.36, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.34, - "English,Chinese,Vietnamese,Filipino,Malay": 0.35333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.44666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.35333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.37333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3466666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.32, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.32, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.34, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664 - } - }, - "AC3_2": 0.5400542494989798, - "AC3_3": 0.4801223760866704, - "AC3_4": 0.4404736275067279, - "AC3_5": 0.41249999995078124, - "AC3_6": 0.39053254433017054, - "AC3_7": 0.37160342712482447 + "accuracy": 0.2523743740286652, + "category_acc": { + "agronomy": 0.25443786982248523, + "anatomy": 0.28378378378378377, + "ancient_chinese": 0.2804878048780488, + "arts": 0.26875, + "astronomy": 0.30303030303030304, + "business_ethics": 0.22966507177033493, + "chinese_civil_service_exam": 0.24375, + "chinese_driving_rule": 0.2366412213740458, + "chinese_food_culture": 0.29411764705882354, + "chinese_foreign_policy": 0.21495327102803738, + "chinese_history": 0.23219814241486067, + "chinese_literature": 0.23529411764705882, + "chinese_teacher_qualification": 0.22905027932960895, + "clinical_knowledge": 0.25738396624472576, + "college_actuarial_science": 0.2358490566037736, + "college_education": 0.24299065420560748, + "college_engineering_hydrology": 0.330188679245283, + "college_law": 0.3425925925925926, + "college_mathematics": 0.3238095238095238, + "college_medical_statistics": 0.19811320754716982, + "college_medicine": 0.23809523809523808, + "computer_science": 0.24509803921568626, + "computer_security": 0.22807017543859648, + "conceptual_physics": 0.25170068027210885, + "construction_project_management": 0.2517985611510791, + "economics": 0.22641509433962265, + "education": 0.3067484662576687, + "electrical_engineering": 0.20348837209302326, + "elementary_chinese": 0.2698412698412698, + "elementary_commonsense": 0.22727272727272727, + "elementary_information_and_technology": 0.20588235294117646, + "elementary_mathematics": 0.24347826086956523, + "ethnology": 0.22962962962962963, + "food_science": 0.22377622377622378, + "genetics": 0.2727272727272727, + "global_facts": 0.3087248322147651, + "high_school_biology": 0.2603550295857988, + "high_school_chemistry": 0.22727272727272727, + "high_school_geography": 0.2627118644067797, + "high_school_mathematics": 0.21341463414634146, + "high_school_physics": 0.24545454545454545, + "high_school_politics": 0.36363636363636365, + "human_sexuality": 0.30158730158730157, + "international_law": 0.23783783783783785, + "journalism": 0.2616279069767442, + "jurisprudence": 0.2384428223844282, + "legal_and_moral_basis": 0.22429906542056074, + "logical": 0.2764227642276423, + "machine_learning": 0.21311475409836064, + "management": 0.24285714285714285, + "marketing": 0.2611111111111111, + "marxist_theory": 0.26455026455026454, + "modern_chinese": 0.28448275862068967, + "nutrition": 0.2827586206896552, + "philosophy": 0.34285714285714286, + "professional_accounting": 0.2571428571428571, + "professional_law": 0.21800947867298578, + "professional_medicine": 0.2393617021276596, + "professional_psychology": 0.24568965517241378, + "public_relations": 0.27011494252873564, + "security_study": 0.2222222222222222, + "sociology": 0.21238938053097345, + "sports_science": 0.2727272727272727, + "traditional_chinese_medicine": 0.24864864864864866, + "virology": 0.2781065088757396, + "world_history": 0.2608695652173913, + "world_religions": 0.25 + } }, "prompt_2": { - "overall_acc": 0.479047619047619, - "language_acc": { - "Spanish": 0.44666666666666666, - "English": 0.5333333333333333, - "Chinese": 0.49333333333333335, - "Vietnamese": 0.49333333333333335, - "Filipino": 0.36, - "Indonesian": 0.49333333333333335, - "Malay": 0.5333333333333333 - }, - "consistency_score_2": 0.64, - "consistency_score_3": 0.4992380952380953, - "consistency_score_4": 0.4180952380952382, - "consistency_score_5": 0.3628571428571429, - "consistency_score_6": 0.32095238095238093, - "consistency_score_7": 0.2866666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6666666666666666, - "Spanish,Chinese": 0.6266666666666667, - "Spanish,Vietnamese": 0.5733333333333334, - "Spanish,Filipino": 0.47333333333333333, - "Spanish,Indonesian": 0.66, - "Spanish,Malay": 0.6266666666666667, - "English,Chinese": 0.76, - "English,Vietnamese": 0.6866666666666666, - "English,Filipino": 0.5733333333333334, - "English,Indonesian": 0.76, - "English,Malay": 0.7133333333333334, - "Chinese,Vietnamese": 0.6466666666666666, - "Chinese,Filipino": 0.5133333333333333, - "Chinese,Indonesian": 0.7466666666666667, - "Chinese,Malay": 0.7133333333333334, - "Vietnamese,Filipino": 0.5, - "Vietnamese,Indonesian": 0.68, - "Vietnamese,Malay": 0.6866666666666666, - "Filipino,Indonesian": 0.5466666666666666, - "Filipino,Malay": 0.5333333333333333, - "Indonesian,Malay": 0.7533333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5466666666666666, - "Spanish,English,Vietnamese": 0.5, - "Spanish,English,Filipino": 0.4, - "Spanish,English,Indonesian": 0.5666666666666667, - "Spanish,English,Malay": 0.5466666666666666, - "Spanish,Chinese,Vietnamese": 0.47333333333333333, - "Spanish,Chinese,Filipino": 0.36666666666666664, - "Spanish,Chinese,Indonesian": 0.54, - "Spanish,Chinese,Malay": 0.52, - "Spanish,Vietnamese,Filipino": 0.34, - "Spanish,Vietnamese,Indonesian": 0.5, - "Spanish,Vietnamese,Malay": 0.49333333333333335, - "Spanish,Filipino,Indonesian": 0.38666666666666666, - "Spanish,Filipino,Malay": 0.37333333333333335, - "Spanish,Indonesian,Malay": 0.54, - "English,Chinese,Vietnamese": 0.5866666666666667, - "English,Chinese,Filipino": 0.4533333333333333, - "English,Chinese,Indonesian": 0.6533333333333333, - "English,Chinese,Malay": 0.6133333333333333, - "English,Vietnamese,Filipino": 0.42, - "English,Vietnamese,Indonesian": 0.5866666666666667, - "English,Vietnamese,Malay": 0.5866666666666667, - "English,Filipino,Indonesian": 0.4666666666666667, - "English,Filipino,Malay": 0.4533333333333333, - "English,Indonesian,Malay": 0.6333333333333333, - "Chinese,Vietnamese,Filipino": 0.41333333333333333, - "Chinese,Vietnamese,Indonesian": 0.5733333333333334, - "Chinese,Vietnamese,Malay": 0.5666666666666667, - "Chinese,Filipino,Indonesian": 0.44, - "Chinese,Filipino,Malay": 0.43333333333333335, - "Chinese,Indonesian,Malay": 0.62, - "Vietnamese,Filipino,Indonesian": 0.41333333333333333, - "Vietnamese,Filipino,Malay": 0.41333333333333333, - "Vietnamese,Indonesian,Malay": 0.6, - "Filipino,Indonesian,Malay": 0.4533333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.46, - "Spanish,English,Chinese,Filipino": 0.34, - "Spanish,English,Chinese,Indonesian": 0.49333333333333335, - "Spanish,English,Chinese,Malay": 0.49333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.32, - "Spanish,English,Vietnamese,Indonesian": 0.4666666666666667, - "Spanish,English,Vietnamese,Malay": 0.4666666666666667, - "Spanish,English,Filipino,Indonesian": 0.36666666666666664, - "Spanish,English,Filipino,Malay": 0.3466666666666667, - "Spanish,English,Indonesian,Malay": 0.5, - "Spanish,Chinese,Vietnamese,Filipino": 0.32, - "Spanish,Chinese,Vietnamese,Indonesian": 0.44666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.44, - "Spanish,Chinese,Filipino,Indonesian": 0.34, - "Spanish,Chinese,Filipino,Malay": 0.32666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.47333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.46, - "Spanish,Filipino,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese,Filipino": 0.38, - "English,Chinese,Vietnamese,Indonesian": 0.5333333333333333, - "English,Chinese,Vietnamese,Malay": 0.52, - "English,Chinese,Filipino,Indonesian": 0.41333333333333333, - "English,Chinese,Filipino,Malay": 0.4066666666666667, - "English,Chinese,Indonesian,Malay": 0.5533333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.37333333333333335, - "English,Vietnamese,Filipino,Malay": 0.38, - "English,Vietnamese,Indonesian,Malay": 0.5466666666666666, - "English,Filipino,Indonesian,Malay": 0.41333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.38, - "Chinese,Vietnamese,Indonesian,Malay": 0.5266666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.4, - "Vietnamese,Filipino,Indonesian,Malay": 0.37333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.43333333333333335, - "Spanish,English,Chinese,Vietnamese,Malay": 0.43333333333333335, - "Spanish,English,Chinese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.32, - "Spanish,English,Chinese,Indonesian,Malay": 0.44666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,English,Vietnamese,Filipino,Malay": 0.30666666666666664, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.44666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.3333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.3, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.42, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.36, - "English,Chinese,Vietnamese,Filipino,Malay": 0.35333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.49333333333333335, - "English,Chinese,Filipino,Indonesian,Malay": 0.38, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.35333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.36 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.41333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.30666666666666664, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.3, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.34 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667 - } - }, - "AC3_2": 0.5479489361212471, - "AC3_3": 0.4889345052163023, - "AC3_4": 0.4465008593173322, - "AC3_5": 0.4129347122974293, - "AC3_6": 0.3843786847592089, - "AC3_7": 0.3586898838669207 + "accuracy": 0.2562597133482991, + "category_acc": { + "agronomy": 0.2485207100591716, + "anatomy": 0.28378378378378377, + "ancient_chinese": 0.24390243902439024, + "arts": 0.3125, + "astronomy": 0.23636363636363636, + "business_ethics": 0.23923444976076555, + "chinese_civil_service_exam": 0.26875, + "chinese_driving_rule": 0.21374045801526717, + "chinese_food_culture": 0.2647058823529412, + "chinese_foreign_policy": 0.2616822429906542, + "chinese_history": 0.25386996904024767, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.2681564245810056, + "clinical_knowledge": 0.2742616033755274, + "college_actuarial_science": 0.18867924528301888, + "college_education": 0.18691588785046728, + "college_engineering_hydrology": 0.3113207547169811, + "college_law": 0.32407407407407407, + "college_mathematics": 0.2761904761904762, + "college_medical_statistics": 0.19811320754716982, + "college_medicine": 0.2454212454212454, + "computer_science": 0.27941176470588236, + "computer_security": 0.21052631578947367, + "conceptual_physics": 0.25170068027210885, + "construction_project_management": 0.26618705035971224, + "economics": 0.2389937106918239, + "education": 0.24539877300613497, + "electrical_engineering": 0.20348837209302326, + "elementary_chinese": 0.23015873015873015, + "elementary_commonsense": 0.24242424242424243, + "elementary_information_and_technology": 0.23109243697478993, + "elementary_mathematics": 0.24782608695652175, + "ethnology": 0.31851851851851853, + "food_science": 0.27972027972027974, + "genetics": 0.26136363636363635, + "global_facts": 0.26174496644295303, + "high_school_biology": 0.21301775147928995, + "high_school_chemistry": 0.25, + "high_school_geography": 0.2966101694915254, + "high_school_mathematics": 0.22560975609756098, + "high_school_physics": 0.23636363636363636, + "high_school_politics": 0.36363636363636365, + "human_sexuality": 0.2857142857142857, + "international_law": 0.20540540540540542, + "journalism": 0.26744186046511625, + "jurisprudence": 0.2384428223844282, + "legal_and_moral_basis": 0.26635514018691586, + "logical": 0.25203252032520324, + "machine_learning": 0.23770491803278687, + "management": 0.24285714285714285, + "marketing": 0.3, + "marxist_theory": 0.25925925925925924, + "modern_chinese": 0.2672413793103448, + "nutrition": 0.27586206896551724, + "philosophy": 0.29523809523809524, + "professional_accounting": 0.29714285714285715, + "professional_law": 0.21800947867298578, + "professional_medicine": 0.3058510638297872, + "professional_psychology": 0.27155172413793105, + "public_relations": 0.22413793103448276, + "security_study": 0.25925925925925924, + "sociology": 0.2610619469026549, + "sports_science": 0.2606060606060606, + "traditional_chinese_medicine": 0.23783783783783785, + "virology": 0.2603550295857988, + "world_history": 0.2919254658385093, + "world_religions": 0.225 + } }, "prompt_3": { - "overall_acc": 0.46761904761904766, - "language_acc": { - "Spanish": 0.43333333333333335, - "English": 0.5533333333333333, - "Chinese": 0.5, - "Vietnamese": 0.4666666666666667, - "Filipino": 0.30666666666666664, - "Indonesian": 0.5, - "Malay": 0.5133333333333333 - }, - "consistency_score_2": 0.6279365079365079, - "consistency_score_3": 0.48438095238095236, - "consistency_score_4": 0.40380952380952373, - "consistency_score_5": 0.35015873015873017, - "consistency_score_6": 0.31047619047619046, - "consistency_score_7": 0.28, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6266666666666667, - "Spanish,Chinese": 0.6333333333333333, - "Spanish,Vietnamese": 0.6066666666666667, - "Spanish,Filipino": 0.5133333333333333, - "Spanish,Indonesian": 0.66, - "Spanish,Malay": 0.6066666666666667, - "English,Chinese": 0.72, - "English,Vietnamese": 0.66, - "English,Filipino": 0.5533333333333333, - "English,Indonesian": 0.7, - "English,Malay": 0.7133333333333334, - "Chinese,Vietnamese": 0.6266666666666667, - "Chinese,Filipino": 0.5533333333333333, - "Chinese,Indonesian": 0.68, - "Chinese,Malay": 0.6933333333333334, - "Vietnamese,Filipino": 0.47333333333333333, - "Vietnamese,Indonesian": 0.6666666666666666, - "Vietnamese,Malay": 0.68, - "Filipino,Indonesian": 0.5533333333333333, - "Filipino,Malay": 0.5266666666666666, - "Indonesian,Malay": 0.74 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5066666666666667, - "Spanish,English,Vietnamese": 0.4866666666666667, - "Spanish,English,Filipino": 0.3933333333333333, - "Spanish,English,Indonesian": 0.5066666666666667, - "Spanish,English,Malay": 0.52, - "Spanish,Chinese,Vietnamese": 0.49333333333333335, - "Spanish,Chinese,Filipino": 0.4066666666666667, - "Spanish,Chinese,Indonesian": 0.5333333333333333, - "Spanish,Chinese,Malay": 0.5133333333333333, - "Spanish,Vietnamese,Filipino": 0.35333333333333333, - "Spanish,Vietnamese,Indonesian": 0.5133333333333333, - "Spanish,Vietnamese,Malay": 0.4866666666666667, - "Spanish,Filipino,Indonesian": 0.3933333333333333, - "Spanish,Filipino,Malay": 0.38666666666666666, - "Spanish,Indonesian,Malay": 0.54, - "English,Chinese,Vietnamese": 0.5333333333333333, - "English,Chinese,Filipino": 0.4533333333333333, - "English,Chinese,Indonesian": 0.5866666666666667, - "English,Chinese,Malay": 0.6066666666666667, - "English,Vietnamese,Filipino": 0.38666666666666666, - "English,Vietnamese,Indonesian": 0.54, - "English,Vietnamese,Malay": 0.5666666666666667, - "English,Filipino,Indonesian": 0.43333333333333335, - "English,Filipino,Malay": 0.44, - "English,Indonesian,Malay": 0.6066666666666667, - "Chinese,Vietnamese,Filipino": 0.4, - "Chinese,Vietnamese,Indonesian": 0.52, - "Chinese,Vietnamese,Malay": 0.5466666666666666, - "Chinese,Filipino,Indonesian": 0.43333333333333335, - "Chinese,Filipino,Malay": 0.44, - "Chinese,Indonesian,Malay": 0.5866666666666667, - "Vietnamese,Filipino,Indonesian": 0.3933333333333333, - "Vietnamese,Filipino,Malay": 0.41333333333333333, - "Vietnamese,Indonesian,Malay": 0.5866666666666667, - "Filipino,Indonesian,Malay": 0.44666666666666666 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.43333333333333335, - "Spanish,English,Chinese,Filipino": 0.34, - "Spanish,English,Chinese,Indonesian": 0.46, - "Spanish,English,Chinese,Malay": 0.4666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.3, - "Spanish,English,Vietnamese,Indonesian": 0.44, - "Spanish,English,Vietnamese,Malay": 0.4533333333333333, - "Spanish,English,Filipino,Indonesian": 0.3333333333333333, - "Spanish,English,Filipino,Malay": 0.34, - "Spanish,English,Indonesian,Malay": 0.48, - "Spanish,Chinese,Vietnamese,Filipino": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian": 0.44666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.44666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.36, - "Spanish,Chinese,Filipino,Malay": 0.35333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.47333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.46, - "Spanish,Filipino,Indonesian,Malay": 0.36, - "English,Chinese,Vietnamese,Filipino": 0.35333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.4666666666666667, - "English,Chinese,Vietnamese,Malay": 0.5, - "English,Chinese,Filipino,Indonesian": 0.38666666666666666, - "English,Chinese,Filipino,Malay": 0.4, - "English,Chinese,Indonesian,Malay": 0.5266666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.34, - "English,Vietnamese,Filipino,Malay": 0.36666666666666664, - "English,Vietnamese,Indonesian,Malay": 0.5066666666666667, - "English,Filipino,Indonesian,Malay": 0.3933333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.36, - "Chinese,Vietnamese,Filipino,Malay": 0.37333333333333335, - "Chinese,Vietnamese,Indonesian,Malay": 0.4866666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.3933333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.36666666666666664 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.4066666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.42, - "Spanish,English,Chinese,Filipino,Indonesian": 0.32, - "Spanish,English,Chinese,Filipino,Malay": 0.32, - "Spanish,English,Chinese,Indonesian,Malay": 0.43333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.42, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.34, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.32, - "English,Chinese,Vietnamese,Filipino,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.44666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.36, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.34 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.28, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.2866666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.3933333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.31333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.31333333333333335 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.28 - } - }, - "AC3_2": 0.5360477983807874, - "AC3_3": 0.4758524361625958, - "AC3_4": 0.43337808998409577, - "AC3_5": 0.4004532682148588, - "AC3_6": 0.3731794602306428, - "AC3_7": 0.35026751587671584 + "accuracy": 0.2538421688827491, + "category_acc": { + "agronomy": 0.20710059171597633, + "anatomy": 0.31756756756756754, + "ancient_chinese": 0.23170731707317074, + "arts": 0.2875, + "astronomy": 0.24242424242424243, + "business_ethics": 0.22488038277511962, + "chinese_civil_service_exam": 0.21875, + "chinese_driving_rule": 0.2366412213740458, + "chinese_food_culture": 0.23529411764705882, + "chinese_foreign_policy": 0.2523364485981308, + "chinese_history": 0.24148606811145512, + "chinese_literature": 0.27450980392156865, + "chinese_teacher_qualification": 0.26256983240223464, + "clinical_knowledge": 0.22784810126582278, + "college_actuarial_science": 0.24528301886792453, + "college_education": 0.22429906542056074, + "college_engineering_hydrology": 0.3490566037735849, + "college_law": 0.3148148148148148, + "college_mathematics": 0.3142857142857143, + "college_medical_statistics": 0.19811320754716982, + "college_medicine": 0.27472527472527475, + "computer_science": 0.24509803921568626, + "computer_security": 0.29239766081871343, + "conceptual_physics": 0.2653061224489796, + "construction_project_management": 0.2158273381294964, + "economics": 0.2138364779874214, + "education": 0.2392638036809816, + "electrical_engineering": 0.22674418604651161, + "elementary_chinese": 0.24603174603174602, + "elementary_commonsense": 0.23737373737373738, + "elementary_information_and_technology": 0.23949579831932774, + "elementary_mathematics": 0.27391304347826084, + "ethnology": 0.21481481481481482, + "food_science": 0.2727272727272727, + "genetics": 0.30113636363636365, + "global_facts": 0.28859060402684567, + "high_school_biology": 0.2781065088757396, + "high_school_chemistry": 0.26515151515151514, + "high_school_geography": 0.2711864406779661, + "high_school_mathematics": 0.2682926829268293, + "high_school_physics": 0.2545454545454545, + "high_school_politics": 0.27972027972027974, + "human_sexuality": 0.23809523809523808, + "international_law": 0.24864864864864866, + "journalism": 0.26744186046511625, + "jurisprudence": 0.24087591240875914, + "legal_and_moral_basis": 0.21962616822429906, + "logical": 0.3170731707317073, + "machine_learning": 0.20491803278688525, + "management": 0.26666666666666666, + "marketing": 0.2111111111111111, + "marxist_theory": 0.24867724867724866, + "modern_chinese": 0.23275862068965517, + "nutrition": 0.30344827586206896, + "philosophy": 0.3047619047619048, + "professional_accounting": 0.26857142857142857, + "professional_law": 0.2559241706161137, + "professional_medicine": 0.26063829787234044, + "professional_psychology": 0.23275862068965517, + "public_relations": 0.25287356321839083, + "security_study": 0.25925925925925924, + "sociology": 0.24778761061946902, + "sports_science": 0.2909090909090909, + "traditional_chinese_medicine": 0.25405405405405407, + "virology": 0.23076923076923078, + "world_history": 0.2795031055900621, + "world_religions": 0.2125 + } }, "prompt_4": { - "overall_acc": 0.4761904761904762, - "language_acc": { - "Spanish": 0.4866666666666667, - "English": 0.54, - "Chinese": 0.47333333333333333, - "Vietnamese": 0.46, - "Filipino": 0.36666666666666664, - "Indonesian": 0.5, - "Malay": 0.5066666666666667 - }, - "consistency_score_2": 0.6482539682539681, - "consistency_score_3": 0.5087619047619047, - "consistency_score_4": 0.43200000000000005, - "consistency_score_5": 0.3822222222222222, - "consistency_score_6": 0.3447619047619047, - "consistency_score_7": 0.31333333333333335, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6933333333333334, - "Spanish,Chinese": 0.6533333333333333, - "Spanish,Vietnamese": 0.6666666666666666, - "Spanish,Filipino": 0.5333333333333333, - "Spanish,Indonesian": 0.7, - "Spanish,Malay": 0.6533333333333333, - "English,Chinese": 0.7533333333333333, - "English,Vietnamese": 0.6666666666666666, - "English,Filipino": 0.5466666666666666, - "English,Indonesian": 0.7266666666666667, - "English,Malay": 0.7533333333333333, - "Chinese,Vietnamese": 0.64, - "Chinese,Filipino": 0.5133333333333333, - "Chinese,Indonesian": 0.7, - "Chinese,Malay": 0.72, - "Vietnamese,Filipino": 0.5333333333333333, - "Vietnamese,Indonesian": 0.6466666666666666, - "Vietnamese,Malay": 0.6666666666666666, - "Filipino,Indonesian": 0.5466666666666666, - "Filipino,Malay": 0.5866666666666667, - "Indonesian,Malay": 0.7133333333333334 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5666666666666667, - "Spanish,English,Vietnamese": 0.5533333333333333, - "Spanish,English,Filipino": 0.44, - "Spanish,English,Indonesian": 0.5866666666666667, - "Spanish,English,Malay": 0.58, - "Spanish,Chinese,Vietnamese": 0.5133333333333333, - "Spanish,Chinese,Filipino": 0.4066666666666667, - "Spanish,Chinese,Indonesian": 0.5466666666666666, - "Spanish,Chinese,Malay": 0.54, - "Spanish,Vietnamese,Filipino": 0.4066666666666667, - "Spanish,Vietnamese,Indonesian": 0.5466666666666666, - "Spanish,Vietnamese,Malay": 0.54, - "Spanish,Filipino,Indonesian": 0.44, - "Spanish,Filipino,Malay": 0.44666666666666666, - "Spanish,Indonesian,Malay": 0.5533333333333333, - "English,Chinese,Vietnamese": 0.56, - "English,Chinese,Filipino": 0.44, - "English,Chinese,Indonesian": 0.6, - "English,Chinese,Malay": 0.6333333333333333, - "English,Vietnamese,Filipino": 0.42, - "English,Vietnamese,Indonesian": 0.5466666666666666, - "English,Vietnamese,Malay": 0.5733333333333334, - "English,Filipino,Indonesian": 0.44, - "English,Filipino,Malay": 0.4866666666666667, - "English,Indonesian,Malay": 0.62, - "Chinese,Vietnamese,Filipino": 0.4066666666666667, - "Chinese,Vietnamese,Indonesian": 0.52, - "Chinese,Vietnamese,Malay": 0.5466666666666666, - "Chinese,Filipino,Indonesian": 0.41333333333333333, - "Chinese,Filipino,Malay": 0.4533333333333333, - "Chinese,Indonesian,Malay": 0.5866666666666667, - "Vietnamese,Filipino,Indonesian": 0.42, - "Vietnamese,Filipino,Malay": 0.4533333333333333, - "Vietnamese,Indonesian,Malay": 0.5533333333333333, - "Filipino,Indonesian,Malay": 0.4666666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.48, - "Spanish,English,Chinese,Filipino": 0.37333333333333335, - "Spanish,English,Chinese,Indonesian": 0.5066666666666667, - "Spanish,English,Chinese,Malay": 0.5133333333333333, - "Spanish,English,Vietnamese,Filipino": 0.36, - "Spanish,English,Vietnamese,Indonesian": 0.4866666666666667, - "Spanish,English,Vietnamese,Malay": 0.5, - "Spanish,English,Filipino,Indonesian": 0.3933333333333333, - "Spanish,English,Filipino,Malay": 0.4066666666666667, - "Spanish,English,Indonesian,Malay": 0.5266666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.35333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.46, - "Spanish,Chinese,Vietnamese,Malay": 0.47333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.36, - "Spanish,Chinese,Filipino,Malay": 0.38, - "Spanish,Chinese,Indonesian,Malay": 0.4866666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.36666666666666664, - "Spanish,Vietnamese,Filipino,Malay": 0.37333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.48, - "Spanish,Filipino,Indonesian,Malay": 0.3933333333333333, - "English,Chinese,Vietnamese,Filipino": 0.38, - "English,Chinese,Vietnamese,Indonesian": 0.48, - "English,Chinese,Vietnamese,Malay": 0.5, - "English,Chinese,Filipino,Indonesian": 0.37333333333333335, - "English,Chinese,Filipino,Malay": 0.42, - "English,Chinese,Indonesian,Malay": 0.54, - "English,Vietnamese,Filipino,Indonesian": 0.38, - "English,Vietnamese,Filipino,Malay": 0.4, - "English,Vietnamese,Indonesian,Malay": 0.52, - "English,Filipino,Indonesian,Malay": 0.42, - "Chinese,Vietnamese,Filipino,Indonesian": 0.36, - "Chinese,Vietnamese,Filipino,Malay": 0.38666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.49333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.4, - "Vietnamese,Filipino,Indonesian,Malay": 0.3933333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.44, - "Spanish,English,Chinese,Vietnamese,Malay": 0.4533333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.34, - "Spanish,English,Chinese,Filipino,Malay": 0.36666666666666664, - "Spanish,English,Chinese,Indonesian,Malay": 0.47333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.34, - "Spanish,English,Vietnamese,Filipino,Malay": 0.3466666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.4666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.37333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.34, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.44, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.35333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.36666666666666664, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.4666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.37333333333333335, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.37333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.36 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.32666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.34, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.3333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3466666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.31333333333333335 - } - }, - "AC3_2": 0.5490575676422788, - "AC3_3": 0.491937637507495, - "AC3_4": 0.45302013417830633, - "AC3_5": 0.42406311632140786, - "AC3_6": 0.3999558059395701, - "AC3_7": 0.3779654201370893 + "accuracy": 0.25962700742531514, + "category_acc": { + "agronomy": 0.2781065088757396, + "anatomy": 0.25675675675675674, + "ancient_chinese": 0.2804878048780488, + "arts": 0.3, + "astronomy": 0.23636363636363636, + "business_ethics": 0.22009569377990432, + "chinese_civil_service_exam": 0.2375, + "chinese_driving_rule": 0.2366412213740458, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.2616822429906542, + "chinese_history": 0.24458204334365324, + "chinese_literature": 0.24019607843137256, + "chinese_teacher_qualification": 0.26256983240223464, + "clinical_knowledge": 0.21518987341772153, + "college_actuarial_science": 0.25471698113207547, + "college_education": 0.22429906542056074, + "college_engineering_hydrology": 0.330188679245283, + "college_law": 0.3148148148148148, + "college_mathematics": 0.3047619047619048, + "college_medical_statistics": 0.2358490566037736, + "college_medicine": 0.30036630036630035, + "computer_science": 0.2549019607843137, + "computer_security": 0.28654970760233917, + "conceptual_physics": 0.19047619047619047, + "construction_project_management": 0.26618705035971224, + "economics": 0.1761006289308176, + "education": 0.25153374233128833, + "electrical_engineering": 0.1744186046511628, + "elementary_chinese": 0.2976190476190476, + "elementary_commonsense": 0.23232323232323232, + "elementary_information_and_technology": 0.2647058823529412, + "elementary_mathematics": 0.2782608695652174, + "ethnology": 0.23703703703703705, + "food_science": 0.24475524475524477, + "genetics": 0.26704545454545453, + "global_facts": 0.2751677852348993, + "high_school_biology": 0.26627218934911245, + "high_school_chemistry": 0.30303030303030304, + "high_school_geography": 0.3220338983050847, + "high_school_mathematics": 0.25, + "high_school_physics": 0.24545454545454545, + "high_school_politics": 0.3076923076923077, + "human_sexuality": 0.2857142857142857, + "international_law": 0.24324324324324326, + "journalism": 0.28488372093023256, + "jurisprudence": 0.2749391727493917, + "legal_and_moral_basis": 0.2616822429906542, + "logical": 0.2682926829268293, + "machine_learning": 0.2786885245901639, + "management": 0.2619047619047619, + "marketing": 0.26666666666666666, + "marxist_theory": 0.2698412698412698, + "modern_chinese": 0.2413793103448276, + "nutrition": 0.27586206896551724, + "philosophy": 0.2857142857142857, + "professional_accounting": 0.26857142857142857, + "professional_law": 0.27014218009478674, + "professional_medicine": 0.2473404255319149, + "professional_psychology": 0.22413793103448276, + "public_relations": 0.26436781609195403, + "security_study": 0.18518518518518517, + "sociology": 0.23008849557522124, + "sports_science": 0.3333333333333333, + "traditional_chinese_medicine": 0.23783783783783785, + "virology": 0.27218934911242604, + "world_history": 0.2857142857142857, + "world_religions": 0.275 + } }, "prompt_5": { - "overall_acc": 0.4638095238095238, - "language_acc": { - "Spanish": 0.4533333333333333, - "English": 0.56, - "Chinese": 0.47333333333333333, - "Vietnamese": 0.46, - "Filipino": 0.3333333333333333, - "Indonesian": 0.49333333333333335, - "Malay": 0.47333333333333333 - }, - "consistency_score_2": 0.6333333333333334, - "consistency_score_3": 0.4916190476190476, - "consistency_score_4": 0.4133333333333333, - "consistency_score_5": 0.36063492063492053, - "consistency_score_6": 0.32, - "consistency_score_7": 0.2866666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6333333333333333, - "Spanish,Chinese": 0.5933333333333334, - "Spanish,Vietnamese": 0.6266666666666667, - "Spanish,Filipino": 0.49333333333333335, - "Spanish,Indonesian": 0.64, - "Spanish,Malay": 0.62, - "English,Chinese": 0.7133333333333334, - "English,Vietnamese": 0.68, - "English,Filipino": 0.5266666666666666, - "English,Indonesian": 0.7133333333333334, - "English,Malay": 0.7733333333333333, - "Chinese,Vietnamese": 0.66, - "Chinese,Filipino": 0.5333333333333333, - "Chinese,Indonesian": 0.7066666666666667, - "Chinese,Malay": 0.6866666666666666, - "Vietnamese,Filipino": 0.5133333333333333, - "Vietnamese,Indonesian": 0.68, - "Vietnamese,Malay": 0.7, - "Filipino,Indonesian": 0.5133333333333333, - "Filipino,Malay": 0.5333333333333333, - "Indonesian,Malay": 0.76 - }, - "3_combine": { - "Spanish,English,Chinese": 0.49333333333333335, - "Spanish,English,Vietnamese": 0.52, - "Spanish,English,Filipino": 0.37333333333333335, - "Spanish,English,Indonesian": 0.5333333333333333, - "Spanish,English,Malay": 0.56, - "Spanish,Chinese,Vietnamese": 0.4866666666666667, - "Spanish,Chinese,Filipino": 0.38, - "Spanish,Chinese,Indonesian": 0.5, - "Spanish,Chinese,Malay": 0.5, - "Spanish,Vietnamese,Filipino": 0.38, - "Spanish,Vietnamese,Indonesian": 0.5066666666666667, - "Spanish,Vietnamese,Malay": 0.5266666666666666, - "Spanish,Filipino,Indonesian": 0.38, - "Spanish,Filipino,Malay": 0.4, - "Spanish,Indonesian,Malay": 0.54, - "English,Chinese,Vietnamese": 0.5466666666666666, - "English,Chinese,Filipino": 0.43333333333333335, - "English,Chinese,Indonesian": 0.5866666666666667, - "English,Chinese,Malay": 0.62, - "English,Vietnamese,Filipino": 0.4, - "English,Vietnamese,Indonesian": 0.56, - "English,Vietnamese,Malay": 0.6, - "English,Filipino,Indonesian": 0.41333333333333333, - "English,Filipino,Malay": 0.4533333333333333, - "English,Indonesian,Malay": 0.64, - "Chinese,Vietnamese,Filipino": 0.41333333333333333, - "Chinese,Vietnamese,Indonesian": 0.5466666666666666, - "Chinese,Vietnamese,Malay": 0.5533333333333333, - "Chinese,Filipino,Indonesian": 0.41333333333333333, - "Chinese,Filipino,Malay": 0.43333333333333335, - "Chinese,Indonesian,Malay": 0.6066666666666667, - "Vietnamese,Filipino,Indonesian": 0.41333333333333333, - "Vietnamese,Filipino,Malay": 0.42, - "Vietnamese,Indonesian,Malay": 0.6066666666666667, - "Filipino,Indonesian,Malay": 0.4666666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.43333333333333335, - "Spanish,English,Chinese,Filipino": 0.3333333333333333, - "Spanish,English,Chinese,Indonesian": 0.4533333333333333, - "Spanish,English,Chinese,Malay": 0.4666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.31333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.4533333333333333, - "Spanish,English,Vietnamese,Malay": 0.4866666666666667, - "Spanish,English,Filipino,Indonesian": 0.34, - "Spanish,English,Filipino,Malay": 0.36, - "Spanish,English,Indonesian,Malay": 0.5066666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.43333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.4533333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.34, - "Spanish,Chinese,Filipino,Malay": 0.35333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.4666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.3466666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.47333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.37333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.36, - "English,Chinese,Vietnamese,Indonesian": 0.49333333333333335, - "English,Chinese,Vietnamese,Malay": 0.5, - "English,Chinese,Filipino,Indonesian": 0.36666666666666664, - "English,Chinese,Filipino,Malay": 0.3933333333333333, - "English,Chinese,Indonesian,Malay": 0.5533333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.36, - "English,Vietnamese,Filipino,Malay": 0.38, - "English,Vietnamese,Indonesian,Malay": 0.54, - "English,Filipino,Indonesian,Malay": 0.4066666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.36666666666666664, - "Chinese,Vietnamese,Filipino,Malay": 0.37333333333333335, - "Chinese,Vietnamese,Indonesian,Malay": 0.52, - "Chinese,Filipino,Indonesian,Malay": 0.4066666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.3933333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.29333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.4066666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.42, - "Spanish,English,Chinese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,English,Chinese,Filipino,Malay": 0.32666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.44, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,English,Vietnamese,Filipino,Malay": 0.31333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.44666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.34, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.32, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.34, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.34, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.48, - "English,Chinese,Filipino,Indonesian,Malay": 0.36666666666666664, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.36, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.36666666666666664 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.4, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.31333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.3, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667 - } - }, - "AC3_2": 0.5354745369882308, - "AC3_3": 0.4773095617332883, - "AC3_4": 0.43711907342103057, - "AC3_5": 0.4057663604033198, - "AC3_6": 0.37871202911328705, - "AC3_7": 0.3543316412387418 + "accuracy": 0.25461923674667586, + "category_acc": { + "agronomy": 0.28402366863905326, + "anatomy": 0.2972972972972973, + "ancient_chinese": 0.2621951219512195, + "arts": 0.25625, + "astronomy": 0.20606060606060606, + "business_ethics": 0.24880382775119617, + "chinese_civil_service_exam": 0.275, + "chinese_driving_rule": 0.2366412213740458, + "chinese_food_culture": 0.2647058823529412, + "chinese_foreign_policy": 0.2523364485981308, + "chinese_history": 0.23219814241486067, + "chinese_literature": 0.23039215686274508, + "chinese_teacher_qualification": 0.2346368715083799, + "clinical_knowledge": 0.24050632911392406, + "college_actuarial_science": 0.2169811320754717, + "college_education": 0.19626168224299065, + "college_engineering_hydrology": 0.2830188679245283, + "college_law": 0.37962962962962965, + "college_mathematics": 0.24761904761904763, + "college_medical_statistics": 0.19811320754716982, + "college_medicine": 0.23809523809523808, + "computer_science": 0.24509803921568626, + "computer_security": 0.2573099415204678, + "conceptual_physics": 0.2857142857142857, + "construction_project_management": 0.2446043165467626, + "economics": 0.2389937106918239, + "education": 0.26993865030674846, + "electrical_engineering": 0.21511627906976744, + "elementary_chinese": 0.26587301587301587, + "elementary_commonsense": 0.23232323232323232, + "elementary_information_and_technology": 0.24369747899159663, + "elementary_mathematics": 0.20434782608695654, + "ethnology": 0.2518518518518518, + "food_science": 0.25874125874125875, + "genetics": 0.26136363636363635, + "global_facts": 0.3288590604026846, + "high_school_biology": 0.3254437869822485, + "high_school_chemistry": 0.2727272727272727, + "high_school_geography": 0.3050847457627119, + "high_school_mathematics": 0.25609756097560976, + "high_school_physics": 0.21818181818181817, + "high_school_politics": 0.3076923076923077, + "human_sexuality": 0.30952380952380953, + "international_law": 0.2702702702702703, + "journalism": 0.20930232558139536, + "jurisprudence": 0.2384428223844282, + "legal_and_moral_basis": 0.24766355140186916, + "logical": 0.3333333333333333, + "machine_learning": 0.2540983606557377, + "management": 0.2714285714285714, + "marketing": 0.3111111111111111, + "marxist_theory": 0.2804232804232804, + "modern_chinese": 0.29310344827586204, + "nutrition": 0.22758620689655173, + "philosophy": 0.29523809523809524, + "professional_accounting": 0.25142857142857145, + "professional_law": 0.22748815165876776, + "professional_medicine": 0.2579787234042553, + "professional_psychology": 0.2025862068965517, + "public_relations": 0.2471264367816092, + "security_study": 0.2518518518518518, + "sociology": 0.23008849557522124, + "sports_science": 0.32727272727272727, + "traditional_chinese_medicine": 0.21621621621621623, + "virology": 0.2485207100591716, + "world_history": 0.22981366459627328, + "world_religions": 0.25625 + } } }, - "cross_logiqa": { + "zbench": { "prompt_1": { - "overall_acc": 0.3668831168831169, - "language_acc": { - "English": 0.3806818181818182, - "Filipino": 0.2840909090909091, - "Spanish": 0.3465909090909091, - "Malay": 0.38636363636363635, - "Chinese": 0.4602272727272727, - "Indonesian": 0.375, - "Vietnamese": 0.3352272727272727 - }, - "consistency_score_2": 0.5286796536796539, - "consistency_score_3": 0.34448051948051944, - "consistency_score_4": 0.2423701298701299, - "consistency_score_5": 0.1774891774891775, - "consistency_score_6": 0.1314935064935065, - "consistency_score_7": 0.09659090909090909, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4715909090909091, - "English,Spanish": 0.6079545454545454, - "English,Malay": 0.5681818181818182, - "English,Chinese": 0.5511363636363636, - "English,Indonesian": 0.5625, - "English,Vietnamese": 0.5965909090909091, - "Filipino,Spanish": 0.5, - "Filipino,Malay": 0.4659090909090909, - "Filipino,Chinese": 0.3522727272727273, - "Filipino,Indonesian": 0.4431818181818182, - "Filipino,Vietnamese": 0.4431818181818182, - "Spanish,Malay": 0.5909090909090909, - "Spanish,Chinese": 0.5, - "Spanish,Indonesian": 0.5852272727272727, - "Spanish,Vietnamese": 0.6022727272727273, - "Malay,Chinese": 0.5284090909090909, - "Malay,Indonesian": 0.6534090909090909, - "Malay,Vietnamese": 0.5681818181818182, - "Chinese,Indonesian": 0.5227272727272727, - "Chinese,Vietnamese": 0.4659090909090909, - "Indonesian,Vietnamese": 0.5227272727272727 - }, - "3_combine": { - "English,Filipino,Spanish": 0.3409090909090909, - "English,Filipino,Malay": 0.3125, - "English,Filipino,Chinese": 0.24431818181818182, - "English,Filipino,Indonesian": 0.3068181818181818, - "English,Filipino,Vietnamese": 0.3068181818181818, - "English,Spanish,Malay": 0.4147727272727273, - "English,Spanish,Chinese": 0.39204545454545453, - "English,Spanish,Indonesian": 0.4090909090909091, - "English,Spanish,Vietnamese": 0.4375, - "English,Malay,Chinese": 0.375, - "English,Malay,Indonesian": 0.4318181818181818, - "English,Malay,Vietnamese": 0.39204545454545453, - "English,Chinese,Indonesian": 0.3693181818181818, - "English,Chinese,Vietnamese": 0.3409090909090909, - "English,Indonesian,Vietnamese": 0.3806818181818182, - "Filipino,Spanish,Malay": 0.3409090909090909, - "Filipino,Spanish,Chinese": 0.24431818181818182, - "Filipino,Spanish,Indonesian": 0.32954545454545453, - "Filipino,Spanish,Vietnamese": 0.32386363636363635, - "Filipino,Malay,Chinese": 0.23863636363636365, - "Filipino,Malay,Indonesian": 0.3409090909090909, - "Filipino,Malay,Vietnamese": 0.30113636363636365, - "Filipino,Chinese,Indonesian": 0.23863636363636365, - "Filipino,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Chinese": 0.3693181818181818, - "Spanish,Malay,Indonesian": 0.4602272727272727, - "Spanish,Malay,Vietnamese": 0.4034090909090909, - "Spanish,Chinese,Indonesian": 0.36363636363636365, - "Spanish,Chinese,Vietnamese": 0.32954545454545453, - "Spanish,Indonesian,Vietnamese": 0.38636363636363635, - "Malay,Chinese,Indonesian": 0.4090909090909091, - "Malay,Chinese,Vietnamese": 0.32954545454545453, - "Malay,Indonesian,Vietnamese": 0.4090909090909091, - "Chinese,Indonesian,Vietnamese": 0.30113636363636365 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.24431818181818182, - "English,Filipino,Spanish,Chinese": 0.18181818181818182, - "English,Filipino,Spanish,Indonesian": 0.24431818181818182, - "English,Filipino,Spanish,Vietnamese": 0.24431818181818182, - "English,Filipino,Malay,Chinese": 0.18181818181818182, - "English,Filipino,Malay,Indonesian": 0.24431818181818182, - "English,Filipino,Malay,Vietnamese": 0.22727272727272727, - "English,Filipino,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay,Chinese": 0.30113636363636365, - "English,Spanish,Malay,Indonesian": 0.3352272727272727, - "English,Spanish,Malay,Vietnamese": 0.3068181818181818, - "English,Spanish,Chinese,Indonesian": 0.29545454545454547, - "English,Spanish,Chinese,Vietnamese": 0.2840909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.30113636363636365, - "English,Malay,Chinese,Indonesian": 0.3068181818181818, - "English,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "English,Chinese,Indonesian,Vietnamese": 0.25, - "Filipino,Spanish,Malay,Chinese": 0.19886363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.2784090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese,Indonesian": 0.1875, - "Filipino,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.30113636363636365, - "Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.32386363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.20454545454545456, - "English,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese,Indonesian": 0.25, - "English,Spanish,Malay,Chinese,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.25, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - } - }, - "AC3_2": 0.4331659276794933, - "AC3_3": 0.3553290615776051, - "AC3_4": 0.29190327386975085, - "AC3_5": 0.23923988531919801, - "AC3_6": 0.19359956001022544, - "AC3_7": 0.15292150928080891 + "accuracy": 0.3333333333333333 }, "prompt_2": { - "overall_acc": 0.3806818181818182, - "language_acc": { - "English": 0.42045454545454547, - "Filipino": 0.2727272727272727, - "Spanish": 0.32954545454545453, - "Malay": 0.4034090909090909, - "Chinese": 0.4375, - "Indonesian": 0.4318181818181818, - "Vietnamese": 0.3693181818181818 - }, - "consistency_score_2": 0.5367965367965367, - "consistency_score_3": 0.3540584415584417, - "consistency_score_4": 0.2548701298701299, - "consistency_score_5": 0.1929112554112554, - "consistency_score_6": 0.15016233766233764, - "consistency_score_7": 0.11931818181818182, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.42613636363636365, - "English,Spanish": 0.5454545454545454, - "English,Malay": 0.5852272727272727, - "English,Chinese": 0.5454545454545454, - "English,Indonesian": 0.5965909090909091, - "English,Vietnamese": 0.5340909090909091, - "Filipino,Spanish": 0.48295454545454547, - "Filipino,Malay": 0.48295454545454547, - "Filipino,Chinese": 0.39204545454545453, - "Filipino,Indonesian": 0.5056818181818182, - "Filipino,Vietnamese": 0.42613636363636365, - "Spanish,Malay": 0.6363636363636364, - "Spanish,Chinese": 0.4602272727272727, - "Spanish,Indonesian": 0.5681818181818182, - "Spanish,Vietnamese": 0.5909090909090909, - "Malay,Chinese": 0.5681818181818182, - "Malay,Indonesian": 0.6875, - "Malay,Vietnamese": 0.6136363636363636, - "Chinese,Indonesian": 0.5284090909090909, - "Chinese,Vietnamese": 0.5511363636363636, - "Indonesian,Vietnamese": 0.5454545454545454 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2840909090909091, - "English,Filipino,Malay": 0.3068181818181818, - "English,Filipino,Chinese": 0.24431818181818182, - "English,Filipino,Indonesian": 0.32954545454545453, - "English,Filipino,Vietnamese": 0.2727272727272727, - "English,Spanish,Malay": 0.42045454545454547, - "English,Spanish,Chinese": 0.32954545454545453, - "English,Spanish,Indonesian": 0.39204545454545453, - "English,Spanish,Vietnamese": 0.3806818181818182, - "English,Malay,Chinese": 0.4034090909090909, - "English,Malay,Indonesian": 0.4659090909090909, - "English,Malay,Vietnamese": 0.4090909090909091, - "English,Chinese,Indonesian": 0.375, - "English,Chinese,Vietnamese": 0.35795454545454547, - "English,Indonesian,Vietnamese": 0.36363636363636365, - "Filipino,Spanish,Malay": 0.3409090909090909, - "Filipino,Spanish,Chinese": 0.23863636363636365, - "Filipino,Spanish,Indonesian": 0.32954545454545453, - "Filipino,Spanish,Vietnamese": 0.30113636363636365, - "Filipino,Malay,Chinese": 0.2784090909090909, - "Filipino,Malay,Indonesian": 0.38636363636363635, - "Filipino,Malay,Vietnamese": 0.3181818181818182, - "Filipino,Chinese,Indonesian": 0.2727272727272727, - "Filipino,Chinese,Vietnamese": 0.24431818181818182, - "Filipino,Indonesian,Vietnamese": 0.3125, - "Spanish,Malay,Chinese": 0.375, - "Spanish,Malay,Indonesian": 0.4659090909090909, - "Spanish,Malay,Vietnamese": 0.4602272727272727, - "Spanish,Chinese,Indonesian": 0.3409090909090909, - "Spanish,Chinese,Vietnamese": 0.3409090909090909, - "Spanish,Indonesian,Vietnamese": 0.3977272727272727, - "Malay,Chinese,Indonesian": 0.4375, - "Malay,Chinese,Vietnamese": 0.3977272727272727, - "Malay,Indonesian,Vietnamese": 0.4659090909090909, - "Chinese,Indonesian,Vietnamese": 0.3522727272727273 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.2215909090909091, - "English,Filipino,Spanish,Chinese": 0.17045454545454544, - "English,Filipino,Spanish,Indonesian": 0.23863636363636365, - "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, - "English,Filipino,Malay,Chinese": 0.21022727272727273, - "English,Filipino,Malay,Indonesian": 0.2784090909090909, - "English,Filipino,Malay,Vietnamese": 0.23295454545454544, - "English,Filipino,Chinese,Indonesian": 0.19886363636363635, - "English,Filipino,Chinese,Vietnamese": 0.18181818181818182, - "English,Filipino,Indonesian,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay,Chinese": 0.2840909090909091, - "English,Spanish,Malay,Indonesian": 0.3409090909090909, - "English,Spanish,Malay,Vietnamese": 0.3181818181818182, - "English,Spanish,Chinese,Indonesian": 0.26704545454545453, - "English,Spanish,Chinese,Vietnamese": 0.2556818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.2784090909090909, - "English,Malay,Chinese,Indonesian": 0.32954545454545453, - "English,Malay,Chinese,Vietnamese": 0.29545454545454547, - "English,Malay,Indonesian,Vietnamese": 0.3352272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay,Chinese": 0.19886363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Chinese,Indonesian": 0.1875, - "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Malay,Chinese,Indonesian": 0.24431818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.32386363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.2897727272727273, - "Spanish,Malay,Indonesian,Vietnamese": 0.3693181818181818, - "Spanish,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.3125 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "English,Filipino,Malay,Chinese,Indonesian": 0.1875, - "English,Filipino,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.2159090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.25, - "English,Spanish,Malay,Chinese,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.25, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1875, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - } - }, - "AC3_2": 0.4454572262992667, - "AC3_3": 0.3668877796471693, - "AC3_4": 0.3053233483787561, - "AC3_5": 0.2560623927512415, - "AC3_6": 0.2153704475546469, - "AC3_7": 0.18168904955043905 + "accuracy": 0.3333333333333333 }, "prompt_3": { - "overall_acc": 0.39610389610389607, - "language_acc": { - "English": 0.3977272727272727, - "Filipino": 0.3068181818181818, - "Spanish": 0.32954545454545453, - "Malay": 0.4431818181818182, - "Chinese": 0.4375, - "Indonesian": 0.45454545454545453, - "Vietnamese": 0.4034090909090909 - }, - "consistency_score_2": 0.5373376623376623, - "consistency_score_3": 0.35974025974025986, - "consistency_score_4": 0.26818181818181813, - "consistency_score_5": 0.21401515151515152, - "consistency_score_6": 0.17857142857142858, - "consistency_score_7": 0.1534090909090909, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4147727272727273, - "English,Spanish": 0.5568181818181818, - "English,Malay": 0.5738636363636364, - "English,Chinese": 0.5454545454545454, - "English,Indonesian": 0.625, - "English,Vietnamese": 0.5511363636363636, - "Filipino,Spanish": 0.4431818181818182, - "Filipino,Malay": 0.45454545454545453, - "Filipino,Chinese": 0.4147727272727273, - "Filipino,Indonesian": 0.44886363636363635, - "Filipino,Vietnamese": 0.4431818181818182, - "Spanish,Malay": 0.5681818181818182, - "Spanish,Chinese": 0.5284090909090909, - "Spanish,Indonesian": 0.5454545454545454, - "Spanish,Vietnamese": 0.5965909090909091, - "Malay,Chinese": 0.5681818181818182, - "Malay,Indonesian": 0.7272727272727273, - "Malay,Vietnamese": 0.5625, - "Chinese,Indonesian": 0.5738636363636364, - "Chinese,Vietnamese": 0.5681818181818182, - "Indonesian,Vietnamese": 0.5738636363636364 - }, - "3_combine": { - "English,Filipino,Spanish": 0.26136363636363635, - "English,Filipino,Malay": 0.29545454545454547, - "English,Filipino,Chinese": 0.25, - "English,Filipino,Indonesian": 0.32954545454545453, - "English,Filipino,Vietnamese": 0.2727272727272727, - "English,Spanish,Malay": 0.3806818181818182, - "English,Spanish,Chinese": 0.36363636363636365, - "English,Spanish,Indonesian": 0.3977272727272727, - "English,Spanish,Vietnamese": 0.4090909090909091, - "English,Malay,Chinese": 0.3806818181818182, - "English,Malay,Indonesian": 0.48863636363636365, - "English,Malay,Vietnamese": 0.38636363636363635, - "English,Chinese,Indonesian": 0.4090909090909091, - "English,Chinese,Vietnamese": 0.3806818181818182, - "English,Indonesian,Vietnamese": 0.4147727272727273, - "Filipino,Spanish,Malay": 0.30113636363636365, - "Filipino,Spanish,Chinese": 0.26136363636363635, - "Filipino,Spanish,Indonesian": 0.29545454545454547, - "Filipino,Spanish,Vietnamese": 0.3068181818181818, - "Filipino,Malay,Chinese": 0.2784090909090909, - "Filipino,Malay,Indonesian": 0.3693181818181818, - "Filipino,Malay,Vietnamese": 0.3125, - "Filipino,Chinese,Indonesian": 0.2840909090909091, - "Filipino,Chinese,Vietnamese": 0.2784090909090909, - "Filipino,Indonesian,Vietnamese": 0.32386363636363635, - "Spanish,Malay,Chinese": 0.39204545454545453, - "Spanish,Malay,Indonesian": 0.4431818181818182, - "Spanish,Malay,Vietnamese": 0.42613636363636365, - "Spanish,Chinese,Indonesian": 0.3806818181818182, - "Spanish,Chinese,Vietnamese": 0.3977272727272727, - "Spanish,Indonesian,Vietnamese": 0.39204545454545453, - "Malay,Chinese,Indonesian": 0.4772727272727273, - "Malay,Chinese,Vietnamese": 0.3806818181818182, - "Malay,Indonesian,Vietnamese": 0.4659090909090909, - "Chinese,Indonesian,Vietnamese": 0.4034090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.20454545454545456, - "English,Filipino,Spanish,Chinese": 0.1875, - "English,Filipino,Spanish,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Vietnamese": 0.20454545454545456, - "English,Filipino,Malay,Chinese": 0.21022727272727273, - "English,Filipino,Malay,Indonesian": 0.2840909090909091, - "English,Filipino,Malay,Vietnamese": 0.23295454545454544, - "English,Filipino,Chinese,Indonesian": 0.2159090909090909, - "English,Filipino,Chinese,Vietnamese": 0.20454545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay,Chinese": 0.2784090909090909, - "English,Spanish,Malay,Indonesian": 0.32386363636363635, - "English,Spanish,Malay,Vietnamese": 0.3125, - "English,Spanish,Chinese,Indonesian": 0.29545454545454547, - "English,Spanish,Chinese,Vietnamese": 0.3068181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.3068181818181818, - "English,Malay,Chinese,Indonesian": 0.35795454545454547, - "English,Malay,Chinese,Vietnamese": 0.2897727272727273, - "English,Malay,Indonesian,Vietnamese": 0.3465909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.3125, - "Filipino,Spanish,Malay,Chinese": 0.21022727272727273, - "Filipino,Spanish,Malay,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Malay,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.26136363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.29545454545454547, - "Filipino,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Chinese,Indonesian": 0.3409090909090909, - "Spanish,Malay,Chinese,Vietnamese": 0.3125, - "Spanish,Malay,Indonesian,Vietnamese": 0.35795454545454547, - "Spanish,Chinese,Indonesian,Vietnamese": 0.3068181818181818, - "Malay,Chinese,Indonesian,Vietnamese": 0.3465909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.21022727272727273, - "English,Filipino,Malay,Chinese,Vietnamese": 0.1875, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.1875, - "English,Spanish,Malay,Chinese,Indonesian": 0.26136363636363635, - "English,Spanish,Malay,Chinese,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.20454545454545456, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2840909090909091 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1875, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.18181818181818182 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 - } - }, - "AC3_2": 0.4560361377264128, - "AC3_3": 0.37704735122427113, - "AC3_4": 0.31982582417650535, - "AC3_5": 0.2778875226313332, - "AC3_6": 0.24616626307257994, - "AC3_7": 0.22116288434272593 + "accuracy": 0.3333333333333333 }, "prompt_4": { - "overall_acc": 0.38149350649350655, - "language_acc": { - "English": 0.4034090909090909, - "Filipino": 0.30113636363636365, - "Spanish": 0.3465909090909091, - "Malay": 0.4147727272727273, - "Chinese": 0.42045454545454547, - "Indonesian": 0.42045454545454547, - "Vietnamese": 0.36363636363636365 - }, - "consistency_score_2": 0.5346320346320347, - "consistency_score_3": 0.35503246753246753, - "consistency_score_4": 0.2587662337662337, - "consistency_score_5": 0.1975108225108225, - "consistency_score_6": 0.1534090909090909, - "consistency_score_7": 0.11931818181818182, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4318181818181818, - "English,Spanish": 0.5681818181818182, - "English,Malay": 0.5568181818181818, - "English,Chinese": 0.5795454545454546, - "English,Indonesian": 0.6193181818181818, - "English,Vietnamese": 0.5681818181818182, - "Filipino,Spanish": 0.4375, - "Filipino,Malay": 0.4375, - "Filipino,Chinese": 0.4034090909090909, - "Filipino,Indonesian": 0.5056818181818182, - "Filipino,Vietnamese": 0.42045454545454547, - "Spanish,Malay": 0.5568181818181818, - "Spanish,Chinese": 0.5340909090909091, - "Spanish,Indonesian": 0.5738636363636364, - "Spanish,Vietnamese": 0.5852272727272727, - "Malay,Chinese": 0.5397727272727273, - "Malay,Indonesian": 0.6931818181818182, - "Malay,Vietnamese": 0.5909090909090909, - "Chinese,Indonesian": 0.5397727272727273, - "Chinese,Vietnamese": 0.5113636363636364, - "Indonesian,Vietnamese": 0.5738636363636364 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Malay": 0.2784090909090909, - "English,Filipino,Chinese": 0.26704545454545453, - "English,Filipino,Indonesian": 0.3409090909090909, - "English,Filipino,Vietnamese": 0.2727272727272727, - "English,Spanish,Malay": 0.375, - "English,Spanish,Chinese": 0.4034090909090909, - "English,Spanish,Indonesian": 0.42045454545454547, - "English,Spanish,Vietnamese": 0.4090909090909091, - "English,Malay,Chinese": 0.3806818181818182, - "English,Malay,Indonesian": 0.4659090909090909, - "English,Malay,Vietnamese": 0.3977272727272727, - "English,Chinese,Indonesian": 0.4090909090909091, - "English,Chinese,Vietnamese": 0.3806818181818182, - "English,Indonesian,Vietnamese": 0.42613636363636365, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.26704545454545453, - "Filipino,Spanish,Indonesian": 0.3352272727272727, - "Filipino,Spanish,Vietnamese": 0.2897727272727273, - "Filipino,Malay,Chinese": 0.26704545454545453, - "Filipino,Malay,Indonesian": 0.3693181818181818, - "Filipino,Malay,Vietnamese": 0.29545454545454547, - "Filipino,Chinese,Indonesian": 0.2897727272727273, - "Filipino,Chinese,Vietnamese": 0.23295454545454544, - "Filipino,Indonesian,Vietnamese": 0.3181818181818182, - "Spanish,Malay,Chinese": 0.3693181818181818, - "Spanish,Malay,Indonesian": 0.4431818181818182, - "Spanish,Malay,Vietnamese": 0.3977272727272727, - "Spanish,Chinese,Indonesian": 0.375, - "Spanish,Chinese,Vietnamese": 0.36363636363636365, - "Spanish,Indonesian,Vietnamese": 0.3977272727272727, - "Malay,Chinese,Indonesian": 0.42613636363636365, - "Malay,Chinese,Vietnamese": 0.3693181818181818, - "Malay,Indonesian,Vietnamese": 0.4659090909090909, - "Chinese,Indonesian,Vietnamese": 0.3522727272727273 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.19318181818181818, - "English,Filipino,Spanish,Chinese": 0.19886363636363635, - "English,Filipino,Spanish,Indonesian": 0.25, - "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, - "English,Filipino,Malay,Chinese": 0.1875, - "English,Filipino,Malay,Indonesian": 0.2556818181818182, - "English,Filipino,Malay,Vietnamese": 0.21022727272727273, - "English,Filipino,Chinese,Indonesian": 0.2159090909090909, - "English,Filipino,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Indonesian,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay,Chinese": 0.2897727272727273, - "English,Spanish,Malay,Indonesian": 0.32954545454545453, - "English,Spanish,Malay,Vietnamese": 0.32386363636363635, - "English,Spanish,Chinese,Indonesian": 0.3125, - "English,Spanish,Chinese,Vietnamese": 0.3125, - "English,Spanish,Indonesian,Vietnamese": 0.3352272727272727, - "English,Malay,Chinese,Indonesian": 0.32386363636363635, - "English,Malay,Chinese,Vietnamese": 0.29545454545454547, - "English,Malay,Indonesian,Vietnamese": 0.35795454545454547, - "English,Chinese,Indonesian,Vietnamese": 0.29545454545454547, - "Filipino,Spanish,Malay,Chinese": 0.19318181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.26136363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Chinese,Indonesian": 0.20454545454545456, - "Filipino,Spanish,Chinese,Vietnamese": 0.1875, - "Filipino,Spanish,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.23863636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1875, - "Spanish,Malay,Chinese,Indonesian": 0.3181818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.29545454545454547, - "Spanish,Malay,Indonesian,Vietnamese": 0.3465909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.3181818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Indonesian": 0.1875, - "English,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.17613636363636365, - "English,Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.2556818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.26704545454545453 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - } - }, - "AC3_2": 0.44526353735905744, - "AC3_3": 0.36778765641966943, - "AC3_4": 0.3083674691965883, - "AC3_5": 0.26027127074033796, - "AC3_6": 0.2188232859295697, - "AC3_7": 0.1817813466559083 + "accuracy": 0.36363636363636365 }, "prompt_5": { - "overall_acc": 0.3766233766233767, - "language_acc": { - "English": 0.4147727272727273, - "Filipino": 0.26136363636363635, - "Spanish": 0.3409090909090909, - "Malay": 0.3977272727272727, - "Chinese": 0.4375, - "Indonesian": 0.42045454545454547, - "Vietnamese": 0.36363636363636365 - }, - "consistency_score_2": 0.5427489177489178, - "consistency_score_3": 0.3647727272727273, - "consistency_score_4": 0.2693181818181818, - "consistency_score_5": 0.20914502164502166, - "consistency_score_6": 0.16558441558441556, - "consistency_score_7": 0.13068181818181818, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4431818181818182, - "English,Spanish": 0.5113636363636364, - "English,Malay": 0.5738636363636364, - "English,Chinese": 0.5795454545454546, - "English,Indonesian": 0.5965909090909091, - "English,Vietnamese": 0.5965909090909091, - "Filipino,Spanish": 0.4375, - "Filipino,Malay": 0.48863636363636365, - "Filipino,Chinese": 0.4034090909090909, - "Filipino,Indonesian": 0.5, - "Filipino,Vietnamese": 0.4772727272727273, - "Spanish,Malay": 0.5909090909090909, - "Spanish,Chinese": 0.5170454545454546, - "Spanish,Indonesian": 0.5454545454545454, - "Spanish,Vietnamese": 0.5568181818181818, - "Malay,Chinese": 0.5852272727272727, - "Malay,Indonesian": 0.6875, - "Malay,Vietnamese": 0.5738636363636364, - "Chinese,Indonesian": 0.6079545454545454, - "Chinese,Vietnamese": 0.5625, - "Indonesian,Vietnamese": 0.5625 - }, - "3_combine": { - "English,Filipino,Spanish": 0.26704545454545453, - "English,Filipino,Malay": 0.3068181818181818, - "English,Filipino,Chinese": 0.2727272727272727, - "English,Filipino,Indonesian": 0.3409090909090909, - "English,Filipino,Vietnamese": 0.3125, - "English,Spanish,Malay": 0.3693181818181818, - "English,Spanish,Chinese": 0.35795454545454547, - "English,Spanish,Indonesian": 0.3693181818181818, - "English,Spanish,Vietnamese": 0.375, - "English,Malay,Chinese": 0.4034090909090909, - "English,Malay,Indonesian": 0.4772727272727273, - "English,Malay,Vietnamese": 0.4090909090909091, - "English,Chinese,Indonesian": 0.4375, - "English,Chinese,Vietnamese": 0.4147727272727273, - "English,Indonesian,Vietnamese": 0.4147727272727273, - "Filipino,Spanish,Malay": 0.3181818181818182, - "Filipino,Spanish,Chinese": 0.2556818181818182, - "Filipino,Spanish,Indonesian": 0.3068181818181818, - "Filipino,Spanish,Vietnamese": 0.29545454545454547, - "Filipino,Malay,Chinese": 0.29545454545454547, - "Filipino,Malay,Indonesian": 0.3806818181818182, - "Filipino,Malay,Vietnamese": 0.3409090909090909, - "Filipino,Chinese,Indonesian": 0.32386363636363635, - "Filipino,Chinese,Vietnamese": 0.29545454545454547, - "Filipino,Indonesian,Vietnamese": 0.3465909090909091, - "Spanish,Malay,Chinese": 0.3806818181818182, - "Spanish,Malay,Indonesian": 0.44886363636363635, - "Spanish,Malay,Vietnamese": 0.3977272727272727, - "Spanish,Chinese,Indonesian": 0.38636363636363635, - "Spanish,Chinese,Vietnamese": 0.3465909090909091, - "Spanish,Indonesian,Vietnamese": 0.3806818181818182, - "Malay,Chinese,Indonesian": 0.4772727272727273, - "Malay,Chinese,Vietnamese": 0.4034090909090909, - "Malay,Indonesian,Vietnamese": 0.44886363636363635, - "Chinese,Indonesian,Vietnamese": 0.4090909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.20454545454545456, - "English,Filipino,Spanish,Chinese": 0.1875, - "English,Filipino,Spanish,Indonesian": 0.22727272727272727, - "English,Filipino,Spanish,Vietnamese": 0.2159090909090909, - "English,Filipino,Malay,Chinese": 0.2159090909090909, - "English,Filipino,Malay,Indonesian": 0.2784090909090909, - "English,Filipino,Malay,Vietnamese": 0.25, - "English,Filipino,Chinese,Indonesian": 0.24431818181818182, - "English,Filipino,Chinese,Vietnamese": 0.22727272727272727, - "English,Filipino,Indonesian,Vietnamese": 0.26704545454545453, - "English,Spanish,Malay,Chinese": 0.2727272727272727, - "English,Spanish,Malay,Indonesian": 0.3068181818181818, - "English,Spanish,Malay,Vietnamese": 0.29545454545454547, - "English,Spanish,Chinese,Indonesian": 0.29545454545454547, - "English,Spanish,Chinese,Vietnamese": 0.2727272727272727, - "English,Spanish,Indonesian,Vietnamese": 0.2897727272727273, - "English,Malay,Chinese,Indonesian": 0.36363636363636365, - "English,Malay,Chinese,Vietnamese": 0.3181818181818182, - "English,Malay,Indonesian,Vietnamese": 0.35795454545454547, - "English,Chinese,Indonesian,Vietnamese": 0.3352272727272727, - "Filipino,Spanish,Malay,Chinese": 0.20454545454545456, - "Filipino,Spanish,Malay,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.26136363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.25, - "Filipino,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Spanish,Malay,Chinese,Indonesian": 0.32954545454545453, - "Spanish,Malay,Chinese,Vietnamese": 0.2727272727272727, - "Spanish,Malay,Indonesian,Vietnamese": 0.32954545454545453, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.35795454545454547 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Indonesian": 0.19318181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Chinese,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese,Indonesian": 0.21022727272727273, - "English,Filipino,Malay,Chinese,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2556818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.29545454545454547, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1875, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - } - }, - "AC3_2": 0.4446771591667553, - "AC3_3": 0.3706033401176184, - "AC3_4": 0.3140578948084705, - "AC3_5": 0.2689421432611267, - "AC3_6": 0.2300334395718526, - "AC3_7": 0.19403636359811519 + "accuracy": 0.21212121212121213 } }, - "sg_eval": { + "ind_emotion": { "prompt_1": { - "accuracy": 0.46601941747572817 + "accuracy": 0.225 }, "prompt_2": { - "accuracy": 0.49514563106796117 + "accuracy": 0.24772727272727274 }, "prompt_3": { - "accuracy": 0.4854368932038835 + "accuracy": 0.22272727272727272 }, "prompt_4": { - "accuracy": 0.4563106796116505 + "accuracy": 0.2 }, "prompt_5": { - "accuracy": 0.47572815533980584 + "accuracy": 0.20227272727272727 } }, - "cn_eval": { + "ocnli": { "prompt_1": { - "accuracy": 0.5619047619047619 + "accuracy": 0.32949152542372884 }, "prompt_2": { - "accuracy": 0.5333333333333333 + "accuracy": 0.3488135593220339 }, "prompt_3": { - "accuracy": 0.5333333333333333 + "accuracy": 0.32033898305084746 }, "prompt_4": { - "accuracy": 0.5142857142857142 + "accuracy": 0.3416949152542373 }, "prompt_5": { - "accuracy": 0.49523809523809526 + "accuracy": 0.3396610169491525 } }, - "us_eval": { + "c3": { "prompt_1": { - "accuracy": 0.4766355140186916 + "accuracy": 0.3051608077786088 }, "prompt_2": { - "accuracy": 0.48598130841121495 + "accuracy": 0.30254300673148843 }, "prompt_3": { - "accuracy": 0.4766355140186916 + "accuracy": 0.31787584143605085 }, "prompt_4": { - "accuracy": 0.4766355140186916 + "accuracy": 0.3070306656694091 }, "prompt_5": { - "accuracy": 0.48598130841121495 + "accuracy": 0.31787584143605085 } }, - "ph_eval": { + "dream": { "prompt_1": { - "accuracy": 0.35, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.2, - "politics": 0.3, - "culture": 0.3, - "film": 0.5, - "law": 0.3, - "geography": 0.7 - } + "accuracy": 0.37236648701616853 }, "prompt_2": { - "accuracy": 0.32, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.1, - "politics": 0.2, - "culture": 0.3, - "film": 0.3, - "law": 0.3, - "geography": 0.7 - } + "accuracy": 0.37677609015188634 }, "prompt_3": { - "accuracy": 0.38, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.1, - "politics": 0.3, - "culture": 0.4, - "film": 0.5, - "law": 0.4, - "geography": 0.7 - } + "accuracy": 0.37677609015188634 }, "prompt_4": { - "accuracy": 0.31, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.4, - "history": 0.2, - "literature": 0.0, - "politics": 0.2, - "culture": 0.3, - "film": 0.5, - "law": 0.3, - "geography": 0.7 - } + "accuracy": 0.3728564429201372 }, "prompt_5": { - "accuracy": 0.35, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.5, - "history": 0.3333333333333333, - "literature": 0.1, - "politics": 0.3, - "culture": 0.3, - "film": 0.4, - "law": 0.4, - "geography": 0.6 - } + "accuracy": 0.36893679568838805 } }, - "sing2eng": { + "samsum": { "prompt_1": { - "bleu_score": 0.19122569165761827 + "rouge1": 0.2850073990198992, + "rouge2": 0.08352220743518561, + "rougeL": 0.21206530932751466, + "avg_rouge": 0.19353163859419983 }, "prompt_2": { - "bleu_score": 0.1839165378163868 + "rouge1": 0.30658379785153117, + "rouge2": 0.09320374513029613, + "rougeL": 0.22341544203058414, + "avg_rouge": 0.2077343283374705 }, "prompt_3": { - "bleu_score": 0.17952537283353398 + "rouge1": 0.28776691352486655, + "rouge2": 0.08167280160298225, + "rougeL": 0.20838776666273462, + "avg_rouge": 0.19260916059686115 }, "prompt_4": { - "bleu_score": 0.1933676065669291 + "rouge1": 0.2756527589777085, + "rouge2": 0.08054095290677721, + "rougeL": 0.20134716056134175, + "avg_rouge": 0.1858469574819425 }, "prompt_5": { - "bleu_score": 0.15086335060190453 + "rouge1": 0.3053455922803706, + "rouge2": 0.0909102250678557, + "rougeL": 0.22428544015406174, + "avg_rouge": 0.20684708583409603 } }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { + "dialogsum": { "prompt_1": { - "bleu_score": 0.324339395459608 + "rouge1": 0.22417868322315565, + "rouge2": 0.06099633594873171, + "rougeL": 0.1645146116712034, + "avg_rouge": 0.14989654361436358 }, "prompt_2": { - "bleu_score": 0.3273967748185027 + "rouge1": 0.2196654150670897, + "rouge2": 0.06032763846133467, + "rougeL": 0.16056995602370785, + "avg_rouge": 0.14685433651737742 }, "prompt_3": { - "bleu_score": 0.3196010701000885 + "rouge1": 0.21832150041066453, + "rouge2": 0.059156306603515486, + "rougeL": 0.16139471401530808, + "avg_rouge": 0.1462908403431627 }, "prompt_4": { - "bleu_score": 0.32767706196989815 + "rouge1": 0.21922502072906003, + "rouge2": 0.05878269252032803, + "rougeL": 0.1592118241412318, + "avg_rouge": 0.1457398457968733 }, "prompt_5": { - "bleu_score": 0.3187991929832196 + "rouge1": 0.21060609015887038, + "rouge2": 0.057833836582119554, + "rougeL": 0.15643443484914793, + "avg_rouge": 0.14162478719671262 } }, - "flores_vie2eng": { + "sst2": { "prompt_1": { - "bleu_score": 0.27085509796010654 + "accuracy": 0.6731651376146789 }, "prompt_2": { - "bleu_score": 0.2707474488773402 + "accuracy": 0.5756880733944955 }, "prompt_3": { - "bleu_score": 0.26784999147889677 + "accuracy": 0.6043577981651376 }, "prompt_4": { - "bleu_score": 0.27258411084179224 + "accuracy": 0.5665137614678899 }, "prompt_5": { - "bleu_score": 0.2631681201816227 + "accuracy": 0.6903669724770642 } }, - "flores_zho2eng": { + "cola": { "prompt_1": { - "bleu_score": 0.18604303674060185 + "accuracy": 0.5244487056567594 }, "prompt_2": { - "bleu_score": 0.18635068134758903 + "accuracy": 0.5225311601150527 }, "prompt_3": { - "bleu_score": 0.18146892556399355 + "accuracy": 0.5445829338446788 }, "prompt_4": { - "bleu_score": 0.18755908746696348 + "accuracy": 0.6442953020134228 }, "prompt_5": { - "bleu_score": 0.183939869933563 + "accuracy": 0.5349952061361457 } }, - "flores_zsm2eng": { + "qqp": { "prompt_1": { - "bleu_score": 0.33225067496360916 + "accuracy": 0.4965 }, "prompt_2": { - "bleu_score": 0.33180912980852667 + "accuracy": 0.498 }, "prompt_3": { - "bleu_score": 0.3262570609619245 + "accuracy": 0.495 }, "prompt_4": { - "bleu_score": 0.3330672950263117 + "accuracy": 0.5025 }, "prompt_5": { - "bleu_score": 0.3202455551875229 + "accuracy": 0.496 } }, - "mmlu": { + "mnli": { "prompt_1": { - "accuracy": 0.44807467911318555 + "accuracy": 0.5 }, "prompt_2": { - "accuracy": 0.4632438739789965 + "accuracy": 0.5 }, "prompt_3": { - "accuracy": 0.47374562427071176 + "accuracy": 0.5 }, "prompt_4": { - "accuracy": 0.45040840140023336 + "accuracy": 0.5 }, "prompt_5": { - "accuracy": 0.45040840140023336 + "accuracy": 0.5 } }, - "mmlu_full": { + "qnli": { "prompt_1": { - "accuracy": 0.4281730425455846, - "category_acc": { - "high_school_european_history": 0.5670731707317073, - "business_ethics": 0.46464646464646464, - "clinical_knowledge": 0.42803030303030304, - "medical_genetics": 0.5050505050505051, - "high_school_us_history": 0.5270935960591133, - "high_school_physics": 0.28, - "high_school_world_history": 0.5889830508474576, - "virology": 0.3878787878787879, - "high_school_microeconomics": 0.37130801687763715, - "econometrics": 0.2743362831858407, - "college_computer_science": 0.29292929292929293, - "high_school_biology": 0.4563106796116505, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.3416370106761566, - "philosophy": 0.4870967741935484, - "professional_medicine": 0.3062730627306273, - "nutrition": 0.42295081967213116, - "global_facts": 0.2727272727272727, - "machine_learning": 0.36936936936936937, - "security_studies": 0.4057377049180328, - "public_relations": 0.5045871559633027, - "professional_psychology": 0.41407528641571195, - "prehistory": 0.47368421052631576, - "anatomy": 0.47761194029850745, - "human_sexuality": 0.47692307692307695, - "college_medicine": 0.4069767441860465, - "high_school_government_and_politics": 0.6041666666666666, - "college_chemistry": 0.3939393939393939, - "logical_fallacies": 0.5493827160493827, - "high_school_geography": 0.5888324873096447, - "elementary_mathematics": 0.32891246684350134, - "human_aging": 0.509009009009009, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.5845588235294118, - "formal_logic": 0.248, - "high_school_statistics": 0.2837209302325581, - "international_law": 0.6083333333333333, - "high_school_mathematics": 0.275092936802974, - "high_school_computer_science": 0.48484848484848486, - "conceptual_physics": 0.36324786324786323, - "miscellaneous": 0.6214833759590793, - "high_school_chemistry": 0.33663366336633666, - "marketing": 0.6909871244635193, - "professional_law": 0.32159165035877363, - "management": 0.5980392156862745, - "college_physics": 0.18811881188118812, - "jurisprudence": 0.5046728971962616, - "world_religions": 0.5588235294117647, - "sociology": 0.585, - "us_foreign_policy": 0.6262626262626263, - "high_school_macroeconomics": 0.3856041131105398, - "computer_security": 0.5656565656565656, - "moral_scenarios": 0.2639821029082774, - "moral_disputes": 0.4608695652173913, - "electrical_engineering": 0.4513888888888889, - "astronomy": 0.48344370860927155, - "college_biology": 0.45454545454545453 - } + "accuracy": 0.8 }, "prompt_2": { - "accuracy": 0.44368966750089384, - "category_acc": { - "high_school_european_history": 0.5487804878048781, - "business_ethics": 0.4444444444444444, - "clinical_knowledge": 0.4583333333333333, - "medical_genetics": 0.5252525252525253, - "high_school_us_history": 0.541871921182266, - "high_school_physics": 0.28, - "high_school_world_history": 0.6016949152542372, - "virology": 0.41818181818181815, - "high_school_microeconomics": 0.41350210970464135, - "econometrics": 0.24778761061946902, - "college_computer_science": 0.31313131313131315, - "high_school_biology": 0.47896440129449835, - "abstract_algebra": 0.20202020202020202, - "professional_accounting": 0.36298932384341637, - "philosophy": 0.4935483870967742, - "professional_medicine": 0.33210332103321033, - "nutrition": 0.46229508196721314, - "global_facts": 0.36363636363636365, - "machine_learning": 0.3333333333333333, - "security_studies": 0.4672131147540984, - "public_relations": 0.5321100917431193, - "professional_psychology": 0.4369885433715221, - "prehistory": 0.5201238390092879, - "anatomy": 0.5074626865671642, - "human_sexuality": 0.5307692307692308, - "college_medicine": 0.37209302325581395, - "high_school_government_and_politics": 0.625, - "college_chemistry": 0.32323232323232326, - "logical_fallacies": 0.6049382716049383, - "high_school_geography": 0.583756345177665, - "elementary_mathematics": 0.29973474801061006, - "human_aging": 0.5135135135135135, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.6231617647058824, - "formal_logic": 0.312, - "high_school_statistics": 0.28837209302325584, - "international_law": 0.6083333333333333, - "high_school_mathematics": 0.2899628252788104, - "high_school_computer_science": 0.5353535353535354, - "conceptual_physics": 0.3717948717948718, - "miscellaneous": 0.639386189258312, - "high_school_chemistry": 0.35148514851485146, - "marketing": 0.7167381974248928, - "professional_law": 0.32811480756686234, - "management": 0.6470588235294118, - "college_physics": 0.19801980198019803, - "jurisprudence": 0.514018691588785, - "world_religions": 0.5647058823529412, - "sociology": 0.63, - "us_foreign_policy": 0.6767676767676768, - "high_school_macroeconomics": 0.39845758354755784, - "computer_security": 0.5555555555555556, - "moral_scenarios": 0.2550335570469799, - "moral_disputes": 0.48985507246376814, - "electrical_engineering": 0.4166666666666667, - "astronomy": 0.5298013245033113, - "college_biology": 0.4965034965034965 - } + "accuracy": 0.6 }, "prompt_3": { - "accuracy": 0.44955309259921344, - "category_acc": { - "high_school_european_history": 0.6097560975609756, - "business_ethics": 0.45454545454545453, - "clinical_knowledge": 0.45075757575757575, - "medical_genetics": 0.5050505050505051, - "high_school_us_history": 0.5960591133004927, - "high_school_physics": 0.24666666666666667, - "high_school_world_history": 0.6016949152542372, - "virology": 0.3939393939393939, - "high_school_microeconomics": 0.42616033755274263, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.32323232323232326, - "high_school_biology": 0.5016181229773463, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.34519572953736655, - "philosophy": 0.4870967741935484, - "professional_medicine": 0.35793357933579334, - "nutrition": 0.4557377049180328, - "global_facts": 0.3434343434343434, - "machine_learning": 0.32432432432432434, - "security_studies": 0.5, - "public_relations": 0.4954128440366973, - "professional_psychology": 0.43044189852700493, - "prehistory": 0.5046439628482973, - "anatomy": 0.47761194029850745, - "human_sexuality": 0.5, - "college_medicine": 0.4011627906976744, - "high_school_government_and_politics": 0.6354166666666666, - "college_chemistry": 0.31313131313131315, - "logical_fallacies": 0.5802469135802469, - "high_school_geography": 0.5634517766497462, - "elementary_mathematics": 0.3156498673740053, - "human_aging": 0.49099099099099097, - "college_mathematics": 0.35353535353535354, - "high_school_psychology": 0.625, - "formal_logic": 0.304, - "high_school_statistics": 0.2930232558139535, - "international_law": 0.6166666666666667, - "high_school_mathematics": 0.3271375464684015, - "high_school_computer_science": 0.47474747474747475, - "conceptual_physics": 0.3888888888888889, - "miscellaneous": 0.6419437340153452, - "high_school_chemistry": 0.3465346534653465, - "marketing": 0.7339055793991416, - "professional_law": 0.3496412263535551, - "management": 0.6666666666666666, - "college_physics": 0.24752475247524752, - "jurisprudence": 0.5046728971962616, - "world_religions": 0.5823529411764706, - "sociology": 0.635, - "us_foreign_policy": 0.6868686868686869, - "high_school_macroeconomics": 0.41131105398457585, - "computer_security": 0.5656565656565656, - "moral_scenarios": 0.2662192393736018, - "moral_disputes": 0.48405797101449277, - "electrical_engineering": 0.4305555555555556, - "astronomy": 0.5231788079470199, - "college_biology": 0.44755244755244755 - } + "accuracy": 0.4 }, "prompt_4": { - "accuracy": 0.44504826599928493, - "category_acc": { - "high_school_european_history": 0.5853658536585366, - "business_ethics": 0.494949494949495, - "clinical_knowledge": 0.45075757575757575, - "medical_genetics": 0.46464646464646464, - "high_school_us_history": 0.5467980295566502, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.6313559322033898, - "virology": 0.4, - "high_school_microeconomics": 0.4219409282700422, - "econometrics": 0.21238938053097345, - "college_computer_science": 0.37373737373737376, - "high_school_biology": 0.5145631067961165, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.3594306049822064, - "philosophy": 0.4967741935483871, - "professional_medicine": 0.3210332103321033, - "nutrition": 0.4557377049180328, - "global_facts": 0.3434343434343434, - "machine_learning": 0.26126126126126126, - "security_studies": 0.4713114754098361, - "public_relations": 0.5137614678899083, - "professional_psychology": 0.44680851063829785, - "prehistory": 0.5139318885448917, - "anatomy": 0.5074626865671642, - "human_sexuality": 0.5076923076923077, - "college_medicine": 0.36046511627906974, - "high_school_government_and_politics": 0.6197916666666666, - "college_chemistry": 0.32323232323232326, - "logical_fallacies": 0.5802469135802469, - "high_school_geography": 0.5736040609137056, - "elementary_mathematics": 0.3050397877984085, - "human_aging": 0.5045045045045045, - "college_mathematics": 0.3434343434343434, - "high_school_psychology": 0.6213235294117647, - "formal_logic": 0.264, - "high_school_statistics": 0.2837209302325581, - "international_law": 0.6166666666666667, - "high_school_mathematics": 0.2862453531598513, - "high_school_computer_science": 0.45454545454545453, - "conceptual_physics": 0.3888888888888889, - "miscellaneous": 0.6368286445012787, - "high_school_chemistry": 0.3811881188118812, - "marketing": 0.7124463519313304, - "professional_law": 0.3385518590998043, - "management": 0.6274509803921569, - "college_physics": 0.25742574257425743, - "jurisprudence": 0.4953271028037383, - "world_religions": 0.5764705882352941, - "sociology": 0.65, - "us_foreign_policy": 0.6666666666666666, - "high_school_macroeconomics": 0.39331619537275064, - "computer_security": 0.5959595959595959, - "moral_scenarios": 0.2505592841163311, - "moral_disputes": 0.5072463768115942, - "electrical_engineering": 0.4652777777777778, - "astronomy": 0.4768211920529801, - "college_biology": 0.4405594405594406 - } + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.43560958169467284, - "category_acc": { - "high_school_european_history": 0.5853658536585366, - "business_ethics": 0.47474747474747475, - "clinical_knowledge": 0.4393939393939394, - "medical_genetics": 0.46464646464646464, - "high_school_us_history": 0.5812807881773399, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.6228813559322034, - "virology": 0.41818181818181815, - "high_school_microeconomics": 0.3924050632911392, - "econometrics": 0.24778761061946902, - "college_computer_science": 0.35353535353535354, - "high_school_biology": 0.4854368932038835, - "abstract_algebra": 0.25252525252525254, - "professional_accounting": 0.3416370106761566, - "philosophy": 0.47419354838709676, - "professional_medicine": 0.34317343173431736, - "nutrition": 0.4426229508196721, - "global_facts": 0.3434343434343434, - "machine_learning": 0.25225225225225223, - "security_studies": 0.45081967213114754, - "public_relations": 0.5596330275229358, - "professional_psychology": 0.4238952536824877, - "prehistory": 0.5139318885448917, - "anatomy": 0.47761194029850745, - "human_sexuality": 0.4846153846153846, - "college_medicine": 0.38372093023255816, - "high_school_government_and_politics": 0.6145833333333334, - "college_chemistry": 0.3333333333333333, - "logical_fallacies": 0.5555555555555556, - "high_school_geography": 0.5583756345177665, - "elementary_mathematics": 0.3129973474801061, - "human_aging": 0.509009009009009, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.5992647058823529, - "formal_logic": 0.28, - "high_school_statistics": 0.2651162790697674, - "international_law": 0.5583333333333333, - "high_school_mathematics": 0.27137546468401486, - "high_school_computer_science": 0.5151515151515151, - "conceptual_physics": 0.358974358974359, - "miscellaneous": 0.6265984654731458, - "high_school_chemistry": 0.3564356435643564, - "marketing": 0.703862660944206, - "professional_law": 0.32354859752120024, - "management": 0.6078431372549019, - "college_physics": 0.22772277227722773, - "jurisprudence": 0.5233644859813084, - "world_religions": 0.5941176470588235, - "sociology": 0.625, - "us_foreign_policy": 0.6464646464646465, - "high_school_macroeconomics": 0.37017994858611825, - "computer_security": 0.5959595959595959, - "moral_scenarios": 0.24720357941834453, - "moral_disputes": 0.4956521739130435, - "electrical_engineering": 0.4444444444444444, - "astronomy": 0.46357615894039733, - "college_biology": 0.4825174825174825 - } + "accuracy": 0.8 } }, - "c_eval": { + "wnli": { "prompt_1": { - "accuracy": 0.42867756315007427 + "accuracy": 0.4 }, "prompt_2": { - "accuracy": 0.4390787518573551 + "accuracy": 0.3 }, "prompt_3": { - "accuracy": 0.42570579494799404 + "accuracy": 0.3 }, "prompt_4": { - "accuracy": 0.4138187221396731 + "accuracy": 0.5 }, "prompt_5": { - "accuracy": 0.43684992570579495 + "accuracy": 0.4 } }, - "c_eval_full": { + "rte": { "prompt_1": { - "accuracy": 0.4352428393524284, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.30952380952380953, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.4482758620689655, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.4583333333333333, - "high_school_biology": 0.4583333333333333, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.5, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.38333333333333336, - "business_administration": 0.4473684210526316, - "marxism": 0.5833333333333334, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.6176470588235294, - "teacher_qualification": 0.6122448979591837, - "high_school_politics": 0.625, - "high_school_geography": 0.5, - "middle_school_politics": 0.6923076923076923, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.3333333333333333, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.47368421052631576, - "professional_tour_guide": 0.4117647058823529, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.4583333333333333, - "high_school_history": 0.64, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.34615384615384615, - "sports_science": 0.4166666666666667, - "plant_protection": 0.48148148148148145, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.5882352941176471, - "accountant": 0.3888888888888889, - "fire_engineer": 0.4166666666666667, - "environmental_impact_assessment_engineer": 0.5, - "tax_accountant": 0.42592592592592593, - "physician": 0.5185185185185185 - } + "accuracy": 0.4 }, "prompt_2": { - "accuracy": 0.4539227895392279, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.375, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.35714285714285715, - "college_physics": 0.5, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.13043478260869565, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.4583333333333333, - "high_school_chemistry": 0.5, - "high_school_biology": 0.4166666666666667, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.5416666666666666, - "middle_school_chemistry": 0.4, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.4, - "business_administration": 0.42105263157894735, - "marxism": 0.5416666666666666, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.6176470588235294, - "teacher_qualification": 0.673469387755102, - "high_school_politics": 0.625, - "high_school_geography": 0.5, - "middle_school_politics": 0.6923076923076923, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.5357142857142857, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.4074074074074074, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.5263157894736842, - "professional_tour_guide": 0.5, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.5416666666666666, - "high_school_history": 0.64, - "middle_school_history": 0.5925925925925926, - "civil_servant": 0.3269230769230769, - "sports_science": 0.4583333333333333, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.5490196078431373, - "accountant": 0.3888888888888889, - "fire_engineer": 0.4722222222222222, - "environmental_impact_assessment_engineer": 0.5555555555555556, - "tax_accountant": 0.48148148148148145, - "physician": 0.5370370370370371 - } + "accuracy": 0.4 }, "prompt_3": { - "accuracy": 0.45143212951432127, - "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.38095238095238093, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.4827586206896552, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.375, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.6153846153846154, - "middle_school_physics": 0.5416666666666666, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.3333333333333333, - "business_administration": 0.34210526315789475, - "marxism": 0.5833333333333334, - "mao_zedong_thought": 0.6551724137931034, - "education_science": 0.6176470588235294, - "teacher_qualification": 0.673469387755102, - "high_school_politics": 0.625, - "high_school_geography": 0.5416666666666666, - "middle_school_politics": 0.6538461538461539, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.5357142857142857, - "ideological_and_moral_cultivation": 0.5416666666666666, - "logic": 0.37037037037037035, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.47368421052631576, - "professional_tour_guide": 0.47058823529411764, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.5833333333333334, - "high_school_history": 0.64, - "middle_school_history": 0.5555555555555556, - "civil_servant": 0.36538461538461536, - "sports_science": 0.4583333333333333, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.5, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.6078431372549019, - "accountant": 0.3888888888888889, - "fire_engineer": 0.5, - "environmental_impact_assessment_engineer": 0.5833333333333334, - "tax_accountant": 0.46296296296296297, - "physician": 0.5555555555555556 - } + "accuracy": 0.4 }, "prompt_4": { - "accuracy": 0.4259028642590286, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.375, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.2619047619047619, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.08333333333333333, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.5172413793103449, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.4583333333333333, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.5416666666666666, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.36666666666666664, - "business_administration": 0.39473684210526316, - "marxism": 0.5416666666666666, - "mao_zedong_thought": 0.6551724137931034, - "education_science": 0.5588235294117647, - "teacher_qualification": 0.5918367346938775, - "high_school_politics": 0.625, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.7692307692307693, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.4444444444444444, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.5526315789473685, - "professional_tour_guide": 0.5, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.64, - "middle_school_history": 0.4444444444444444, - "civil_servant": 0.38461538461538464, - "sports_science": 0.4166666666666667, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.5882352941176471, - "accountant": 0.37037037037037035, - "fire_engineer": 0.5, - "environmental_impact_assessment_engineer": 0.4722222222222222, - "tax_accountant": 0.3888888888888889, - "physician": 0.4444444444444444 - } + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.4396014943960149, - "category_acc": { - "computer_network": 0.375, - "operating_system": 0.375, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.35714285714285715, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.125, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.4166666666666667, - "high_school_biology": 0.5, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.46153846153846156, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.38333333333333336, - "business_administration": 0.5, - "marxism": 0.5416666666666666, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.6176470588235294, - "teacher_qualification": 0.673469387755102, - "high_school_politics": 0.5833333333333334, - "high_school_geography": 0.5, - "middle_school_politics": 0.6923076923076923, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.37037037037037035, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.4411764705882353, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.5, - "high_school_history": 0.68, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.3269230769230769, - "sports_science": 0.4583333333333333, - "plant_protection": 0.48148148148148145, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.5686274509803921, - "accountant": 0.37037037037037035, - "fire_engineer": 0.4166666666666667, - "environmental_impact_assessment_engineer": 0.5833333333333334, - "tax_accountant": 0.4444444444444444, - "physician": 0.5 - } + "accuracy": 0.5 } }, - "cmmlu": { + "mrpc": { "prompt_1": { - "accuracy": 0.48028673835125446 + "accuracy": 0.6 }, "prompt_2": { - "accuracy": 0.4767025089605735 + "accuracy": 0.6 }, "prompt_3": { - "accuracy": 0.48028673835125446 + "accuracy": 0.6 }, "prompt_4": { - "accuracy": 0.4659498207885305 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.45161290322580644 + "accuracy": 0.6 } + } + }, + "five_shot": { + "cross_xquad": { + "prompt_1": -1 + }, + "cross_mmlu": { + "prompt_1": -1 + }, + "cross_logiqa": { + "prompt_1": -1 + }, + "sg_eval": { + "prompt_1": -1 + }, + "cn_eval": { + "prompt_1": -1 + }, + "us_eval": { + "prompt_1": -1 + }, + "ph_eval": { + "prompt_1": -1 + }, + "sing2eng": { + "prompt_1": -1 + }, + "indommlu": { + "prompt_1": -1 + }, + "flores_ind2eng": { + "prompt_1": -1 + }, + "flores_vie2eng": { + "prompt_1": -1 + }, + "flores_zho2eng": { + "prompt_1": -1 + }, + "flores_zsm2eng": { + "prompt_1": -1 + }, + "mmlu": { + "prompt_1": -1 + }, + "mmlu_full": { + "prompt_1": -1 + }, + "c_eval": { + "prompt_1": -1 + }, + "c_eval_full": { + "prompt_1": -1 + }, + "cmmlu": { + "prompt_1": -1 }, "cmmlu_full": { + "prompt_1": -1 + }, + "zbench": { + "prompt_1": -1 + }, + "ind_emotion": { + "prompt_1": -1 + }, + "ocnli": { + "prompt_1": -1 + }, + "c3": { + "prompt_1": -1 + }, + "dream": { + "prompt_1": -1 + }, + "samsum": { + "prompt_1": -1 + }, + "dialogsum": { + "prompt_1": -1 + }, + "sst2": { + "prompt_1": -1 + }, + "cola": { + "prompt_1": -1 + }, + "qqp": { + "prompt_1": -1 + }, + "mnli": { + "prompt_1": -1 + }, + "qnli": { + "prompt_1": -1 + }, + "wnli": { + "prompt_1": -1 + }, + "rte": { + "prompt_1": -1 + }, + "mrpc": { + "prompt_1": -1 + } + } + }, + "sailor_1_8b_chat": { + "model_size": "1.8B", + "model_link": "https://huggingface.co/sail/Sailor-1.8B-Chat", + "zero_shot": { + "cross_xquad": { "prompt_1": { - "accuracy": 0.43740286651700916, - "category_acc": { - "agronomy": 0.38461538461538464, - "anatomy": 0.35135135135135137, - "ancient_chinese": 0.27439024390243905, - "arts": 0.60625, - "astronomy": 0.2909090909090909, - "business_ethics": 0.4880382775119617, - "chinese_civil_service_exam": 0.4125, - "chinese_driving_rule": 0.6717557251908397, - "chinese_food_culture": 0.4264705882352941, - "chinese_foreign_policy": 0.4953271028037383, - "chinese_history": 0.47678018575851394, - "chinese_literature": 0.36764705882352944, - "chinese_teacher_qualification": 0.547486033519553, - "clinical_knowledge": 0.379746835443038, - "college_actuarial_science": 0.19811320754716982, - "college_education": 0.5981308411214953, - "college_engineering_hydrology": 0.4056603773584906, - "college_law": 0.37962962962962965, - "college_mathematics": 0.3142857142857143, - "college_medical_statistics": 0.37735849056603776, - "college_medicine": 0.38461538461538464, - "computer_science": 0.4362745098039216, - "computer_security": 0.47953216374269003, - "conceptual_physics": 0.46938775510204084, - "construction_project_management": 0.381294964028777, - "economics": 0.4716981132075472, - "education": 0.5153374233128835, - "electrical_engineering": 0.43023255813953487, - "elementary_chinese": 0.4087301587301587, - "elementary_commonsense": 0.48484848484848486, - "elementary_information_and_technology": 0.6512605042016807, - "elementary_mathematics": 0.2956521739130435, - "ethnology": 0.4074074074074074, - "food_science": 0.4825174825174825, - "genetics": 0.375, - "global_facts": 0.3959731543624161, - "high_school_biology": 0.34911242603550297, - "high_school_chemistry": 0.3333333333333333, - "high_school_geography": 0.5, - "high_school_mathematics": 0.3170731707317073, - "high_school_physics": 0.4, - "high_school_politics": 0.3916083916083916, - "human_sexuality": 0.4444444444444444, - "international_law": 0.3837837837837838, - "journalism": 0.5174418604651163, - "jurisprudence": 0.43552311435523117, - "legal_and_moral_basis": 0.780373831775701, - "logical": 0.43089430894308944, - "machine_learning": 0.38524590163934425, - "management": 0.5142857142857142, - "marketing": 0.55, - "marxist_theory": 0.5132275132275133, - "modern_chinese": 0.3275862068965517, - "nutrition": 0.42758620689655175, - "philosophy": 0.49523809523809526, - "professional_accounting": 0.44, - "professional_law": 0.3744075829383886, - "professional_medicine": 0.3271276595744681, - "professional_psychology": 0.5129310344827587, - "public_relations": 0.4540229885057471, - "security_study": 0.4888888888888889, - "sociology": 0.4336283185840708, - "sports_science": 0.46060606060606063, - "traditional_chinese_medicine": 0.3675675675675676, - "virology": 0.39644970414201186, - "world_history": 0.45962732919254656, - "world_religions": 0.45625 - } + "overall_acc": 0.6323529411764706, + "language_acc": { + "Spanish": 0.6109243697478992, + "English": 0.6672268907563025, + "Chinese": 0.6386554621848739, + "Vietnamese": 0.6126050420168068 + }, + "consistency_score_2": 0.6215686274509804, + "consistency_score_3": 0.4714285714285714, + "consistency_score_4": 0.37815126050420167, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.6546218487394958, + "Spanish,Chinese": 0.6025210084033613, + "Spanish,Vietnamese": 0.6210084033613446, + "English,Chinese": 0.6235294117647059, + "English,Vietnamese": 0.6369747899159663, + "Chinese,Vietnamese": 0.5907563025210084 + }, + "3_combine": { + "Spanish,English,Chinese": 0.47983193277310926, + "Spanish,English,Vietnamese": 0.4899159663865546, + "Spanish,Chinese,Vietnamese": 0.4504201680672269, + "English,Chinese,Vietnamese": 0.46554621848739497 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.37815126050420167 + } + }, + "AC3_2": 0.6269144091851598, + "AC3_3": 0.5401598781391099, + "AC3_4": 0.4732787085259903 }, "prompt_2": { - "accuracy": 0.440338456225177, - "category_acc": { - "agronomy": 0.40236686390532544, - "anatomy": 0.32432432432432434, - "ancient_chinese": 0.31097560975609756, - "arts": 0.6375, - "astronomy": 0.30303030303030304, - "business_ethics": 0.48325358851674644, - "chinese_civil_service_exam": 0.3875, - "chinese_driving_rule": 0.6870229007633588, - "chinese_food_culture": 0.45588235294117646, - "chinese_foreign_policy": 0.45794392523364486, - "chinese_history": 0.48297213622291024, - "chinese_literature": 0.3431372549019608, - "chinese_teacher_qualification": 0.553072625698324, - "clinical_knowledge": 0.37130801687763715, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.6261682242990654, - "college_engineering_hydrology": 0.4056603773584906, - "college_law": 0.37037037037037035, - "college_mathematics": 0.34285714285714286, - "college_medical_statistics": 0.3867924528301887, - "college_medicine": 0.40293040293040294, - "computer_science": 0.45588235294117646, - "computer_security": 0.43859649122807015, - "conceptual_physics": 0.48299319727891155, - "construction_project_management": 0.381294964028777, - "economics": 0.4968553459119497, - "education": 0.49693251533742333, - "electrical_engineering": 0.4011627906976744, - "elementary_chinese": 0.4126984126984127, - "elementary_commonsense": 0.4797979797979798, - "elementary_information_and_technology": 0.6722689075630253, - "elementary_mathematics": 0.2782608695652174, - "ethnology": 0.43703703703703706, - "food_science": 0.4405594405594406, - "genetics": 0.375, - "global_facts": 0.436241610738255, - "high_school_biology": 0.33727810650887574, - "high_school_chemistry": 0.3181818181818182, - "high_school_geography": 0.5084745762711864, - "high_school_mathematics": 0.2926829268292683, - "high_school_physics": 0.42727272727272725, - "high_school_politics": 0.40559440559440557, - "human_sexuality": 0.46825396825396826, - "international_law": 0.4, - "journalism": 0.5058139534883721, - "jurisprudence": 0.44038929440389296, - "legal_and_moral_basis": 0.7990654205607477, - "logical": 0.4146341463414634, - "machine_learning": 0.38524590163934425, - "management": 0.5333333333333333, - "marketing": 0.5444444444444444, - "marxist_theory": 0.5079365079365079, - "modern_chinese": 0.33620689655172414, - "nutrition": 0.46206896551724136, - "philosophy": 0.44761904761904764, - "professional_accounting": 0.44, - "professional_law": 0.3791469194312796, - "professional_medicine": 0.30851063829787234, - "professional_psychology": 0.47844827586206895, - "public_relations": 0.47126436781609193, - "security_study": 0.4962962962962963, - "sociology": 0.46017699115044247, - "sports_science": 0.42424242424242425, - "traditional_chinese_medicine": 0.41081081081081083, - "virology": 0.42011834319526625, - "world_history": 0.4472049689440994, - "world_religions": 0.46875 - } + "overall_acc": 0.6466386554621849, + "language_acc": { + "Spanish": 0.6571428571428571, + "English": 0.6663865546218487, + "Chinese": 0.6504201680672269, + "Vietnamese": 0.6126050420168068 + }, + "consistency_score_2": 0.6168067226890757, + "consistency_score_3": 0.47016806722689075, + "consistency_score_4": 0.38235294117647056, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.66890756302521, + "Spanish,Chinese": 0.5949579831932773, + "Spanish,Vietnamese": 0.6184873949579832, + "English,Chinese": 0.5974789915966386, + "English,Vietnamese": 0.6378151260504201, + "Chinese,Vietnamese": 0.5831932773109244 + }, + "3_combine": { + "Spanish,English,Chinese": 0.473109243697479, + "Spanish,English,Vietnamese": 0.5033613445378151, + "Spanish,Chinese,Vietnamese": 0.4504201680672269, + "English,Chinese,Vietnamese": 0.453781512605042 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.38235294117647056 + } + }, + "AC3_2": 0.6313704995965111, + "AC3_3": 0.5444609897647811, + "AC3_4": 0.48055628943519973 }, "prompt_3": { - "accuracy": 0.44163356933172165, - "category_acc": { - "agronomy": 0.4260355029585799, - "anatomy": 0.33783783783783783, - "ancient_chinese": 0.24390243902439024, - "arts": 0.64375, - "astronomy": 0.32727272727272727, - "business_ethics": 0.4784688995215311, - "chinese_civil_service_exam": 0.4125, - "chinese_driving_rule": 0.7175572519083969, - "chinese_food_culture": 0.4485294117647059, - "chinese_foreign_policy": 0.48598130841121495, - "chinese_history": 0.47678018575851394, - "chinese_literature": 0.3872549019607843, - "chinese_teacher_qualification": 0.5586592178770949, - "clinical_knowledge": 0.3755274261603376, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.5981308411214953, - "college_engineering_hydrology": 0.4056603773584906, - "college_law": 0.3888888888888889, - "college_mathematics": 0.2857142857142857, - "college_medical_statistics": 0.44339622641509435, - "college_medicine": 0.4175824175824176, - "computer_science": 0.46078431372549017, - "computer_security": 0.4444444444444444, - "conceptual_physics": 0.4557823129251701, - "construction_project_management": 0.37410071942446044, - "economics": 0.4591194968553459, - "education": 0.5153374233128835, - "electrical_engineering": 0.4418604651162791, - "elementary_chinese": 0.4246031746031746, - "elementary_commonsense": 0.4595959595959596, - "elementary_information_and_technology": 0.6722689075630253, - "elementary_mathematics": 0.30434782608695654, - "ethnology": 0.43703703703703706, - "food_science": 0.4405594405594406, - "genetics": 0.35795454545454547, - "global_facts": 0.40268456375838924, - "high_school_biology": 0.35502958579881655, - "high_school_chemistry": 0.3333333333333333, - "high_school_geography": 0.4661016949152542, - "high_school_mathematics": 0.24390243902439024, - "high_school_physics": 0.4090909090909091, - "high_school_politics": 0.4125874125874126, - "human_sexuality": 0.4523809523809524, - "international_law": 0.3783783783783784, - "journalism": 0.5406976744186046, - "jurisprudence": 0.4257907542579075, - "legal_and_moral_basis": 0.7850467289719626, - "logical": 0.4065040650406504, - "machine_learning": 0.4016393442622951, - "management": 0.5333333333333333, - "marketing": 0.5222222222222223, - "marxist_theory": 0.5291005291005291, - "modern_chinese": 0.41379310344827586, - "nutrition": 0.4827586206896552, - "philosophy": 0.45714285714285713, - "professional_accounting": 0.4228571428571429, - "professional_law": 0.3981042654028436, - "professional_medicine": 0.324468085106383, - "professional_psychology": 0.4827586206896552, - "public_relations": 0.4885057471264368, - "security_study": 0.5111111111111111, - "sociology": 0.4247787610619469, - "sports_science": 0.44242424242424244, - "traditional_chinese_medicine": 0.3783783783783784, - "virology": 0.40236686390532544, - "world_history": 0.45962732919254656, - "world_religions": 0.4625 - } + "overall_acc": 0.6630252100840336, + "language_acc": { + "Spanish": 0.6747899159663866, + "English": 0.7025210084033613, + "Chinese": 0.634453781512605, + "Vietnamese": 0.6403361344537815 + }, + "consistency_score_2": 0.6438375350140056, + "consistency_score_3": 0.5039915966386554, + "consistency_score_4": 0.42016806722689076, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.7151260504201681, + "Spanish,Chinese": 0.6201680672268908, + "Spanish,Vietnamese": 0.6655462184873949, + "English,Chinese": 0.6025210084033613, + "English,Vietnamese": 0.6512605042016807, + "Chinese,Vietnamese": 0.6084033613445378 + }, + "3_combine": { + "Spanish,English,Chinese": 0.5058823529411764, + "Spanish,English,Vietnamese": 0.5445378151260504, + "Spanish,Chinese,Vietnamese": 0.4865546218487395, + "English,Chinese,Vietnamese": 0.4789915966386555 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.42016806722689076 + } + }, + "AC3_2": 0.6532905134547375, + "AC3_3": 0.5726723596238706, + "AC3_4": 0.5143717688307397 }, "prompt_4": { - "accuracy": 0.428250733897427, - "category_acc": { - "agronomy": 0.40236686390532544, - "anatomy": 0.33783783783783783, - "ancient_chinese": 0.25, - "arts": 0.60625, - "astronomy": 0.30303030303030304, - "business_ethics": 0.49760765550239233, - "chinese_civil_service_exam": 0.36875, - "chinese_driving_rule": 0.7022900763358778, - "chinese_food_culture": 0.4264705882352941, - "chinese_foreign_policy": 0.4953271028037383, - "chinese_history": 0.47987616099071206, - "chinese_literature": 0.3627450980392157, - "chinese_teacher_qualification": 0.5307262569832403, - "clinical_knowledge": 0.35443037974683544, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.5981308411214953, - "college_engineering_hydrology": 0.3867924528301887, - "college_law": 0.39814814814814814, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.4056603773584906, - "college_medicine": 0.39194139194139194, - "computer_science": 0.4264705882352941, - "computer_security": 0.45614035087719296, - "conceptual_physics": 0.47619047619047616, - "construction_project_management": 0.41007194244604317, - "economics": 0.44025157232704404, - "education": 0.5030674846625767, - "electrical_engineering": 0.4186046511627907, - "elementary_chinese": 0.4365079365079365, - "elementary_commonsense": 0.4393939393939394, - "elementary_information_and_technology": 0.634453781512605, - "elementary_mathematics": 0.24347826086956523, - "ethnology": 0.4074074074074074, - "food_science": 0.43356643356643354, - "genetics": 0.3409090909090909, - "global_facts": 0.40939597315436244, - "high_school_biology": 0.3136094674556213, - "high_school_chemistry": 0.3484848484848485, - "high_school_geography": 0.4491525423728814, - "high_school_mathematics": 0.25, - "high_school_physics": 0.35454545454545455, - "high_school_politics": 0.40559440559440557, - "human_sexuality": 0.4523809523809524, - "international_law": 0.372972972972973, - "journalism": 0.5058139534883721, - "jurisprudence": 0.43552311435523117, - "legal_and_moral_basis": 0.7616822429906542, - "logical": 0.4634146341463415, - "machine_learning": 0.38524590163934425, - "management": 0.49047619047619045, - "marketing": 0.5388888888888889, - "marxist_theory": 0.4656084656084656, - "modern_chinese": 0.35344827586206895, - "nutrition": 0.41379310344827586, - "philosophy": 0.44761904761904764, - "professional_accounting": 0.4228571428571429, - "professional_law": 0.35545023696682465, - "professional_medicine": 0.324468085106383, - "professional_psychology": 0.49137931034482757, - "public_relations": 0.4425287356321839, - "security_study": 0.4666666666666667, - "sociology": 0.4469026548672566, - "sports_science": 0.43636363636363634, - "traditional_chinese_medicine": 0.3891891891891892, - "virology": 0.39644970414201186, - "world_history": 0.4720496894409938, - "world_religions": 0.41875 - } + "overall_acc": 0.6373949579831933, + "language_acc": { + "Spanish": 0.6411764705882353, + "English": 0.6823529411764706, + "Chinese": 0.6277310924369748, + "Vietnamese": 0.5983193277310924 + }, + "consistency_score_2": 0.6127450980392157, + "consistency_score_3": 0.4607142857142857, + "consistency_score_4": 0.3663865546218487, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.6697478991596638, + "Spanish,Chinese": 0.6033613445378151, + "Spanish,Vietnamese": 0.6159663865546219, + "English,Chinese": 0.6050420168067226, + "English,Vietnamese": 0.6142857142857143, + "Chinese,Vietnamese": 0.5680672268907563 + }, + "3_combine": { + "Spanish,English,Chinese": 0.4756302521008403, + "Spanish,English,Vietnamese": 0.4890756302521008, + "Spanish,Chinese,Vietnamese": 0.4369747899159664, + "English,Chinese,Vietnamese": 0.4411764705882353 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.3663865546218487 + } + }, + "AC3_2": 0.6248270089521416, + "AC3_3": 0.534841072404764, + "AC3_4": 0.46530632340391215 }, "prompt_5": { - "accuracy": 0.43438093593507165, - "category_acc": { - "agronomy": 0.378698224852071, - "anatomy": 0.33783783783783783, - "ancient_chinese": 0.22560975609756098, - "arts": 0.6375, - "astronomy": 0.3090909090909091, - "business_ethics": 0.49760765550239233, - "chinese_civil_service_exam": 0.41875, - "chinese_driving_rule": 0.6793893129770993, - "chinese_food_culture": 0.4338235294117647, - "chinese_foreign_policy": 0.4766355140186916, - "chinese_history": 0.4674922600619195, - "chinese_literature": 0.3627450980392157, - "chinese_teacher_qualification": 0.5363128491620112, - "clinical_knowledge": 0.3628691983122363, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.5887850467289719, - "college_engineering_hydrology": 0.39622641509433965, - "college_law": 0.3611111111111111, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.4339622641509434, - "college_medicine": 0.3882783882783883, - "computer_science": 0.47549019607843135, - "computer_security": 0.4678362573099415, - "conceptual_physics": 0.4965986394557823, - "construction_project_management": 0.39568345323741005, - "economics": 0.4339622641509434, - "education": 0.5153374233128835, - "electrical_engineering": 0.4186046511627907, - "elementary_chinese": 0.42063492063492064, - "elementary_commonsense": 0.45454545454545453, - "elementary_information_and_technology": 0.6554621848739496, - "elementary_mathematics": 0.29130434782608694, - "ethnology": 0.4666666666666667, - "food_science": 0.45454545454545453, - "genetics": 0.3693181818181818, - "global_facts": 0.3959731543624161, - "high_school_biology": 0.33136094674556216, - "high_school_chemistry": 0.3181818181818182, - "high_school_geography": 0.4576271186440678, - "high_school_mathematics": 0.27439024390243905, - "high_school_physics": 0.39090909090909093, - "high_school_politics": 0.40559440559440557, - "human_sexuality": 0.4444444444444444, - "international_law": 0.34594594594594597, - "journalism": 0.5174418604651163, - "jurisprudence": 0.43552311435523117, - "legal_and_moral_basis": 0.7757009345794392, - "logical": 0.4065040650406504, - "machine_learning": 0.3442622950819672, - "management": 0.5047619047619047, - "marketing": 0.5277777777777778, - "marxist_theory": 0.5079365079365079, - "modern_chinese": 0.3620689655172414, - "nutrition": 0.4068965517241379, - "philosophy": 0.42857142857142855, - "professional_accounting": 0.44, - "professional_law": 0.38388625592417064, - "professional_medicine": 0.324468085106383, - "professional_psychology": 0.5086206896551724, - "public_relations": 0.45977011494252873, - "security_study": 0.5259259259259259, - "sociology": 0.43805309734513276, - "sports_science": 0.42424242424242425, - "traditional_chinese_medicine": 0.3783783783783784, - "virology": 0.40828402366863903, - "world_history": 0.45962732919254656, - "world_religions": 0.475 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.30303030303030304 - }, - "prompt_2": { - "accuracy": 0.21212121212121213 - }, - "prompt_3": { - "accuracy": 0.30303030303030304 - }, - "prompt_4": { - "accuracy": 0.21212121212121213 - }, - "prompt_5": { - "accuracy": 0.30303030303030304 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.6431818181818182 - }, - "prompt_2": { - "accuracy": 0.6477272727272727 - }, - "prompt_3": { - "accuracy": 0.6704545454545454 - }, - "prompt_4": { - "accuracy": 0.5840909090909091 - }, - "prompt_5": { - "accuracy": 0.6068181818181818 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3976271186440678 - }, - "prompt_2": { - "accuracy": 0.3928813559322034 - }, - "prompt_3": { - "accuracy": 0.4023728813559322 - }, - "prompt_4": { - "accuracy": 0.3806779661016949 - }, - "prompt_5": { - "accuracy": 0.3616949152542373 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.7778608825729244 - }, - "prompt_2": { - "accuracy": 0.7696335078534031 - }, - "prompt_3": { - "accuracy": 0.7778608825729244 - }, - "prompt_4": { - "accuracy": 0.7729992520568437 - }, - "prompt_5": { - "accuracy": 0.7718773373223635 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.7143557079862812 - }, - "prompt_2": { - "accuracy": 0.718765311121999 - }, - "prompt_3": { - "accuracy": 0.7241548260656541 - }, - "prompt_4": { - "accuracy": 0.7153356197942186 - }, - "prompt_5": { - "accuracy": 0.7109260166585007 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.3282850882209375, - "rouge2": 0.10397893538900248, - "rougeL": 0.23947917154228332, - "avg_rouge": 0.22391439838407443 - }, - "prompt_2": { - "rouge1": 0.3330434286177517, - "rouge2": 0.10727973676869428, - "rougeL": 0.2442139509277289, - "avg_rouge": 0.22817903877139165 - }, - "prompt_3": { - "rouge1": 0.34627792405695745, - "rouge2": 0.11043052401338135, - "rougeL": 0.25588382522436004, - "avg_rouge": 0.23753075776489962 - }, - "prompt_4": { - "rouge1": 0.33563520400875785, - "rouge2": 0.11037814048617912, - "rougeL": 0.2469015131690465, - "avg_rouge": 0.2309716192213278 - }, - "prompt_5": { - "rouge1": 0.34338330367375025, - "rouge2": 0.10810859926877506, - "rougeL": 0.2527894506228756, - "avg_rouge": 0.23476045118846697 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.21952370642540356, - "rouge2": 0.05735534682293868, - "rougeL": 0.16000096395591165, - "avg_rouge": 0.14562667240141794 - }, - "prompt_2": { - "rouge1": 0.21759519249923093, - "rouge2": 0.05500692448173594, - "rougeL": 0.1581552632193168, - "avg_rouge": 0.14358579340009456 - }, - "prompt_3": { - "rouge1": 0.2176202907199696, - "rouge2": 0.054628780758061206, - "rougeL": 0.15805702711780686, - "avg_rouge": 0.14343536619861255 - }, - "prompt_4": { - "rouge1": 0.21898425174121633, - "rouge2": 0.05678643072422844, - "rougeL": 0.15927762262092648, - "avg_rouge": 0.1450161016954571 - }, - "prompt_5": { - "rouge1": 0.2220267684607781, - "rouge2": 0.05898808646026801, - "rougeL": 0.16292247919841782, - "avg_rouge": 0.14797911137315464 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.911697247706422 - }, - "prompt_2": { - "accuracy": 0.8990825688073395 - }, - "prompt_3": { - "accuracy": 0.9094036697247706 - }, - "prompt_4": { - "accuracy": 0.9197247706422018 - }, - "prompt_5": { - "accuracy": 0.9105504587155964 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.5752636625119847 - }, - "prompt_2": { - "accuracy": 0.6030680728667306 - }, - "prompt_3": { - "accuracy": 0.6519654841802492 - }, - "prompt_4": { - "accuracy": 0.6155321188878236 - }, - "prompt_5": { - "accuracy": 0.6529242569511026 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.5645 - }, - "prompt_2": { - "accuracy": 0.6945 - }, - "prompt_3": { - "accuracy": 0.604 - }, - "prompt_4": { - "accuracy": 0.629 - }, - "prompt_5": { - "accuracy": 0.7105 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.6115 - }, - "prompt_2": { - "accuracy": 0.6325 - }, - "prompt_3": { - "accuracy": 0.622 - }, - "prompt_4": { - "accuracy": 0.6165 - }, - "prompt_5": { - "accuracy": 0.589 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.654 - }, - "prompt_2": { - "accuracy": 0.6225 - }, - "prompt_3": { - "accuracy": 0.6505 - }, - "prompt_4": { - "accuracy": 0.571 - }, - "prompt_5": { - "accuracy": 0.649 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.4647887323943662 - }, - "prompt_2": { - "accuracy": 0.5492957746478874 - }, - "prompt_3": { - "accuracy": 0.4647887323943662 - }, - "prompt_4": { - "accuracy": 0.4507042253521127 - }, - "prompt_5": { - "accuracy": 0.49295774647887325 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.6714801444043321 - }, - "prompt_2": { - "accuracy": 0.7148014440433214 - }, - "prompt_3": { - "accuracy": 0.6823104693140795 - }, - "prompt_4": { - "accuracy": 0.6028880866425993 - }, - "prompt_5": { - "accuracy": 0.6787003610108303 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.7205882352941176 - }, - "prompt_2": { - "accuracy": 0.7450980392156863 - }, - "prompt_3": { - "accuracy": 0.7034313725490197 - }, - "prompt_4": { - "accuracy": 0.7377450980392157 - }, - "prompt_5": { - "accuracy": 0.7401960784313726 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.719747899159664, + "overall_acc": 0.6640756302521008, "language_acc": { - "English": 0.7663865546218488, - "Chinese": 0.711764705882353, - "Spanish": 0.6915966386554622, - "Vietnamese": 0.7092436974789916 - }, - "consistency_score_2": 0.7605042016806722, - "consistency_score_3": 0.657563025210084, - "consistency_score_4": 0.5949579831932773, + "Spanish": 0.6697478991596638, + "English": 0.6966386554621848, + "Chinese": 0.6630252100840336, + "Vietnamese": 0.626890756302521 + }, + "consistency_score_2": 0.6662464985994397, + "consistency_score_3": 0.5294117647058824, + "consistency_score_4": 0.44537815126050423, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.780672268907563, - "English,Spanish": 0.7655462184873949, - "English,Vietnamese": 0.7647058823529411, - "Chinese,Spanish": 0.7512605042016807, - "Chinese,Vietnamese": 0.7571428571428571, - "Spanish,Vietnamese": 0.7436974789915967 + "Spanish,English": 0.7151260504201681, + "Spanish,Chinese": 0.6512605042016807, + "Spanish,Vietnamese": 0.6899159663865546, + "English,Chinese": 0.646218487394958, + "English,Vietnamese": 0.6840336134453782, + "Chinese,Vietnamese": 0.6109243697478992 }, "3_combine": { - "English,Chinese,Spanish": 0.6638655462184874, - "English,Chinese,Vietnamese": 0.6663865546218487, - "English,Spanish,Vietnamese": 0.6554621848739496, - "Chinese,Spanish,Vietnamese": 0.6445378151260505 + "Spanish,English,Chinese": 0.5369747899159664, + "Spanish,English,Vietnamese": 0.5672268907563025, + "Spanish,Chinese,Vietnamese": 0.5100840336134453, + "English,Chinese,Vietnamese": 0.5033613445378151 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5949579831932773 + "Spanish,English,Chinese,Vietnamese": 0.44537815126050423 } }, - "AC3_2": 0.7395649715532123, - "AC3_3": 0.687251654730379, - "AC3_4": 0.6514305050383917 + "AC3_2": 0.6651592931245526, + "AC3_3": 0.589146484103178, + "AC3_4": 0.5331718750265324 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.46952380952380957, + "overall_acc": 0.3647619047619047, "language_acc": { - "Spanish": 0.43333333333333335, - "English": 0.5733333333333334, - "Chinese": 0.5, - "Vietnamese": 0.4666666666666667, - "Filipino": 0.36666666666666664, - "Indonesian": 0.4666666666666667, - "Malay": 0.48 - }, - "consistency_score_2": 0.5498412698412698, - "consistency_score_3": 0.3769523809523809, - "consistency_score_4": 0.2864761904761905, - "consistency_score_5": 0.23079365079365083, - "consistency_score_6": 0.19333333333333333, - "consistency_score_7": 0.16666666666666666, + "Filipino": 0.23333333333333334, + "Vietnamese": 0.35333333333333333, + "Chinese": 0.38, + "Spanish": 0.36666666666666664, + "Malay": 0.3933333333333333, + "Indonesian": 0.41333333333333333, + "English": 0.41333333333333333 + }, + "consistency_score_2": 0.45968253968253964, + "consistency_score_3": 0.2721904761904761, + "consistency_score_4": 0.18666666666666668, + "consistency_score_5": 0.13999999999999999, + "consistency_score_6": 0.10952380952380951, + "consistency_score_7": 0.08666666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5866666666666667, - "Spanish,Chinese": 0.5866666666666667, - "Spanish,Vietnamese": 0.54, - "Spanish,Filipino": 0.5, - "Spanish,Indonesian": 0.5133333333333333, - "Spanish,Malay": 0.6066666666666667, - "English,Chinese": 0.6333333333333333, - "English,Vietnamese": 0.6, - "English,Filipino": 0.47333333333333333, - "English,Indonesian": 0.58, - "English,Malay": 0.5933333333333334, - "Chinese,Vietnamese": 0.56, - "Chinese,Filipino": 0.42, - "Chinese,Indonesian": 0.5266666666666666, - "Chinese,Malay": 0.5933333333333334, - "Vietnamese,Filipino": 0.52, - "Vietnamese,Indonesian": 0.5933333333333334, - "Vietnamese,Malay": 0.5666666666666667, - "Filipino,Indonesian": 0.44666666666666666, - "Filipino,Malay": 0.5, - "Indonesian,Malay": 0.6066666666666667 + "Filipino,Vietnamese": 0.49333333333333335, + "Filipino,Chinese": 0.4, + "Filipino,Spanish": 0.4266666666666667, + "Filipino,Malay": 0.43333333333333335, + "Filipino,Indonesian": 0.4266666666666667, + "Filipino,English": 0.38666666666666666, + "Vietnamese,Chinese": 0.48, + "Vietnamese,Spanish": 0.48, + "Vietnamese,Malay": 0.4066666666666667, + "Vietnamese,Indonesian": 0.48, + "Vietnamese,English": 0.4866666666666667, + "Chinese,Spanish": 0.43333333333333335, + "Chinese,Malay": 0.41333333333333333, + "Chinese,Indonesian": 0.44666666666666666, + "Chinese,English": 0.44, + "Spanish,Malay": 0.44, + "Spanish,Indonesian": 0.46, + "Spanish,English": 0.49333333333333335, + "Malay,Indonesian": 0.5866666666666667, + "Malay,English": 0.52, + "Indonesian,English": 0.52 }, "3_combine": { - "Spanish,English,Chinese": 0.4533333333333333, - "Spanish,English,Vietnamese": 0.41333333333333333, - "Spanish,English,Filipino": 0.32, - "Spanish,English,Indonesian": 0.3933333333333333, - "Spanish,English,Malay": 0.43333333333333335, - "Spanish,Chinese,Vietnamese": 0.3933333333333333, - "Spanish,Chinese,Filipino": 0.32, - "Spanish,Chinese,Indonesian": 0.37333333333333335, - "Spanish,Chinese,Malay": 0.44666666666666666, - "Spanish,Vietnamese,Filipino": 0.32, - "Spanish,Vietnamese,Indonesian": 0.37333333333333335, - "Spanish,Vietnamese,Malay": 0.4, - "Spanish,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Filipino,Malay": 0.36, - "Spanish,Indonesian,Malay": 0.41333333333333333, - "English,Chinese,Vietnamese": 0.4533333333333333, - "English,Chinese,Filipino": 0.32666666666666666, - "English,Chinese,Indonesian": 0.42, - "English,Chinese,Malay": 0.4533333333333333, - "English,Vietnamese,Filipino": 0.3466666666666667, - "English,Vietnamese,Indonesian": 0.42, - "English,Vietnamese,Malay": 0.43333333333333335, - "English,Filipino,Indonesian": 0.32, - "English,Filipino,Malay": 0.3466666666666667, - "English,Indonesian,Malay": 0.44, - "Chinese,Vietnamese,Filipino": 0.29333333333333333, - "Chinese,Vietnamese,Indonesian": 0.3933333333333333, - "Chinese,Vietnamese,Malay": 0.41333333333333333, - "Chinese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Filipino,Malay": 0.32666666666666666, - "Chinese,Indonesian,Malay": 0.41333333333333333, - "Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Vietnamese,Filipino,Malay": 0.3333333333333333, - "Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Filipino,Indonesian,Malay": 0.3333333333333333 + "Filipino,Vietnamese,Chinese": 0.25333333333333335, + "Filipino,Vietnamese,Spanish": 0.2733333333333333, + "Filipino,Vietnamese,Malay": 0.25333333333333335, + "Filipino,Vietnamese,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,English": 0.26, + "Filipino,Chinese,Spanish": 0.21333333333333335, + "Filipino,Chinese,Malay": 0.22666666666666666, + "Filipino,Chinese,Indonesian": 0.22, + "Filipino,Chinese,English": 0.20666666666666667, + "Filipino,Spanish,Malay": 0.26666666666666666, + "Filipino,Spanish,Indonesian": 0.24, + "Filipino,Spanish,English": 0.25333333333333335, + "Filipino,Malay,Indonesian": 0.2866666666666667, + "Filipino,Malay,English": 0.26, + "Filipino,Indonesian,English": 0.24666666666666667, + "Vietnamese,Chinese,Spanish": 0.2866666666666667, + "Vietnamese,Chinese,Malay": 0.23333333333333334, + "Vietnamese,Chinese,Indonesian": 0.29333333333333333, + "Vietnamese,Chinese,English": 0.29333333333333333, + "Vietnamese,Spanish,Malay": 0.2733333333333333, + "Vietnamese,Spanish,Indonesian": 0.3, + "Vietnamese,Spanish,English": 0.30666666666666664, + "Vietnamese,Malay,Indonesian": 0.30666666666666664, + "Vietnamese,Malay,English": 0.2866666666666667, + "Vietnamese,Indonesian,English": 0.32, + "Chinese,Spanish,Malay": 0.25333333333333335, + "Chinese,Spanish,Indonesian": 0.25333333333333335, + "Chinese,Spanish,English": 0.25333333333333335, + "Chinese,Malay,Indonesian": 0.30666666666666664, + "Chinese,Malay,English": 0.2733333333333333, + "Chinese,Indonesian,English": 0.2733333333333333, + "Spanish,Malay,Indonesian": 0.30666666666666664, + "Spanish,Malay,English": 0.28, + "Spanish,Indonesian,English": 0.30666666666666664, + "Malay,Indonesian,English": 0.37333333333333335 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.3333333333333333, - "Spanish,English,Chinese,Filipino": 0.26, - "Spanish,English,Chinese,Indonesian": 0.32, - "Spanish,English,Chinese,Malay": 0.36, - "Spanish,English,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,English,Vietnamese,Malay": 0.3333333333333333, - "Spanish,English,Filipino,Indonesian": 0.24, - "Spanish,English,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Indonesian,Malay": 0.3466666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.32666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.22, - "Spanish,Chinese,Filipino,Malay": 0.2733333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.32, - "Spanish,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,Vietnamese,Filipino,Malay": 0.26, - "Spanish,Vietnamese,Indonesian,Malay": 0.32, - "Spanish,Filipino,Indonesian,Malay": 0.26, - "English,Chinese,Vietnamese,Filipino": 0.26, - "English,Chinese,Vietnamese,Indonesian": 0.34, - "English,Chinese,Vietnamese,Malay": 0.35333333333333333, - "English,Chinese,Filipino,Indonesian": 0.24, - "English,Chinese,Filipino,Malay": 0.26, - "English,Chinese,Indonesian,Malay": 0.36, - "English,Vietnamese,Filipino,Indonesian": 0.26, - "English,Vietnamese,Filipino,Malay": 0.2866666666666667, - "English,Vietnamese,Indonesian,Malay": 0.36666666666666664, - "English,Filipino,Indonesian,Malay": 0.26, - "Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.22, - "Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 + "Filipino,Vietnamese,Chinese,Spanish": 0.14, + "Filipino,Vietnamese,Chinese,Malay": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Indonesian": 0.18, + "Filipino,Vietnamese,Chinese,English": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Malay": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,English": 0.18, + "Filipino,Vietnamese,Malay,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Malay,English": 0.18666666666666668, + "Filipino,Vietnamese,Indonesian,English": 0.18666666666666668, + "Filipino,Chinese,Spanish,Malay": 0.16, + "Filipino,Chinese,Spanish,Indonesian": 0.14, + "Filipino,Chinese,Spanish,English": 0.14, + "Filipino,Chinese,Malay,Indonesian": 0.17333333333333334, + "Filipino,Chinese,Malay,English": 0.15333333333333332, + "Filipino,Chinese,Indonesian,English": 0.15333333333333332, + "Filipino,Spanish,Malay,Indonesian": 0.19333333333333333, + "Filipino,Spanish,Malay,English": 0.18666666666666668, + "Filipino,Spanish,Indonesian,English": 0.18, + "Filipino,Malay,Indonesian,English": 0.19333333333333333, + "Vietnamese,Chinese,Spanish,Malay": 0.18666666666666668, + "Vietnamese,Chinese,Spanish,Indonesian": 0.21333333333333335, + "Vietnamese,Chinese,Spanish,English": 0.18666666666666668, + "Vietnamese,Chinese,Malay,Indonesian": 0.20666666666666667, + "Vietnamese,Chinese,Malay,English": 0.18, + "Vietnamese,Chinese,Indonesian,English": 0.20666666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.22, + "Vietnamese,Spanish,Malay,English": 0.21333333333333335, + "Vietnamese,Spanish,Indonesian,English": 0.23333333333333334, + "Vietnamese,Malay,Indonesian,English": 0.23333333333333334, + "Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Chinese,Spanish,Malay,English": 0.18, + "Chinese,Spanish,Indonesian,English": 0.19333333333333333, + "Chinese,Malay,Indonesian,English": 0.23333333333333334, + "Spanish,Malay,Indonesian,English": 0.22 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.2, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.2866666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.2, - "Spanish,English,Chinese,Filipino,Malay": 0.22, - "Spanish,English,Chinese,Indonesian,Malay": 0.29333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.26, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.22, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Filipino,Indonesian,Malay": 0.2, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.1, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.14, + "Filipino,Vietnamese,Chinese,Malay,English": 0.12, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Malay,English": 0.14666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14666666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,English": 0.11333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.12, + "Filipino,Chinese,Malay,Indonesian,English": 0.13333333333333333, + "Filipino,Spanish,Malay,Indonesian,English": 0.14666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16666666666666666, + "Vietnamese,Chinese,Spanish,Malay,English": 0.14, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16666666666666666, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.16666666666666666, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Chinese,Spanish,Malay,Indonesian,English": 0.16 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.1, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.11333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667 } }, - "AC3_2": 0.5065183668735356, - "AC3_3": 0.41817624614664617, - "AC3_4": 0.35583966209644835, - "AC3_5": 0.30946854893478537, - "AC3_6": 0.2738888888475694, - "AC3_7": 0.2460079839932672 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.36444805194805197, + "AC3_2": 0.4067579807710232, + "AC3_3": 0.3117492594634734, + "AC3_4": 0.246954519241341, + "AC3_5": 0.20233962260142327, + "AC3_6": 0.16846433348905487, + "AC3_7": 0.14005625875941088 + }, + "prompt_2": { + "overall_acc": 0.33714285714285713, "language_acc": { - "English": 0.38636363636363635, - "Filipino": 0.3068181818181818, - "Spanish": 0.3522727272727273, - "Malay": 0.3522727272727273, - "Chinese": 0.38636363636363635, - "Indonesian": 0.4090909090909091, - "Vietnamese": 0.35795454545454547 + "Filipino": 0.22666666666666666, + "Vietnamese": 0.32666666666666666, + "Chinese": 0.35333333333333333, + "Spanish": 0.32666666666666666, + "Malay": 0.37333333333333335, + "Indonesian": 0.36666666666666664, + "English": 0.38666666666666666 }, - "consistency_score_2": 0.5627705627705628, - "consistency_score_3": 0.40129870129870127, - "consistency_score_4": 0.3126623376623377, - "consistency_score_5": 0.25243506493506496, - "consistency_score_6": 0.20535714285714285, - "consistency_score_7": 0.16477272727272727, + "consistency_score_2": 0.4333333333333334, + "consistency_score_3": 0.24723809523809526, + "consistency_score_4": 0.16971428571428573, + "consistency_score_5": 0.1304761904761905, + "consistency_score_6": 0.10571428571428573, + "consistency_score_7": 0.08666666666666667, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4431818181818182, - "English,Spanish": 0.6477272727272727, - "English,Malay": 0.6022727272727273, - "English,Chinese": 0.6136363636363636, - "English,Indonesian": 0.6136363636363636, - "English,Vietnamese": 0.5965909090909091, - "Filipino,Spanish": 0.4715909090909091, - "Filipino,Malay": 0.44886363636363635, - "Filipino,Chinese": 0.4375, - "Filipino,Indonesian": 0.4659090909090909, - "Filipino,Vietnamese": 0.4431818181818182, - "Spanish,Malay": 0.6306818181818182, - "Spanish,Chinese": 0.6079545454545454, - "Spanish,Indonesian": 0.6363636363636364, - "Spanish,Vietnamese": 0.5965909090909091, - "Malay,Chinese": 0.5738636363636364, - "Malay,Indonesian": 0.6534090909090909, - "Malay,Vietnamese": 0.5795454545454546, - "Chinese,Indonesian": 0.5852272727272727, - "Chinese,Vietnamese": 0.5625, - "Indonesian,Vietnamese": 0.6079545454545454 + "Filipino,Vietnamese": 0.4, + "Filipino,Chinese": 0.4, + "Filipino,Spanish": 0.4266666666666667, + "Filipino,Malay": 0.36666666666666664, + "Filipino,Indonesian": 0.38666666666666666, + "Filipino,English": 0.38666666666666666, + "Vietnamese,Chinese": 0.4266666666666667, + "Vietnamese,Spanish": 0.42, + "Vietnamese,Malay": 0.5, + "Vietnamese,Indonesian": 0.48, + "Vietnamese,English": 0.4666666666666667, + "Chinese,Spanish": 0.44666666666666666, + "Chinese,Malay": 0.42, + "Chinese,Indonesian": 0.35333333333333333, + "Chinese,English": 0.5066666666666667, + "Spanish,Malay": 0.4266666666666667, + "Spanish,Indonesian": 0.43333333333333335, + "Spanish,English": 0.3933333333333333, + "Malay,Indonesian": 0.5133333333333333, + "Malay,English": 0.48, + "Indonesian,English": 0.4666666666666667 }, "3_combine": { - "English,Filipino,Spanish": 0.35795454545454547, - "English,Filipino,Malay": 0.3181818181818182, - "English,Filipino,Chinese": 0.3068181818181818, - "English,Filipino,Indonesian": 0.32954545454545453, - "English,Filipino,Vietnamese": 0.3409090909090909, - "English,Spanish,Malay": 0.48295454545454547, - "English,Spanish,Chinese": 0.4715909090909091, - "English,Spanish,Indonesian": 0.5, - "English,Spanish,Vietnamese": 0.4659090909090909, - "English,Malay,Chinese": 0.42613636363636365, - "English,Malay,Indonesian": 0.48295454545454547, - "English,Malay,Vietnamese": 0.44886363636363635, - "English,Chinese,Indonesian": 0.4602272727272727, - "English,Chinese,Vietnamese": 0.42613636363636365, - "English,Indonesian,Vietnamese": 0.4715909090909091, - "Filipino,Spanish,Malay": 0.32954545454545453, - "Filipino,Spanish,Chinese": 0.32954545454545453, - "Filipino,Spanish,Indonesian": 0.3522727272727273, - "Filipino,Spanish,Vietnamese": 0.32954545454545453, - "Filipino,Malay,Chinese": 0.30113636363636365, - "Filipino,Malay,Indonesian": 0.3352272727272727, - "Filipino,Malay,Vietnamese": 0.3068181818181818, - "Filipino,Chinese,Indonesian": 0.32386363636363635, - "Filipino,Chinese,Vietnamese": 0.3068181818181818, - "Filipino,Indonesian,Vietnamese": 0.32386363636363635, - "Spanish,Malay,Chinese": 0.4431818181818182, - "Spanish,Malay,Indonesian": 0.4943181818181818, - "Spanish,Malay,Vietnamese": 0.44886363636363635, - "Spanish,Chinese,Indonesian": 0.4659090909090909, - "Spanish,Chinese,Vietnamese": 0.42045454545454547, - "Spanish,Indonesian,Vietnamese": 0.4772727272727273, - "Malay,Chinese,Indonesian": 0.45454545454545453, - "Malay,Chinese,Vietnamese": 0.4090909090909091, - "Malay,Indonesian,Vietnamese": 0.4659090909090909, - "Chinese,Indonesian,Vietnamese": 0.4375 + "Filipino,Vietnamese,Chinese": 0.21333333333333335, + "Filipino,Vietnamese,Spanish": 0.24, + "Filipino,Vietnamese,Malay": 0.21333333333333335, + "Filipino,Vietnamese,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,English": 0.21333333333333335, + "Filipino,Chinese,Spanish": 0.24, + "Filipino,Chinese,Malay": 0.2, + "Filipino,Chinese,Indonesian": 0.2, + "Filipino,Chinese,English": 0.23333333333333334, + "Filipino,Spanish,Malay": 0.23333333333333334, + "Filipino,Spanish,Indonesian": 0.24666666666666667, + "Filipino,Spanish,English": 0.22666666666666666, + "Filipino,Malay,Indonesian": 0.23333333333333334, + "Filipino,Malay,English": 0.2, + "Filipino,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish": 0.2733333333333333, + "Vietnamese,Chinese,Malay": 0.26666666666666666, + "Vietnamese,Chinese,Indonesian": 0.24666666666666667, + "Vietnamese,Chinese,English": 0.2866666666666667, + "Vietnamese,Spanish,Malay": 0.26666666666666666, + "Vietnamese,Spanish,Indonesian": 0.26, + "Vietnamese,Spanish,English": 0.23333333333333334, + "Vietnamese,Malay,Indonesian": 0.30666666666666664, + "Vietnamese,Malay,English": 0.3, + "Vietnamese,Indonesian,English": 0.2866666666666667, + "Chinese,Spanish,Malay": 0.25333333333333335, + "Chinese,Spanish,Indonesian": 0.22, + "Chinese,Spanish,English": 0.2733333333333333, + "Chinese,Malay,Indonesian": 0.25333333333333335, + "Chinese,Malay,English": 0.2733333333333333, + "Chinese,Indonesian,English": 0.25333333333333335, + "Spanish,Malay,Indonesian": 0.28, + "Spanish,Malay,English": 0.22666666666666666, + "Spanish,Indonesian,English": 0.23333333333333334, + "Malay,Indonesian,English": 0.3 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.2784090909090909, - "English,Filipino,Spanish,Chinese": 0.26136363636363635, - "English,Filipino,Spanish,Indonesian": 0.30113636363636365, - "English,Filipino,Spanish,Vietnamese": 0.2897727272727273, - "English,Filipino,Malay,Chinese": 0.23295454545454544, - "English,Filipino,Malay,Indonesian": 0.26704545454545453, - "English,Filipino,Malay,Vietnamese": 0.26704545454545453, - "English,Filipino,Chinese,Indonesian": 0.2556818181818182, - "English,Filipino,Chinese,Vietnamese": 0.24431818181818182, - "English,Filipino,Indonesian,Vietnamese": 0.2727272727272727, - "English,Spanish,Malay,Chinese": 0.35795454545454547, - "English,Spanish,Malay,Indonesian": 0.42045454545454547, - "English,Spanish,Malay,Vietnamese": 0.39204545454545453, - "English,Spanish,Chinese,Indonesian": 0.3977272727272727, - "English,Spanish,Chinese,Vietnamese": 0.35795454545454547, - "English,Spanish,Indonesian,Vietnamese": 0.4090909090909091, - "English,Malay,Chinese,Indonesian": 0.3693181818181818, - "English,Malay,Chinese,Vietnamese": 0.3352272727272727, - "English,Malay,Indonesian,Vietnamese": 0.3977272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.375, - "Filipino,Spanish,Malay,Chinese": 0.24431818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.2897727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Chinese,Indonesian": 0.2784090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.26136363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.25, - "Spanish,Malay,Chinese,Indonesian": 0.38636363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.3409090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.3977272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.375, - "Malay,Chinese,Indonesian,Vietnamese": 0.3522727272727273 + "Filipino,Vietnamese,Chinese,Spanish": 0.18, + "Filipino,Vietnamese,Chinese,Malay": 0.14, + "Filipino,Vietnamese,Chinese,Indonesian": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,English": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Malay": 0.16, + "Filipino,Vietnamese,Spanish,Indonesian": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,English": 0.15333333333333332, + "Filipino,Vietnamese,Malay,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Malay,English": 0.14666666666666667, + "Filipino,Vietnamese,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Malay": 0.16, + "Filipino,Chinese,Spanish,Indonesian": 0.15333333333333332, + "Filipino,Chinese,Spanish,English": 0.16666666666666666, + "Filipino,Chinese,Malay,Indonesian": 0.14, + "Filipino,Chinese,Malay,English": 0.14666666666666667, + "Filipino,Chinese,Indonesian,English": 0.16, + "Filipino,Spanish,Malay,Indonesian": 0.18, + "Filipino,Spanish,Malay,English": 0.14666666666666667, + "Filipino,Spanish,Indonesian,English": 0.14666666666666667, + "Filipino,Malay,Indonesian,English": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,Malay": 0.19333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.18, + "Vietnamese,Chinese,Spanish,English": 0.2, + "Vietnamese,Chinese,Malay,Indonesian": 0.18, + "Vietnamese,Chinese,Malay,English": 0.2, + "Vietnamese,Chinese,Indonesian,English": 0.2, + "Vietnamese,Spanish,Malay,Indonesian": 0.18, + "Vietnamese,Spanish,Malay,English": 0.18, + "Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, + "Vietnamese,Malay,Indonesian,English": 0.20666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, + "Chinese,Spanish,Malay,English": 0.19333333333333333, + "Chinese,Spanish,Indonesian,English": 0.18666666666666668, + "Chinese,Malay,Indonesian,English": 0.19333333333333333, + "Spanish,Malay,Indonesian,English": 0.18 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.20454545454545456, - "English,Filipino,Spanish,Malay,Indonesian": 0.2556818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, - "English,Filipino,Spanish,Chinese,Indonesian": 0.23295454545454544, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.2556818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.21022727272727273, - "English,Filipino,Malay,Chinese,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Chinese,Indonesian": 0.3352272727272727, - "English,Spanish,Malay,Chinese,Vietnamese": 0.30113636363636365, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.35795454545454547, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.3352272727272727, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.3125, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.3181818181818182 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.1, + "Filipino,Vietnamese,Chinese,Malay,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.12, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.12, + "Filipino,Vietnamese,Spanish,Malay,English": 0.12666666666666668, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12666666666666668, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.12, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12, + "Filipino,Chinese,Spanish,Malay,English": 0.12666666666666668, + "Filipino,Chinese,Spanish,Indonesian,English": 0.12666666666666668, + "Filipino,Chinese,Malay,Indonesian,English": 0.12, + "Filipino,Spanish,Malay,Indonesian,English": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, + "Vietnamese,Chinese,Spanish,Malay,English": 0.16, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.15333333333333332, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.14666666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.19886363636363635, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1875 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.09333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667 } }, - "AC3_2": 0.4423997361950351, - "AC3_3": 0.3819866797956042, - "AC3_4": 0.3365748971221688, - "AC3_5": 0.2982719582565016, - "AC3_6": 0.2626933251472229, - "AC3_7": 0.2269415783174078 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.42718446601941745 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.3904761904761905 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.4205607476635514 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.31, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.2, - "culture": 0.4, - "film": 0.3, - "law": 0.2, - "geography": 0.6 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.1686934227644769 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.308138258845429 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.24492026033306286 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.1742210037946195 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.31367051196725554 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.4469078179696616 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.4154451197711834, - "category_acc": { - "high_school_european_history": 0.5426829268292683, - "business_ethics": 0.37373737373737376, - "clinical_knowledge": 0.4659090909090909, - "medical_genetics": 0.5050505050505051, - "high_school_us_history": 0.5665024630541872, - "high_school_physics": 0.31333333333333335, - "high_school_world_history": 0.5720338983050848, - "virology": 0.4121212121212121, - "high_school_microeconomics": 0.41350210970464135, - "econometrics": 0.3008849557522124, - "college_computer_science": 0.37373737373737376, - "high_school_biology": 0.4854368932038835, - "abstract_algebra": 0.1919191919191919, - "professional_accounting": 0.3202846975088968, - "philosophy": 0.4774193548387097, - "professional_medicine": 0.34317343173431736, - "nutrition": 0.45901639344262296, - "global_facts": 0.3434343434343434, - "machine_learning": 0.2972972972972973, - "security_studies": 0.47540983606557374, - "public_relations": 0.4954128440366973, - "professional_psychology": 0.41080196399345337, - "prehistory": 0.48606811145510836, - "anatomy": 0.417910447761194, - "human_sexuality": 0.43846153846153846, - "college_medicine": 0.36046511627906974, - "high_school_government_and_politics": 0.5729166666666666, - "college_chemistry": 0.35353535353535354, - "logical_fallacies": 0.4506172839506173, - "high_school_geography": 0.5380710659898477, - "elementary_mathematics": 0.26790450928381965, - "human_aging": 0.4594594594594595, - "college_mathematics": 0.32323232323232326, - "high_school_psychology": 0.5533088235294118, - "formal_logic": 0.264, - "high_school_statistics": 0.25116279069767444, - "international_law": 0.4666666666666667, - "high_school_mathematics": 0.2342007434944238, - "high_school_computer_science": 0.41414141414141414, - "conceptual_physics": 0.3162393162393162, - "miscellaneous": 0.5959079283887468, - "high_school_chemistry": 0.3316831683168317, - "marketing": 0.648068669527897, - "professional_law": 0.3176777560339204, - "management": 0.5980392156862745, - "college_physics": 0.21782178217821782, - "jurisprudence": 0.5046728971962616, - "world_religions": 0.5117647058823529, - "sociology": 0.555, - "us_foreign_policy": 0.5757575757575758, - "high_school_macroeconomics": 0.40616966580976865, - "computer_security": 0.5050505050505051, - "moral_scenarios": 0.25838926174496646, - "moral_disputes": 0.4434782608695652, - "electrical_engineering": 0.3541666666666667, - "astronomy": 0.46357615894039733, - "college_biology": 0.4195804195804196 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.40713224368499257 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.4016189290161893, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.2916666666666667, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.38095238095238093, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.375, - "high_school_chemistry": 0.4166666666666667, - "high_school_biology": 0.4583333333333333, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.625, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.36666666666666664, - "business_administration": 0.42105263157894735, - "marxism": 0.5, - "mao_zedong_thought": 0.5517241379310345, - "education_science": 0.5294117647058824, - "teacher_qualification": 0.6122448979591837, - "high_school_politics": 0.4583333333333333, - "high_school_geography": 0.375, - "middle_school_politics": 0.5384615384615384, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.4583333333333333, - "logic": 0.37037037037037035, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.375, - "high_school_history": 0.64, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.3269230769230769, - "sports_science": 0.5416666666666666, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.375, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.5098039215686274, - "accountant": 0.3148148148148148, - "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.4444444444444444, - "tax_accountant": 0.46296296296296297, - "physician": 0.4444444444444444 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.40860215053763443 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.4056294249697807, - "category_acc": { - "agronomy": 0.35502958579881655, - "anatomy": 0.2905405405405405, - "ancient_chinese": 0.2621951219512195, - "arts": 0.55, - "astronomy": 0.2727272727272727, - "business_ethics": 0.4354066985645933, - "chinese_civil_service_exam": 0.38125, - "chinese_driving_rule": 0.6641221374045801, - "chinese_food_culture": 0.45588235294117646, - "chinese_foreign_policy": 0.48598130841121495, - "chinese_history": 0.4582043343653251, - "chinese_literature": 0.30392156862745096, - "chinese_teacher_qualification": 0.553072625698324, - "clinical_knowledge": 0.3291139240506329, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.48598130841121495, - "college_engineering_hydrology": 0.41509433962264153, - "college_law": 0.42592592592592593, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.36792452830188677, - "college_medicine": 0.3772893772893773, - "computer_science": 0.4166666666666667, - "computer_security": 0.45614035087719296, - "conceptual_physics": 0.40816326530612246, - "construction_project_management": 0.3669064748201439, - "economics": 0.3710691823899371, - "education": 0.4601226993865031, - "electrical_engineering": 0.38953488372093026, - "elementary_chinese": 0.36904761904761907, - "elementary_commonsense": 0.494949494949495, - "elementary_information_and_technology": 0.5252100840336135, - "elementary_mathematics": 0.25217391304347825, - "ethnology": 0.4074074074074074, - "food_science": 0.3916083916083916, - "genetics": 0.32954545454545453, - "global_facts": 0.4429530201342282, - "high_school_biology": 0.28402366863905326, - "high_school_chemistry": 0.3181818181818182, - "high_school_geography": 0.4406779661016949, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.34545454545454546, - "high_school_politics": 0.3776223776223776, - "human_sexuality": 0.3968253968253968, - "international_law": 0.41621621621621624, - "journalism": 0.436046511627907, - "jurisprudence": 0.45742092457420924, - "legal_and_moral_basis": 0.7102803738317757, - "logical": 0.36585365853658536, - "machine_learning": 0.319672131147541, - "management": 0.4857142857142857, - "marketing": 0.4444444444444444, - "marxist_theory": 0.4444444444444444, - "modern_chinese": 0.3448275862068966, - "nutrition": 0.4, - "philosophy": 0.42857142857142855, - "professional_accounting": 0.4114285714285714, - "professional_law": 0.33175355450236965, - "professional_medicine": 0.31648936170212766, - "professional_psychology": 0.43103448275862066, - "public_relations": 0.3850574712643678, - "security_study": 0.5111111111111111, - "sociology": 0.42920353982300885, - "sports_science": 0.38181818181818183, - "traditional_chinese_medicine": 0.372972972972973, - "virology": 0.378698224852071, - "world_history": 0.4720496894409938, - "world_religions": 0.45 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.30303030303030304 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.30454545454545456 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.34983050847457625 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.6907255048616305 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.631063204311612 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.32583409599848656, - "rouge2": 0.10551433593882852, - "rougeL": 0.24922056116762198, - "avg_rouge": 0.22685633103497901 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.36252321016376693, - "rouge2": 0.11439366048975183, - "rougeL": 0.28084530444119976, - "avg_rouge": 0.2525873916982395 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.5848623853211009 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.5043144774688398 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.536 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.3875 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.4955 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.5211267605633803 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.4981949458483754 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.47794117647058826 - } - } - } - }, - "sailor_7b_chat": { - "model_size": "7B", - "model_link": "https://huggingface.co/sail/Sailor-7B-Chat", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.9147058823529411, - "language_acc": { - "English": 0.9470588235294117, - "Chinese": 0.9016806722689076, - "Spanish": 0.9092436974789916, - "Vietnamese": 0.9008403361344538 - }, - "consistency_score_2": 0.8911764705882353, - "consistency_score_3": 0.8415966386554621, - "consistency_score_4": 0.8067226890756303, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8915966386554622, - "English,Spanish": 0.9184873949579831, - "English,Vietnamese": 0.9168067226890756, - "Chinese,Spanish": 0.8714285714285714, - "Chinese,Vietnamese": 0.8605042016806723, - "Spanish,Vietnamese": 0.888235294117647 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8445378151260504, - "English,Chinese,Vietnamese": 0.8403361344537815, - "English,Spanish,Vietnamese": 0.8647058823529412, - "Chinese,Spanish,Vietnamese": 0.8168067226890756 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8067226890756303 - } - }, - "AC3_2": 0.9027878903504684, - "AC3_3": 0.8766296087313981, - "AC3_4": 0.8573274561950818 - }, - "prompt_2": { - "overall_acc": 0.9199579831932773, - "language_acc": { - "English": 0.9470588235294117, - "Chinese": 0.9033613445378151, - "Spanish": 0.9193277310924369, - "Vietnamese": 0.9100840336134454 - }, - "consistency_score_2": 0.8983193277310925, - "consistency_score_3": 0.8521008403361344, - "consistency_score_4": 0.8168067226890756, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8974789915966387, - "English,Spanish": 0.9218487394957983, - "English,Vietnamese": 0.9201680672268907, - "Chinese,Spanish": 0.8848739495798319, - "Chinese,Vietnamese": 0.8756302521008403, - "Spanish,Vietnamese": 0.8899159663865546 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8554621848739495, - "English,Chinese,Vietnamese": 0.8529411764705882, - "English,Spanish,Vietnamese": 0.8697478991596639, - "Chinese,Spanish,Vietnamese": 0.8302521008403362 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8168067226890756 - } - }, - "AC3_2": 0.9090098985368248, - "AC3_3": 0.8847301907815925, - "AC3_4": 0.8653191335307888 + "AC3_2": 0.37923362170603275, + "AC3_3": 0.2852747252259087, + "AC3_4": 0.2257754871514616, + "AC3_5": 0.18814082044274558, + "AC3_6": 0.16095852530927668, + "AC3_7": 0.13788764041690293 }, "prompt_3": { - "overall_acc": 0.9252100840336134, + "overall_acc": 0.35333333333333333, "language_acc": { - "English": 0.9470588235294117, - "Chinese": 0.9176470588235294, - "Spanish": 0.9193277310924369, - "Vietnamese": 0.9168067226890756 + "Filipino": 0.24666666666666667, + "Vietnamese": 0.35333333333333333, + "Chinese": 0.38, + "Spanish": 0.3333333333333333, + "Malay": 0.3466666666666667, + "Indonesian": 0.35333333333333333, + "English": 0.46 }, - "consistency_score_2": 0.907703081232493, - "consistency_score_3": 0.8649159663865547, - "consistency_score_4": 0.8327731092436975, + "consistency_score_2": 0.5107936507936508, + "consistency_score_3": 0.33923809523809517, + "consistency_score_4": 0.259047619047619, + "consistency_score_5": 0.2146031746031746, + "consistency_score_6": 0.1885714285714286, + "consistency_score_7": 0.17333333333333334, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.915126050420168, - "English,Spanish": 0.9218487394957983, - "English,Vietnamese": 0.9285714285714286, - "Chinese,Spanish": 0.8899159663865546, - "Chinese,Vietnamese": 0.8915966386554622, - "Spanish,Vietnamese": 0.8991596638655462 + "Filipino,Vietnamese": 0.48, + "Filipino,Chinese": 0.48, + "Filipino,Spanish": 0.4533333333333333, + "Filipino,Malay": 0.54, + "Filipino,Indonesian": 0.46, + "Filipino,English": 0.49333333333333335, + "Vietnamese,Chinese": 0.5066666666666667, + "Vietnamese,Spanish": 0.4866666666666667, + "Vietnamese,Malay": 0.5666666666666667, + "Vietnamese,Indonesian": 0.5, + "Vietnamese,English": 0.5666666666666667, + "Chinese,Spanish": 0.4666666666666667, + "Chinese,Malay": 0.5266666666666666, + "Chinese,Indonesian": 0.47333333333333333, + "Chinese,English": 0.5333333333333333, + "Spanish,Malay": 0.4666666666666667, + "Spanish,Indonesian": 0.48, + "Spanish,English": 0.49333333333333335, + "Malay,Indonesian": 0.62, + "Malay,English": 0.6, + "Indonesian,English": 0.5333333333333333 }, "3_combine": { - "English,Chinese,Spanish": 0.8672268907563025, - "English,Chinese,Vietnamese": 0.8714285714285714, - "English,Spanish,Vietnamese": 0.8773109243697479, - "Chinese,Spanish,Vietnamese": 0.8436974789915966 + "Filipino,Vietnamese,Chinese": 0.29333333333333333, + "Filipino,Vietnamese,Spanish": 0.3, + "Filipino,Vietnamese,Malay": 0.37333333333333335, + "Filipino,Vietnamese,Indonesian": 0.29333333333333333, + "Filipino,Vietnamese,English": 0.3333333333333333, + "Filipino,Chinese,Spanish": 0.29333333333333333, + "Filipino,Chinese,Malay": 0.32666666666666666, + "Filipino,Chinese,Indonesian": 0.2733333333333333, + "Filipino,Chinese,English": 0.30666666666666664, + "Filipino,Spanish,Malay": 0.32666666666666666, + "Filipino,Spanish,Indonesian": 0.29333333333333333, + "Filipino,Spanish,English": 0.31333333333333335, + "Filipino,Malay,Indonesian": 0.36666666666666664, + "Filipino,Malay,English": 0.35333333333333333, + "Filipino,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish": 0.3333333333333333, + "Vietnamese,Chinese,Malay": 0.36666666666666664, + "Vietnamese,Chinese,Indonesian": 0.3, + "Vietnamese,Chinese,English": 0.36, + "Vietnamese,Spanish,Malay": 0.37333333333333335, + "Vietnamese,Spanish,Indonesian": 0.32666666666666666, + "Vietnamese,Spanish,English": 0.35333333333333333, + "Vietnamese,Malay,Indonesian": 0.41333333333333333, + "Vietnamese,Malay,English": 0.43333333333333335, + "Vietnamese,Indonesian,English": 0.37333333333333335, + "Chinese,Spanish,Malay": 0.32, + "Chinese,Spanish,Indonesian": 0.30666666666666664, + "Chinese,Spanish,English": 0.3333333333333333, + "Chinese,Malay,Indonesian": 0.36666666666666664, + "Chinese,Malay,English": 0.36, + "Chinese,Indonesian,English": 0.34, + "Spanish,Malay,Indonesian": 0.37333333333333335, + "Spanish,Malay,English": 0.3466666666666667, + "Spanish,Indonesian,English": 0.31333333333333335, + "Malay,Indonesian,English": 0.4266666666666667 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8327731092436975 - } - }, - "AC3_2": 0.9163729738358224, - "AC3_3": 0.8940476272159613, - "AC3_4": 0.8765613701961333 - }, - "prompt_4": { - "overall_acc": 0.9199579831932773, - "language_acc": { - "English": 0.9504201680672268, - "Chinese": 0.9092436974789916, - "Spanish": 0.9184873949579831, - "Vietnamese": 0.9016806722689076 - }, - "consistency_score_2": 0.8980392156862745, - "consistency_score_3": 0.8521008403361345, - "consistency_score_4": 0.8184873949579832, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.9058823529411765, - "English,Spanish": 0.9235294117647059, - "English,Vietnamese": 0.9193277310924369, - "Chinese,Spanish": 0.880672268907563, - "Chinese,Vietnamese": 0.8756302521008403, - "Spanish,Vietnamese": 0.8831932773109243 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8596638655462185, - "English,Chinese,Vietnamese": 0.8554621848739495, - "English,Spanish,Vietnamese": 0.865546218487395, - "Chinese,Spanish,Vietnamese": 0.8277310924369747 + "Filipino,Vietnamese,Chinese,Spanish": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Malay": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,English": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Malay": 0.28, + "Filipino,Vietnamese,Spanish,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,English": 0.24666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Malay,English": 0.3, + "Filipino,Vietnamese,Indonesian,English": 0.24666666666666667, + "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, + "Filipino,Chinese,Spanish,Indonesian": 0.21333333333333335, + "Filipino,Chinese,Spanish,English": 0.24, + "Filipino,Chinese,Malay,Indonesian": 0.23333333333333334, + "Filipino,Chinese,Malay,English": 0.24, + "Filipino,Chinese,Indonesian,English": 0.22666666666666666, + "Filipino,Spanish,Malay,Indonesian": 0.26, + "Filipino,Spanish,Malay,English": 0.25333333333333335, + "Filipino,Spanish,Indonesian,English": 0.22666666666666666, + "Filipino,Malay,Indonesian,English": 0.26, + "Vietnamese,Chinese,Spanish,Malay": 0.2733333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,English": 0.26, + "Vietnamese,Chinese,Malay,Indonesian": 0.2866666666666667, + "Vietnamese,Chinese,Malay,English": 0.28, + "Vietnamese,Chinese,Indonesian,English": 0.26, + "Vietnamese,Spanish,Malay,Indonesian": 0.3, + "Vietnamese,Spanish,Malay,English": 0.30666666666666664, + "Vietnamese,Spanish,Indonesian,English": 0.26, + "Vietnamese,Malay,Indonesian,English": 0.34, + "Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Chinese,Spanish,Malay,English": 0.26666666666666666, + "Chinese,Spanish,Indonesian,English": 0.25333333333333335, + "Chinese,Malay,Indonesian,English": 0.30666666666666664, + "Spanish,Malay,Indonesian,English": 0.2866666666666667 }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8184873949579832 - } - }, - "AC3_2": 0.908866467071504, - "AC3_3": 0.8847301907815927, - "AC3_4": 0.8662613419492056 - }, - "prompt_5": { - "overall_acc": 0.9172268907563025, - "language_acc": { - "English": 0.9420168067226891, - "Chinese": 0.9134453781512605, - "Spanish": 0.9008403361344538, - "Vietnamese": 0.9126050420168067 - }, - "consistency_score_2": 0.8984593837535014, - "consistency_score_3": 0.852310924369748, - "consistency_score_4": 0.8184873949579832, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.9084033613445378, - "English,Spanish": 0.9117647058823529, - "English,Vietnamese": 0.9193277310924369, - "Chinese,Spanish": 0.8823529411764706, - "Chinese,Vietnamese": 0.8840336134453781, - "Spanish,Vietnamese": 0.8848739495798319 + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2, + "Filipino,Vietnamese,Chinese,Malay,English": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22, + "Filipino,Vietnamese,Spanish,Malay,English": 0.24, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.23333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Filipino,Chinese,Spanish,Malay,English": 0.2, + "Filipino,Chinese,Spanish,Indonesian,English": 0.19333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.20666666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.21333333333333335, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Malay,English": 0.22666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.21333333333333335, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.25333333333333335, + "Chinese,Spanish,Malay,Indonesian,English": 0.24 }, - "3_combine": { - "English,Chinese,Spanish": 0.8563025210084033, - "English,Chinese,Vietnamese": 0.8596638655462185, - "English,Spanish,Vietnamese": 0.8630252100840337, - "Chinese,Spanish,Vietnamese": 0.8302521008403362 + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.2, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667 }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8184873949579832 + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334 } }, - "AC3_2": 0.9077461437639024, - "AC3_3": 0.8835781777485523, - "AC3_4": 0.86504864831292 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.5247619047619048, + "AC3_2": 0.41771736463449144, + "AC3_3": 0.34614228084477683, + "AC3_4": 0.29893208911655594, + "AC3_5": 0.2670244083748205, + "AC3_6": 0.24590509661543056, + "AC3_7": 0.2325738396182877 + }, + "prompt_4": { + "overall_acc": 0.3628571428571429, "language_acc": { - "Spanish": 0.52, - "English": 0.6333333333333333, - "Chinese": 0.56, - "Vietnamese": 0.5266666666666666, - "Filipino": 0.4, - "Indonesian": 0.5, - "Malay": 0.5333333333333333 + "Filipino": 0.26, + "Vietnamese": 0.3466666666666667, + "Chinese": 0.38, + "Spanish": 0.3466666666666667, + "Malay": 0.4, + "Indonesian": 0.38, + "English": 0.4266666666666667 }, - "consistency_score_2": 0.5888888888888888, - "consistency_score_3": 0.43009523809523814, - "consistency_score_4": 0.3396190476190477, - "consistency_score_5": 0.2803174603174603, - "consistency_score_6": 0.2380952380952381, - "consistency_score_7": 0.20666666666666667, + "consistency_score_2": 0.47555555555555556, + "consistency_score_3": 0.2939047619047619, + "consistency_score_4": 0.2106666666666667, + "consistency_score_5": 0.16412698412698412, + "consistency_score_6": 0.13238095238095238, + "consistency_score_7": 0.10666666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6733333333333333, - "Spanish,Chinese": 0.5866666666666667, - "Spanish,Vietnamese": 0.6333333333333333, - "Spanish,Filipino": 0.4533333333333333, - "Spanish,Indonesian": 0.6466666666666666, - "Spanish,Malay": 0.62, - "English,Chinese": 0.66, - "English,Vietnamese": 0.64, - "English,Filipino": 0.48, - "English,Indonesian": 0.66, - "English,Malay": 0.6666666666666666, - "Chinese,Vietnamese": 0.6733333333333333, - "Chinese,Filipino": 0.49333333333333335, - "Chinese,Indonesian": 0.5933333333333334, - "Chinese,Malay": 0.5933333333333334, - "Vietnamese,Filipino": 0.4533333333333333, - "Vietnamese,Indonesian": 0.6533333333333333, - "Vietnamese,Malay": 0.6, - "Filipino,Indonesian": 0.4266666666666667, - "Filipino,Malay": 0.44666666666666666, - "Indonesian,Malay": 0.7133333333333334 + "Filipino,Vietnamese": 0.46, + "Filipino,Chinese": 0.42, + "Filipino,Spanish": 0.47333333333333333, + "Filipino,Malay": 0.41333333333333333, + "Filipino,Indonesian": 0.4533333333333333, + "Filipino,English": 0.4066666666666667, + "Vietnamese,Chinese": 0.44, + "Vietnamese,Spanish": 0.46, + "Vietnamese,Malay": 0.5066666666666667, + "Vietnamese,Indonesian": 0.49333333333333335, + "Vietnamese,English": 0.44666666666666666, + "Chinese,Spanish": 0.46, + "Chinese,Malay": 0.49333333333333335, + "Chinese,Indonesian": 0.4266666666666667, + "Chinese,English": 0.5466666666666666, + "Spanish,Malay": 0.43333333333333335, + "Spanish,Indonesian": 0.44666666666666666, + "Spanish,English": 0.5133333333333333, + "Malay,Indonesian": 0.56, + "Malay,English": 0.5866666666666667, + "Indonesian,English": 0.5466666666666666 }, "3_combine": { - "Spanish,English,Chinese": 0.49333333333333335, - "Spanish,English,Vietnamese": 0.5, - "Spanish,English,Filipino": 0.35333333333333333, - "Spanish,English,Indonesian": 0.5266666666666666, - "Spanish,English,Malay": 0.5066666666666667, - "Spanish,Chinese,Vietnamese": 0.48, - "Spanish,Chinese,Filipino": 0.3333333333333333, - "Spanish,Chinese,Indonesian": 0.46, - "Spanish,Chinese,Malay": 0.4533333333333333, - "Spanish,Vietnamese,Filipino": 0.34, - "Spanish,Vietnamese,Indonesian": 0.5066666666666667, - "Spanish,Vietnamese,Malay": 0.4666666666666667, - "Spanish,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Filipino,Malay": 0.32, - "Spanish,Indonesian,Malay": 0.5266666666666666, - "English,Chinese,Vietnamese": 0.52, - "English,Chinese,Filipino": 0.38, - "English,Chinese,Indonesian": 0.5, - "English,Chinese,Malay": 0.49333333333333335, - "English,Vietnamese,Filipino": 0.3466666666666667, - "English,Vietnamese,Indonesian": 0.52, - "English,Vietnamese,Malay": 0.48, - "English,Filipino,Indonesian": 0.3333333333333333, - "English,Filipino,Malay": 0.34, - "English,Indonesian,Malay": 0.5666666666666667, - "Chinese,Vietnamese,Filipino": 0.35333333333333333, - "Chinese,Vietnamese,Indonesian": 0.49333333333333335, - "Chinese,Vietnamese,Malay": 0.4666666666666667, - "Chinese,Filipino,Indonesian": 0.30666666666666664, - "Chinese,Filipino,Malay": 0.3333333333333333, - "Chinese,Indonesian,Malay": 0.4866666666666667, - "Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "Vietnamese,Filipino,Malay": 0.34, - "Vietnamese,Indonesian,Malay": 0.5333333333333333, - "Filipino,Indonesian,Malay": 0.3333333333333333 + "Filipino,Vietnamese,Chinese": 0.25333333333333335, + "Filipino,Vietnamese,Spanish": 0.2733333333333333, + "Filipino,Vietnamese,Malay": 0.2866666666666667, + "Filipino,Vietnamese,Indonesian": 0.28, + "Filipino,Vietnamese,English": 0.24, + "Filipino,Chinese,Spanish": 0.26, + "Filipino,Chinese,Malay": 0.24666666666666667, + "Filipino,Chinese,Indonesian": 0.25333333333333335, + "Filipino,Chinese,English": 0.26666666666666666, + "Filipino,Spanish,Malay": 0.25333333333333335, + "Filipino,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Spanish,English": 0.28, + "Filipino,Malay,Indonesian": 0.29333333333333333, + "Filipino,Malay,English": 0.26666666666666666, + "Filipino,Indonesian,English": 0.29333333333333333, + "Vietnamese,Chinese,Spanish": 0.25333333333333335, + "Vietnamese,Chinese,Malay": 0.30666666666666664, + "Vietnamese,Chinese,Indonesian": 0.2733333333333333, + "Vietnamese,Chinese,English": 0.3, + "Vietnamese,Spanish,Malay": 0.2733333333333333, + "Vietnamese,Spanish,Indonesian": 0.28, + "Vietnamese,Spanish,English": 0.29333333333333333, + "Vietnamese,Malay,Indonesian": 0.3466666666666667, + "Vietnamese,Malay,English": 0.3333333333333333, + "Vietnamese,Indonesian,English": 0.30666666666666664, + "Chinese,Spanish,Malay": 0.2866666666666667, + "Chinese,Spanish,Indonesian": 0.26, + "Chinese,Spanish,English": 0.3466666666666667, + "Chinese,Malay,Indonesian": 0.32666666666666666, + "Chinese,Malay,English": 0.36666666666666664, + "Chinese,Indonesian,English": 0.3333333333333333, + "Spanish,Malay,Indonesian": 0.32, + "Spanish,Malay,English": 0.32666666666666666, + "Spanish,Indonesian,English": 0.32, + "Malay,Indonesian,English": 0.42 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4066666666666667, - "Spanish,English,Chinese,Filipino": 0.2866666666666667, - "Spanish,English,Chinese,Indonesian": 0.42, - "Spanish,English,Chinese,Malay": 0.4, - "Spanish,English,Vietnamese,Filipino": 0.29333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.44, - "Spanish,English,Vietnamese,Malay": 0.3933333333333333, - "Spanish,English,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Indonesian,Malay": 0.44666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.28, - "Spanish,Chinese,Vietnamese,Indonesian": 0.3933333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.38, - "Spanish,Chinese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.26666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.4, - "Spanish,Vietnamese,Filipino,Indonesian": 0.28, - "Spanish,Vietnamese,Filipino,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino": 0.3, - "English,Chinese,Vietnamese,Indonesian": 0.42, - "English,Chinese,Vietnamese,Malay": 0.4, - "English,Chinese,Filipino,Indonesian": 0.29333333333333333, - "English,Chinese,Filipino,Malay": 0.3, - "English,Chinese,Indonesian,Malay": 0.44, - "English,Vietnamese,Filipino,Indonesian": 0.3, - "English,Vietnamese,Filipino,Malay": 0.28, - "English,Vietnamese,Indonesian,Malay": 0.46, - "English,Filipino,Indonesian,Malay": 0.2866666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.2866666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.42, - "Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667 + "Filipino,Vietnamese,Chinese,Spanish": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,Malay": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,English": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Malay": 0.18, + "Filipino,Vietnamese,Spanish,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,English": 0.17333333333333334, + "Filipino,Vietnamese,Malay,Indonesian": 0.21333333333333335, + "Filipino,Vietnamese,Malay,English": 0.19333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.2, + "Filipino,Chinese,Spanish,Malay": 0.18, + "Filipino,Chinese,Spanish,Indonesian": 0.18666666666666668, + "Filipino,Chinese,Spanish,English": 0.21333333333333335, + "Filipino,Chinese,Malay,Indonesian": 0.2, + "Filipino,Chinese,Malay,English": 0.19333333333333333, + "Filipino,Chinese,Indonesian,English": 0.2, + "Filipino,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Spanish,Malay,English": 0.18666666666666668, + "Filipino,Spanish,Indonesian,English": 0.2, + "Filipino,Malay,Indonesian,English": 0.24, + "Vietnamese,Chinese,Spanish,Malay": 0.19333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.19333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.2, + "Vietnamese,Chinese,Malay,Indonesian": 0.24, + "Vietnamese,Chinese,Malay,English": 0.24666666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, + "Vietnamese,Spanish,Malay,Indonesian": 0.22, + "Vietnamese,Spanish,Malay,English": 0.21333333333333335, + "Vietnamese,Spanish,Indonesian,English": 0.22666666666666666, + "Vietnamese,Malay,Indonesian,English": 0.2733333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, + "Chinese,Spanish,Malay,English": 0.26, + "Chinese,Spanish,Indonesian,English": 0.23333333333333334, + "Chinese,Malay,Indonesian,English": 0.28, + "Spanish,Malay,Indonesian,English": 0.26666666666666666 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.36, - "Spanish,English,Chinese,Vietnamese,Malay": 0.3333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.24, - "Spanish,English,Chinese,Indonesian,Malay": 0.36666666666666664, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.24, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.38, - "Spanish,English,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.3466666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "English,Chinese,Vietnamese,Filipino,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.38, - "English,Chinese,Filipino,Indonesian,Malay": 0.26, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.14, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16, + "Filipino,Vietnamese,Chinese,Malay,English": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14666666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.18, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.16, + "Filipino,Chinese,Spanish,Malay,English": 0.16, + "Filipino,Chinese,Spanish,Indonesian,English": 0.16, + "Filipino,Chinese,Malay,Indonesian,English": 0.18, + "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Malay,English": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.21333333333333335, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, + "Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.32, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.22, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.12, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, - "AC3_2": 0.554979095351154, - "AC3_3": 0.4727358391495264, - "AC3_4": 0.41236248394985947, - "AC3_5": 0.36542962290789177, - "AC3_6": 0.32756673201817466, - "AC3_7": 0.2965451388483456 + "AC3_2": 0.41163195754272464, + "AC3_3": 0.3247613522873095, + "AC3_4": 0.26656924605116283, + "AC3_5": 0.22602065400186097, + "AC3_6": 0.19398901094984014, + "AC3_7": 0.1648681541231011 }, - "prompt_2": { - "overall_acc": 0.5390476190476191, + "prompt_5": { + "overall_acc": 0.3380952380952381, "language_acc": { - "Spanish": 0.54, - "English": 0.6466666666666666, - "Chinese": 0.58, - "Vietnamese": 0.54, - "Filipino": 0.38666666666666666, - "Indonesian": 0.54, - "Malay": 0.54 + "Filipino": 0.23333333333333334, + "Vietnamese": 0.3466666666666667, + "Chinese": 0.35333333333333333, + "Spanish": 0.32666666666666666, + "Malay": 0.32666666666666666, + "Indonesian": 0.37333333333333335, + "English": 0.4066666666666667 }, - "consistency_score_2": 0.599047619047619, - "consistency_score_3": 0.4384761904761905, - "consistency_score_4": 0.3445714285714286, - "consistency_score_5": 0.28253968253968254, - "consistency_score_6": 0.23904761904761904, - "consistency_score_7": 0.20666666666666667, + "consistency_score_2": 0.5336507936507936, + "consistency_score_3": 0.3683809523809523, + "consistency_score_4": 0.2862857142857142, + "consistency_score_5": 0.23682539682539688, + "consistency_score_6": 0.20380952380952383, + "consistency_score_7": 0.18, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6933333333333334, - "Spanish,Chinese": 0.62, - "Spanish,Vietnamese": 0.7133333333333334, - "Spanish,Filipino": 0.42, - "Spanish,Indonesian": 0.6666666666666666, - "Spanish,Malay": 0.6533333333333333, - "English,Chinese": 0.6533333333333333, - "English,Vietnamese": 0.6866666666666666, - "English,Filipino": 0.47333333333333333, - "English,Indonesian": 0.6933333333333334, - "English,Malay": 0.7066666666666667, - "Chinese,Vietnamese": 0.6533333333333333, - "Chinese,Filipino": 0.44666666666666666, - "Chinese,Indonesian": 0.6333333333333333, - "Chinese,Malay": 0.6, - "Vietnamese,Filipino": 0.4066666666666667, - "Vietnamese,Indonesian": 0.6733333333333333, - "Vietnamese,Malay": 0.64, - "Filipino,Indonesian": 0.4533333333333333, - "Filipino,Malay": 0.38666666666666666, - "Indonesian,Malay": 0.7066666666666667 + "Filipino,Vietnamese": 0.49333333333333335, + "Filipino,Chinese": 0.4866666666666667, + "Filipino,Spanish": 0.43333333333333335, + "Filipino,Malay": 0.4866666666666667, + "Filipino,Indonesian": 0.5066666666666667, + "Filipino,English": 0.47333333333333333, + "Vietnamese,Chinese": 0.5133333333333333, + "Vietnamese,Spanish": 0.5333333333333333, + "Vietnamese,Malay": 0.5866666666666667, + "Vietnamese,Indonesian": 0.5733333333333334, + "Vietnamese,English": 0.5933333333333334, + "Chinese,Spanish": 0.52, + "Chinese,Malay": 0.5733333333333334, + "Chinese,Indonesian": 0.5333333333333333, + "Chinese,English": 0.6066666666666667, + "Spanish,Malay": 0.5066666666666667, + "Spanish,Indonesian": 0.4866666666666667, + "Spanish,English": 0.5666666666666667, + "Malay,Indonesian": 0.6266666666666667, + "Malay,English": 0.5333333333333333, + "Indonesian,English": 0.5733333333333334 }, "3_combine": { - "Spanish,English,Chinese": 0.5066666666666667, - "Spanish,English,Vietnamese": 0.5666666666666667, - "Spanish,English,Filipino": 0.3466666666666667, - "Spanish,English,Indonesian": 0.5533333333333333, - "Spanish,English,Malay": 0.5333333333333333, - "Spanish,Chinese,Vietnamese": 0.52, - "Spanish,Chinese,Filipino": 0.3, - "Spanish,Chinese,Indonesian": 0.5, - "Spanish,Chinese,Malay": 0.4666666666666667, - "Spanish,Vietnamese,Filipino": 0.31333333333333335, - "Spanish,Vietnamese,Indonesian": 0.5466666666666666, - "Spanish,Vietnamese,Malay": 0.5266666666666666, - "Spanish,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Filipino,Malay": 0.29333333333333333, - "Spanish,Indonesian,Malay": 0.5466666666666666, - "English,Chinese,Vietnamese": 0.5133333333333333, - "English,Chinese,Filipino": 0.3466666666666667, - "English,Chinese,Indonesian": 0.5333333333333333, - "English,Chinese,Malay": 0.5, - "English,Vietnamese,Filipino": 0.32666666666666666, - "English,Vietnamese,Indonesian": 0.56, - "English,Vietnamese,Malay": 0.5333333333333333, - "English,Filipino,Indonesian": 0.35333333333333333, - "English,Filipino,Malay": 0.3333333333333333, - "English,Indonesian,Malay": 0.5866666666666667, - "Chinese,Vietnamese,Filipino": 0.3, - "Chinese,Vietnamese,Indonesian": 0.52, - "Chinese,Vietnamese,Malay": 0.48, - "Chinese,Filipino,Indonesian": 0.32, - "Chinese,Filipino,Malay": 0.29333333333333333, - "Chinese,Indonesian,Malay": 0.5133333333333333, - "Vietnamese,Filipino,Indonesian": 0.32, - "Vietnamese,Filipino,Malay": 0.28, - "Vietnamese,Indonesian,Malay": 0.5466666666666666, - "Filipino,Indonesian,Malay": 0.3333333333333333 + "Filipino,Vietnamese,Chinese": 0.31333333333333335, + "Filipino,Vietnamese,Spanish": 0.31333333333333335, + "Filipino,Vietnamese,Malay": 0.36666666666666664, + "Filipino,Vietnamese,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,English": 0.3466666666666667, + "Filipino,Chinese,Spanish": 0.2866666666666667, + "Filipino,Chinese,Malay": 0.32666666666666666, + "Filipino,Chinese,Indonesian": 0.32666666666666666, + "Filipino,Chinese,English": 0.34, + "Filipino,Spanish,Malay": 0.30666666666666664, + "Filipino,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Spanish,English": 0.3, + "Filipino,Malay,Indonesian": 0.38, + "Filipino,Malay,English": 0.31333333333333335, + "Filipino,Indonesian,English": 0.32, + "Vietnamese,Chinese,Spanish": 0.36666666666666664, + "Vietnamese,Chinese,Malay": 0.41333333333333333, + "Vietnamese,Chinese,Indonesian": 0.38, + "Vietnamese,Chinese,English": 0.41333333333333333, + "Vietnamese,Spanish,Malay": 0.3933333333333333, + "Vietnamese,Spanish,Indonesian": 0.36666666666666664, + "Vietnamese,Spanish,English": 0.41333333333333333, + "Vietnamese,Malay,Indonesian": 0.46, + "Vietnamese,Malay,English": 0.44, + "Vietnamese,Indonesian,English": 0.44, + "Chinese,Spanish,Malay": 0.38, + "Chinese,Spanish,Indonesian": 0.35333333333333333, + "Chinese,Spanish,English": 0.4066666666666667, + "Chinese,Malay,Indonesian": 0.4266666666666667, + "Chinese,Malay,English": 0.4066666666666667, + "Chinese,Indonesian,English": 0.4066666666666667, + "Spanish,Malay,Indonesian": 0.3933333333333333, + "Spanish,Malay,English": 0.38, + "Spanish,Indonesian,English": 0.37333333333333335, + "Malay,Indonesian,English": 0.42 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.44666666666666666, - "Spanish,English,Chinese,Filipino": 0.2733333333333333, - "Spanish,English,Chinese,Indonesian": 0.43333333333333335, - "Spanish,English,Chinese,Malay": 0.4, - "Spanish,English,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.4866666666666667, - "Spanish,English,Vietnamese,Malay": 0.4533333333333333, - "Spanish,English,Filipino,Indonesian": 0.3, - "Spanish,English,Filipino,Malay": 0.28, - "Spanish,English,Indonesian,Malay": 0.4866666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.44, - "Spanish,Chinese,Vietnamese,Malay": 0.4066666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Chinese,Filipino,Malay": 0.24, - "Spanish,Chinese,Indonesian,Malay": 0.42, - "Spanish,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.24, - "Spanish,Vietnamese,Indonesian,Malay": 0.46, - "Spanish,Filipino,Indonesian,Malay": 0.28, - "English,Chinese,Vietnamese,Filipino": 0.26666666666666666, - "English,Chinese,Vietnamese,Indonesian": 0.4533333333333333, - "English,Chinese,Vietnamese,Malay": 0.41333333333333333, - "English,Chinese,Filipino,Indonesian": 0.3, - "English,Chinese,Filipino,Malay": 0.28, - "English,Chinese,Indonesian,Malay": 0.44666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "English,Vietnamese,Filipino,Malay": 0.26, - "English,Vietnamese,Indonesian,Malay": 0.47333333333333333, - "English,Filipino,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "Chinese,Vietnamese,Filipino,Malay": 0.24, - "Chinese,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666 + "Filipino,Vietnamese,Chinese,Spanish": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Malay": 0.28, + "Filipino,Vietnamese,Chinese,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay": 0.26666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,English": 0.24666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.32, + "Filipino,Vietnamese,Malay,English": 0.29333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.28, + "Filipino,Chinese,Spanish,Malay": 0.24666666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.21333333333333335, + "Filipino,Chinese,Spanish,English": 0.24, + "Filipino,Chinese,Malay,Indonesian": 0.2733333333333333, + "Filipino,Chinese,Malay,English": 0.26, + "Filipino,Chinese,Indonesian,English": 0.25333333333333335, + "Filipino,Spanish,Malay,Indonesian": 0.25333333333333335, + "Filipino,Spanish,Malay,English": 0.24, + "Filipino,Spanish,Indonesian,English": 0.20666666666666667, + "Filipino,Malay,Indonesian,English": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,Malay": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian": 0.29333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.31333333333333335, + "Vietnamese,Chinese,Malay,Indonesian": 0.3466666666666667, + "Vietnamese,Chinese,Malay,English": 0.3466666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.32666666666666666, + "Vietnamese,Spanish,Malay,Indonesian": 0.3333333333333333, + "Vietnamese,Spanish,Malay,English": 0.3333333333333333, + "Vietnamese,Spanish,Indonesian,English": 0.31333333333333335, + "Vietnamese,Malay,Indonesian,English": 0.37333333333333335, + "Chinese,Spanish,Malay,Indonesian": 0.31333333333333335, + "Chinese,Spanish,Malay,English": 0.3333333333333333, + "Chinese,Spanish,Indonesian,English": 0.31333333333333335, + "Chinese,Malay,Indonesian,English": 0.3333333333333333, + "Spanish,Malay,Indonesian,English": 0.31333333333333335 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.24, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.4, - "Spanish,English,Chinese,Vietnamese,Malay": 0.36, - "Spanish,English,Chinese,Filipino,Indonesian": 0.26, - "Spanish,English,Chinese,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.37333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,English,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.42, - "Spanish,English,Filipino,Indonesian,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.24, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.38, - "English,Chinese,Filipino,Indonesian,Malay": 0.26, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.22, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.2, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.24, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,Malay,English": 0.22, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.26, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.22, + "Filipino,Chinese,Spanish,Indonesian,English": 0.18666666666666668, + "Filipino,Chinese,Malay,Indonesian,English": 0.22, + "Filipino,Spanish,Malay,Indonesian,English": 0.2, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.28, + "Vietnamese,Chinese,Spanish,Malay,English": 0.29333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.26666666666666666, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.3, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.2866666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.2866666666666667 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.34, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.2, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.26 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18 } }, - "AC3_2": 0.5674660290395992, - "AC3_3": 0.4835883160252054, - "AC3_4": 0.4204083391880947, - "AC3_5": 0.37075145355022227, - "AC3_6": 0.3312140816733504, - "AC3_7": 0.29878246058147445 - }, - "prompt_3": { - "overall_acc": 0.5361904761904762, + "AC3_2": 0.41393888941944634, + "AC3_3": 0.35258894062207374, + "AC3_4": 0.310040965696008, + "AC3_5": 0.2785411900333268, + "AC3_6": 0.2543141685027398, + "AC3_7": 0.234926470542891 + } + }, + "cross_logiqa": { + "prompt_1": { + "overall_acc": 0.3262987012987013, "language_acc": { - "Spanish": 0.54, - "English": 0.6533333333333333, - "Chinese": 0.5666666666666667, - "Vietnamese": 0.54, - "Filipino": 0.36666666666666664, - "Indonesian": 0.5266666666666666, - "Malay": 0.56 + "Indonesian": 0.3352272727272727, + "English": 0.32386363636363635, + "Filipino": 0.2897727272727273, + "Spanish": 0.32954545454545453, + "Chinese": 0.35795454545454547, + "Malay": 0.3352272727272727, + "Vietnamese": 0.3125 }, - "consistency_score_2": 0.6047619047619047, - "consistency_score_3": 0.4466666666666667, - "consistency_score_4": 0.35390476190476194, - "consistency_score_5": 0.2904761904761905, - "consistency_score_6": 0.24380952380952375, - "consistency_score_7": 0.20666666666666667, + "consistency_score_2": 0.4897186147186148, + "consistency_score_3": 0.30584415584415586, + "consistency_score_4": 0.2103896103896104, + "consistency_score_5": 0.15313852813852813, + "consistency_score_6": 0.11607142857142858, + "consistency_score_7": 0.09090909090909091, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6666666666666666, - "Spanish,Chinese": 0.66, - "Spanish,Vietnamese": 0.6933333333333334, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.6733333333333333, - "Spanish,Malay": 0.6733333333333333, - "English,Chinese": 0.7, - "English,Vietnamese": 0.7266666666666667, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.6533333333333333, - "English,Malay": 0.7, - "Chinese,Vietnamese": 0.6733333333333333, - "Chinese,Filipino": 0.47333333333333333, - "Chinese,Indonesian": 0.6266666666666667, - "Chinese,Malay": 0.6266666666666667, - "Vietnamese,Filipino": 0.4066666666666667, - "Vietnamese,Indonesian": 0.6933333333333334, - "Vietnamese,Malay": 0.6666666666666666, - "Filipino,Indonesian": 0.42, - "Filipino,Malay": 0.4, - "Indonesian,Malay": 0.66 + "Indonesian,English": 0.5852272727272727, + "Indonesian,Filipino": 0.3465909090909091, + "Indonesian,Spanish": 0.5227272727272727, + "Indonesian,Chinese": 0.48295454545454547, + "Indonesian,Malay": 0.6022727272727273, + "Indonesian,Vietnamese": 0.5568181818181818, + "English,Filipino": 0.3352272727272727, + "English,Spanish": 0.5170454545454546, + "English,Chinese": 0.5056818181818182, + "English,Malay": 0.5454545454545454, + "English,Vietnamese": 0.5965909090909091, + "Filipino,Spanish": 0.4772727272727273, + "Filipino,Chinese": 0.2840909090909091, + "Filipino,Malay": 0.4034090909090909, + "Filipino,Vietnamese": 0.3522727272727273, + "Spanish,Chinese": 0.4375, + "Spanish,Malay": 0.6420454545454546, + "Spanish,Vietnamese": 0.5511363636363636, + "Chinese,Malay": 0.4659090909090909, + "Chinese,Vietnamese": 0.48863636363636365, + "Malay,Vietnamese": 0.5852272727272727 }, "3_combine": { - "Spanish,English,Chinese": 0.54, - "Spanish,English,Vietnamese": 0.56, - "Spanish,English,Filipino": 0.34, - "Spanish,English,Indonesian": 0.5266666666666666, - "Spanish,English,Malay": 0.5333333333333333, - "Spanish,Chinese,Vietnamese": 0.54, - "Spanish,Chinese,Filipino": 0.3333333333333333, - "Spanish,Chinese,Indonesian": 0.52, - "Spanish,Chinese,Malay": 0.5266666666666666, - "Spanish,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian": 0.5533333333333333, - "Spanish,Vietnamese,Malay": 0.5466666666666666, - "Spanish,Filipino,Indonesian": 0.32, - "Spanish,Filipino,Malay": 0.3, - "Spanish,Indonesian,Malay": 0.54, - "English,Chinese,Vietnamese": 0.5666666666666667, - "English,Chinese,Filipino": 0.36, - "English,Chinese,Indonesian": 0.52, - "English,Chinese,Malay": 0.54, - "English,Vietnamese,Filipino": 0.35333333333333333, - "English,Vietnamese,Indonesian": 0.5733333333333334, - "English,Vietnamese,Malay": 0.5666666666666667, - "English,Filipino,Indonesian": 0.32666666666666666, - "English,Filipino,Malay": 0.3333333333333333, - "English,Indonesian,Malay": 0.5333333333333333, - "Chinese,Vietnamese,Filipino": 0.32, - "Chinese,Vietnamese,Indonesian": 0.5266666666666666, - "Chinese,Vietnamese,Malay": 0.52, - "Chinese,Filipino,Indonesian": 0.32, - "Chinese,Filipino,Malay": 0.31333333333333335, - "Chinese,Indonesian,Malay": 0.4866666666666667, - "Vietnamese,Filipino,Indonesian": 0.32, - "Vietnamese,Filipino,Malay": 0.29333333333333333, - "Vietnamese,Indonesian,Malay": 0.5466666666666666, - "Filipino,Indonesian,Malay": 0.3 + "Indonesian,English,Filipino": 0.2215909090909091, + "Indonesian,English,Spanish": 0.36363636363636365, + "Indonesian,English,Chinese": 0.3465909090909091, + "Indonesian,English,Malay": 0.4147727272727273, + "Indonesian,English,Vietnamese": 0.42613636363636365, + "Indonesian,Filipino,Spanish": 0.26136363636363635, + "Indonesian,Filipino,Chinese": 0.16477272727272727, + "Indonesian,Filipino,Malay": 0.24431818181818182, + "Indonesian,Filipino,Vietnamese": 0.21022727272727273, + "Indonesian,Spanish,Chinese": 0.29545454545454547, + "Indonesian,Spanish,Malay": 0.4431818181818182, + "Indonesian,Spanish,Vietnamese": 0.3806818181818182, + "Indonesian,Chinese,Malay": 0.3522727272727273, + "Indonesian,Chinese,Vietnamese": 0.3465909090909091, + "Indonesian,Malay,Vietnamese": 0.4147727272727273, + "English,Filipino,Spanish": 0.23295454545454544, + "English,Filipino,Chinese": 0.1875, + "English,Filipino,Malay": 0.20454545454545456, + "English,Filipino,Vietnamese": 0.21022727272727273, + "English,Spanish,Chinese": 0.2897727272727273, + "English,Spanish,Malay": 0.39204545454545453, + "English,Spanish,Vietnamese": 0.38636363636363635, + "English,Chinese,Malay": 0.3181818181818182, + "English,Chinese,Vietnamese": 0.3522727272727273, + "English,Malay,Vietnamese": 0.4090909090909091, + "Filipino,Spanish,Chinese": 0.19886363636363635, + "Filipino,Spanish,Malay": 0.32954545454545453, + "Filipino,Spanish,Vietnamese": 0.25, + "Filipino,Chinese,Malay": 0.20454545454545456, + "Filipino,Chinese,Vietnamese": 0.18181818181818182, + "Filipino,Malay,Vietnamese": 0.24431818181818182, + "Spanish,Chinese,Malay": 0.32954545454545453, + "Spanish,Chinese,Vietnamese": 0.3181818181818182, + "Spanish,Malay,Vietnamese": 0.42613636363636365, + "Chinese,Malay,Vietnamese": 0.3522727272727273 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.47333333333333333, - "Spanish,English,Chinese,Filipino": 0.29333333333333333, - "Spanish,English,Chinese,Indonesian": 0.4533333333333333, - "Spanish,English,Chinese,Malay": 0.4533333333333333, - "Spanish,English,Vietnamese,Filipino": 0.29333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.4666666666666667, - "Spanish,English,Vietnamese,Malay": 0.4666666666666667, - "Spanish,English,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Indonesian,Malay": 0.4533333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.44666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.4533333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Chinese,Filipino,Malay": 0.2733333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.44, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.47333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino": 0.3, - "English,Chinese,Vietnamese,Indonesian": 0.46, - "English,Chinese,Vietnamese,Malay": 0.4666666666666667, - "English,Chinese,Filipino,Indonesian": 0.28, - "English,Chinese,Filipino,Malay": 0.29333333333333333, - "English,Chinese,Indonesian,Malay": 0.43333333333333335, - "English,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "English,Vietnamese,Filipino,Malay": 0.2733333333333333, - "English,Vietnamese,Indonesian,Malay": 0.47333333333333333, - "English,Filipino,Indonesian,Malay": 0.2733333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.43333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.26 + "Indonesian,English,Filipino,Spanish": 0.17045454545454544, + "Indonesian,English,Filipino,Chinese": 0.14204545454545456, + "Indonesian,English,Filipino,Malay": 0.1590909090909091, + "Indonesian,English,Filipino,Vietnamese": 0.16477272727272727, + "Indonesian,English,Spanish,Chinese": 0.23295454545454544, + "Indonesian,English,Spanish,Malay": 0.3181818181818182, + "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, + "Indonesian,English,Chinese,Malay": 0.26136363636363635, + "Indonesian,English,Chinese,Vietnamese": 0.2897727272727273, + "Indonesian,English,Malay,Vietnamese": 0.32386363636363635, + "Indonesian,Filipino,Spanish,Chinese": 0.13068181818181818, + "Indonesian,Filipino,Spanish,Malay": 0.2215909090909091, + "Indonesian,Filipino,Spanish,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Chinese,Malay": 0.14204545454545456, + "Indonesian,Filipino,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,Spanish,Chinese,Malay": 0.2556818181818182, + "Indonesian,Spanish,Chinese,Vietnamese": 0.25, + "Indonesian,Spanish,Malay,Vietnamese": 0.32954545454545453, + "Indonesian,Chinese,Malay,Vietnamese": 0.2840909090909091, + "English,Filipino,Spanish,Chinese": 0.13636363636363635, + "English,Filipino,Spanish,Malay": 0.17045454545454544, + "English,Filipino,Spanish,Vietnamese": 0.16477272727272727, + "English,Filipino,Chinese,Malay": 0.14772727272727273, + "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, + "English,Filipino,Malay,Vietnamese": 0.1534090909090909, + "English,Spanish,Chinese,Malay": 0.23863636363636365, + "English,Spanish,Chinese,Vietnamese": 0.24431818181818182, + "English,Spanish,Malay,Vietnamese": 0.3125, + "English,Chinese,Malay,Vietnamese": 0.2727272727272727, + "Filipino,Spanish,Chinese,Malay": 0.16477272727272727, + "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, + "Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, + "Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, + "Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.4, - "Spanish,English,Chinese,Vietnamese,Malay": 0.4066666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Chinese,Indonesian,Malay": 0.3933333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.3933333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24 + "Indonesian,English,Filipino,Spanish,Chinese": 0.11363636363636363, + "Indonesian,English,Filipino,Spanish,Malay": 0.14204545454545456, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.13068181818181818, + "Indonesian,English,Filipino,Chinese,Malay": 0.11931818181818182, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.11363636363636363, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese,Malay": 0.20454545454545456, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.20454545454545456, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.125, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, + "English,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, + "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, + "English,Filipino,Chinese,Malay,Vietnamese": 0.125, + "English,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1875, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 } }, - "AC3_2": 0.5684156132739742, - "AC3_3": 0.4873514211390454, - "AC3_4": 0.42638215478278607, - "AC3_5": 0.3768158875997982, - "AC3_6": 0.33520088372933776, - "AC3_7": 0.29834188030171893 + "AC3_2": 0.39164499289010174, + "AC3_3": 0.31574049981998215, + "AC3_4": 0.25582765686891834, + "AC3_5": 0.20844815451077167, + "AC3_6": 0.17123197899143633, + "AC3_7": 0.14220021220503895 }, - "prompt_4": { - "overall_acc": 0.5304761904761904, + "prompt_2": { + "overall_acc": 0.3092532467532468, "language_acc": { - "Spanish": 0.5, - "English": 0.6466666666666666, - "Chinese": 0.5866666666666667, - "Vietnamese": 0.5266666666666666, - "Filipino": 0.42, - "Indonesian": 0.49333333333333335, - "Malay": 0.54 + "Indonesian": 0.2840909090909091, + "English": 0.38636363636363635, + "Filipino": 0.26704545454545453, + "Spanish": 0.29545454545454547, + "Chinese": 0.3068181818181818, + "Malay": 0.3068181818181818, + "Vietnamese": 0.3181818181818182 }, - "consistency_score_2": 0.5984126984126985, - "consistency_score_3": 0.4417142857142857, - "consistency_score_4": 0.35390476190476194, - "consistency_score_5": 0.2965079365079366, - "consistency_score_6": 0.2561904761904762, - "consistency_score_7": 0.22666666666666666, + "consistency_score_2": 0.45670995670995673, + "consistency_score_3": 0.26120129870129866, + "consistency_score_4": 0.1642857142857143, + "consistency_score_5": 0.10876623376623377, + "consistency_score_6": 0.07386363636363637, + "consistency_score_7": 0.05113636363636364, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6466666666666666, - "Spanish,Chinese": 0.56, - "Spanish,Vietnamese": 0.66, - "Spanish,Filipino": 0.4666666666666667, - "Spanish,Indonesian": 0.66, - "Spanish,Malay": 0.6266666666666667, - "English,Chinese": 0.68, - "English,Vietnamese": 0.6466666666666666, - "English,Filipino": 0.5, - "English,Indonesian": 0.68, - "English,Malay": 0.6666666666666666, - "Chinese,Vietnamese": 0.6666666666666666, - "Chinese,Filipino": 0.47333333333333333, - "Chinese,Indonesian": 0.6333333333333333, - "Chinese,Malay": 0.66, - "Vietnamese,Filipino": 0.48, - "Vietnamese,Indonesian": 0.6, - "Vietnamese,Malay": 0.64, - "Filipino,Indonesian": 0.4666666666666667, - "Filipino,Malay": 0.44666666666666666, - "Indonesian,Malay": 0.7066666666666667 + "Indonesian,English": 0.5454545454545454, + "Indonesian,Filipino": 0.2784090909090909, + "Indonesian,Spanish": 0.48295454545454547, + "Indonesian,Chinese": 0.4602272727272727, + "Indonesian,Malay": 0.5738636363636364, + "Indonesian,Vietnamese": 0.5681818181818182, + "English,Filipino": 0.30113636363636365, + "English,Spanish": 0.48295454545454547, + "English,Chinese": 0.4943181818181818, + "English,Malay": 0.5056818181818182, + "English,Vietnamese": 0.5852272727272727, + "Filipino,Spanish": 0.4375, + "Filipino,Chinese": 0.3352272727272727, + "Filipino,Malay": 0.30113636363636365, + "Filipino,Vietnamese": 0.3181818181818182, + "Spanish,Chinese": 0.42613636363636365, + "Spanish,Malay": 0.5056818181818182, + "Spanish,Vietnamese": 0.5227272727272727, + "Chinese,Malay": 0.44886363636363635, + "Chinese,Vietnamese": 0.5, + "Malay,Vietnamese": 0.5170454545454546 }, "3_combine": { - "Spanish,English,Chinese": 0.48, - "Spanish,English,Vietnamese": 0.5066666666666667, - "Spanish,English,Filipino": 0.36666666666666664, - "Spanish,English,Indonesian": 0.5266666666666666, - "Spanish,English,Malay": 0.49333333333333335, - "Spanish,Chinese,Vietnamese": 0.48, - "Spanish,Chinese,Filipino": 0.30666666666666664, - "Spanish,Chinese,Indonesian": 0.47333333333333333, - "Spanish,Chinese,Malay": 0.47333333333333333, - "Spanish,Vietnamese,Filipino": 0.3466666666666667, - "Spanish,Vietnamese,Indonesian": 0.49333333333333335, - "Spanish,Vietnamese,Malay": 0.5, - "Spanish,Filipino,Indonesian": 0.36, - "Spanish,Filipino,Malay": 0.32, - "Spanish,Indonesian,Malay": 0.54, - "English,Chinese,Vietnamese": 0.5133333333333333, - "English,Chinese,Filipino": 0.37333333333333335, - "English,Chinese,Indonesian": 0.54, - "English,Chinese,Malay": 0.5333333333333333, - "English,Vietnamese,Filipino": 0.36666666666666664, - "English,Vietnamese,Indonesian": 0.5, - "English,Vietnamese,Malay": 0.5, - "English,Filipino,Indonesian": 0.37333333333333335, - "English,Filipino,Malay": 0.36, - "English,Indonesian,Malay": 0.5666666666666667, - "Chinese,Vietnamese,Filipino": 0.36, - "Chinese,Vietnamese,Indonesian": 0.49333333333333335, - "Chinese,Vietnamese,Malay": 0.52, - "Chinese,Filipino,Indonesian": 0.3333333333333333, - "Chinese,Filipino,Malay": 0.3466666666666667, - "Chinese,Indonesian,Malay": 0.5533333333333333, - "Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "Vietnamese,Filipino,Malay": 0.3333333333333333, - "Vietnamese,Indonesian,Malay": 0.5266666666666666, - "Filipino,Indonesian,Malay": 0.35333333333333333 + "Indonesian,English,Filipino": 0.1590909090909091, + "Indonesian,English,Spanish": 0.32954545454545453, + "Indonesian,English,Chinese": 0.3125, + "Indonesian,English,Malay": 0.375, + "Indonesian,English,Vietnamese": 0.38636363636363635, + "Indonesian,Filipino,Spanish": 0.19886363636363635, + "Indonesian,Filipino,Chinese": 0.1534090909090909, + "Indonesian,Filipino,Malay": 0.17045454545454544, + "Indonesian,Filipino,Vietnamese": 0.16477272727272727, + "Indonesian,Spanish,Chinese": 0.2556818181818182, + "Indonesian,Spanish,Malay": 0.3409090909090909, + "Indonesian,Spanish,Vietnamese": 0.36363636363636365, + "Indonesian,Chinese,Malay": 0.3068181818181818, + "Indonesian,Chinese,Vietnamese": 0.3181818181818182, + "Indonesian,Malay,Vietnamese": 0.38636363636363635, + "English,Filipino,Spanish": 0.1875, + "English,Filipino,Chinese": 0.17613636363636365, + "English,Filipino,Malay": 0.1590909090909091, + "English,Filipino,Vietnamese": 0.1875, + "English,Spanish,Chinese": 0.26136363636363635, + "English,Spanish,Malay": 0.30113636363636365, + "English,Spanish,Vietnamese": 0.36363636363636365, + "English,Chinese,Malay": 0.29545454545454547, + "English,Chinese,Vietnamese": 0.3522727272727273, + "English,Malay,Vietnamese": 0.3409090909090909, + "Filipino,Spanish,Chinese": 0.21022727272727273, + "Filipino,Spanish,Malay": 0.2159090909090909, + "Filipino,Spanish,Vietnamese": 0.2159090909090909, + "Filipino,Chinese,Malay": 0.1534090909090909, + "Filipino,Chinese,Vietnamese": 0.16477272727272727, + "Filipino,Malay,Vietnamese": 0.1590909090909091, + "Spanish,Chinese,Malay": 0.26704545454545453, + "Spanish,Chinese,Vietnamese": 0.2840909090909091, + "Spanish,Malay,Vietnamese": 0.32954545454545453, + "Chinese,Malay,Vietnamese": 0.29545454545454547 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.41333333333333333, - "Spanish,English,Chinese,Filipino": 0.2866666666666667, - "Spanish,English,Chinese,Indonesian": 0.43333333333333335, - "Spanish,English,Chinese,Malay": 0.41333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,English,Vietnamese,Indonesian": 0.4266666666666667, - "Spanish,English,Vietnamese,Malay": 0.41333333333333333, - "Spanish,English,Filipino,Indonesian": 0.31333333333333335, - "Spanish,English,Filipino,Malay": 0.2866666666666667, - "Spanish,English,Indonesian,Malay": 0.4666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian": 0.41333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.41333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Chinese,Filipino,Malay": 0.26, - "Spanish,Chinese,Indonesian,Malay": 0.44, - "Spanish,Vietnamese,Filipino,Indonesian": 0.28, - "Spanish,Vietnamese,Filipino,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.44666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino": 0.3, - "English,Chinese,Vietnamese,Indonesian": 0.44, - "English,Chinese,Vietnamese,Malay": 0.4266666666666667, - "English,Chinese,Filipino,Indonesian": 0.31333333333333335, - "English,Chinese,Filipino,Malay": 0.31333333333333335, - "English,Chinese,Indonesian,Malay": 0.4866666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "English,Vietnamese,Filipino,Malay": 0.2866666666666667, - "English,Vietnamese,Indonesian,Malay": 0.44666666666666666, - "English,Filipino,Indonesian,Malay": 0.32, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.46, - "Chinese,Filipino,Indonesian,Malay": 0.3, - "Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333 + "Indonesian,English,Filipino,Spanish": 0.125, + "Indonesian,English,Filipino,Chinese": 0.10795454545454546, + "Indonesian,English,Filipino,Malay": 0.11363636363636363, + "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, + "Indonesian,English,Spanish,Chinese": 0.20454545454545456, + "Indonesian,English,Spanish,Malay": 0.23863636363636365, + "Indonesian,English,Spanish,Vietnamese": 0.26136363636363635, + "Indonesian,English,Chinese,Malay": 0.22727272727272727, + "Indonesian,English,Chinese,Vietnamese": 0.2556818181818182, + "Indonesian,English,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,Filipino,Spanish,Chinese": 0.10795454545454546, + "Indonesian,Filipino,Spanish,Malay": 0.13068181818181818, + "Indonesian,Filipino,Spanish,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Chinese,Malay": 0.10227272727272728, + "Indonesian,Filipino,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Malay,Vietnamese": 0.125, + "Indonesian,Spanish,Chinese,Malay": 0.19886363636363635, + "Indonesian,Spanish,Chinese,Vietnamese": 0.20454545454545456, + "Indonesian,Spanish,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,Chinese,Malay,Vietnamese": 0.23295454545454544, + "English,Filipino,Spanish,Chinese": 0.11931818181818182, + "English,Filipino,Spanish,Malay": 0.125, + "English,Filipino,Spanish,Vietnamese": 0.14772727272727273, + "English,Filipino,Chinese,Malay": 0.10795454545454546, + "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, + "English,Filipino,Malay,Vietnamese": 0.10227272727272728, + "English,Spanish,Chinese,Malay": 0.1875, + "English,Spanish,Chinese,Vietnamese": 0.22727272727272727, + "English,Spanish,Malay,Vietnamese": 0.2215909090909091, + "English,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Filipino,Spanish,Chinese,Malay": 0.11363636363636363, + "Filipino,Spanish,Chinese,Vietnamese": 0.125, + "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, + "Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, + "Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.38, - "Spanish,English,Chinese,Vietnamese,Malay": 0.36, - "Spanish,English,Chinese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.4066666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.3933333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.2733333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.3933333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.26, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666 + "Indonesian,English,Filipino,Spanish,Chinese": 0.08522727272727272, + "Indonesian,English,Filipino,Spanish,Malay": 0.09090909090909091, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.08522727272727272, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Spanish,Chinese,Malay": 0.1590909090909091, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.17613636363636365, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.1875, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.07954545454545454, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, + "English,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, + "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, + "English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, + "English,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.36, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.06818181818181818, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0625, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364 } }, - "AC3_2": 0.5624002356350062, - "AC3_3": 0.48204321535570116, - "AC3_4": 0.42456375696459786, - "AC3_5": 0.38039521062944315, - "AC3_6": 0.3455159690555827, - "AC3_7": 0.3176184485953669 + "AC3_2": 0.36878804694548106, + "AC3_3": 0.28320345699416466, + "AC3_4": 0.21457955820925284, + "AC3_5": 0.16093178662144336, + "AC3_6": 0.11924595528474537, + "AC3_7": 0.08776105648670479 }, - "prompt_5": { - "overall_acc": 0.5380952380952381, + "prompt_3": { + "overall_acc": 0.32711038961038963, "language_acc": { - "Spanish": 0.5533333333333333, - "English": 0.66, - "Chinese": 0.5866666666666667, - "Vietnamese": 0.5333333333333333, - "Filipino": 0.36666666666666664, - "Indonesian": 0.5266666666666666, - "Malay": 0.54 + "Indonesian": 0.30113636363636365, + "English": 0.39204545454545453, + "Filipino": 0.3068181818181818, + "Spanish": 0.3068181818181818, + "Chinese": 0.375, + "Malay": 0.3181818181818182, + "Vietnamese": 0.2897727272727273 }, - "consistency_score_2": 0.606984126984127, - "consistency_score_3": 0.44952380952380955, - "consistency_score_4": 0.35733333333333334, - "consistency_score_5": 0.2933333333333334, - "consistency_score_6": 0.24285714285714283, - "consistency_score_7": 0.2, + "consistency_score_2": 0.49323593073593064, + "consistency_score_3": 0.3066558441558441, + "consistency_score_4": 0.20925324675324675, + "consistency_score_5": 0.14989177489177488, + "consistency_score_6": 0.10957792207792207, + "consistency_score_7": 0.07954545454545454, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.72, - "Spanish,Chinese": 0.6066666666666667, - "Spanish,Vietnamese": 0.6666666666666666, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.7, - "Spanish,Malay": 0.64, - "English,Chinese": 0.6866666666666666, - "English,Vietnamese": 0.6933333333333334, - "English,Filipino": 0.46, - "English,Indonesian": 0.74, - "English,Malay": 0.7, - "Chinese,Vietnamese": 0.6266666666666667, - "Chinese,Filipino": 0.44, - "Chinese,Indonesian": 0.6533333333333333, - "Chinese,Malay": 0.6333333333333333, - "Vietnamese,Filipino": 0.38666666666666666, - "Vietnamese,Indonesian": 0.68, - "Vietnamese,Malay": 0.6533333333333333, - "Filipino,Indonesian": 0.46, - "Filipino,Malay": 0.43333333333333335, - "Indonesian,Malay": 0.7266666666666667 + "Indonesian,English": 0.5681818181818182, + "Indonesian,Filipino": 0.32386363636363635, + "Indonesian,Spanish": 0.5227272727272727, + "Indonesian,Chinese": 0.44886363636363635, + "Indonesian,Malay": 0.6022727272727273, + "Indonesian,Vietnamese": 0.5909090909090909, + "English,Filipino": 0.32386363636363635, + "English,Spanish": 0.5568181818181818, + "English,Chinese": 0.5340909090909091, + "English,Malay": 0.5625, + "English,Vietnamese": 0.5568181818181818, + "Filipino,Spanish": 0.4715909090909091, + "Filipino,Chinese": 0.3522727272727273, + "Filipino,Malay": 0.42613636363636365, + "Filipino,Vietnamese": 0.32954545454545453, + "Spanish,Chinese": 0.5056818181818182, + "Spanish,Malay": 0.5795454545454546, + "Spanish,Vietnamese": 0.5625, + "Chinese,Malay": 0.4943181818181818, + "Chinese,Vietnamese": 0.5, + "Malay,Vietnamese": 0.5454545454545454 }, "3_combine": { - "Spanish,English,Chinese": 0.5266666666666666, - "Spanish,English,Vietnamese": 0.5733333333333334, - "Spanish,English,Filipino": 0.34, - "Spanish,English,Indonesian": 0.6066666666666667, - "Spanish,English,Malay": 0.5466666666666666, - "Spanish,Chinese,Vietnamese": 0.49333333333333335, - "Spanish,Chinese,Filipino": 0.3, - "Spanish,Chinese,Indonesian": 0.52, - "Spanish,Chinese,Malay": 0.4666666666666667, - "Spanish,Vietnamese,Filipino": 0.3, - "Spanish,Vietnamese,Indonesian": 0.5533333333333333, - "Spanish,Vietnamese,Malay": 0.52, - "Spanish,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Filipino,Malay": 0.30666666666666664, - "Spanish,Indonesian,Malay": 0.56, - "English,Chinese,Vietnamese": 0.5266666666666666, - "English,Chinese,Filipino": 0.34, - "English,Chinese,Indonesian": 0.5733333333333334, - "English,Chinese,Malay": 0.5333333333333333, - "English,Vietnamese,Filipino": 0.32, - "English,Vietnamese,Indonesian": 0.5933333333333334, - "English,Vietnamese,Malay": 0.5466666666666666, - "English,Filipino,Indonesian": 0.36666666666666664, - "English,Filipino,Malay": 0.3466666666666667, - "English,Indonesian,Malay": 0.6133333333333333, - "Chinese,Vietnamese,Filipino": 0.2733333333333333, - "Chinese,Vietnamese,Indonesian": 0.5133333333333333, - "Chinese,Vietnamese,Malay": 0.5, - "Chinese,Filipino,Indonesian": 0.32, - "Chinese,Filipino,Malay": 0.31333333333333335, - "Chinese,Indonesian,Malay": 0.54, - "Vietnamese,Filipino,Indonesian": 0.32, - "Vietnamese,Filipino,Malay": 0.30666666666666664, - "Vietnamese,Indonesian,Malay": 0.5666666666666667, - "Filipino,Indonesian,Malay": 0.35333333333333333 + "Indonesian,English,Filipino": 0.20454545454545456, + "Indonesian,English,Spanish": 0.36363636363636365, + "Indonesian,English,Chinese": 0.32954545454545453, + "Indonesian,English,Malay": 0.4147727272727273, + "Indonesian,English,Vietnamese": 0.4090909090909091, + "Indonesian,Filipino,Spanish": 0.24431818181818182, + "Indonesian,Filipino,Chinese": 0.16477272727272727, + "Indonesian,Filipino,Malay": 0.25, + "Indonesian,Filipino,Vietnamese": 0.19318181818181818, + "Indonesian,Spanish,Chinese": 0.30113636363636365, + "Indonesian,Spanish,Malay": 0.38636363636363635, + "Indonesian,Spanish,Vietnamese": 0.3806818181818182, + "Indonesian,Chinese,Malay": 0.3352272727272727, + "Indonesian,Chinese,Vietnamese": 0.3409090909090909, + "Indonesian,Malay,Vietnamese": 0.42045454545454547, + "English,Filipino,Spanish": 0.23863636363636365, + "English,Filipino,Chinese": 0.19318181818181818, + "English,Filipino,Malay": 0.23295454545454544, + "English,Filipino,Vietnamese": 0.19318181818181818, + "English,Spanish,Chinese": 0.3522727272727273, + "English,Spanish,Malay": 0.3977272727272727, + "English,Spanish,Vietnamese": 0.3977272727272727, + "English,Chinese,Malay": 0.35795454545454547, + "English,Chinese,Vietnamese": 0.3522727272727273, + "English,Malay,Vietnamese": 0.3977272727272727, + "Filipino,Spanish,Chinese": 0.23863636363636365, + "Filipino,Spanish,Malay": 0.3181818181818182, + "Filipino,Spanish,Vietnamese": 0.24431818181818182, + "Filipino,Chinese,Malay": 0.2215909090909091, + "Filipino,Chinese,Vietnamese": 0.19318181818181818, + "Filipino,Malay,Vietnamese": 0.23295454545454544, + "Spanish,Chinese,Malay": 0.3522727272727273, + "Spanish,Chinese,Vietnamese": 0.3465909090909091, + "Spanish,Malay,Vietnamese": 0.3977272727272727, + "Chinese,Malay,Vietnamese": 0.3352272727272727 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4533333333333333, - "Spanish,English,Chinese,Filipino": 0.26666666666666666, - "Spanish,English,Chinese,Indonesian": 0.48, - "Spanish,English,Chinese,Malay": 0.44, - "Spanish,English,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.5133333333333333, - "Spanish,English,Vietnamese,Malay": 0.4666666666666667, - "Spanish,English,Filipino,Indonesian": 0.30666666666666664, - "Spanish,English,Filipino,Malay": 0.28, - "Spanish,English,Indonesian,Malay": 0.5266666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.4533333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.41333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Chinese,Filipino,Malay": 0.23333333333333334, - "Spanish,Chinese,Indonesian,Malay": 0.44666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.4866666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino": 0.24666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.47333333333333333, - "English,Chinese,Vietnamese,Malay": 0.43333333333333335, - "English,Chinese,Filipino,Indonesian": 0.29333333333333333, - "English,Chinese,Filipino,Malay": 0.28, - "English,Chinese,Indonesian,Malay": 0.4866666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.3, - "English,Vietnamese,Filipino,Malay": 0.28, - "English,Vietnamese,Indonesian,Malay": 0.52, - "English,Filipino,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.4533333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667 + "Indonesian,English,Filipino,Spanish": 0.1590909090909091, + "Indonesian,English,Filipino,Chinese": 0.11931818181818182, + "Indonesian,English,Filipino,Malay": 0.17045454545454544, + "Indonesian,English,Filipino,Vietnamese": 0.14204545454545456, + "Indonesian,English,Spanish,Chinese": 0.24431818181818182, + "Indonesian,English,Spanish,Malay": 0.2840909090909091, + "Indonesian,English,Spanish,Vietnamese": 0.29545454545454547, + "Indonesian,English,Chinese,Malay": 0.26136363636363635, + "Indonesian,English,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,English,Malay,Vietnamese": 0.32386363636363635, + "Indonesian,Filipino,Spanish,Chinese": 0.125, + "Indonesian,Filipino,Spanish,Malay": 0.21022727272727273, + "Indonesian,Filipino,Spanish,Vietnamese": 0.1590909090909091, + "Indonesian,Filipino,Chinese,Malay": 0.13068181818181818, + "Indonesian,Filipino,Chinese,Vietnamese": 0.125, + "Indonesian,Filipino,Malay,Vietnamese": 0.17045454545454544, + "Indonesian,Spanish,Chinese,Malay": 0.24431818181818182, + "Indonesian,Spanish,Chinese,Vietnamese": 0.2556818181818182, + "Indonesian,Spanish,Malay,Vietnamese": 0.3125, + "Indonesian,Chinese,Malay,Vietnamese": 0.2727272727272727, + "English,Filipino,Spanish,Chinese": 0.1534090909090909, + "English,Filipino,Spanish,Malay": 0.19886363636363635, + "English,Filipino,Spanish,Vietnamese": 0.17045454545454544, + "English,Filipino,Chinese,Malay": 0.1534090909090909, + "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, + "English,Filipino,Malay,Vietnamese": 0.1590909090909091, + "English,Spanish,Chinese,Malay": 0.2727272727272727, + "English,Spanish,Chinese,Vietnamese": 0.26704545454545453, + "English,Spanish,Malay,Vietnamese": 0.32386363636363635, + "English,Chinese,Malay,Vietnamese": 0.26704545454545453, + "Filipino,Spanish,Chinese,Malay": 0.1875, + "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, + "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, + "Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, + "Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.22, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.4266666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.38666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Chinese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.4266666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Vietnamese,Filipino,Malay": 0.24, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.46, - "Spanish,English,Filipino,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.24, - "English,Chinese,Vietnamese,Filipino,Malay": 0.22666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.42, - "English,Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.28, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 + "Indonesian,English,Filipino,Spanish,Chinese": 0.09659090909090909, + "Indonesian,English,Filipino,Spanish,Malay": 0.14204545454545456, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.125, + "Indonesian,English,Filipino,Chinese,Malay": 0.10227272727272728, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09659090909090909, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese,Malay": 0.19886363636363635, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.21022727272727273, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, + "English,Filipino,Spanish,Chinese,Malay": 0.13636363636363635, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, + "English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, + "English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, + "English,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.125, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454 } }, - "AC3_2": 0.5704674772100065, - "AC3_3": 0.48983790232446656, - "AC3_4": 0.42946890727672027, - "AC3_5": 0.3796869033525769, - "AC3_6": 0.33466898950418433, - "AC3_7": 0.29161290318629757 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.44724025974025977, + "AC3_2": 0.393352401178792, + "AC3_3": 0.31655303579740146, + "AC3_4": 0.25523322768110196, + "AC3_5": 0.2055804376700676, + "AC3_6": 0.16416320664420125, + "AC3_7": 0.12797133003566927 + }, + "prompt_4": { + "overall_acc": 0.31818181818181823, "language_acc": { - "English": 0.5056818181818182, - "Filipino": 0.32386363636363635, - "Spanish": 0.375, - "Malay": 0.48863636363636365, - "Chinese": 0.5795454545454546, - "Indonesian": 0.4431818181818182, - "Vietnamese": 0.4147727272727273 + "Indonesian": 0.32386363636363635, + "English": 0.32954545454545453, + "Filipino": 0.2897727272727273, + "Spanish": 0.3068181818181818, + "Chinese": 0.36363636363636365, + "Malay": 0.3465909090909091, + "Vietnamese": 0.26704545454545453 }, - "consistency_score_2": 0.509469696969697, - "consistency_score_3": 0.33896103896103896, - "consistency_score_4": 0.24788961038961033, - "consistency_score_5": 0.18993506493506493, - "consistency_score_6": 0.14935064935064934, - "consistency_score_7": 0.11931818181818182, + "consistency_score_2": 0.47781385281385286, + "consistency_score_3": 0.2883116883116883, + "consistency_score_4": 0.18863636363636363, + "consistency_score_5": 0.12662337662337664, + "consistency_score_6": 0.08360389610389611, + "consistency_score_7": 0.05113636363636364, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.5340909090909091, - "English,Malay": 0.6306818181818182, - "English,Chinese": 0.6136363636363636, - "English,Indonesian": 0.6022727272727273, - "English,Vietnamese": 0.5568181818181818, - "Filipino,Spanish": 0.375, - "Filipino,Malay": 0.3693181818181818, - "Filipino,Chinese": 0.36363636363636365, - "Filipino,Indonesian": 0.375, - "Filipino,Vietnamese": 0.30113636363636365, - "Spanish,Malay": 0.5625, - "Spanish,Chinese": 0.48295454545454547, - "Spanish,Indonesian": 0.5681818181818182, - "Spanish,Vietnamese": 0.5, - "Malay,Chinese": 0.625, - "Malay,Indonesian": 0.6477272727272727, - "Malay,Vietnamese": 0.5568181818181818, - "Chinese,Indonesian": 0.5909090909090909, - "Chinese,Vietnamese": 0.5340909090909091, - "Indonesian,Vietnamese": 0.5738636363636364 + "Indonesian,English": 0.5454545454545454, + "Indonesian,Filipino": 0.3125, + "Indonesian,Spanish": 0.5227272727272727, + "Indonesian,Chinese": 0.45454545454545453, + "Indonesian,Malay": 0.5454545454545454, + "Indonesian,Vietnamese": 0.5340909090909091, + "English,Filipino": 0.2840909090909091, + "English,Spanish": 0.5397727272727273, + "English,Chinese": 0.5113636363636364, + "English,Malay": 0.5738636363636364, + "English,Vietnamese": 0.5397727272727273, + "Filipino,Spanish": 0.48863636363636365, + "Filipino,Chinese": 0.3068181818181818, + "Filipino,Malay": 0.4034090909090909, + "Filipino,Vietnamese": 0.3522727272727273, + "Spanish,Chinese": 0.44886363636363635, + "Spanish,Malay": 0.5511363636363636, + "Spanish,Vietnamese": 0.5511363636363636, + "Chinese,Malay": 0.5, + "Chinese,Vietnamese": 0.48295454545454547, + "Malay,Vietnamese": 0.5852272727272727 }, "3_combine": { - "English,Filipino,Spanish": 0.23863636363636365, - "English,Filipino,Malay": 0.26136363636363635, - "English,Filipino,Chinese": 0.23863636363636365, - "English,Filipino,Indonesian": 0.25, - "English,Filipino,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay": 0.4147727272727273, - "English,Spanish,Chinese": 0.3806818181818182, - "English,Spanish,Indonesian": 0.4090909090909091, - "English,Spanish,Vietnamese": 0.36363636363636365, - "English,Malay,Chinese": 0.48295454545454547, - "English,Malay,Indonesian": 0.4943181818181818, - "English,Malay,Vietnamese": 0.4318181818181818, - "English,Chinese,Indonesian": 0.4602272727272727, - "English,Chinese,Vietnamese": 0.42045454545454547, - "English,Indonesian,Vietnamese": 0.4318181818181818, - "Filipino,Spanish,Malay": 0.2556818181818182, - "Filipino,Spanish,Chinese": 0.21022727272727273, - "Filipino,Spanish,Indonesian": 0.26136363636363635, - "Filipino,Spanish,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Chinese": 0.25, - "Filipino,Malay,Indonesian": 0.26136363636363635, - "Filipino,Malay,Vietnamese": 0.21022727272727273, - "Filipino,Chinese,Indonesian": 0.23863636363636365, - "Filipino,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Chinese": 0.3977272727272727, - "Spanish,Malay,Indonesian": 0.44886363636363635, - "Spanish,Malay,Vietnamese": 0.3693181818181818, - "Spanish,Chinese,Indonesian": 0.3693181818181818, - "Spanish,Chinese,Vietnamese": 0.3352272727272727, - "Spanish,Indonesian,Vietnamese": 0.3806818181818182, - "Malay,Chinese,Indonesian": 0.4772727272727273, - "Malay,Chinese,Vietnamese": 0.4147727272727273, - "Malay,Indonesian,Vietnamese": 0.44886363636363635, - "Chinese,Indonesian,Vietnamese": 0.42613636363636365 + "Indonesian,English,Filipino": 0.16477272727272727, + "Indonesian,English,Spanish": 0.36363636363636365, + "Indonesian,English,Chinese": 0.32386363636363635, + "Indonesian,English,Malay": 0.3806818181818182, + "Indonesian,English,Vietnamese": 0.36363636363636365, + "Indonesian,Filipino,Spanish": 0.22727272727272727, + "Indonesian,Filipino,Chinese": 0.1534090909090909, + "Indonesian,Filipino,Malay": 0.20454545454545456, + "Indonesian,Filipino,Vietnamese": 0.18181818181818182, + "Indonesian,Spanish,Chinese": 0.29545454545454547, + "Indonesian,Spanish,Malay": 0.3693181818181818, + "Indonesian,Spanish,Vietnamese": 0.36363636363636365, + "Indonesian,Chinese,Malay": 0.3125, + "Indonesian,Chinese,Vietnamese": 0.3125, + "Indonesian,Malay,Vietnamese": 0.4034090909090909, + "English,Filipino,Spanish": 0.22727272727272727, + "English,Filipino,Chinese": 0.1534090909090909, + "English,Filipino,Malay": 0.20454545454545456, + "English,Filipino,Vietnamese": 0.18181818181818182, + "English,Spanish,Chinese": 0.30113636363636365, + "English,Spanish,Malay": 0.375, + "English,Spanish,Vietnamese": 0.375, + "English,Chinese,Malay": 0.3465909090909091, + "English,Chinese,Vietnamese": 0.3352272727272727, + "English,Malay,Vietnamese": 0.3977272727272727, + "Filipino,Spanish,Chinese": 0.22727272727272727, + "Filipino,Spanish,Malay": 0.2897727272727273, + "Filipino,Spanish,Vietnamese": 0.24431818181818182, + "Filipino,Chinese,Malay": 0.20454545454545456, + "Filipino,Chinese,Vietnamese": 0.17613636363636365, + "Filipino,Malay,Vietnamese": 0.24431818181818182, + "Spanish,Chinese,Malay": 0.3125, + "Spanish,Chinese,Vietnamese": 0.32386363636363635, + "Spanish,Malay,Vietnamese": 0.38636363636363635, + "Chinese,Malay,Vietnamese": 0.36363636363636365 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.2159090909090909, - "English,Filipino,Spanish,Chinese": 0.17045454545454544, - "English,Filipino,Spanish,Indonesian": 0.20454545454545456, + "Indonesian,English,Filipino,Spanish": 0.13636363636363635, + "Indonesian,English,Filipino,Chinese": 0.09090909090909091, + "Indonesian,English,Filipino,Malay": 0.11931818181818182, + "Indonesian,English,Filipino,Vietnamese": 0.11931818181818182, + "Indonesian,English,Spanish,Chinese": 0.22727272727272727, + "Indonesian,English,Spanish,Malay": 0.2840909090909091, + "Indonesian,English,Spanish,Vietnamese": 0.2784090909090909, + "Indonesian,English,Chinese,Malay": 0.24431818181818182, + "Indonesian,English,Chinese,Vietnamese": 0.24431818181818182, + "Indonesian,English,Malay,Vietnamese": 0.29545454545454547, + "Indonesian,Filipino,Spanish,Chinese": 0.13068181818181818, + "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "Indonesian,Filipino,Chinese,Malay": 0.11363636363636363, + "Indonesian,Filipino,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Malay": 0.22727272727272727, + "Indonesian,Spanish,Chinese,Vietnamese": 0.22727272727272727, + "Indonesian,Spanish,Malay,Vietnamese": 0.29545454545454547, + "Indonesian,Chinese,Malay,Vietnamese": 0.26704545454545453, + "English,Filipino,Spanish,Chinese": 0.13068181818181818, + "English,Filipino,Spanish,Malay": 0.17045454545454544, "English,Filipino,Spanish,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Chinese": 0.19886363636363635, - "English,Filipino,Malay,Indonesian": 0.2159090909090909, - "English,Filipino,Malay,Vietnamese": 0.17045454545454544, - "English,Filipino,Chinese,Indonesian": 0.1875, - "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay,Chinese": 0.3352272727272727, - "English,Spanish,Malay,Indonesian": 0.36363636363636365, - "English,Spanish,Malay,Vietnamese": 0.3068181818181818, - "English,Spanish,Chinese,Indonesian": 0.32386363636363635, - "English,Spanish,Chinese,Vietnamese": 0.2897727272727273, - "English,Spanish,Indonesian,Vietnamese": 0.3125, - "English,Malay,Chinese,Indonesian": 0.3977272727272727, - "English,Malay,Chinese,Vietnamese": 0.35795454545454547, - "English,Malay,Indonesian,Vietnamese": 0.375, - "English,Chinese,Indonesian,Vietnamese": 0.36363636363636365, - "Filipino,Spanish,Malay,Chinese": 0.17613636363636365, - "Filipino,Spanish,Malay,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Indonesian,Vietnamese": 0.1875, - "Filipino,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Chinese,Indonesian": 0.3352272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.2897727272727273, - "Spanish,Malay,Indonesian,Vietnamese": 0.3181818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.36363636363636365 + "English,Filipino,Chinese,Malay": 0.11931818181818182, + "English,Filipino,Chinese,Vietnamese": 0.10227272727272728, + "English,Filipino,Malay,Vietnamese": 0.14204545454545456, + "English,Spanish,Chinese,Malay": 0.22727272727272727, + "English,Spanish,Chinese,Vietnamese": 0.23863636363636365, + "English,Spanish,Malay,Vietnamese": 0.2840909090909091, + "English,Chinese,Malay,Vietnamese": 0.26704545454545453, + "Filipino,Spanish,Chinese,Malay": 0.16477272727272727, + "Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, + "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, + "Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, + "Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.19318181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Indonesian": 0.17045454545454544, - "English,Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.29545454545454547, - "English,Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.32386363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182 + "Indonesian,English,Filipino,Spanish,Chinese": 0.07954545454545454, + "Indonesian,English,Filipino,Spanish,Malay": 0.10795454545454546, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, + "Indonesian,English,Filipino,Chinese,Malay": 0.06818181818181818, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.06818181818181818, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,English,Spanish,Chinese,Malay": 0.17613636363636365, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1875, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.125, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635, + "English,Filipino,Spanish,Chinese,Malay": 0.10795454545454546, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, + "English,Filipino,Spanish,Malay,Vietnamese": 0.125, + "English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, + "English,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0625, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364 } }, - "AC3_2": 0.47633111368941355, - "AC3_3": 0.38564429577488496, - "AC3_4": 0.31897985825554914, - "AC3_5": 0.26663495735744225, - "AC3_6": 0.2239243749072491, - "AC3_7": 0.1883791351582064 + "AC3_2": 0.3819912242273957, + "AC3_3": 0.3025111932536058, + "AC3_4": 0.23685283321865602, + "AC3_5": 0.1811546117708194, + "AC3_6": 0.13241505965483033, + "AC3_7": 0.08811188808803014 }, - "prompt_2": { - "overall_acc": 0.44967532467532473, + "prompt_5": { + "overall_acc": 0.32224025974025977, "language_acc": { - "English": 0.5056818181818182, - "Filipino": 0.32954545454545453, - "Spanish": 0.3977272727272727, - "Malay": 0.48863636363636365, - "Chinese": 0.5795454545454546, - "Indonesian": 0.4375, - "Vietnamese": 0.4090909090909091 + "Indonesian": 0.3409090909090909, + "English": 0.32954545454545453, + "Filipino": 0.2897727272727273, + "Spanish": 0.3125, + "Chinese": 0.3125, + "Malay": 0.35795454545454547, + "Vietnamese": 0.3125 }, - "consistency_score_2": 0.5192099567099567, - "consistency_score_3": 0.34318181818181814, - "consistency_score_4": 0.24707792207792204, - "consistency_score_5": 0.18452380952380953, - "consistency_score_6": 0.14042207792207792, - "consistency_score_7": 0.10795454545454546, + "consistency_score_2": 0.4783549783549783, + "consistency_score_3": 0.287012987012987, + "consistency_score_4": 0.1910714285714286, + "consistency_score_5": 0.13419913419913423, + "consistency_score_6": 0.09659090909090909, + "consistency_score_7": 0.06818181818181818, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.39204545454545453, - "English,Spanish": 0.5681818181818182, - "English,Malay": 0.6363636363636364, - "English,Chinese": 0.5511363636363636, - "English,Indonesian": 0.625, - "English,Vietnamese": 0.5284090909090909, - "Filipino,Spanish": 0.4431818181818182, - "Filipino,Malay": 0.4375, - "Filipino,Chinese": 0.3465909090909091, - "Filipino,Indonesian": 0.375, - "Filipino,Vietnamese": 0.3068181818181818, - "Spanish,Malay": 0.6022727272727273, - "Spanish,Chinese": 0.5, - "Spanish,Indonesian": 0.625, - "Spanish,Vietnamese": 0.5, - "Malay,Chinese": 0.6022727272727273, - "Malay,Indonesian": 0.6931818181818182, - "Malay,Vietnamese": 0.5227272727272727, - "Chinese,Indonesian": 0.5625, - "Chinese,Vietnamese": 0.4943181818181818, - "Indonesian,Vietnamese": 0.5909090909090909 + "Indonesian,English": 0.5738636363636364, + "Indonesian,Filipino": 0.32954545454545453, + "Indonesian,Spanish": 0.5113636363636364, + "Indonesian,Chinese": 0.4318181818181818, + "Indonesian,Malay": 0.5340909090909091, + "Indonesian,Vietnamese": 0.5681818181818182, + "English,Filipino": 0.3352272727272727, + "English,Spanish": 0.5625, + "English,Chinese": 0.5397727272727273, + "English,Malay": 0.5284090909090909, + "English,Vietnamese": 0.5170454545454546, + "Filipino,Spanish": 0.4659090909090909, + "Filipino,Chinese": 0.3125, + "Filipino,Malay": 0.375, + "Filipino,Vietnamese": 0.3977272727272727, + "Spanish,Chinese": 0.4943181818181818, + "Spanish,Malay": 0.5681818181818182, + "Spanish,Vietnamese": 0.5625, + "Chinese,Malay": 0.44886363636363635, + "Chinese,Vietnamese": 0.45454545454545453, + "Malay,Vietnamese": 0.5340909090909091 }, "3_combine": { - "English,Filipino,Spanish": 0.2727272727272727, - "English,Filipino,Malay": 0.2784090909090909, - "English,Filipino,Chinese": 0.2159090909090909, - "English,Filipino,Indonesian": 0.2784090909090909, - "English,Filipino,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay": 0.42613636363636365, - "English,Spanish,Chinese": 0.3693181818181818, - "English,Spanish,Indonesian": 0.44886363636363635, - "English,Spanish,Vietnamese": 0.3522727272727273, - "English,Malay,Chinese": 0.44886363636363635, - "English,Malay,Indonesian": 0.5113636363636364, - "English,Malay,Vietnamese": 0.3977272727272727, - "English,Chinese,Indonesian": 0.4318181818181818, - "English,Chinese,Vietnamese": 0.3693181818181818, - "English,Indonesian,Vietnamese": 0.4090909090909091, - "Filipino,Spanish,Malay": 0.3181818181818182, - "Filipino,Spanish,Chinese": 0.23863636363636365, - "Filipino,Spanish,Indonesian": 0.3068181818181818, - "Filipino,Spanish,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Chinese": 0.26704545454545453, - "Filipino,Malay,Indonesian": 0.3181818181818182, - "Filipino,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Chinese,Indonesian": 0.23863636363636365, - "Filipino,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Indonesian,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Chinese": 0.4090909090909091, - "Spanish,Malay,Indonesian": 0.5113636363636364, - "Spanish,Malay,Vietnamese": 0.36363636363636365, - "Spanish,Chinese,Indonesian": 0.39204545454545453, - "Spanish,Chinese,Vietnamese": 0.3125, - "Spanish,Indonesian,Vietnamese": 0.4034090909090909, - "Malay,Chinese,Indonesian": 0.4715909090909091, - "Malay,Chinese,Vietnamese": 0.36363636363636365, - "Malay,Indonesian,Vietnamese": 0.4431818181818182, - "Chinese,Indonesian,Vietnamese": 0.38636363636363635 + "Indonesian,English,Filipino": 0.21022727272727273, + "Indonesian,English,Spanish": 0.375, + "Indonesian,English,Chinese": 0.3181818181818182, + "Indonesian,English,Malay": 0.375, + "Indonesian,English,Vietnamese": 0.3806818181818182, + "Indonesian,Filipino,Spanish": 0.23295454545454544, + "Indonesian,Filipino,Chinese": 0.14204545454545456, + "Indonesian,Filipino,Malay": 0.19886363636363635, + "Indonesian,Filipino,Vietnamese": 0.21022727272727273, + "Indonesian,Spanish,Chinese": 0.2897727272727273, + "Indonesian,Spanish,Malay": 0.35795454545454547, + "Indonesian,Spanish,Vietnamese": 0.35795454545454547, + "Indonesian,Chinese,Malay": 0.2784090909090909, + "Indonesian,Chinese,Vietnamese": 0.29545454545454547, + "Indonesian,Malay,Vietnamese": 0.375, + "English,Filipino,Spanish": 0.26136363636363635, + "English,Filipino,Chinese": 0.18181818181818182, + "English,Filipino,Malay": 0.2215909090909091, + "English,Filipino,Vietnamese": 0.2215909090909091, + "English,Spanish,Chinese": 0.3409090909090909, + "English,Spanish,Malay": 0.39204545454545453, + "English,Spanish,Vietnamese": 0.36363636363636365, + "English,Chinese,Malay": 0.32386363636363635, + "English,Chinese,Vietnamese": 0.32954545454545453, + "English,Malay,Vietnamese": 0.3409090909090909, + "Filipino,Spanish,Chinese": 0.23295454545454544, + "Filipino,Spanish,Malay": 0.2784090909090909, + "Filipino,Spanish,Vietnamese": 0.2727272727272727, + "Filipino,Chinese,Malay": 0.14772727272727273, + "Filipino,Chinese,Vietnamese": 0.1875, + "Filipino,Malay,Vietnamese": 0.23863636363636365, + "Spanish,Chinese,Malay": 0.3068181818181818, + "Spanish,Chinese,Vietnamese": 0.32386363636363635, + "Spanish,Malay,Vietnamese": 0.375, + "Chinese,Malay,Vietnamese": 0.3068181818181818 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.21022727272727273, - "English,Filipino,Spanish,Chinese": 0.17045454545454544, - "English,Filipino,Spanish,Indonesian": 0.22727272727272727, - "English,Filipino,Spanish,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese": 0.1875, - "English,Filipino,Malay,Indonesian": 0.23295454545454544, + "Indonesian,English,Filipino,Spanish": 0.17613636363636365, + "Indonesian,English,Filipino,Chinese": 0.125, + "Indonesian,English,Filipino,Malay": 0.1590909090909091, + "Indonesian,English,Filipino,Vietnamese": 0.16477272727272727, + "Indonesian,English,Spanish,Chinese": 0.23295454545454544, + "Indonesian,English,Spanish,Malay": 0.2840909090909091, + "Indonesian,English,Spanish,Vietnamese": 0.2784090909090909, + "Indonesian,English,Chinese,Malay": 0.2215909090909091, + "Indonesian,English,Chinese,Vietnamese": 0.23295454545454544, + "Indonesian,English,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,Filipino,Spanish,Chinese": 0.125, + "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Chinese,Malay": 0.09090909090909091, + "Indonesian,Filipino,Chinese,Vietnamese": 0.11363636363636363, + "Indonesian,Filipino,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,Spanish,Chinese,Malay": 0.20454545454545456, + "Indonesian,Spanish,Chinese,Vietnamese": 0.2159090909090909, + "Indonesian,Spanish,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,Chinese,Malay,Vietnamese": 0.2215909090909091, + "English,Filipino,Spanish,Chinese": 0.1590909090909091, + "English,Filipino,Spanish,Malay": 0.19886363636363635, + "English,Filipino,Spanish,Vietnamese": 0.1875, + "English,Filipino,Chinese,Malay": 0.11363636363636363, + "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, "English,Filipino,Malay,Vietnamese": 0.16477272727272727, - "English,Filipino,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Indonesian,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Chinese": 0.32386363636363635, - "English,Spanish,Malay,Indonesian": 0.39204545454545453, - "English,Spanish,Malay,Vietnamese": 0.2840909090909091, - "English,Spanish,Chinese,Indonesian": 0.32954545454545453, - "English,Spanish,Chinese,Vietnamese": 0.2727272727272727, - "English,Spanish,Indonesian,Vietnamese": 0.3125, - "English,Malay,Chinese,Indonesian": 0.38636363636363635, - "English,Malay,Chinese,Vietnamese": 0.3181818181818182, - "English,Malay,Indonesian,Vietnamese": 0.35795454545454547, - "English,Chinese,Indonesian,Vietnamese": 0.32386363636363635, - "Filipino,Spanish,Malay,Chinese": 0.21022727272727273, - "Filipino,Spanish,Malay,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.1875, - "Filipino,Spanish,Chinese,Indonesian": 0.20454545454545456, - "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "Filipino,Malay,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.3522727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.2727272727272727, - "Spanish,Malay,Indonesian,Vietnamese": 0.3465909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.32954545454545453 + "English,Spanish,Chinese,Malay": 0.25, + "English,Spanish,Chinese,Vietnamese": 0.2556818181818182, + "English,Spanish,Malay,Vietnamese": 0.26704545454545453, + "English,Chinese,Malay,Vietnamese": 0.22727272727272727, + "Filipino,Spanish,Chinese,Malay": 0.13068181818181818, + "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, + "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, + "Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, + "Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese,Indonesian": 0.30113636363636365, - "English,Spanish,Malay,Chinese,Vietnamese": 0.23863636363636365, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.29545454545454547, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635 + "Indonesian,English,Filipino,Spanish,Chinese": 0.11363636363636363, + "Indonesian,English,Filipino,Spanish,Malay": 0.14204545454545456, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "Indonesian,English,Filipino,Chinese,Malay": 0.08522727272727272, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.125, + "Indonesian,English,Spanish,Chinese,Malay": 0.18181818181818182, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1875, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544, + "English,Filipino,Spanish,Chinese,Malay": 0.10795454545454546, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, + "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, + "English,Filipino,Chinese,Malay,Vietnamese": 0.09659090909090909, + "English,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.07954545454545454, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 } }, - "AC3_2": 0.4819474716352327, - "AC3_3": 0.3892766733184845, - "AC3_4": 0.3189216421517317, - "AC3_5": 0.26167113598994424, - "AC3_6": 0.21401329066203012, - "AC3_7": 0.1741100965679563 - }, - "prompt_3": { - "overall_acc": 0.45941558441558444, - "language_acc": { - "English": 0.48863636363636365, - "Filipino": 0.3522727272727273, - "Spanish": 0.42045454545454547, - "Malay": 0.48863636363636365, - "Chinese": 0.5795454545454546, - "Indonesian": 0.4602272727272727, - "Vietnamese": 0.42613636363636365 - }, - "consistency_score_2": 0.5243506493506493, - "consistency_score_3": 0.3506493506493505, - "consistency_score_4": 0.25844155844155847, - "consistency_score_5": 0.19994588744588743, - "consistency_score_6": 0.1599025974025974, - "consistency_score_7": 0.13068181818181818, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4318181818181818, - "English,Spanish": 0.5795454545454546, - "English,Malay": 0.5852272727272727, - "English,Chinese": 0.5909090909090909, - "English,Indonesian": 0.6079545454545454, - "English,Vietnamese": 0.5227272727272727, - "Filipino,Spanish": 0.4375, - "Filipino,Malay": 0.42613636363636365, - "Filipino,Chinese": 0.3806818181818182, - "Filipino,Indonesian": 0.4034090909090909, - "Filipino,Vietnamese": 0.3693181818181818, - "Spanish,Malay": 0.6193181818181818, - "Spanish,Chinese": 0.5568181818181818, - "Spanish,Indonesian": 0.6136363636363636, - "Spanish,Vietnamese": 0.48863636363636365, - "Malay,Chinese": 0.5625, - "Malay,Indonesian": 0.6306818181818182, - "Malay,Vietnamese": 0.5113636363636364, - "Chinese,Indonesian": 0.5738636363636364, - "Chinese,Vietnamese": 0.5454545454545454, - "Indonesian,Vietnamese": 0.5738636363636364 - }, - "3_combine": { - "English,Filipino,Spanish": 0.29545454545454547, - "English,Filipino,Malay": 0.2840909090909091, - "English,Filipino,Chinese": 0.25, - "English,Filipino,Indonesian": 0.2840909090909091, - "English,Filipino,Vietnamese": 0.23863636363636365, - "English,Spanish,Malay": 0.4318181818181818, - "English,Spanish,Chinese": 0.4147727272727273, - "English,Spanish,Indonesian": 0.4375, - "English,Spanish,Vietnamese": 0.3522727272727273, - "English,Malay,Chinese": 0.4318181818181818, - "English,Malay,Indonesian": 0.4602272727272727, - "English,Malay,Vietnamese": 0.36363636363636365, - "English,Chinese,Indonesian": 0.4431818181818182, - "English,Chinese,Vietnamese": 0.4034090909090909, - "English,Indonesian,Vietnamese": 0.3977272727272727, - "Filipino,Spanish,Malay": 0.32386363636363635, - "Filipino,Spanish,Chinese": 0.2727272727272727, - "Filipino,Spanish,Indonesian": 0.3125, - "Filipino,Spanish,Vietnamese": 0.24431818181818182, - "Filipino,Malay,Chinese": 0.2727272727272727, - "Filipino,Malay,Indonesian": 0.30113636363636365, - "Filipino,Malay,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Indonesian": 0.2727272727272727, - "Filipino,Chinese,Vietnamese": 0.25, - "Filipino,Indonesian,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Chinese": 0.42613636363636365, - "Spanish,Malay,Indonesian": 0.4772727272727273, - "Spanish,Malay,Vietnamese": 0.35795454545454547, - "Spanish,Chinese,Indonesian": 0.42045454545454547, - "Spanish,Chinese,Vietnamese": 0.35795454545454547, - "Spanish,Indonesian,Vietnamese": 0.39204545454545453, - "Malay,Chinese,Indonesian": 0.42613636363636365, - "Malay,Chinese,Vietnamese": 0.3522727272727273, - "Malay,Indonesian,Vietnamese": 0.4034090909090909, - "Chinese,Indonesian,Vietnamese": 0.4147727272727273 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.23295454545454544, - "English,Filipino,Spanish,Chinese": 0.20454545454545456, - "English,Filipino,Spanish,Indonesian": 0.23863636363636365, - "English,Filipino,Spanish,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Chinese": 0.21022727272727273, - "English,Filipino,Malay,Indonesian": 0.22727272727272727, - "English,Filipino,Malay,Vietnamese": 0.1875, - "English,Filipino,Chinese,Indonesian": 0.21022727272727273, - "English,Filipino,Chinese,Vietnamese": 0.1875, - "English,Filipino,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay,Chinese": 0.3465909090909091, - "English,Spanish,Malay,Indonesian": 0.35795454545454547, - "English,Spanish,Malay,Vietnamese": 0.29545454545454547, - "English,Spanish,Chinese,Indonesian": 0.3409090909090909, - "English,Spanish,Chinese,Vietnamese": 0.2840909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.3068181818181818, - "English,Malay,Chinese,Indonesian": 0.35795454545454547, - "English,Malay,Chinese,Vietnamese": 0.3068181818181818, - "English,Malay,Indonesian,Vietnamese": 0.3068181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.3352272727272727, - "Filipino,Spanish,Malay,Chinese": 0.22727272727272727, - "Filipino,Spanish,Malay,Indonesian": 0.26136363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.1875, - "Filipino,Spanish,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.3522727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.3125, - "Spanish,Chinese,Indonesian,Vietnamese": 0.3125, - "Malay,Chinese,Indonesian,Vietnamese": 0.3068181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.18181818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay,Chinese,Indonesian": 0.29545454545454547, - "English,Spanish,Malay,Chinese,Vietnamese": 0.25, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2556818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - } - }, - "AC3_2": 0.48974004538333255, - "AC3_3": 0.3977305259379403, - "AC3_4": 0.33079584363985765, - "AC3_5": 0.27862791697302125, - "AC3_6": 0.23723425980407994, - "AC3_7": 0.20348255592400155 - }, - "prompt_4": { - "overall_acc": 0.45454545454545453, - "language_acc": { - "English": 0.48295454545454547, - "Filipino": 0.32386363636363635, - "Spanish": 0.4375, - "Malay": 0.48295454545454547, - "Chinese": 0.5795454545454546, - "Indonesian": 0.44886363636363635, - "Vietnamese": 0.42613636363636365 - }, - "consistency_score_2": 0.5137987012987013, - "consistency_score_3": 0.3334415584415584, - "consistency_score_4": 0.2311688311688312, - "consistency_score_5": 0.16396103896103897, - "consistency_score_6": 0.1176948051948052, - "consistency_score_7": 0.08522727272727272, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3806818181818182, - "English,Spanish": 0.6136363636363636, - "English,Malay": 0.6136363636363636, - "English,Chinese": 0.5568181818181818, - "English,Indonesian": 0.6022727272727273, - "English,Vietnamese": 0.5, - "Filipino,Spanish": 0.36363636363636365, - "Filipino,Malay": 0.39204545454545453, - "Filipino,Chinese": 0.3352272727272727, - "Filipino,Indonesian": 0.36363636363636365, - "Filipino,Vietnamese": 0.32954545454545453, - "Spanish,Malay": 0.6590909090909091, - "Spanish,Chinese": 0.5284090909090909, - "Spanish,Indonesian": 0.6477272727272727, - "Spanish,Vietnamese": 0.5511363636363636, - "Malay,Chinese": 0.5681818181818182, - "Malay,Indonesian": 0.6931818181818182, - "Malay,Vietnamese": 0.5113636363636364, - "Chinese,Indonesian": 0.5170454545454546, - "Chinese,Vietnamese": 0.48863636363636365, - "Indonesian,Vietnamese": 0.5738636363636364 - }, - "3_combine": { - "English,Filipino,Spanish": 0.25, - "English,Filipino,Malay": 0.26136363636363635, - "English,Filipino,Chinese": 0.2215909090909091, - "English,Filipino,Indonesian": 0.2727272727272727, - "English,Filipino,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay": 0.4715909090909091, - "English,Spanish,Chinese": 0.4034090909090909, - "English,Spanish,Indonesian": 0.4602272727272727, - "English,Spanish,Vietnamese": 0.38636363636363635, - "English,Malay,Chinese": 0.4147727272727273, - "English,Malay,Indonesian": 0.48295454545454547, - "English,Malay,Vietnamese": 0.3693181818181818, - "English,Chinese,Indonesian": 0.3977272727272727, - "English,Chinese,Vietnamese": 0.3352272727272727, - "English,Indonesian,Vietnamese": 0.39204545454545453, - "Filipino,Spanish,Malay": 0.2727272727272727, - "Filipino,Spanish,Chinese": 0.21022727272727273, - "Filipino,Spanish,Indonesian": 0.2784090909090909, - "Filipino,Spanish,Vietnamese": 0.21022727272727273, - "Filipino,Malay,Chinese": 0.2215909090909091, - "Filipino,Malay,Indonesian": 0.2840909090909091, - "Filipino,Malay,Vietnamese": 0.2159090909090909, - "Filipino,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Chinese,Vietnamese": 0.1875, - "Filipino,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Chinese": 0.4090909090909091, - "Spanish,Malay,Indonesian": 0.5227272727272727, - "Spanish,Malay,Vietnamese": 0.4034090909090909, - "Spanish,Chinese,Indonesian": 0.38636363636363635, - "Spanish,Chinese,Vietnamese": 0.3352272727272727, - "Spanish,Indonesian,Vietnamese": 0.4375, - "Malay,Chinese,Indonesian": 0.42613636363636365, - "Malay,Chinese,Vietnamese": 0.32386363636363635, - "Malay,Indonesian,Vietnamese": 0.4375, - "Chinese,Indonesian,Vietnamese": 0.3465909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.19886363636363635, - "English,Filipino,Spanish,Chinese": 0.16477272727272727, - "English,Filipino,Spanish,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Chinese": 0.17045454545454544, - "English,Filipino,Malay,Indonesian": 0.21022727272727273, - "English,Filipino,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese": 0.32954545454545453, - "English,Spanish,Malay,Indonesian": 0.3806818181818182, - "English,Spanish,Malay,Vietnamese": 0.3125, - "English,Spanish,Chinese,Indonesian": 0.3181818181818182, - "English,Spanish,Chinese,Vietnamese": 0.26704545454545453, - "English,Spanish,Indonesian,Vietnamese": 0.32954545454545453, - "English,Malay,Chinese,Indonesian": 0.3352272727272727, - "English,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Malay,Indonesian,Vietnamese": 0.32386363636363635, - "English,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Malay,Chinese": 0.17045454545454544, - "Filipino,Spanish,Malay,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.32386363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.26704545454545453, - "Spanish,Malay,Indonesian,Vietnamese": 0.3693181818181818, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.2784090909090909 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Malay,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.125, - "English,Filipino,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian": 0.26704545454545453, - "English,Spanish,Malay,Chinese,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.125, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 - } - }, - "AC3_2": 0.4823592165927232, - "AC3_3": 0.3846874180131355, - "AC3_4": 0.3064738291564078, - "AC3_5": 0.240992603158364, - "AC3_6": 0.18697614439027851, - "AC3_7": 0.14354066982986652 - }, - "prompt_5": { - "overall_acc": 0.4594155844155844, - "language_acc": { - "English": 0.5113636363636364, - "Filipino": 0.36363636363636365, - "Spanish": 0.4318181818181818, - "Malay": 0.48863636363636365, - "Chinese": 0.5511363636363636, - "Indonesian": 0.4318181818181818, - "Vietnamese": 0.4375 - }, - "consistency_score_2": 0.5308441558441559, - "consistency_score_3": 0.349512987012987, - "consistency_score_4": 0.24821428571428572, - "consistency_score_5": 0.18317099567099568, - "consistency_score_6": 0.13717532467532467, - "consistency_score_7": 0.10227272727272728, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4147727272727273, - "English,Spanish": 0.6306818181818182, - "English,Malay": 0.625, - "English,Chinese": 0.6022727272727273, - "English,Indonesian": 0.6079545454545454, - "English,Vietnamese": 0.5511363636363636, - "Filipino,Spanish": 0.36363636363636365, - "Filipino,Malay": 0.4147727272727273, - "Filipino,Chinese": 0.3693181818181818, - "Filipino,Indonesian": 0.3068181818181818, - "Filipino,Vietnamese": 0.32386363636363635, - "Spanish,Malay": 0.6306818181818182, - "Spanish,Chinese": 0.5852272727272727, - "Spanish,Indonesian": 0.6363636363636364, - "Spanish,Vietnamese": 0.5965909090909091, - "Malay,Chinese": 0.5909090909090909, - "Malay,Indonesian": 0.6534090909090909, - "Malay,Vietnamese": 0.5227272727272727, - "Chinese,Indonesian": 0.5511363636363636, - "Chinese,Vietnamese": 0.5738636363636364, - "Indonesian,Vietnamese": 0.5965909090909091 - }, - "3_combine": { - "English,Filipino,Spanish": 0.26704545454545453, - "English,Filipino,Malay": 0.2840909090909091, - "English,Filipino,Chinese": 0.26136363636363635, - "English,Filipino,Indonesian": 0.23295454545454544, - "English,Filipino,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay": 0.45454545454545453, - "English,Spanish,Chinese": 0.45454545454545453, - "English,Spanish,Indonesian": 0.4602272727272727, - "English,Spanish,Vietnamese": 0.42613636363636365, - "English,Malay,Chinese": 0.44886363636363635, - "English,Malay,Indonesian": 0.4659090909090909, - "English,Malay,Vietnamese": 0.375, - "English,Chinese,Indonesian": 0.42045454545454547, - "English,Chinese,Vietnamese": 0.4090909090909091, - "English,Indonesian,Vietnamese": 0.4147727272727273, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.24431818181818182, - "Filipino,Spanish,Indonesian": 0.23863636363636365, - "Filipino,Spanish,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese": 0.26136363636363635, - "Filipino,Malay,Indonesian": 0.25, - "Filipino,Malay,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian": 0.20454545454545456, - "Filipino,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Indonesian,Vietnamese": 0.21022727272727273, - "Spanish,Malay,Chinese": 0.44886363636363635, - "Spanish,Malay,Indonesian": 0.48863636363636365, - "Spanish,Malay,Vietnamese": 0.4034090909090909, - "Spanish,Chinese,Indonesian": 0.4318181818181818, - "Spanish,Chinese,Vietnamese": 0.42045454545454547, - "Spanish,Indonesian,Vietnamese": 0.45454545454545453, - "Malay,Chinese,Indonesian": 0.42613636363636365, - "Malay,Chinese,Vietnamese": 0.375, - "Malay,Indonesian,Vietnamese": 0.42045454545454547, - "Chinese,Indonesian,Vietnamese": 0.4090909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.2159090909090909, - "English,Filipino,Spanish,Chinese": 0.20454545454545456, - "English,Filipino,Spanish,Indonesian": 0.19318181818181818, - "English,Filipino,Spanish,Vietnamese": 0.18181818181818182, - "English,Filipino,Malay,Chinese": 0.20454545454545456, - "English,Filipino,Malay,Indonesian": 0.1875, - "English,Filipino,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Chinese,Indonesian": 0.17045454545454544, - "English,Filipino,Chinese,Vietnamese": 0.17045454545454544, - "English,Filipino,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese": 0.375, - "English,Spanish,Malay,Indonesian": 0.36363636363636365, - "English,Spanish,Malay,Vietnamese": 0.3068181818181818, - "English,Spanish,Chinese,Indonesian": 0.35795454545454547, - "English,Spanish,Chinese,Vietnamese": 0.32954545454545453, - "English,Spanish,Indonesian,Vietnamese": 0.3409090909090909, - "English,Malay,Chinese,Indonesian": 0.3465909090909091, - "English,Malay,Chinese,Vietnamese": 0.3068181818181818, - "English,Malay,Indonesian,Vietnamese": 0.3181818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.3181818181818182, - "Filipino,Spanish,Malay,Chinese": 0.20454545454545456, - "Filipino,Spanish,Malay,Indonesian": 0.21022727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian": 0.1875, - "Filipino,Malay,Chinese,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.3522727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.3125, - "Spanish,Malay,Indonesian,Vietnamese": 0.3522727272727273, - "Spanish,Chinese,Indonesian,Vietnamese": 0.3409090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.3125 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.125, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese,Indonesian": 0.30113636363636365, - "English,Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2727272727272727, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2727272727272727 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728 - } - }, - "AC3_2": 0.4925537576679523, - "AC3_3": 0.3969984961366556, - "AC3_4": 0.3222970536692641, - "AC3_5": 0.2619152426113252, - "AC3_6": 0.21126866327406696, - "AC3_7": 0.16730162897704584 + "AC3_2": 0.38507656583279287, + "AC3_3": 0.3036081956446938, + "AC3_4": 0.23989676511094088, + "AC3_5": 0.1894856772924148, + "AC3_6": 0.14863019728657087, + "AC3_7": 0.11254961252078487 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.5728155339805825 + "accuracy": 0.3592233009708738 }, "prompt_2": { - "accuracy": 0.5825242718446602 + "accuracy": 0.3883495145631068 }, "prompt_3": { - "accuracy": 0.5825242718446602 + "accuracy": 0.39805825242718446 }, "prompt_4": { - "accuracy": 0.5728155339805825 + "accuracy": 0.3786407766990291 }, "prompt_5": { - "accuracy": 0.5825242718446602 + "accuracy": 0.34951456310679613 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.5714285714285714 + "accuracy": 0.29523809523809524 }, "prompt_2": { - "accuracy": 0.580952380952381 + "accuracy": 0.3523809523809524 }, "prompt_3": { - "accuracy": 0.580952380952381 + "accuracy": 0.3142857142857143 }, "prompt_4": { - "accuracy": 0.5142857142857142 + "accuracy": 0.26666666666666666 }, "prompt_5": { - "accuracy": 0.580952380952381 + "accuracy": 0.3142857142857143 } }, "us_eval": { "prompt_1": { - "accuracy": 0.5327102803738317 + "accuracy": 0.411214953271028 }, "prompt_2": { - "accuracy": 0.5607476635514018 + "accuracy": 0.34579439252336447 }, "prompt_3": { - "accuracy": 0.5514018691588785 + "accuracy": 0.411214953271028 }, "prompt_4": { - "accuracy": 0.5700934579439252 + "accuracy": 0.38317757009345793 }, "prompt_5": { - "accuracy": 0.5514018691588785 + "accuracy": 0.3925233644859813 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.38, + "accuracy": 0.27, "category_acc": { - "brand": 0.3, - "demographics": 0.0, + "brand": 0.2, + "demographics": 0.2, "biology": 0.3, - "history": 0.13333333333333333, + "history": 0.3333333333333333, "literature": 0.4, - "politics": 0.6, - "culture": 0.3, - "film": 0.4, - "law": 0.5, - "geography": 0.8 + "politics": 0.1, + "culture": 0.2, + "film": 0.3, + "law": 0.1, + "geography": 0.5 } }, "prompt_2": { - "accuracy": 0.39, + "accuracy": 0.29, "category_acc": { - "brand": 0.4, - "demographics": 0.0, - "biology": 0.3, - "history": 0.2, - "literature": 0.4, - "politics": 0.6, - "culture": 0.3, - "film": 0.4, - "law": 0.5, - "geography": 0.7 + "brand": 0.1, + "demographics": 0.2, + "biology": 0.5, + "history": 0.26666666666666666, + "literature": 0.5, + "politics": 0.0, + "culture": 0.2, + "film": 0.3, + "law": 0.3, + "geography": 0.5 } }, "prompt_3": { - "accuracy": 0.38, + "accuracy": 0.27, "category_acc": { - "brand": 0.4, - "demographics": 0.4, - "biology": 0.2, - "history": 0.13333333333333333, - "literature": 0.5, - "politics": 0.6, - "culture": 0.3, - "film": 0.3, - "law": 0.5, - "geography": 0.6 + "brand": 0.3, + "demographics": 0.0, + "biology": 0.6, + "history": 0.2, + "literature": 0.3, + "politics": 0.1, + "culture": 0.2, + "film": 0.2, + "law": 0.2, + "geography": 0.5 } }, "prompt_4": { - "accuracy": 0.36, + "accuracy": 0.28, "category_acc": { - "brand": 0.4, + "brand": 0.3, "demographics": 0.0, - "biology": 0.2, - "history": 0.13333333333333333, - "literature": 0.3, - "politics": 0.6, + "biology": 0.4, + "history": 0.2, + "literature": 0.4, + "politics": 0.3, "culture": 0.3, - "film": 0.4, - "law": 0.5, - "geography": 0.7 + "film": 0.2, + "law": 0.2, + "geography": 0.4 } }, "prompt_5": { - "accuracy": 0.37, + "accuracy": 0.24, "category_acc": { - "brand": 0.5, + "brand": 0.2, "demographics": 0.0, - "biology": 0.2, + "biology": 0.6, "history": 0.13333333333333333, "literature": 0.4, - "politics": 0.7, - "culture": 0.2, - "film": 0.2, - "law": 0.5, - "geography": 0.8 + "politics": 0.2, + "culture": 0.3, + "film": 0.1, + "law": 0.0, + "geography": 0.4 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.1881043322037295 + "bleu_score": 0.06305082715215274 }, "prompt_2": { - "bleu_score": 0.17254063490107463 + "bleu_score": 0.06463430550758704 }, "prompt_3": { - "bleu_score": 0.18353318191120205 + "bleu_score": 0.06356003237143983 }, "prompt_4": { - "bleu_score": 0.19351347876654987 + "bleu_score": 0.06539930841182513 }, "prompt_5": { - "bleu_score": 0.14449203821294404 + "bleu_score": 0.06031168781615755 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "prompt_1": { + "accuracy": 0.3378730222311236, + "category_acc": { + "History": 0.3092369477911647, + "Geography": 0.25918367346938775, + "Lampungic": 0.272108843537415, + "Social science": 0.4774624373956594, + "Balinese": 0.28450106157112526, + "Makassarese": 0.2956989247311828, + "Banjarese": 0.3541666666666667, + "Chemistry": 0.20583941605839415, + "Biology": 0.27455621301775146, + "Science": 0.3849329205366357, + "Christian religion": 0.38308457711442784, + "Art": 0.36605657237936773, + "Islam religion": 0.3968705547652916, + "Hindu religion": 0.36666666666666664, + "Madurese": 0.2847457627118644, + "Sport": 0.34459459459459457, + "Indonesian language": 0.37391033623910336, + "Physics": 0.298989898989899, + "Minangkabau culture": 0.3417085427135678, + "Dayak language": 0.3486238532110092, + "Sociology": 0.29435483870967744, + "Economy": 0.305327868852459, + "Sundanese": 0.32757130509939497, + "Javanese": 0.3084677419354839, + "Civic education": 0.38197424892703863 + } + }, + "prompt_2": { + "accuracy": 0.3440817144001602, + "category_acc": { + "History": 0.3172690763052209, + "Geography": 0.2836734693877551, + "Lampungic": 0.2585034013605442, + "Social science": 0.49248747913188645, + "Balinese": 0.29936305732484075, + "Makassarese": 0.3333333333333333, + "Banjarese": 0.3125, + "Chemistry": 0.21897810218978103, + "Biology": 0.25088757396449707, + "Science": 0.39628482972136225, + "Christian religion": 0.39800995024875624, + "Art": 0.37603993344425957, + "Islam religion": 0.4167852062588905, + "Hindu religion": 0.35333333333333333, + "Madurese": 0.31864406779661014, + "Sport": 0.39864864864864863, + "Indonesian language": 0.3866749688667497, + "Physics": 0.27070707070707073, + "Minangkabau culture": 0.33668341708542715, + "Dayak language": 0.3211009174311927, + "Sociology": 0.2923387096774194, + "Economy": 0.2581967213114754, + "Sundanese": 0.3327571305099395, + "Javanese": 0.3215725806451613, + "Civic education": 0.38912732474964234 + } + }, + "prompt_3": { + "accuracy": 0.3464183189799052, + "category_acc": { + "History": 0.3032128514056225, + "Geography": 0.2510204081632653, + "Lampungic": 0.2653061224489796, + "Social science": 0.49081803005008345, + "Balinese": 0.2823779193205945, + "Makassarese": 0.3333333333333333, + "Banjarese": 0.3263888888888889, + "Chemistry": 0.22043795620437956, + "Biology": 0.2603550295857988, + "Science": 0.40970072239422084, + "Christian religion": 0.40298507462686567, + "Art": 0.39267886855241263, + "Islam religion": 0.41109530583214793, + "Hindu religion": 0.35333333333333333, + "Madurese": 0.3016949152542373, + "Sport": 0.3918918918918919, + "Indonesian language": 0.3866749688667497, + "Physics": 0.30303030303030304, + "Minangkabau culture": 0.3316582914572864, + "Dayak language": 0.3302752293577982, + "Sociology": 0.29838709677419356, + "Economy": 0.29918032786885246, + "Sundanese": 0.34485738980121, + "Javanese": 0.3064516129032258, + "Civic education": 0.39341917024320455 + } + }, + "prompt_4": { + "accuracy": 0.3415448294278657, + "category_acc": { + "History": 0.3232931726907631, + "Geography": 0.2673469387755102, + "Lampungic": 0.32653061224489793, + "Social science": 0.48414023372287146, + "Balinese": 0.2760084925690021, + "Makassarese": 0.24731182795698925, + "Banjarese": 0.3125, + "Chemistry": 0.24817518248175183, + "Biology": 0.2733727810650888, + "Science": 0.391124871001032, + "Christian religion": 0.39800995024875624, + "Art": 0.37104825291181365, + "Islam religion": 0.39544807965860596, + "Hindu religion": 0.35333333333333333, + "Madurese": 0.2711864406779661, + "Sport": 0.3581081081081081, + "Indonesian language": 0.3941469489414695, + "Physics": 0.2606060606060606, + "Minangkabau culture": 0.2914572864321608, + "Dayak language": 0.22935779816513763, + "Sociology": 0.2862903225806452, + "Economy": 0.29713114754098363, + "Sundanese": 0.31892826274848746, + "Javanese": 0.30443548387096775, + "Civic education": 0.4034334763948498 + } + }, + "prompt_5": { + "accuracy": 0.34681887976500436, + "category_acc": { + "History": 0.3473895582329317, + "Geography": 0.2816326530612245, + "Lampungic": 0.35374149659863946, + "Social science": 0.5008347245409015, + "Balinese": 0.2781316348195329, + "Makassarese": 0.27956989247311825, + "Banjarese": 0.3472222222222222, + "Chemistry": 0.24817518248175183, + "Biology": 0.2591715976331361, + "Science": 0.4004127966976264, + "Christian religion": 0.3781094527363184, + "Art": 0.3826955074875208, + "Islam religion": 0.40825035561877665, + "Hindu religion": 0.38, + "Madurese": 0.28135593220338984, + "Sport": 0.40540540540540543, + "Indonesian language": 0.37951432129514323, + "Physics": 0.298989898989899, + "Minangkabau culture": 0.36180904522613067, + "Dayak language": 0.26605504587155965, + "Sociology": 0.2903225806451613, + "Economy": 0.2766393442622951, + "Sundanese": 0.34312878133102853, + "Javanese": 0.3155241935483871, + "Civic education": 0.38912732474964234 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.340437967088031 + "bleu_score": 0.07908898359419463 }, "prompt_2": { - "bleu_score": 0.34571633176359734 + "bleu_score": 0.07897628396673814 }, "prompt_3": { - "bleu_score": 0.3365556165930653 + "bleu_score": 0.07899232385079807 }, "prompt_4": { - "bleu_score": 0.33927115982892103 + "bleu_score": 0.07891920764416933 }, "prompt_5": { - "bleu_score": 0.3160342413615289 + "bleu_score": 0.07468075077628368 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.285086458985313 + "bleu_score": 0.06631314380326508 }, "prompt_2": { - "bleu_score": 0.29477309084540837 + "bleu_score": 0.06715912321244885 }, "prompt_3": { - "bleu_score": 0.2852903895678898 + "bleu_score": 0.0669243196705807 }, "prompt_4": { - "bleu_score": 0.28796662021350056 + "bleu_score": 0.06705363387875442 }, "prompt_5": { - "bleu_score": 0.2751661127414082 + "bleu_score": 0.06463296313543722 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.20998932116501742 + "bleu_score": 0.04429631011508256 }, "prompt_2": { - "bleu_score": 0.21458722737916455 + "bleu_score": 0.04422337129052579 }, "prompt_3": { - "bleu_score": 0.20771000406029358 + "bleu_score": 0.044872517066552334 }, "prompt_4": { - "bleu_score": 0.2064550003969712 + "bleu_score": 0.04557580925559713 }, "prompt_5": { - "bleu_score": 0.20607546621812284 + "bleu_score": 0.04355200215183761 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.35182120748040147 + "bleu_score": 0.07958207120800556 }, "prompt_2": { - "bleu_score": 0.36230412611683754 + "bleu_score": 0.07948625371017644 }, "prompt_3": { - "bleu_score": 0.3503262414576083 + "bleu_score": 0.07995669494424623 }, "prompt_4": { - "bleu_score": 0.3542996136643981 + "bleu_score": 0.07923605117741923 }, "prompt_5": { - "bleu_score": 0.3283253615686442 + "bleu_score": 0.07715463343756887 } }, "mmlu": { "prompt_1": { - "accuracy": 0.5122520420070011 + "accuracy": 0.35939323220536756 }, "prompt_2": { - "accuracy": 0.5204200700116686 + "accuracy": 0.35122520420070014 }, "prompt_3": { - "accuracy": 0.5542590431738623 + "accuracy": 0.35822637106184363 }, "prompt_4": { - "accuracy": 0.5495915985997666 + "accuracy": 0.34072345390898484 }, "prompt_5": { - "accuracy": 0.5390898483080513 + "accuracy": 0.367561260210035 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.49731855559528065, + "accuracy": 0.36467643904183056, "category_acc": { - "high_school_european_history": 0.6402439024390244, - "business_ethics": 0.6363636363636364, - "clinical_knowledge": 0.5227272727272727, - "medical_genetics": 0.6464646464646465, - "high_school_us_history": 0.5665024630541872, - "high_school_physics": 0.30666666666666664, - "high_school_world_history": 0.6101694915254238, - "virology": 0.42424242424242425, - "high_school_microeconomics": 0.5147679324894515, - "econometrics": 0.36283185840707965, - "college_computer_science": 0.46464646464646464, - "high_school_biology": 0.5954692556634305, - "abstract_algebra": 0.3333333333333333, - "professional_accounting": 0.3487544483985765, - "philosophy": 0.567741935483871, - "professional_medicine": 0.4907749077490775, - "nutrition": 0.521311475409836, - "global_facts": 0.31313131313131315, - "machine_learning": 0.4144144144144144, - "security_studies": 0.38934426229508196, - "public_relations": 0.5596330275229358, - "professional_psychology": 0.4533551554828151, - "prehistory": 0.5201238390092879, - "anatomy": 0.5074626865671642, - "human_sexuality": 0.5692307692307692, - "college_medicine": 0.4883720930232558, - "high_school_government_and_politics": 0.6875, - "college_chemistry": 0.36363636363636365, - "logical_fallacies": 0.6234567901234568, - "high_school_geography": 0.5989847715736041, - "elementary_mathematics": 0.4403183023872679, - "human_aging": 0.6261261261261262, - "college_mathematics": 0.3838383838383838, - "high_school_psychology": 0.7279411764705882, - "formal_logic": 0.336, - "high_school_statistics": 0.386046511627907, - "international_law": 0.6583333333333333, - "high_school_mathematics": 0.27137546468401486, - "high_school_computer_science": 0.5050505050505051, - "conceptual_physics": 0.4700854700854701, - "miscellaneous": 0.7378516624040921, - "high_school_chemistry": 0.37623762376237624, - "marketing": 0.7081545064377682, - "professional_law": 0.365296803652968, - "management": 0.6078431372549019, - "college_physics": 0.3069306930693069, - "jurisprudence": 0.4953271028037383, - "world_religions": 0.6941176470588235, - "sociology": 0.735, - "us_foreign_policy": 0.6868686868686869, - "high_school_macroeconomics": 0.48586118251928023, - "computer_security": 0.6666666666666666, - "moral_scenarios": 0.2651006711409396, - "moral_disputes": 0.5478260869565217, - "electrical_engineering": 0.4652777777777778, - "astronomy": 0.46357615894039733, - "college_biology": 0.5314685314685315 + "high_school_european_history": 0.45121951219512196, + "business_ethics": 0.3838383838383838, + "clinical_knowledge": 0.4128787878787879, + "medical_genetics": 0.3939393939393939, + "high_school_us_history": 0.4482758620689655, + "high_school_physics": 0.26666666666666666, + "high_school_world_history": 0.4957627118644068, + "virology": 0.3151515151515151, + "high_school_microeconomics": 0.3459915611814346, + "econometrics": 0.2831858407079646, + "college_computer_science": 0.3333333333333333, + "high_school_biology": 0.36245954692556637, + "abstract_algebra": 0.25252525252525254, + "professional_accounting": 0.3167259786476868, + "philosophy": 0.3774193548387097, + "professional_medicine": 0.2988929889298893, + "nutrition": 0.4163934426229508, + "global_facts": 0.3434343434343434, + "machine_learning": 0.2972972972972973, + "security_studies": 0.29918032786885246, + "public_relations": 0.46788990825688076, + "professional_psychology": 0.3617021276595745, + "prehistory": 0.3993808049535604, + "anatomy": 0.4253731343283582, + "human_sexuality": 0.45384615384615384, + "college_medicine": 0.38372093023255816, + "high_school_government_and_politics": 0.4583333333333333, + "college_chemistry": 0.2828282828282828, + "logical_fallacies": 0.4012345679012346, + "high_school_geography": 0.4619289340101523, + "elementary_mathematics": 0.30238726790450926, + "human_aging": 0.35135135135135137, + "college_mathematics": 0.3434343434343434, + "high_school_psychology": 0.4375, + "formal_logic": 0.36, + "high_school_statistics": 0.25116279069767444, + "international_law": 0.5333333333333333, + "high_school_mathematics": 0.22304832713754646, + "high_school_computer_science": 0.32323232323232326, + "conceptual_physics": 0.3333333333333333, + "miscellaneous": 0.4782608695652174, + "high_school_chemistry": 0.3069306930693069, + "marketing": 0.5708154506437768, + "professional_law": 0.299412915851272, + "management": 0.45098039215686275, + "college_physics": 0.26732673267326734, + "jurisprudence": 0.34579439252336447, + "world_religions": 0.47058823529411764, + "sociology": 0.46, + "us_foreign_policy": 0.5757575757575758, + "high_school_macroeconomics": 0.3496143958868895, + "computer_security": 0.42424242424242425, + "moral_scenarios": 0.2360178970917226, + "moral_disputes": 0.37681159420289856, + "electrical_engineering": 0.3611111111111111, + "astronomy": 0.423841059602649, + "college_biology": 0.3356643356643357 } }, "prompt_2": { - "accuracy": 0.5038255273507329, - "category_acc": { - "high_school_european_history": 0.6402439024390244, - "business_ethics": 0.6868686868686869, - "clinical_knowledge": 0.553030303030303, - "medical_genetics": 0.6161616161616161, - "high_school_us_history": 0.5763546798029556, - "high_school_physics": 0.32666666666666666, - "high_school_world_history": 0.614406779661017, - "virology": 0.3939393939393939, - "high_school_microeconomics": 0.48523206751054854, - "econometrics": 0.36283185840707965, - "college_computer_science": 0.4444444444444444, - "high_school_biology": 0.6213592233009708, - "abstract_algebra": 0.3434343434343434, - "professional_accounting": 0.3309608540925267, - "philosophy": 0.5806451612903226, - "professional_medicine": 0.5129151291512916, - "nutrition": 0.5704918032786885, - "global_facts": 0.23232323232323232, - "machine_learning": 0.43243243243243246, - "security_studies": 0.5040983606557377, - "public_relations": 0.5137614678899083, - "professional_psychology": 0.46808510638297873, - "prehistory": 0.5294117647058824, - "anatomy": 0.4552238805970149, - "human_sexuality": 0.5538461538461539, - "college_medicine": 0.46511627906976744, - "high_school_government_and_politics": 0.7395833333333334, - "college_chemistry": 0.40404040404040403, - "logical_fallacies": 0.6296296296296297, - "high_school_geography": 0.6294416243654822, - "elementary_mathematics": 0.4350132625994695, - "human_aging": 0.5675675675675675, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.7481617647058824, - "formal_logic": 0.416, - "high_school_statistics": 0.40930232558139534, - "international_law": 0.6833333333333333, - "high_school_mathematics": 0.2825278810408922, - "high_school_computer_science": 0.5252525252525253, - "conceptual_physics": 0.48717948717948717, - "miscellaneous": 0.7416879795396419, - "high_school_chemistry": 0.4306930693069307, - "marketing": 0.6952789699570815, - "professional_law": 0.3607305936073059, - "management": 0.6568627450980392, - "college_physics": 0.31683168316831684, - "jurisprudence": 0.5420560747663551, - "world_religions": 0.711764705882353, - "sociology": 0.7, - "us_foreign_policy": 0.6363636363636364, - "high_school_macroeconomics": 0.48586118251928023, - "computer_security": 0.6464646464646465, - "moral_scenarios": 0.2751677852348993, - "moral_disputes": 0.5130434782608696, - "electrical_engineering": 0.4861111111111111, - "astronomy": 0.5033112582781457, - "college_biology": 0.5314685314685315 - } - }, - "prompt_3": { - "accuracy": 0.5228459063282088, + "accuracy": 0.3643189131212013, "category_acc": { - "high_school_european_history": 0.5914634146341463, - "business_ethics": 0.6464646464646465, - "clinical_knowledge": 0.5568181818181818, - "medical_genetics": 0.6161616161616161, - "high_school_us_history": 0.6059113300492611, - "high_school_physics": 0.3466666666666667, - "high_school_world_history": 0.6016949152542372, - "virology": 0.4666666666666667, - "high_school_microeconomics": 0.5316455696202531, - "econometrics": 0.3805309734513274, - "college_computer_science": 0.40404040404040403, - "high_school_biology": 0.6181229773462783, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.4199288256227758, - "philosophy": 0.5838709677419355, - "professional_medicine": 0.4981549815498155, - "nutrition": 0.5606557377049181, - "global_facts": 0.23232323232323232, - "machine_learning": 0.42342342342342343, - "security_studies": 0.5819672131147541, - "public_relations": 0.5779816513761468, - "professional_psychology": 0.49918166939443537, - "prehistory": 0.5758513931888545, - "anatomy": 0.4552238805970149, - "human_sexuality": 0.5923076923076923, - "college_medicine": 0.46511627906976744, - "high_school_government_and_politics": 0.7552083333333334, - "college_chemistry": 0.3939393939393939, - "logical_fallacies": 0.6111111111111112, - "high_school_geography": 0.6954314720812182, - "elementary_mathematics": 0.4509283819628647, - "human_aging": 0.6126126126126126, - "college_mathematics": 0.3434343434343434, - "high_school_psychology": 0.7647058823529411, - "formal_logic": 0.408, - "high_school_statistics": 0.4232558139534884, - "international_law": 0.6416666666666667, - "high_school_mathematics": 0.31970260223048325, - "high_school_computer_science": 0.5454545454545454, - "conceptual_physics": 0.5213675213675214, - "miscellaneous": 0.7583120204603581, - "high_school_chemistry": 0.44554455445544555, - "marketing": 0.759656652360515, - "professional_law": 0.3731245923026745, - "management": 0.6568627450980392, - "college_physics": 0.3069306930693069, - "jurisprudence": 0.5981308411214953, - "world_religions": 0.7, - "sociology": 0.735, - "us_foreign_policy": 0.7575757575757576, - "high_school_macroeconomics": 0.5218508997429306, - "computer_security": 0.6767676767676768, - "moral_scenarios": 0.25838926174496646, - "moral_disputes": 0.591304347826087, - "electrical_engineering": 0.4930555555555556, - "astronomy": 0.543046357615894, - "college_biology": 0.5454545454545454 + "high_school_european_history": 0.4817073170731707, + "business_ethics": 0.42424242424242425, + "clinical_knowledge": 0.3939393939393939, + "medical_genetics": 0.3333333333333333, + "high_school_us_history": 0.4630541871921182, + "high_school_physics": 0.26, + "high_school_world_history": 0.5550847457627118, + "virology": 0.3090909090909091, + "high_school_microeconomics": 0.34177215189873417, + "econometrics": 0.21238938053097345, + "college_computer_science": 0.3333333333333333, + "high_school_biology": 0.36893203883495146, + "abstract_algebra": 0.23232323232323232, + "professional_accounting": 0.3202846975088968, + "philosophy": 0.3903225806451613, + "professional_medicine": 0.3247232472324723, + "nutrition": 0.4065573770491803, + "global_facts": 0.21212121212121213, + "machine_learning": 0.2972972972972973, + "security_studies": 0.3155737704918033, + "public_relations": 0.43119266055045874, + "professional_psychology": 0.33387888707037644, + "prehistory": 0.38699690402476783, + "anatomy": 0.3880597014925373, + "human_sexuality": 0.4230769230769231, + "college_medicine": 0.3546511627906977, + "high_school_government_and_politics": 0.4010416666666667, + "college_chemistry": 0.29292929292929293, + "logical_fallacies": 0.4074074074074074, + "high_school_geography": 0.4416243654822335, + "elementary_mathematics": 0.29708222811671087, + "human_aging": 0.4009009009009009, + "college_mathematics": 0.31313131313131315, + "high_school_psychology": 0.4227941176470588, + "formal_logic": 0.36, + "high_school_statistics": 0.2651162790697674, + "international_law": 0.49166666666666664, + "high_school_mathematics": 0.21561338289962825, + "high_school_computer_science": 0.41414141414141414, + "conceptual_physics": 0.3162393162393162, + "miscellaneous": 0.46547314578005117, + "high_school_chemistry": 0.2722772277227723, + "marketing": 0.5278969957081545, + "professional_law": 0.32615786040443573, + "management": 0.5, + "college_physics": 0.24752475247524752, + "jurisprudence": 0.34579439252336447, + "world_religions": 0.4117647058823529, + "sociology": 0.5, + "us_foreign_policy": 0.6161616161616161, + "high_school_macroeconomics": 0.33676092544987146, + "computer_security": 0.47474747474747475, + "moral_scenarios": 0.26174496644295303, + "moral_disputes": 0.36231884057971014, + "electrical_engineering": 0.4166666666666667, + "astronomy": 0.423841059602649, + "college_biology": 0.32867132867132864 } }, - "prompt_4": { - "accuracy": 0.5169109760457633, + "prompt_3": { + "accuracy": 0.3584554880228817, "category_acc": { - "high_school_european_history": 0.6463414634146342, - "business_ethics": 0.6767676767676768, - "clinical_knowledge": 0.553030303030303, - "medical_genetics": 0.6464646464646465, - "high_school_us_history": 0.6354679802955665, - "high_school_physics": 0.3, - "high_school_world_history": 0.597457627118644, - "virology": 0.42424242424242425, - "high_school_microeconomics": 0.5063291139240507, - "econometrics": 0.3805309734513274, - "college_computer_science": 0.43434343434343436, - "high_school_biology": 0.627831715210356, + "high_school_european_history": 0.4695121951219512, + "business_ethics": 0.43434343434343436, + "clinical_knowledge": 0.39015151515151514, + "medical_genetics": 0.30303030303030304, + "high_school_us_history": 0.4433497536945813, + "high_school_physics": 0.26, + "high_school_world_history": 0.5296610169491526, + "virology": 0.36363636363636365, + "high_school_microeconomics": 0.34177215189873417, + "econometrics": 0.24778761061946902, + "college_computer_science": 0.30303030303030304, + "high_school_biology": 0.36893203883495146, "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.39501779359430605, - "philosophy": 0.5870967741935483, - "professional_medicine": 0.46863468634686345, - "nutrition": 0.5573770491803278, - "global_facts": 0.36363636363636365, - "machine_learning": 0.4774774774774775, - "security_studies": 0.5778688524590164, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.4795417348608838, - "prehistory": 0.5789473684210527, - "anatomy": 0.48507462686567165, - "human_sexuality": 0.6153846153846154, - "college_medicine": 0.5290697674418605, - "high_school_government_and_politics": 0.7135416666666666, - "college_chemistry": 0.35353535353535354, - "logical_fallacies": 0.6111111111111112, - "high_school_geography": 0.6548223350253807, - "elementary_mathematics": 0.4297082228116711, - "human_aging": 0.6216216216216216, + "professional_accounting": 0.33451957295373663, + "philosophy": 0.3903225806451613, + "professional_medicine": 0.3062730627306273, + "nutrition": 0.419672131147541, + "global_facts": 0.2727272727272727, + "machine_learning": 0.3063063063063063, + "security_studies": 0.29918032786885246, + "public_relations": 0.44036697247706424, + "professional_psychology": 0.3469721767594108, + "prehistory": 0.4055727554179567, + "anatomy": 0.44029850746268656, + "human_sexuality": 0.4461538461538462, + "college_medicine": 0.3488372093023256, + "high_school_government_and_politics": 0.4010416666666667, + "college_chemistry": 0.24242424242424243, + "logical_fallacies": 0.37037037037037035, + "high_school_geography": 0.47715736040609136, + "elementary_mathematics": 0.27851458885941643, + "human_aging": 0.3918918918918919, "college_mathematics": 0.29292929292929293, - "high_school_psychology": 0.7463235294117647, - "formal_logic": 0.384, - "high_school_statistics": 0.4046511627906977, - "international_law": 0.625, - "high_school_mathematics": 0.2788104089219331, - "high_school_computer_science": 0.5454545454545454, - "conceptual_physics": 0.48717948717948717, - "miscellaneous": 0.7416879795396419, - "high_school_chemistry": 0.4504950495049505, - "marketing": 0.759656652360515, - "professional_law": 0.36007827788649704, - "management": 0.6372549019607843, - "college_physics": 0.297029702970297, - "jurisprudence": 0.5607476635514018, - "world_religions": 0.7235294117647059, - "sociology": 0.745, - "us_foreign_policy": 0.7272727272727273, - "high_school_macroeconomics": 0.5141388174807198, - "computer_security": 0.6868686868686869, - "moral_scenarios": 0.27740492170022374, - "moral_disputes": 0.5855072463768116, - "electrical_engineering": 0.5208333333333334, - "astronomy": 0.5033112582781457, - "college_biology": 0.5384615384615384 + "high_school_psychology": 0.4099264705882353, + "formal_logic": 0.304, + "high_school_statistics": 0.2558139534883721, + "international_law": 0.5166666666666667, + "high_school_mathematics": 0.2342007434944238, + "high_school_computer_science": 0.32323232323232326, + "conceptual_physics": 0.2948717948717949, + "miscellaneous": 0.4782608695652174, + "high_school_chemistry": 0.2722772277227723, + "marketing": 0.48497854077253216, + "professional_law": 0.30071754729288974, + "management": 0.5, + "college_physics": 0.25742574257425743, + "jurisprudence": 0.3644859813084112, + "world_religions": 0.4411764705882353, + "sociology": 0.465, + "us_foreign_policy": 0.5858585858585859, + "high_school_macroeconomics": 0.3059125964010283, + "computer_security": 0.37373737373737376, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.3855072463768116, + "electrical_engineering": 0.3541666666666667, + "astronomy": 0.44370860927152317, + "college_biology": 0.36363636363636365 + } + }, + "prompt_4": { + "accuracy": 0.3480872363246335, + "category_acc": { + "high_school_european_history": 0.43902439024390244, + "business_ethics": 0.42424242424242425, + "clinical_knowledge": 0.4015151515151515, + "medical_genetics": 0.32323232323232326, + "high_school_us_history": 0.4729064039408867, + "high_school_physics": 0.2733333333333333, + "high_school_world_history": 0.4872881355932203, + "virology": 0.2909090909090909, + "high_school_microeconomics": 0.31645569620253167, + "econometrics": 0.2831858407079646, + "college_computer_science": 0.31313131313131315, + "high_school_biology": 0.3365695792880259, + "abstract_algebra": 0.2727272727272727, + "professional_accounting": 0.31316725978647686, + "philosophy": 0.3548387096774194, + "professional_medicine": 0.3062730627306273, + "nutrition": 0.4098360655737705, + "global_facts": 0.3333333333333333, + "machine_learning": 0.3063063063063063, + "security_studies": 0.27049180327868855, + "public_relations": 0.43119266055045874, + "professional_psychology": 0.3404255319148936, + "prehistory": 0.38699690402476783, + "anatomy": 0.3880597014925373, + "human_sexuality": 0.3769230769230769, + "college_medicine": 0.4069767441860465, + "high_school_government_and_politics": 0.4322916666666667, + "college_chemistry": 0.36363636363636365, + "logical_fallacies": 0.4012345679012346, + "high_school_geography": 0.45685279187817257, + "elementary_mathematics": 0.2891246684350133, + "human_aging": 0.35585585585585583, + "college_mathematics": 0.30303030303030304, + "high_school_psychology": 0.41544117647058826, + "formal_logic": 0.344, + "high_school_statistics": 0.2558139534883721, + "international_law": 0.48333333333333334, + "high_school_mathematics": 0.20817843866171004, + "high_school_computer_science": 0.32323232323232326, + "conceptual_physics": 0.31196581196581197, + "miscellaneous": 0.4309462915601023, + "high_school_chemistry": 0.2376237623762376, + "marketing": 0.48497854077253216, + "professional_law": 0.2968036529680365, + "management": 0.43137254901960786, + "college_physics": 0.2079207920792079, + "jurisprudence": 0.308411214953271, + "world_religions": 0.4470588235294118, + "sociology": 0.405, + "us_foreign_policy": 0.5353535353535354, + "high_school_macroeconomics": 0.34190231362467866, + "computer_security": 0.41414141414141414, + "moral_scenarios": 0.22706935123042504, + "moral_disputes": 0.36231884057971014, + "electrical_engineering": 0.3263888888888889, + "astronomy": 0.4370860927152318, + "college_biology": 0.32167832167832167 } }, "prompt_5": { - "accuracy": 0.5164819449410082, + "accuracy": 0.35952806578476937, "category_acc": { - "high_school_european_history": 0.6341463414634146, - "business_ethics": 0.6464646464646465, - "clinical_knowledge": 0.5454545454545454, - "medical_genetics": 0.5959595959595959, - "high_school_us_history": 0.6354679802955665, - "high_school_physics": 0.32, - "high_school_world_history": 0.6440677966101694, - "virology": 0.41818181818181815, - "high_school_microeconomics": 0.4810126582278481, - "econometrics": 0.34513274336283184, - "college_computer_science": 0.48484848484848486, - "high_school_biology": 0.627831715210356, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.4234875444839858, - "philosophy": 0.5741935483870968, - "professional_medicine": 0.47601476014760147, - "nutrition": 0.580327868852459, - "global_facts": 0.2828282828282828, - "machine_learning": 0.45045045045045046, - "security_studies": 0.5491803278688525, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.4909983633387889, - "prehistory": 0.5696594427244582, - "anatomy": 0.48507462686567165, - "human_sexuality": 0.6307692307692307, - "college_medicine": 0.48255813953488375, - "high_school_government_and_politics": 0.6979166666666666, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.6234567901234568, - "high_school_geography": 0.6598984771573604, - "elementary_mathematics": 0.4297082228116711, - "human_aging": 0.6081081081081081, - "college_mathematics": 0.29292929292929293, - "high_school_psychology": 0.7426470588235294, - "formal_logic": 0.392, - "high_school_statistics": 0.3813953488372093, - "international_law": 0.6333333333333333, - "high_school_mathematics": 0.275092936802974, - "high_school_computer_science": 0.5050505050505051, - "conceptual_physics": 0.47435897435897434, - "miscellaneous": 0.7583120204603581, - "high_school_chemistry": 0.41089108910891087, - "marketing": 0.759656652360515, - "professional_law": 0.37638617090671883, - "management": 0.6372549019607843, - "college_physics": 0.27722772277227725, - "jurisprudence": 0.6261682242990654, - "world_religions": 0.7176470588235294, - "sociology": 0.74, - "us_foreign_policy": 0.6868686868686869, - "high_school_macroeconomics": 0.5089974293059126, - "computer_security": 0.6565656565656566, - "moral_scenarios": 0.27181208053691275, - "moral_disputes": 0.6057971014492753, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.4966887417218543, - "college_biology": 0.5314685314685315 + "high_school_european_history": 0.4268292682926829, + "business_ethics": 0.40404040404040403, + "clinical_knowledge": 0.375, + "medical_genetics": 0.3838383838383838, + "high_school_us_history": 0.49261083743842365, + "high_school_physics": 0.25333333333333335, + "high_school_world_history": 0.4703389830508475, + "virology": 0.3212121212121212, + "high_school_microeconomics": 0.33755274261603374, + "econometrics": 0.23893805309734514, + "college_computer_science": 0.26262626262626265, + "high_school_biology": 0.37216828478964403, + "abstract_algebra": 0.30303030303030304, + "professional_accounting": 0.3238434163701068, + "philosophy": 0.3870967741935484, + "professional_medicine": 0.31365313653136534, + "nutrition": 0.43278688524590164, + "global_facts": 0.26262626262626265, + "machine_learning": 0.26126126126126126, + "security_studies": 0.30327868852459017, + "public_relations": 0.3853211009174312, + "professional_psychology": 0.32569558101472995, + "prehistory": 0.38390092879256965, + "anatomy": 0.43283582089552236, + "human_sexuality": 0.4461538461538462, + "college_medicine": 0.4127906976744186, + "high_school_government_and_politics": 0.4583333333333333, + "college_chemistry": 0.25252525252525254, + "logical_fallacies": 0.3765432098765432, + "high_school_geography": 0.4619289340101523, + "elementary_mathematics": 0.3103448275862069, + "human_aging": 0.38288288288288286, + "college_mathematics": 0.26262626262626265, + "high_school_psychology": 0.43566176470588236, + "formal_logic": 0.264, + "high_school_statistics": 0.23255813953488372, + "international_law": 0.475, + "high_school_mathematics": 0.23048327137546468, + "high_school_computer_science": 0.3939393939393939, + "conceptual_physics": 0.2777777777777778, + "miscellaneous": 0.4846547314578005, + "high_school_chemistry": 0.2722772277227723, + "marketing": 0.5321888412017167, + "professional_law": 0.2974559686888454, + "management": 0.5, + "college_physics": 0.21782178217821782, + "jurisprudence": 0.38317757009345793, + "world_religions": 0.40588235294117647, + "sociology": 0.495, + "us_foreign_policy": 0.6060606060606061, + "high_school_macroeconomics": 0.3341902313624679, + "computer_security": 0.47474747474747475, + "moral_scenarios": 0.24049217002237136, + "moral_disputes": 0.39420289855072466, + "electrical_engineering": 0.3888888888888889, + "astronomy": 0.41721854304635764, + "college_biology": 0.3706293706293706 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.5460624071322436 + "accuracy": 0.32540861812778604 }, "prompt_2": { - "accuracy": 0.5527488855869243 + "accuracy": 0.31946508172362553 }, "prompt_3": { - "accuracy": 0.5512630014858841 + "accuracy": 0.3573551263001486 }, "prompt_4": { - "accuracy": 0.5356612184249628 + "accuracy": 0.34843982169390786 }, "prompt_5": { - "accuracy": 0.537147102526003 + "accuracy": 0.31426448736998513 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.547945205479452, + "accuracy": 0.3412204234122042, "category_acc": { - "computer_network": 0.5416666666666666, - "operating_system": 0.5, - "computer_architecture": 0.5, - "college_programming": 0.5238095238095238, - "college_physics": 0.4583333333333333, + "computer_network": 0.2916666666666667, + "operating_system": 0.2916666666666667, + "computer_architecture": 0.19230769230769232, + "college_programming": 0.14285714285714285, + "college_physics": 0.2916666666666667, "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.43478260869565216, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.09523809523809523, - "electrical_engineer": 0.40476190476190477, - "metrology_engineer": 0.5172413793103449, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.5416666666666666, - "high_school_chemistry": 0.5416666666666666, - "high_school_biology": 0.625, + "electrical_engineer": 0.35714285714285715, + "metrology_engineer": 0.41379310344827586, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.4583333333333333, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.25, "middle_school_mathematics": 0.375, - "middle_school_biology": 0.8076923076923077, - "middle_school_physics": 0.75, - "middle_school_chemistry": 0.92, - "veterinary_medicine": 0.4642857142857143, - "college_economics": 0.43333333333333335, - "business_administration": 0.47368421052631576, - "marxism": 0.7916666666666666, - "mao_zedong_thought": 0.7241379310344828, - "education_science": 0.6764705882352942, - "teacher_qualification": 0.7551020408163265, - "high_school_politics": 0.8333333333333334, - "high_school_geography": 0.7083333333333334, - "middle_school_politics": 0.8461538461538461, - "middle_school_geography": 0.8823529411764706, - "modern_chinese_history": 0.5, - "ideological_and_moral_cultivation": 0.875, - "logic": 0.5185185185185185, - "law": 0.41379310344827586, - "chinese_language_and_literature": 0.5357142857142857, - "art_studies": 0.5789473684210527, - "professional_tour_guide": 0.6470588235294118, - "legal_professional": 0.5, - "high_school_chinese": 0.5, - "high_school_history": 0.76, - "middle_school_history": 0.7407407407407407, - "civil_servant": 0.46153846153846156, - "sports_science": 0.7083333333333334, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.48148148148148145, - "urban_and_rural_planner": 0.5294117647058824, - "accountant": 0.42592592592592593, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.5555555555555556, - "tax_accountant": 0.42592592592592593, - "physician": 0.5740740740740741 + "middle_school_biology": 0.38461538461538464, + "middle_school_physics": 0.5, + "middle_school_chemistry": 0.44, + "veterinary_medicine": 0.35714285714285715, + "college_economics": 0.4, + "business_administration": 0.2631578947368421, + "marxism": 0.4166666666666667, + "mao_zedong_thought": 0.3448275862068966, + "education_science": 0.4411764705882353, + "teacher_qualification": 0.3673469387755102, + "high_school_politics": 0.2916666666666667, + "high_school_geography": 0.2916666666666667, + "middle_school_politics": 0.4230769230769231, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.25, + "ideological_and_moral_cultivation": 0.375, + "logic": 0.48148148148148145, + "law": 0.2413793103448276, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.34210526315789475, + "professional_tour_guide": 0.4411764705882353, + "legal_professional": 0.25, + "high_school_chinese": 0.16666666666666666, + "high_school_history": 0.6, + "middle_school_history": 0.37037037037037035, + "civil_servant": 0.34615384615384615, + "sports_science": 0.3333333333333333, + "plant_protection": 0.18518518518518517, + "basic_medicine": 0.5, + "clinical_medicine": 0.2222222222222222, + "urban_and_rural_planner": 0.4117647058823529, + "accountant": 0.35185185185185186, + "fire_engineer": 0.3333333333333333, + "environmental_impact_assessment_engineer": 0.3055555555555556, + "tax_accountant": 0.4074074074074074, + "physician": 0.37037037037037035 } }, "prompt_2": { - "accuracy": 0.5529265255292652, + "accuracy": 0.33374844333748444, "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.5, - "college_programming": 0.4523809523809524, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.4523809523809524, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.5, - "high_school_biology": 0.5416666666666666, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.8076923076923077, - "middle_school_physics": 0.8333333333333334, - "middle_school_chemistry": 0.92, - "veterinary_medicine": 0.4642857142857143, - "college_economics": 0.45, - "business_administration": 0.5, - "marxism": 0.875, - "mao_zedong_thought": 0.7241379310344828, - "education_science": 0.6764705882352942, - "teacher_qualification": 0.7551020408163265, - "high_school_politics": 0.7916666666666666, - "high_school_geography": 0.6666666666666666, - "middle_school_politics": 0.8846153846153846, - "middle_school_geography": 0.7647058823529411, - "modern_chinese_history": 0.5, - "ideological_and_moral_cultivation": 0.875, - "logic": 0.48148148148148145, - "law": 0.4482758620689655, - "chinese_language_and_literature": 0.5357142857142857, - "art_studies": 0.5, - "professional_tour_guide": 0.6176470588235294, - "legal_professional": 0.5, - "high_school_chinese": 0.5833333333333334, - "high_school_history": 0.76, - "middle_school_history": 0.8148148148148148, - "civil_servant": 0.4423076923076923, - "sports_science": 0.625, - "plant_protection": 0.7037037037037037, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.5185185185185185, - "urban_and_rural_planner": 0.5490196078431373, - "accountant": 0.42592592592592593, - "fire_engineer": 0.4444444444444444, - "environmental_impact_assessment_engineer": 0.5833333333333334, - "tax_accountant": 0.42592592592592593, - "physician": 0.5555555555555556 + "computer_network": 0.20833333333333334, + "operating_system": 0.2916666666666667, + "computer_architecture": 0.3076923076923077, + "college_programming": 0.23809523809523808, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.391304347826087, + "discrete_mathematics": 0.19047619047619047, + "electrical_engineer": 0.23809523809523808, + "metrology_engineer": 0.3448275862068966, + "high_school_mathematics": 0.17391304347826086, + "high_school_physics": 0.4583333333333333, + "high_school_chemistry": 0.2916666666666667, + "high_school_biology": 0.2916666666666667, + "middle_school_mathematics": 0.125, + "middle_school_biology": 0.5, + "middle_school_physics": 0.375, + "middle_school_chemistry": 0.28, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.4166666666666667, + "business_administration": 0.39473684210526316, + "marxism": 0.375, + "mao_zedong_thought": 0.41379310344827586, + "education_science": 0.4411764705882353, + "teacher_qualification": 0.46938775510204084, + "high_school_politics": 0.25, + "high_school_geography": 0.25, + "middle_school_politics": 0.38461538461538464, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.21428571428571427, + "ideological_and_moral_cultivation": 0.4166666666666667, + "logic": 0.37037037037037035, + "law": 0.20689655172413793, + "chinese_language_and_literature": 0.25, + "art_studies": 0.42105263157894735, + "professional_tour_guide": 0.35294117647058826, + "legal_professional": 0.35714285714285715, + "high_school_chinese": 0.20833333333333334, + "high_school_history": 0.48, + "middle_school_history": 0.4444444444444444, + "civil_servant": 0.28846153846153844, + "sports_science": 0.3333333333333333, + "plant_protection": 0.37037037037037035, + "basic_medicine": 0.5, + "clinical_medicine": 0.2222222222222222, + "urban_and_rural_planner": 0.43137254901960786, + "accountant": 0.2777777777777778, + "fire_engineer": 0.2777777777777778, + "environmental_impact_assessment_engineer": 0.3611111111111111, + "tax_accountant": 0.3888888888888889, + "physician": 0.4074074074074074 } }, "prompt_3": { - "accuracy": 0.5585305105853051, + "accuracy": 0.3343711083437111, "category_acc": { - "computer_network": 0.5416666666666666, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.46153846153846156, - "college_programming": 0.4523809523809524, - "college_physics": 0.375, - "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.42857142857142855, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.4583333333333333, - "high_school_chemistry": 0.5, - "high_school_biology": 0.5416666666666666, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.8076923076923077, - "middle_school_physics": 0.7916666666666666, - "middle_school_chemistry": 0.92, - "veterinary_medicine": 0.4642857142857143, - "college_economics": 0.45, - "business_administration": 0.5, - "marxism": 0.875, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.7352941176470589, - "teacher_qualification": 0.7346938775510204, - "high_school_politics": 0.8333333333333334, - "high_school_geography": 0.6666666666666666, - "middle_school_politics": 0.8461538461538461, - "middle_school_geography": 0.7058823529411765, - "modern_chinese_history": 0.5357142857142857, - "ideological_and_moral_cultivation": 0.8333333333333334, - "logic": 0.5555555555555556, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.5357142857142857, - "art_studies": 0.5526315789473685, - "professional_tour_guide": 0.6176470588235294, - "legal_professional": 0.5714285714285714, - "high_school_chinese": 0.625, - "high_school_history": 0.68, - "middle_school_history": 0.7407407407407407, - "civil_servant": 0.5, - "sports_science": 0.6666666666666666, - "plant_protection": 0.6666666666666666, - "basic_medicine": 0.5833333333333334, - "clinical_medicine": 0.5185185185185185, - "urban_and_rural_planner": 0.5490196078431373, - "accountant": 0.5, - "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.6111111111111112, - "tax_accountant": 0.46296296296296297, - "physician": 0.5740740740740741 + "computer_network": 0.4583333333333333, + "operating_system": 0.25, + "computer_architecture": 0.19230769230769232, + "college_programming": 0.23809523809523808, + "college_physics": 0.375, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.25, + "probability_and_statistics": 0.43478260869565216, + "discrete_mathematics": 0.14285714285714285, + "electrical_engineer": 0.2857142857142857, + "metrology_engineer": 0.41379310344827586, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.4166666666666667, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.20833333333333334, + "middle_school_mathematics": 0.20833333333333334, + "middle_school_biology": 0.46153846153846156, + "middle_school_physics": 0.4166666666666667, + "middle_school_chemistry": 0.4, + "veterinary_medicine": 0.42857142857142855, + "college_economics": 0.4, + "business_administration": 0.39473684210526316, + "marxism": 0.375, + "mao_zedong_thought": 0.3103448275862069, + "education_science": 0.5, + "teacher_qualification": 0.5102040816326531, + "high_school_politics": 0.4166666666666667, + "high_school_geography": 0.2916666666666667, + "middle_school_politics": 0.34615384615384615, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.25, + "ideological_and_moral_cultivation": 0.375, + "logic": 0.4074074074074074, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.39473684210526316, + "professional_tour_guide": 0.3235294117647059, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.25, + "high_school_history": 0.36, + "middle_school_history": 0.37037037037037035, + "civil_servant": 0.2692307692307692, + "sports_science": 0.375, + "plant_protection": 0.4444444444444444, + "basic_medicine": 0.5, + "clinical_medicine": 0.18518518518518517, + "urban_and_rural_planner": 0.3333333333333333, + "accountant": 0.2777777777777778, + "fire_engineer": 0.16666666666666666, + "environmental_impact_assessment_engineer": 0.3333333333333333, + "tax_accountant": 0.3148148148148148, + "physician": 0.3148148148148148 } }, "prompt_4": { - "accuracy": 0.5336239103362391, + "accuracy": 0.3318804483188045, "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.46153846153846156, - "college_programming": 0.5, - "college_physics": 0.5416666666666666, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.4583333333333333, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.047619047619047616, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.6206896551724138, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.5416666666666666, - "high_school_chemistry": 0.4583333333333333, - "high_school_biology": 0.4166666666666667, + "computer_network": 0.5, + "operating_system": 0.25, + "computer_architecture": 0.19230769230769232, + "college_programming": 0.19047619047619047, + "college_physics": 0.16666666666666666, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.09523809523809523, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.4827586206896552, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.4166666666666667, + "high_school_chemistry": 0.375, + "high_school_biology": 0.25, "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.8461538461538461, - "middle_school_physics": 0.7916666666666666, - "middle_school_chemistry": 0.8, + "middle_school_biology": 0.3076923076923077, + "middle_school_physics": 0.4583333333333333, + "middle_school_chemistry": 0.36, "veterinary_medicine": 0.42857142857142855, - "college_economics": 0.4166666666666667, - "business_administration": 0.5, - "marxism": 0.7916666666666666, - "mao_zedong_thought": 0.7241379310344828, - "education_science": 0.7352941176470589, - "teacher_qualification": 0.7551020408163265, - "high_school_politics": 0.8333333333333334, - "high_school_geography": 0.7083333333333334, - "middle_school_politics": 0.8846153846153846, - "middle_school_geography": 0.7647058823529411, - "modern_chinese_history": 0.6071428571428571, - "ideological_and_moral_cultivation": 0.875, - "logic": 0.5185185185185185, - "law": 0.4482758620689655, - "chinese_language_and_literature": 0.5714285714285714, - "art_studies": 0.5789473684210527, - "professional_tour_guide": 0.6764705882352942, - "legal_professional": 0.4642857142857143, - "high_school_chinese": 0.4583333333333333, - "high_school_history": 0.76, - "middle_school_history": 0.8148148148148148, - "civil_servant": 0.36538461538461536, - "sports_science": 0.6666666666666666, - "plant_protection": 0.5925925925925926, - "basic_medicine": 0.5, - "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.47058823529411764, - "accountant": 0.3888888888888889, - "fire_engineer": 0.3611111111111111, - "environmental_impact_assessment_engineer": 0.5555555555555556, - "tax_accountant": 0.4074074074074074, - "physician": 0.48148148148148145 + "college_economics": 0.3333333333333333, + "business_administration": 0.3684210526315789, + "marxism": 0.375, + "mao_zedong_thought": 0.3793103448275862, + "education_science": 0.4411764705882353, + "teacher_qualification": 0.42857142857142855, + "high_school_politics": 0.375, + "high_school_geography": 0.25, + "middle_school_politics": 0.23076923076923078, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.21428571428571427, + "ideological_and_moral_cultivation": 0.4166666666666667, + "logic": 0.48148148148148145, + "law": 0.41379310344827586, + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.3684210526315789, + "professional_tour_guide": 0.4411764705882353, + "legal_professional": 0.25, + "high_school_chinese": 0.16666666666666666, + "high_school_history": 0.52, + "middle_school_history": 0.4074074074074074, + "civil_servant": 0.25, + "sports_science": 0.3333333333333333, + "plant_protection": 0.37037037037037035, + "basic_medicine": 0.4166666666666667, + "clinical_medicine": 0.2222222222222222, + "urban_and_rural_planner": 0.37254901960784315, + "accountant": 0.2962962962962963, + "fire_engineer": 0.3055555555555556, + "environmental_impact_assessment_engineer": 0.2777777777777778, + "tax_accountant": 0.3148148148148148, + "physician": 0.3888888888888889 } }, "prompt_5": { - "accuracy": 0.5417185554171855, + "accuracy": 0.32814445828144456, "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.5476190476190477, - "college_physics": 0.4583333333333333, + "computer_network": 0.375, + "operating_system": 0.375, + "computer_architecture": 0.2692307692307692, + "college_programming": 0.19047619047619047, + "college_physics": 0.3333333333333333, "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.391304347826087, + "advanced_mathematics": 0.25, + "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.4523809523809524, - "metrology_engineer": 0.5862068965517241, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.3103448275862069, "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.4166666666666667, - "high_school_biology": 0.5833333333333334, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.7692307692307693, - "middle_school_physics": 0.75, - "middle_school_chemistry": 0.76, - "veterinary_medicine": 0.42857142857142855, - "college_economics": 0.5, - "business_administration": 0.5526315789473685, - "marxism": 0.8333333333333334, - "mao_zedong_thought": 0.7241379310344828, - "education_science": 0.6764705882352942, - "teacher_qualification": 0.7959183673469388, - "high_school_politics": 0.875, - "high_school_geography": 0.5833333333333334, - "middle_school_politics": 0.9230769230769231, - "middle_school_geography": 0.7647058823529411, - "modern_chinese_history": 0.5, - "ideological_and_moral_cultivation": 0.7916666666666666, - "logic": 0.5185185185185185, - "law": 0.4482758620689655, - "chinese_language_and_literature": 0.5357142857142857, - "art_studies": 0.5263157894736842, - "professional_tour_guide": 0.5882352941176471, - "legal_professional": 0.4642857142857143, - "high_school_chinese": 0.5833333333333334, - "high_school_history": 0.8, - "middle_school_history": 0.7777777777777778, - "civil_servant": 0.4230769230769231, - "sports_science": 0.6666666666666666, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.48148148148148145, - "urban_and_rural_planner": 0.5490196078431373, - "accountant": 0.37037037037037035, - "fire_engineer": 0.3611111111111111, - "environmental_impact_assessment_engineer": 0.5555555555555556, - "tax_accountant": 0.3888888888888889, - "physician": 0.5925925925925926 + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.25, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.4583333333333333, + "middle_school_chemistry": 0.44, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.3333333333333333, + "business_administration": 0.23684210526315788, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.3793103448275862, + "education_science": 0.5, + "teacher_qualification": 0.3469387755102041, + "high_school_politics": 0.4583333333333333, + "high_school_geography": 0.3333333333333333, + "middle_school_politics": 0.23076923076923078, + "middle_school_geography": 0.23529411764705882, + "modern_chinese_history": 0.32142857142857145, + "ideological_and_moral_cultivation": 0.25, + "logic": 0.37037037037037035, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.34210526315789475, + "professional_tour_guide": 0.35294117647058826, + "legal_professional": 0.21428571428571427, + "high_school_chinese": 0.20833333333333334, + "high_school_history": 0.4, + "middle_school_history": 0.48148148148148145, + "civil_servant": 0.34615384615384615, + "sports_science": 0.2916666666666667, + "plant_protection": 0.25925925925925924, + "basic_medicine": 0.4166666666666667, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.39215686274509803, + "accountant": 0.3888888888888889, + "fire_engineer": 0.25, + "environmental_impact_assessment_engineer": 0.3333333333333333, + "tax_accountant": 0.37037037037037035, + "physician": 0.42592592592592593 } } }, "cmmlu": { "prompt_1": { - "accuracy": 0.5448028673835126 + "accuracy": 0.3978494623655914 }, "prompt_2": { - "accuracy": 0.5555555555555556 + "accuracy": 0.3727598566308244 }, "prompt_3": { - "accuracy": 0.5591397849462365 + "accuracy": 0.34050179211469533 }, "prompt_4": { - "accuracy": 0.5555555555555556 + "accuracy": 0.3548387096774194 }, "prompt_5": { - "accuracy": 0.5519713261648745 + "accuracy": 0.3118279569892473 } }, "cmmlu_full": { "prompt_1": { - "accuracy": 0.5457606630979106, + "accuracy": 0.3173027111034364, "category_acc": { - "agronomy": 0.5384615384615384, - "anatomy": 0.42567567567567566, - "ancient_chinese": 0.3597560975609756, - "arts": 0.75, - "astronomy": 0.3575757575757576, - "business_ethics": 0.5550239234449761, - "chinese_civil_service_exam": 0.58125, - "chinese_driving_rule": 0.7175572519083969, - "chinese_food_culture": 0.5661764705882353, - "chinese_foreign_policy": 0.5981308411214953, - "chinese_history": 0.6594427244582043, - "chinese_literature": 0.4166666666666667, - "chinese_teacher_qualification": 0.7262569832402235, - "clinical_knowledge": 0.4936708860759494, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.6448598130841121, - "college_engineering_hydrology": 0.5, - "college_law": 0.4537037037037037, - "college_mathematics": 0.3523809523809524, - "college_medical_statistics": 0.4056603773584906, - "college_medicine": 0.4945054945054945, - "computer_science": 0.5637254901960784, - "computer_security": 0.6900584795321637, - "conceptual_physics": 0.6394557823129252, - "construction_project_management": 0.460431654676259, - "economics": 0.5786163522012578, - "education": 0.6196319018404908, - "electrical_engineering": 0.48255813953488375, - "elementary_chinese": 0.5277777777777778, - "elementary_commonsense": 0.5656565656565656, - "elementary_information_and_technology": 0.7521008403361344, - "elementary_mathematics": 0.33043478260869563, - "ethnology": 0.5777777777777777, - "food_science": 0.5594405594405595, - "genetics": 0.45454545454545453, - "global_facts": 0.5906040268456376, - "high_school_biology": 0.5088757396449705, - "high_school_chemistry": 0.5075757575757576, - "high_school_geography": 0.5423728813559322, - "high_school_mathematics": 0.3170731707317073, - "high_school_physics": 0.43636363636363634, - "high_school_politics": 0.5454545454545454, - "human_sexuality": 0.5555555555555556, - "international_law": 0.4594594594594595, - "journalism": 0.4941860465116279, - "jurisprudence": 0.5279805352798054, - "legal_and_moral_basis": 0.8644859813084113, - "logical": 0.4796747967479675, - "machine_learning": 0.5081967213114754, - "management": 0.6571428571428571, - "marketing": 0.6388888888888888, - "marxist_theory": 0.656084656084656, - "modern_chinese": 0.3879310344827586, - "nutrition": 0.5724137931034483, - "philosophy": 0.638095238095238, - "professional_accounting": 0.6114285714285714, - "professional_law": 0.46919431279620855, - "professional_medicine": 0.39361702127659576, - "professional_psychology": 0.6379310344827587, - "public_relations": 0.5804597701149425, - "security_study": 0.674074074074074, - "sociology": 0.5752212389380531, - "sports_science": 0.503030303030303, - "traditional_chinese_medicine": 0.5513513513513514, - "virology": 0.5266272189349113, - "world_history": 0.6273291925465838, - "world_religions": 0.63125 + "agronomy": 0.3136094674556213, + "anatomy": 0.30405405405405406, + "ancient_chinese": 0.3170731707317073, + "arts": 0.39375, + "astronomy": 0.2727272727272727, + "business_ethics": 0.3062200956937799, + "chinese_civil_service_exam": 0.3125, + "chinese_driving_rule": 0.4351145038167939, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.38317757009345793, + "chinese_history": 0.33436532507739936, + "chinese_literature": 0.3137254901960784, + "chinese_teacher_qualification": 0.36312849162011174, + "clinical_knowledge": 0.2869198312236287, + "college_actuarial_science": 0.2830188679245283, + "college_education": 0.29906542056074764, + "college_engineering_hydrology": 0.32075471698113206, + "college_law": 0.2222222222222222, + "college_mathematics": 0.3238095238095238, + "college_medical_statistics": 0.2358490566037736, + "college_medicine": 0.32967032967032966, + "computer_science": 0.28921568627450983, + "computer_security": 0.3684210526315789, + "conceptual_physics": 0.32653061224489793, + "construction_project_management": 0.2446043165467626, + "economics": 0.3584905660377358, + "education": 0.3987730061349693, + "electrical_engineering": 0.29069767441860467, + "elementary_chinese": 0.27380952380952384, + "elementary_commonsense": 0.29292929292929293, + "elementary_information_and_technology": 0.3907563025210084, + "elementary_mathematics": 0.26521739130434785, + "ethnology": 0.2814814814814815, + "food_science": 0.3146853146853147, + "genetics": 0.2556818181818182, + "global_facts": 0.3422818791946309, + "high_school_biology": 0.23668639053254437, + "high_school_chemistry": 0.26515151515151514, + "high_school_geography": 0.23728813559322035, + "high_school_mathematics": 0.21341463414634146, + "high_school_physics": 0.2636363636363636, + "high_school_politics": 0.4125874125874126, + "human_sexuality": 0.30952380952380953, + "international_law": 0.2648648648648649, + "journalism": 0.3488372093023256, + "jurisprudence": 0.340632603406326, + "legal_and_moral_basis": 0.514018691588785, + "logical": 0.2926829268292683, + "machine_learning": 0.2459016393442623, + "management": 0.3238095238095238, + "marketing": 0.4222222222222222, + "marxist_theory": 0.37566137566137564, + "modern_chinese": 0.27586206896551724, + "nutrition": 0.31724137931034485, + "philosophy": 0.3619047619047619, + "professional_accounting": 0.33714285714285713, + "professional_law": 0.2796208530805687, + "professional_medicine": 0.2579787234042553, + "professional_psychology": 0.33620689655172414, + "public_relations": 0.3505747126436782, + "security_study": 0.32592592592592595, + "sociology": 0.33185840707964603, + "sports_science": 0.2727272727272727, + "traditional_chinese_medicine": 0.2648648648648649, + "virology": 0.3905325443786982, + "world_history": 0.3416149068322981, + "world_religions": 0.33125 } }, "prompt_2": { - "accuracy": 0.5531859782420998, + "accuracy": 0.32274218615092387, "category_acc": { - "agronomy": 0.5443786982248521, - "anatomy": 0.4391891891891892, - "ancient_chinese": 0.3475609756097561, - "arts": 0.75625, - "astronomy": 0.38181818181818183, - "business_ethics": 0.5550239234449761, - "chinese_civil_service_exam": 0.55, - "chinese_driving_rule": 0.7709923664122137, - "chinese_food_culture": 0.5367647058823529, - "chinese_foreign_policy": 0.616822429906542, - "chinese_history": 0.6780185758513931, - "chinese_literature": 0.4166666666666667, - "chinese_teacher_qualification": 0.7430167597765364, - "clinical_knowledge": 0.5063291139240507, - "college_actuarial_science": 0.3113207547169811, - "college_education": 0.6355140186915887, - "college_engineering_hydrology": 0.5, - "college_law": 0.48148148148148145, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.4056603773584906, - "college_medicine": 0.5201465201465202, - "computer_science": 0.5882352941176471, - "computer_security": 0.7134502923976608, - "conceptual_physics": 0.6326530612244898, - "construction_project_management": 0.4748201438848921, - "economics": 0.6163522012578616, - "education": 0.6319018404907976, - "electrical_engineering": 0.5116279069767442, - "elementary_chinese": 0.5476190476190477, - "elementary_commonsense": 0.5555555555555556, - "elementary_information_and_technology": 0.7394957983193278, - "elementary_mathematics": 0.3565217391304348, - "ethnology": 0.5481481481481482, - "food_science": 0.5734265734265734, - "genetics": 0.42613636363636365, - "global_facts": 0.5704697986577181, - "high_school_biology": 0.48520710059171596, - "high_school_chemistry": 0.5303030303030303, - "high_school_geography": 0.5508474576271186, - "high_school_mathematics": 0.27439024390243905, - "high_school_physics": 0.43636363636363634, - "high_school_politics": 0.5664335664335665, - "human_sexuality": 0.5396825396825397, - "international_law": 0.4756756756756757, - "journalism": 0.5406976744186046, - "jurisprudence": 0.5352798053527981, - "legal_and_moral_basis": 0.8457943925233645, - "logical": 0.4634146341463415, - "machine_learning": 0.47540983606557374, - "management": 0.6714285714285714, - "marketing": 0.6444444444444445, - "marxist_theory": 0.671957671957672, - "modern_chinese": 0.43103448275862066, - "nutrition": 0.6344827586206897, - "philosophy": 0.638095238095238, - "professional_accounting": 0.6057142857142858, - "professional_law": 0.46445497630331756, - "professional_medicine": 0.4095744680851064, - "professional_psychology": 0.6422413793103449, - "public_relations": 0.5747126436781609, - "security_study": 0.674074074074074, - "sociology": 0.5796460176991151, - "sports_science": 0.5272727272727272, - "traditional_chinese_medicine": 0.572972972972973, - "virology": 0.5562130177514792, - "world_history": 0.6645962732919255, - "world_religions": 0.63125 + "agronomy": 0.3905325443786982, + "anatomy": 0.30405405405405406, + "ancient_chinese": 0.2621951219512195, + "arts": 0.4375, + "astronomy": 0.2787878787878788, + "business_ethics": 0.3923444976076555, + "chinese_civil_service_exam": 0.31875, + "chinese_driving_rule": 0.40458015267175573, + "chinese_food_culture": 0.27941176470588236, + "chinese_foreign_policy": 0.411214953271028, + "chinese_history": 0.38699690402476783, + "chinese_literature": 0.30392156862745096, + "chinese_teacher_qualification": 0.40782122905027934, + "clinical_knowledge": 0.3291139240506329, + "college_actuarial_science": 0.29245283018867924, + "college_education": 0.3364485981308411, + "college_engineering_hydrology": 0.36792452830188677, + "college_law": 0.28703703703703703, + "college_mathematics": 0.2571428571428571, + "college_medical_statistics": 0.27358490566037735, + "college_medicine": 0.2783882783882784, + "computer_science": 0.2647058823529412, + "computer_security": 0.32748538011695905, + "conceptual_physics": 0.2925170068027211, + "construction_project_management": 0.28776978417266186, + "economics": 0.3522012578616352, + "education": 0.37423312883435583, + "electrical_engineering": 0.31976744186046513, + "elementary_chinese": 0.34523809523809523, + "elementary_commonsense": 0.2676767676767677, + "elementary_information_and_technology": 0.3235294117647059, + "elementary_mathematics": 0.24347826086956523, + "ethnology": 0.2962962962962963, + "food_science": 0.22377622377622378, + "genetics": 0.26136363636363635, + "global_facts": 0.31543624161073824, + "high_school_biology": 0.28994082840236685, + "high_school_chemistry": 0.23484848484848486, + "high_school_geography": 0.2711864406779661, + "high_school_mathematics": 0.23780487804878048, + "high_school_physics": 0.33636363636363636, + "high_school_politics": 0.36363636363636365, + "human_sexuality": 0.3412698412698413, + "international_law": 0.25405405405405407, + "journalism": 0.38372093023255816, + "jurisprudence": 0.35279805352798055, + "legal_and_moral_basis": 0.5046728971962616, + "logical": 0.2845528455284553, + "machine_learning": 0.29508196721311475, + "management": 0.3476190476190476, + "marketing": 0.37222222222222223, + "marxist_theory": 0.3968253968253968, + "modern_chinese": 0.28448275862068967, + "nutrition": 0.31724137931034485, + "philosophy": 0.34285714285714286, + "professional_accounting": 0.30857142857142855, + "professional_law": 0.27488151658767773, + "professional_medicine": 0.2553191489361702, + "professional_psychology": 0.34913793103448276, + "public_relations": 0.3275862068965517, + "security_study": 0.34814814814814815, + "sociology": 0.30973451327433627, + "sports_science": 0.2545454545454545, + "traditional_chinese_medicine": 0.2864864864864865, + "virology": 0.2958579881656805, + "world_history": 0.45962732919254656, + "world_religions": 0.3625 } }, "prompt_3": { - "accuracy": 0.5518045242617855, + "accuracy": 0.3358659989639095, "category_acc": { - "agronomy": 0.5325443786982249, - "anatomy": 0.43243243243243246, - "ancient_chinese": 0.2865853658536585, - "arts": 0.76875, - "astronomy": 0.38181818181818183, - "business_ethics": 0.5598086124401914, - "chinese_civil_service_exam": 0.56875, - "chinese_driving_rule": 0.7480916030534351, - "chinese_food_culture": 0.5220588235294118, - "chinese_foreign_policy": 0.6355140186915887, - "chinese_history": 0.6904024767801857, - "chinese_literature": 0.4362745098039216, - "chinese_teacher_qualification": 0.7374301675977654, - "clinical_knowledge": 0.4978902953586498, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.6355140186915887, - "college_engineering_hydrology": 0.5, - "college_law": 0.46296296296296297, - "college_mathematics": 0.29523809523809524, - "college_medical_statistics": 0.41509433962264153, - "college_medicine": 0.5018315018315018, - "computer_science": 0.5931372549019608, - "computer_security": 0.6900584795321637, - "conceptual_physics": 0.6394557823129252, - "construction_project_management": 0.43884892086330934, - "economics": 0.5911949685534591, - "education": 0.6073619631901841, - "electrical_engineering": 0.5232558139534884, - "elementary_chinese": 0.5476190476190477, - "elementary_commonsense": 0.5757575757575758, - "elementary_information_and_technology": 0.7478991596638656, - "elementary_mathematics": 0.3739130434782609, - "ethnology": 0.5703703703703704, - "food_science": 0.5524475524475524, - "genetics": 0.4090909090909091, - "global_facts": 0.5704697986577181, - "high_school_biology": 0.4911242603550296, - "high_school_chemistry": 0.5606060606060606, - "high_school_geography": 0.5169491525423728, - "high_school_mathematics": 0.31097560975609756, - "high_school_physics": 0.4, - "high_school_politics": 0.5734265734265734, - "human_sexuality": 0.5476190476190477, - "international_law": 0.4756756756756757, - "journalism": 0.5232558139534884, - "jurisprudence": 0.5571776155717761, - "legal_and_moral_basis": 0.8504672897196262, - "logical": 0.4878048780487805, - "machine_learning": 0.48360655737704916, - "management": 0.6571428571428571, - "marketing": 0.6555555555555556, - "marxist_theory": 0.6666666666666666, - "modern_chinese": 0.41379310344827586, - "nutrition": 0.6137931034482759, - "philosophy": 0.6285714285714286, - "professional_accounting": 0.6171428571428571, - "professional_law": 0.45023696682464454, - "professional_medicine": 0.3962765957446808, - "professional_psychology": 0.6681034482758621, - "public_relations": 0.603448275862069, - "security_study": 0.674074074074074, - "sociology": 0.5796460176991151, - "sports_science": 0.5393939393939394, - "traditional_chinese_medicine": 0.5405405405405406, - "virology": 0.5562130177514792, - "world_history": 0.6459627329192547, - "world_religions": 0.63125 + "agronomy": 0.35502958579881655, + "anatomy": 0.2905405405405405, + "ancient_chinese": 0.23170731707317074, + "arts": 0.4, + "astronomy": 0.2909090909090909, + "business_ethics": 0.36363636363636365, + "chinese_civil_service_exam": 0.33125, + "chinese_driving_rule": 0.4198473282442748, + "chinese_food_culture": 0.3088235294117647, + "chinese_foreign_policy": 0.32710280373831774, + "chinese_history": 0.3591331269349845, + "chinese_literature": 0.27941176470588236, + "chinese_teacher_qualification": 0.37988826815642457, + "clinical_knowledge": 0.3291139240506329, + "college_actuarial_science": 0.3018867924528302, + "college_education": 0.32710280373831774, + "college_engineering_hydrology": 0.37735849056603776, + "college_law": 0.3148148148148148, + "college_mathematics": 0.23809523809523808, + "college_medical_statistics": 0.2358490566037736, + "college_medicine": 0.34065934065934067, + "computer_science": 0.3088235294117647, + "computer_security": 0.36257309941520466, + "conceptual_physics": 0.30612244897959184, + "construction_project_management": 0.3237410071942446, + "economics": 0.36477987421383645, + "education": 0.4171779141104294, + "electrical_engineering": 0.31976744186046513, + "elementary_chinese": 0.3134920634920635, + "elementary_commonsense": 0.2878787878787879, + "elementary_information_and_technology": 0.42436974789915966, + "elementary_mathematics": 0.24347826086956523, + "ethnology": 0.3111111111111111, + "food_science": 0.2867132867132867, + "genetics": 0.26704545454545453, + "global_facts": 0.3624161073825503, + "high_school_biology": 0.26627218934911245, + "high_school_chemistry": 0.2727272727272727, + "high_school_geography": 0.3644067796610169, + "high_school_mathematics": 0.24390243902439024, + "high_school_physics": 0.3, + "high_school_politics": 0.3916083916083916, + "human_sexuality": 0.3492063492063492, + "international_law": 0.35135135135135137, + "journalism": 0.4127906976744186, + "jurisprudence": 0.35523114355231145, + "legal_and_moral_basis": 0.5327102803738317, + "logical": 0.3170731707317073, + "machine_learning": 0.32786885245901637, + "management": 0.3523809523809524, + "marketing": 0.3888888888888889, + "marxist_theory": 0.3862433862433862, + "modern_chinese": 0.29310344827586204, + "nutrition": 0.3448275862068966, + "philosophy": 0.3904761904761905, + "professional_accounting": 0.33714285714285713, + "professional_law": 0.2938388625592417, + "professional_medicine": 0.2553191489361702, + "professional_psychology": 0.3620689655172414, + "public_relations": 0.39655172413793105, + "security_study": 0.34074074074074073, + "sociology": 0.3584070796460177, + "sports_science": 0.3151515151515151, + "traditional_chinese_medicine": 0.3081081081081081, + "virology": 0.2958579881656805, + "world_history": 0.40372670807453415, + "world_religions": 0.40625 } }, "prompt_4": { - "accuracy": 0.5390260749438784, + "accuracy": 0.33223968226558453, "category_acc": { - "agronomy": 0.5088757396449705, - "anatomy": 0.43243243243243246, - "ancient_chinese": 0.3048780487804878, - "arts": 0.75625, - "astronomy": 0.3575757575757576, - "business_ethics": 0.5550239234449761, - "chinese_civil_service_exam": 0.525, - "chinese_driving_rule": 0.7404580152671756, - "chinese_food_culture": 0.5147058823529411, - "chinese_foreign_policy": 0.6074766355140186, - "chinese_history": 0.653250773993808, - "chinese_literature": 0.4803921568627451, - "chinese_teacher_qualification": 0.7262569832402235, - "clinical_knowledge": 0.4767932489451477, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.6728971962616822, - "college_engineering_hydrology": 0.49056603773584906, - "college_law": 0.46296296296296297, - "college_mathematics": 0.3142857142857143, - "college_medical_statistics": 0.36792452830188677, - "college_medicine": 0.5018315018315018, - "computer_science": 0.5588235294117647, - "computer_security": 0.6549707602339181, - "conceptual_physics": 0.6054421768707483, - "construction_project_management": 0.45323741007194246, - "economics": 0.5786163522012578, - "education": 0.6319018404907976, - "electrical_engineering": 0.48255813953488375, - "elementary_chinese": 0.5634920634920635, - "elementary_commonsense": 0.5505050505050505, - "elementary_information_and_technology": 0.7310924369747899, - "elementary_mathematics": 0.3782608695652174, - "ethnology": 0.5481481481481482, - "food_science": 0.5524475524475524, - "genetics": 0.4659090909090909, - "global_facts": 0.5704697986577181, - "high_school_biology": 0.47928994082840237, - "high_school_chemistry": 0.5151515151515151, - "high_school_geography": 0.5169491525423728, - "high_school_mathematics": 0.29878048780487804, - "high_school_physics": 0.4090909090909091, - "high_school_politics": 0.5384615384615384, - "human_sexuality": 0.5, - "international_law": 0.4756756756756757, - "journalism": 0.5232558139534884, - "jurisprudence": 0.5109489051094891, - "legal_and_moral_basis": 0.8457943925233645, - "logical": 0.4634146341463415, - "machine_learning": 0.4426229508196721, - "management": 0.638095238095238, - "marketing": 0.6444444444444445, - "marxist_theory": 0.6349206349206349, - "modern_chinese": 0.3448275862068966, - "nutrition": 0.5448275862068965, - "philosophy": 0.6095238095238096, - "professional_accounting": 0.5942857142857143, - "professional_law": 0.4786729857819905, - "professional_medicine": 0.39095744680851063, - "professional_psychology": 0.6422413793103449, - "public_relations": 0.5804597701149425, - "security_study": 0.6592592592592592, - "sociology": 0.5619469026548672, - "sports_science": 0.5272727272727272, - "traditional_chinese_medicine": 0.5351351351351351, - "virology": 0.5384615384615384, - "world_history": 0.6273291925465838, - "world_religions": 0.65625 + "agronomy": 0.3254437869822485, + "anatomy": 0.30405405405405406, + "ancient_chinese": 0.20121951219512196, + "arts": 0.40625, + "astronomy": 0.24848484848484848, + "business_ethics": 0.3253588516746411, + "chinese_civil_service_exam": 0.29375, + "chinese_driving_rule": 0.4198473282442748, + "chinese_food_culture": 0.2867647058823529, + "chinese_foreign_policy": 0.38317757009345793, + "chinese_history": 0.33746130030959753, + "chinese_literature": 0.31862745098039214, + "chinese_teacher_qualification": 0.40782122905027934, + "clinical_knowledge": 0.3080168776371308, + "college_actuarial_science": 0.3018867924528302, + "college_education": 0.34579439252336447, + "college_engineering_hydrology": 0.39622641509433965, + "college_law": 0.25925925925925924, + "college_mathematics": 0.26666666666666666, + "college_medical_statistics": 0.2641509433962264, + "college_medicine": 0.32967032967032966, + "computer_science": 0.3333333333333333, + "computer_security": 0.32748538011695905, + "conceptual_physics": 0.3197278911564626, + "construction_project_management": 0.38848920863309355, + "economics": 0.3333333333333333, + "education": 0.39263803680981596, + "electrical_engineering": 0.3430232558139535, + "elementary_chinese": 0.3253968253968254, + "elementary_commonsense": 0.32323232323232326, + "elementary_information_and_technology": 0.40336134453781514, + "elementary_mathematics": 0.25217391304347825, + "ethnology": 0.2740740740740741, + "food_science": 0.2867132867132867, + "genetics": 0.3125, + "global_facts": 0.31543624161073824, + "high_school_biology": 0.25443786982248523, + "high_school_chemistry": 0.30303030303030304, + "high_school_geography": 0.2542372881355932, + "high_school_mathematics": 0.23170731707317074, + "high_school_physics": 0.21818181818181817, + "high_school_politics": 0.3986013986013986, + "human_sexuality": 0.3333333333333333, + "international_law": 0.32972972972972975, + "journalism": 0.38372093023255816, + "jurisprudence": 0.30900243309002434, + "legal_and_moral_basis": 0.6214953271028038, + "logical": 0.34146341463414637, + "machine_learning": 0.23770491803278687, + "management": 0.35714285714285715, + "marketing": 0.4111111111111111, + "marxist_theory": 0.38095238095238093, + "modern_chinese": 0.3017241379310345, + "nutrition": 0.35172413793103446, + "philosophy": 0.3523809523809524, + "professional_accounting": 0.3485714285714286, + "professional_law": 0.27488151658767773, + "professional_medicine": 0.26063829787234044, + "professional_psychology": 0.375, + "public_relations": 0.3563218390804598, + "security_study": 0.3851851851851852, + "sociology": 0.39823008849557523, + "sports_science": 0.3212121212121212, + "traditional_chinese_medicine": 0.24864864864864866, + "virology": 0.3254437869822485, + "world_history": 0.40993788819875776, + "world_religions": 0.43125 } }, "prompt_5": { - "accuracy": 0.5461060265929891, + "accuracy": 0.3084959419789328, "category_acc": { - "agronomy": 0.47337278106508873, - "anatomy": 0.49324324324324326, - "ancient_chinese": 0.31097560975609756, - "arts": 0.75625, - "astronomy": 0.34545454545454546, - "business_ethics": 0.583732057416268, - "chinese_civil_service_exam": 0.53125, - "chinese_driving_rule": 0.732824427480916, - "chinese_food_culture": 0.5220588235294118, - "chinese_foreign_policy": 0.5794392523364486, - "chinese_history": 0.6904024767801857, - "chinese_literature": 0.4411764705882353, - "chinese_teacher_qualification": 0.7206703910614525, - "clinical_knowledge": 0.5063291139240507, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.6635514018691588, - "college_engineering_hydrology": 0.44339622641509435, - "college_law": 0.4722222222222222, - "college_mathematics": 0.29523809523809524, - "college_medical_statistics": 0.42452830188679247, - "college_medicine": 0.5164835164835165, - "computer_science": 0.5637254901960784, - "computer_security": 0.672514619883041, - "conceptual_physics": 0.6054421768707483, - "construction_project_management": 0.4460431654676259, - "economics": 0.5660377358490566, - "education": 0.6073619631901841, - "electrical_engineering": 0.5116279069767442, - "elementary_chinese": 0.5238095238095238, - "elementary_commonsense": 0.5808080808080808, - "elementary_information_and_technology": 0.7563025210084033, - "elementary_mathematics": 0.3695652173913043, - "ethnology": 0.5703703703703704, - "food_science": 0.5524475524475524, - "genetics": 0.45454545454545453, - "global_facts": 0.5838926174496645, - "high_school_biology": 0.4319526627218935, - "high_school_chemistry": 0.48484848484848486, - "high_school_geography": 0.559322033898305, - "high_school_mathematics": 0.27439024390243905, - "high_school_physics": 0.45454545454545453, - "high_school_politics": 0.5454545454545454, - "human_sexuality": 0.5634920634920635, - "international_law": 0.43783783783783786, - "journalism": 0.5465116279069767, - "jurisprudence": 0.5644768856447688, - "legal_and_moral_basis": 0.8317757009345794, - "logical": 0.4634146341463415, - "machine_learning": 0.48360655737704916, - "management": 0.6476190476190476, - "marketing": 0.6333333333333333, - "marxist_theory": 0.6349206349206349, - "modern_chinese": 0.39655172413793105, - "nutrition": 0.5724137931034483, - "philosophy": 0.6285714285714286, - "professional_accounting": 0.56, - "professional_law": 0.5071090047393365, - "professional_medicine": 0.4308510638297872, - "professional_psychology": 0.6293103448275862, - "public_relations": 0.5977011494252874, - "security_study": 0.6814814814814815, - "sociology": 0.5752212389380531, - "sports_science": 0.5151515151515151, - "traditional_chinese_medicine": 0.5297297297297298, - "virology": 0.5443786982248521, - "world_history": 0.6583850931677019, - "world_religions": 0.6375 + "agronomy": 0.2603550295857988, + "anatomy": 0.32432432432432434, + "ancient_chinese": 0.2865853658536585, + "arts": 0.34375, + "astronomy": 0.23030303030303031, + "business_ethics": 0.3444976076555024, + "chinese_civil_service_exam": 0.36875, + "chinese_driving_rule": 0.4351145038167939, + "chinese_food_culture": 0.21323529411764705, + "chinese_foreign_policy": 0.38317757009345793, + "chinese_history": 0.3653250773993808, + "chinese_literature": 0.3088235294117647, + "chinese_teacher_qualification": 0.31843575418994413, + "clinical_knowledge": 0.2911392405063291, + "college_actuarial_science": 0.33962264150943394, + "college_education": 0.37383177570093457, + "college_engineering_hydrology": 0.2830188679245283, + "college_law": 0.2777777777777778, + "college_mathematics": 0.3047619047619048, + "college_medical_statistics": 0.2358490566037736, + "college_medicine": 0.2967032967032967, + "computer_science": 0.29411764705882354, + "computer_security": 0.30409356725146197, + "conceptual_physics": 0.2925170068027211, + "construction_project_management": 0.302158273381295, + "economics": 0.3710691823899371, + "education": 0.3803680981595092, + "electrical_engineering": 0.26744186046511625, + "elementary_chinese": 0.26587301587301587, + "elementary_commonsense": 0.2878787878787879, + "elementary_information_and_technology": 0.35294117647058826, + "elementary_mathematics": 0.26521739130434785, + "ethnology": 0.31851851851851853, + "food_science": 0.3006993006993007, + "genetics": 0.2727272727272727, + "global_facts": 0.3959731543624161, + "high_school_biology": 0.23076923076923078, + "high_school_chemistry": 0.17424242424242425, + "high_school_geography": 0.211864406779661, + "high_school_mathematics": 0.23780487804878048, + "high_school_physics": 0.2909090909090909, + "high_school_politics": 0.40559440559440557, + "human_sexuality": 0.29365079365079366, + "international_law": 0.2864864864864865, + "journalism": 0.3430232558139535, + "jurisprudence": 0.2798053527980535, + "legal_and_moral_basis": 0.5046728971962616, + "logical": 0.3089430894308943, + "machine_learning": 0.20491803278688525, + "management": 0.319047619047619, + "marketing": 0.3388888888888889, + "marxist_theory": 0.3439153439153439, + "modern_chinese": 0.3448275862068966, + "nutrition": 0.3310344827586207, + "philosophy": 0.3047619047619048, + "professional_accounting": 0.26857142857142857, + "professional_law": 0.27014218009478674, + "professional_medicine": 0.2526595744680851, + "professional_psychology": 0.3103448275862069, + "public_relations": 0.3160919540229885, + "security_study": 0.34074074074074073, + "sociology": 0.3053097345132743, + "sports_science": 0.28484848484848485, + "traditional_chinese_medicine": 0.2756756756756757, + "virology": 0.3254437869822485, + "world_history": 0.391304347826087, + "world_religions": 0.34375 } } }, "zbench": { "prompt_1": { - "accuracy": 0.3333333333333333 + "accuracy": 0.48484848484848486 }, "prompt_2": { - "accuracy": 0.3333333333333333 + "accuracy": 0.42424242424242425 }, "prompt_3": { - "accuracy": 0.30303030303030304 + "accuracy": 0.42424242424242425 }, "prompt_4": { "accuracy": 0.36363636363636365 }, "prompt_5": { - "accuracy": 0.3333333333333333 + "accuracy": 0.3939393939393939 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.6 + "accuracy": 0.5181818181818182 }, "prompt_2": { - "accuracy": 0.5522727272727272 + "accuracy": 0.44545454545454544 }, "prompt_3": { - "accuracy": 0.6068181818181818 + "accuracy": 0.42045454545454547 }, "prompt_4": { - "accuracy": 0.625 + "accuracy": 0.43636363636363634 }, "prompt_5": { - "accuracy": 0.6068181818181818 + "accuracy": 0.41818181818181815 } }, "ocnli": { "prompt_1": { - "accuracy": 0.4888135593220339 + "accuracy": 0.31864406779661014 }, "prompt_2": { - "accuracy": 0.47966101694915253 + "accuracy": 0.31796610169491524 }, "prompt_3": { - "accuracy": 0.4969491525423729 + "accuracy": 0.32033898305084746 }, "prompt_4": { - "accuracy": 0.46372881355932205 + "accuracy": 0.3254237288135593 }, "prompt_5": { - "accuracy": 0.4928813559322034 + "accuracy": 0.32440677966101694 } }, "c3": { "prompt_1": { - "accuracy": 0.8578908002991773 + "accuracy": 0.5474943904263276 }, "prompt_2": { - "accuracy": 0.8590127150336574 + "accuracy": 0.5381451009723262 }, "prompt_3": { - "accuracy": 0.8582647718773373 + "accuracy": 0.5138369483919222 }, "prompt_4": { - "accuracy": 0.8522812266267764 + "accuracy": 0.524682124158564 }, "prompt_5": { - "accuracy": 0.8548990276738968 + "accuracy": 0.5224382946896036 } }, "dream": { "prompt_1": { - "accuracy": 0.7123958843704067 + "accuracy": 0.5972562469377756 }, "prompt_2": { - "accuracy": 0.7545320921117099 + "accuracy": 0.6438020578147967 }, "prompt_3": { - "accuracy": 0.7236648701616855 + "accuracy": 0.6266536011758942 }, "prompt_4": { - "accuracy": 0.7285644292013719 + "accuracy": 0.5658990690837825 }, "prompt_5": { - "accuracy": 0.7153356197942186 + "accuracy": 0.6153846153846154 } }, "samsum": { "prompt_1": { - "rouge1": 0.3385435037042298, - "rouge2": 0.11349753101528316, - "rougeL": 0.25261450255521123, - "avg_rouge": 0.23488517909157472 + "rouge1": 0.19421979959087435, + "rouge2": 0.06983749460043516, + "rougeL": 0.168321567036734, + "avg_rouge": 0.14412628707601452 }, "prompt_2": { - "rouge1": 0.3378272658278295, - "rouge2": 0.11450751352342818, - "rougeL": 0.255180015584705, - "avg_rouge": 0.23583826497865423 + "rouge1": 0.19866330633085436, + "rouge2": 0.07228786591276251, + "rougeL": 0.17140262631047645, + "avg_rouge": 0.14745126618469775 }, "prompt_3": { - "rouge1": 0.3550079859862152, - "rouge2": 0.11685624643443011, - "rougeL": 0.26657073839456313, - "avg_rouge": 0.24614499027173617 + "rouge1": 0.1823010767794734, + "rouge2": 0.062186205910091226, + "rougeL": 0.1637899495200591, + "avg_rouge": 0.13609241073654124 }, "prompt_4": { - "rouge1": 0.3327659363635708, - "rouge2": 0.11078991429605135, - "rougeL": 0.24853552435135012, - "avg_rouge": 0.23069712500365744 + "rouge1": 0.19574582248288472, + "rouge2": 0.07082506606400779, + "rougeL": 0.16926419301155568, + "avg_rouge": 0.14527836051948273 }, "prompt_5": { - "rouge1": 0.3388063709987223, - "rouge2": 0.10924841518363192, - "rougeL": 0.25294384612517107, - "avg_rouge": 0.23366621076917507 + "rouge1": 0.19145248800250056, + "rouge2": 0.06645109075593836, + "rougeL": 0.16682834576905436, + "avg_rouge": 0.1415773081758311 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.21401603013287365, - "rouge2": 0.05754241763422725, - "rougeL": 0.15736832315743748, - "avg_rouge": 0.14297559030817947 + "rouge1": 0.19354612820856404, + "rouge2": 0.05220115551483978, + "rougeL": 0.1475132846001137, + "avg_rouge": 0.13108685610783918 }, "prompt_2": { - "rouge1": 0.21219469676800315, - "rouge2": 0.05494720087280016, - "rougeL": 0.1555814829758187, - "avg_rouge": 0.140907793538874 + "rouge1": 0.18930219636318057, + "rouge2": 0.0509957241945143, + "rougeL": 0.14481650495763934, + "avg_rouge": 0.12837147517177808 }, "prompt_3": { - "rouge1": 0.21233541541522322, - "rouge2": 0.054054666925376894, - "rougeL": 0.15567842442952348, - "avg_rouge": 0.14068950225670787 + "rouge1": 0.18856305511927057, + "rouge2": 0.05047203018596512, + "rougeL": 0.14516669076855662, + "avg_rouge": 0.12806725869126412 }, "prompt_4": { - "rouge1": 0.2139639458237427, - "rouge2": 0.057307755140636366, - "rougeL": 0.15745662620904194, - "avg_rouge": 0.14290944239114034 + "rouge1": 0.19987887059378412, + "rouge2": 0.054264246884584, + "rougeL": 0.15137121232130749, + "avg_rouge": 0.13517144326655853 }, "prompt_5": { - "rouge1": 0.21012619050101916, - "rouge2": 0.05316933177131179, - "rougeL": 0.1583564364726453, - "avg_rouge": 0.14055065291499208 + "rouge1": 0.1887689700181949, + "rouge2": 0.050424274072061896, + "rougeL": 0.14477589018893963, + "avg_rouge": 0.1279897114263988 } }, "sst2": { "prompt_1": { - "accuracy": 0.9094036697247706 + "accuracy": 0.8142201834862385 }, "prompt_2": { - "accuracy": 0.9094036697247706 + "accuracy": 0.7889908256880734 }, "prompt_3": { - "accuracy": 0.8887614678899083 + "accuracy": 0.731651376146789 }, "prompt_4": { - "accuracy": 0.9105504587155964 + "accuracy": 0.7763761467889908 }, "prompt_5": { - "accuracy": 0.8899082568807339 + "accuracy": 0.8027522935779816 } }, "cola": { "prompt_1": { - "accuracy": 0.7027804410354745 + "accuracy": 0.5589645254074784 }, "prompt_2": { - "accuracy": 0.7152444870565676 + "accuracy": 0.5100671140939598 }, "prompt_3": { - "accuracy": 0.7190795781399808 + "accuracy": 0.5982742090124641 }, "prompt_4": { - "accuracy": 0.7363374880153404 + "accuracy": 0.5973154362416108 }, "prompt_5": { - "accuracy": 0.7612655800575263 + "accuracy": 0.6883988494726749 } }, "qqp": { "prompt_1": { - "accuracy": 0.7765 + "accuracy": 0.4865 }, "prompt_2": { - "accuracy": 0.7895 + "accuracy": 0.4955 }, "prompt_3": { - "accuracy": 0.7775 + "accuracy": 0.507 }, "prompt_4": { - "accuracy": 0.737 + "accuracy": 0.4535 }, "prompt_5": { - "accuracy": 0.79 + "accuracy": 0.4795 } }, "mnli": { "prompt_1": { - "accuracy": 0.66 + "accuracy": 0.4 }, "prompt_2": { - "accuracy": 0.669 + "accuracy": 0.4 }, "prompt_3": { - "accuracy": 0.678 + "accuracy": 0.4 }, "prompt_4": { - "accuracy": 0.654 + "accuracy": 0.3 }, "prompt_5": { - "accuracy": 0.62 + "accuracy": 0.5 } }, "qnli": { "prompt_1": { - "accuracy": 0.829 + "accuracy": 0.8 }, "prompt_2": { - "accuracy": 0.8425 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.849 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.823 + "accuracy": 0.7 }, "prompt_5": { - "accuracy": 0.824 + "accuracy": 0.9 } }, "wnli": { "prompt_1": { - "accuracy": 0.5070422535211268 + "accuracy": 0.7 }, "prompt_2": { - "accuracy": 0.5211267605633803 + "accuracy": 0.3 }, "prompt_3": { - "accuracy": 0.5070422535211268 + "accuracy": 0.2 }, "prompt_4": { - "accuracy": 0.4788732394366197 + "accuracy": 0.3 }, "prompt_5": { - "accuracy": 0.5070422535211268 + "accuracy": 0.3 } }, "rte": { "prompt_1": { - "accuracy": 0.776173285198556 + "accuracy": 0.4 }, "prompt_2": { - "accuracy": 0.7653429602888087 + "accuracy": 0.6 }, "prompt_3": { - "accuracy": 0.7472924187725631 + "accuracy": 0.5 }, "prompt_4": { - "accuracy": 0.6606498194945848 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.776173285198556 + "accuracy": 0.5 } }, "mrpc": { "prompt_1": { - "accuracy": 0.6936274509803921 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.6985294117647058 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.7009803921568627 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.6936274509803921 + "accuracy": 0.8 }, "prompt_5": { - "accuracy": 0.696078431372549 + "accuracy": 0.8 } } }, "five_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.8802521008403361, - "language_acc": { - "English": 0.9067226890756303, - "Chinese": 0.8823529411764706, - "Spanish": 0.865546218487395, - "Vietnamese": 0.8663865546218488 - }, - "consistency_score_2": 0.8680672268907562, - "consistency_score_3": 0.8086134453781513, - "consistency_score_4": 0.7697478991596639, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8798319327731092, - "English,Spanish": 0.8764705882352941, - "English,Vietnamese": 0.8831932773109243, - "Chinese,Spanish": 0.8470588235294118, - "Chinese,Vietnamese": 0.8521008403361344, - "Spanish,Vietnamese": 0.8697478991596639 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8109243697478992, - "English,Chinese,Vietnamese": 0.8126050420168067, - "English,Spanish,Vietnamese": 0.8210084033613445, - "Chinese,Spanish,Vietnamese": 0.7899159663865546 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7697478991596639 - } - }, - "AC3_2": 0.8741172027070965, - "AC3_3": 0.8429133812498669, - "AC3_4": 0.8212996428020174 - } + "prompt_1": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.5238095238095238, - "language_acc": { - "Spanish": 0.5266666666666666, - "English": 0.6066666666666667, - "Chinese": 0.5866666666666667, - "Vietnamese": 0.54, - "Filipino": 0.36666666666666664, - "Indonesian": 0.5266666666666666, - "Malay": 0.5133333333333333 - }, - "consistency_score_2": 0.5876190476190476, - "consistency_score_3": 0.4297142857142857, - "consistency_score_4": 0.33504761904761904, - "consistency_score_5": 0.2682539682539683, - "consistency_score_6": 0.2161904761904762, - "consistency_score_7": 0.17333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6333333333333333, - "Spanish,Chinese": 0.66, - "Spanish,Vietnamese": 0.6, - "Spanish,Filipino": 0.52, - "Spanish,Indonesian": 0.6666666666666666, - "Spanish,Malay": 0.66, - "English,Chinese": 0.6466666666666666, - "English,Vietnamese": 0.5733333333333334, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.6666666666666666, - "English,Malay": 0.6866666666666666, - "Chinese,Vietnamese": 0.5466666666666666, - "Chinese,Filipino": 0.48, - "Chinese,Indonesian": 0.6333333333333333, - "Chinese,Malay": 0.6066666666666667, - "Vietnamese,Filipino": 0.36666666666666664, - "Vietnamese,Indonesian": 0.6466666666666666, - "Vietnamese,Malay": 0.5866666666666667, - "Filipino,Indonesian": 0.52, - "Filipino,Malay": 0.4866666666666667, - "Indonesian,Malay": 0.6866666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5133333333333333, - "Spanish,English,Vietnamese": 0.47333333333333333, - "Spanish,English,Filipino": 0.36, - "Spanish,English,Indonesian": 0.5333333333333333, - "Spanish,English,Malay": 0.5133333333333333, - "Spanish,Chinese,Vietnamese": 0.44666666666666666, - "Spanish,Chinese,Filipino": 0.37333333333333335, - "Spanish,Chinese,Indonesian": 0.5133333333333333, - "Spanish,Chinese,Malay": 0.5, - "Spanish,Vietnamese,Filipino": 0.29333333333333333, - "Spanish,Vietnamese,Indonesian": 0.5133333333333333, - "Spanish,Vietnamese,Malay": 0.47333333333333333, - "Spanish,Filipino,Indonesian": 0.38666666666666666, - "Spanish,Filipino,Malay": 0.38666666666666666, - "Spanish,Indonesian,Malay": 0.5533333333333333, - "English,Chinese,Vietnamese": 0.44666666666666666, - "English,Chinese,Filipino": 0.36, - "English,Chinese,Indonesian": 0.52, - "English,Chinese,Malay": 0.5066666666666667, - "English,Vietnamese,Filipino": 0.2866666666666667, - "English,Vietnamese,Indonesian": 0.49333333333333335, - "English,Vietnamese,Malay": 0.47333333333333333, - "English,Filipino,Indonesian": 0.37333333333333335, - "English,Filipino,Malay": 0.36, - "English,Indonesian,Malay": 0.5533333333333333, - "Chinese,Vietnamese,Filipino": 0.26, - "Chinese,Vietnamese,Indonesian": 0.44666666666666666, - "Chinese,Vietnamese,Malay": 0.41333333333333333, - "Chinese,Filipino,Indonesian": 0.36, - "Chinese,Filipino,Malay": 0.3466666666666667, - "Chinese,Indonesian,Malay": 0.5133333333333333, - "Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Vietnamese,Filipino,Malay": 0.2866666666666667, - "Vietnamese,Indonesian,Malay": 0.49333333333333335, - "Filipino,Indonesian,Malay": 0.4 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.3933333333333333, - "Spanish,English,Chinese,Filipino": 0.29333333333333333, - "Spanish,English,Chinese,Indonesian": 0.4533333333333333, - "Spanish,English,Chinese,Malay": 0.4266666666666667, - "Spanish,English,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.4266666666666667, - "Spanish,English,Vietnamese,Malay": 0.4, - "Spanish,English,Filipino,Indonesian": 0.31333333333333335, - "Spanish,English,Filipino,Malay": 0.30666666666666664, - "Spanish,English,Indonesian,Malay": 0.47333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.4, - "Spanish,Chinese,Vietnamese,Malay": 0.36666666666666664, - "Spanish,Chinese,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Chinese,Filipino,Malay": 0.30666666666666664, - "Spanish,Chinese,Indonesian,Malay": 0.46, - "Spanish,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.3333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.22, - "English,Chinese,Vietnamese,Indonesian": 0.3933333333333333, - "English,Chinese,Vietnamese,Malay": 0.36666666666666664, - "English,Chinese,Filipino,Indonesian": 0.30666666666666664, - "English,Chinese,Filipino,Malay": 0.2866666666666667, - "English,Chinese,Indonesian,Malay": 0.44666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.26, - "English,Vietnamese,Filipino,Malay": 0.24666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.42, - "English,Filipino,Indonesian,Malay": 0.32, - "Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.2, - "Chinese,Vietnamese,Indonesian,Malay": 0.38, - "Chinese,Filipino,Indonesian,Malay": 0.3, - "Vietnamese,Filipino,Indonesian,Malay": 0.26 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.36, - "Spanish,English,Chinese,Vietnamese,Malay": 0.3333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Chinese,Indonesian,Malay": 0.4066666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino,Malay": 0.22, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.28, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.18, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.34, - "English,Chinese,Filipino,Indonesian,Malay": 0.26, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.32, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - } - }, - "AC3_2": 0.5538825641155053, - "AC3_3": 0.47211917123066494, - "AC3_4": 0.40868527490515527, - "AC3_5": 0.3548048477460248, - "AC3_6": 0.30606116316266074, - "AC3_7": 0.2604735883050778 - } + "prompt_1": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.41314935064935066, - "language_acc": { - "English": 0.4431818181818182, - "Filipino": 0.2897727272727273, - "Spanish": 0.38636363636363635, - "Malay": 0.42045454545454547, - "Chinese": 0.5170454545454546, - "Indonesian": 0.4147727272727273, - "Vietnamese": 0.42045454545454547 - }, - "consistency_score_2": 0.4843073593073593, - "consistency_score_3": 0.29951298701298706, - "consistency_score_4": 0.20779220779220778, - "consistency_score_5": 0.1534090909090909, - "consistency_score_6": 0.11525974025974026, - "consistency_score_7": 0.08522727272727272, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.48295454545454547, - "English,Malay": 0.5454545454545454, - "English,Chinese": 0.5170454545454546, - "English,Indonesian": 0.5284090909090909, - "English,Vietnamese": 0.4602272727272727, - "Filipino,Spanish": 0.4147727272727273, - "Filipino,Malay": 0.44886363636363635, - "Filipino,Chinese": 0.3806818181818182, - "Filipino,Indonesian": 0.3522727272727273, - "Filipino,Vietnamese": 0.3693181818181818, - "Spanish,Malay": 0.5738636363636364, - "Spanish,Chinese": 0.4943181818181818, - "Spanish,Indonesian": 0.5340909090909091, - "Spanish,Vietnamese": 0.48863636363636365, - "Malay,Chinese": 0.5227272727272727, - "Malay,Indonesian": 0.5795454545454546, - "Malay,Vietnamese": 0.48295454545454547, - "Chinese,Indonesian": 0.5284090909090909, - "Chinese,Vietnamese": 0.5738636363636364, - "Indonesian,Vietnamese": 0.5568181818181818 - }, - "3_combine": { - "English,Filipino,Spanish": 0.20454545454545456, - "English,Filipino,Malay": 0.25, - "English,Filipino,Chinese": 0.20454545454545456, - "English,Filipino,Indonesian": 0.2215909090909091, - "English,Filipino,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay": 0.35795454545454547, - "English,Spanish,Chinese": 0.32954545454545453, - "English,Spanish,Indonesian": 0.32386363636363635, - "English,Spanish,Vietnamese": 0.3068181818181818, - "English,Malay,Chinese": 0.35795454545454547, - "English,Malay,Indonesian": 0.39204545454545453, - "English,Malay,Vietnamese": 0.32386363636363635, - "English,Chinese,Indonesian": 0.35795454545454547, - "English,Chinese,Vietnamese": 0.3409090909090909, - "English,Indonesian,Vietnamese": 0.3465909090909091, - "Filipino,Spanish,Malay": 0.2897727272727273, - "Filipino,Spanish,Chinese": 0.2159090909090909, - "Filipino,Spanish,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Vietnamese": 0.21022727272727273, - "Filipino,Malay,Chinese": 0.25, - "Filipino,Malay,Indonesian": 0.26136363636363635, - "Filipino,Malay,Vietnamese": 0.2556818181818182, - "Filipino,Chinese,Indonesian": 0.21022727272727273, - "Filipino,Chinese,Vietnamese": 0.24431818181818182, - "Filipino,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Chinese": 0.35795454545454547, - "Spanish,Malay,Indonesian": 0.4034090909090909, - "Spanish,Malay,Vietnamese": 0.3352272727272727, - "Spanish,Chinese,Indonesian": 0.3409090909090909, - "Spanish,Chinese,Vietnamese": 0.32386363636363635, - "Spanish,Indonesian,Vietnamese": 0.3522727272727273, - "Malay,Chinese,Indonesian": 0.3806818181818182, - "Malay,Chinese,Vietnamese": 0.3352272727272727, - "Malay,Indonesian,Vietnamese": 0.375, - "Chinese,Indonesian,Vietnamese": 0.3806818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese": 0.17613636363636365, - "English,Filipino,Malay,Indonesian": 0.1875, - "English,Filipino,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese": 0.2784090909090909, - "English,Spanish,Malay,Indonesian": 0.2727272727272727, - "English,Spanish,Malay,Vietnamese": 0.24431818181818182, - "English,Spanish,Chinese,Indonesian": 0.25, - "English,Spanish,Chinese,Vietnamese": 0.25, - "English,Spanish,Indonesian,Vietnamese": 0.24431818181818182, - "English,Malay,Chinese,Indonesian": 0.29545454545454547, - "English,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "English,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "Filipino,Spanish,Malay,Chinese": 0.18181818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Chinese,Indonesian": 0.2784090909090909, - "Spanish,Malay,Chinese,Vietnamese": 0.24431818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.2840909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese,Indonesian": 0.2215909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 - } - }, - "AC3_2": 0.4459073485558842, - "AC3_3": 0.3472713220232767, - "AC3_4": 0.2765130294096747, - "AC3_5": 0.22373990618608852, - "AC3_6": 0.1802371974841941, - "AC3_7": 0.14130515247386782 - } + "prompt_1": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.5922330097087378 - } + "prompt_1": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.5333333333333333 - } + "prompt_1": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.5046728971962616 - } + "prompt_1": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.42, - "category_acc": { - "brand": 0.6, - "demographics": 0.2, - "biology": 0.3, - "history": 0.2, - "literature": 0.2, - "politics": 0.4, - "culture": 0.5, - "film": 0.6, - "law": 0.5, - "geography": 0.7 - } - } + "prompt_1": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.19336186518308476 - } + "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.33285629639027003 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.2794493953638315 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.20056802668301443 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.35421611380882756 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.5052508751458576 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.5027529495888452, - "category_acc": { - "high_school_european_history": 0.6158536585365854, - "business_ethics": 0.6060606060606061, - "clinical_knowledge": 0.5340909090909091, - "medical_genetics": 0.5858585858585859, - "high_school_us_history": 0.6305418719211823, - "high_school_physics": 0.32, - "high_school_world_history": 0.6483050847457628, - "virology": 0.4303030303030303, - "high_school_microeconomics": 0.5611814345991561, - "econometrics": 0.336283185840708, - "college_computer_science": 0.3939393939393939, - "high_school_biology": 0.6084142394822006, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.37722419928825623, - "philosophy": 0.5709677419354838, - "professional_medicine": 0.5202952029520295, - "nutrition": 0.5836065573770491, - "global_facts": 0.2828282828282828, - "machine_learning": 0.36036036036036034, - "security_studies": 0.5655737704918032, - "public_relations": 0.5229357798165137, - "professional_psychology": 0.47299509001636664, - "prehistory": 0.5541795665634675, - "anatomy": 0.4552238805970149, - "human_sexuality": 0.5846153846153846, - "college_medicine": 0.47674418604651164, - "high_school_government_and_politics": 0.7083333333333334, - "college_chemistry": 0.43434343434343436, - "logical_fallacies": 0.5925925925925926, - "high_school_geography": 0.6345177664974619, - "elementary_mathematics": 0.40848806366047746, - "human_aging": 0.5765765765765766, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.7261029411764706, - "formal_logic": 0.352, - "high_school_statistics": 0.4372093023255814, - "international_law": 0.6333333333333333, - "high_school_mathematics": 0.2899628252788104, - "high_school_computer_science": 0.5757575757575758, - "conceptual_physics": 0.4358974358974359, - "miscellaneous": 0.7289002557544757, - "high_school_chemistry": 0.4207920792079208, - "marketing": 0.7682403433476395, - "professional_law": 0.3424657534246575, - "management": 0.6666666666666666, - "college_physics": 0.31683168316831684, - "jurisprudence": 0.5607476635514018, - "world_religions": 0.6647058823529411, - "sociology": 0.72, - "us_foreign_policy": 0.6767676767676768, - "high_school_macroeconomics": 0.4832904884318766, - "computer_security": 0.6060606060606061, - "moral_scenarios": 0.2740492170022371, - "moral_disputes": 0.5217391304347826, - "electrical_engineering": 0.4930555555555556, - "astronomy": 0.5761589403973509, - "college_biology": 0.45454545454545453 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.5118870728083209 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.5211706102117061, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.5, - "college_programming": 0.4523809523809524, - "college_physics": 0.4583333333333333, - "college_chemistry": 0.4482758620689655, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.5862068965517241, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.5416666666666666, - "high_school_chemistry": 0.5, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.4166666666666667, - "middle_school_biology": 0.8076923076923077, - "middle_school_physics": 0.75, - "middle_school_chemistry": 0.68, - "veterinary_medicine": 0.5357142857142857, - "college_economics": 0.45, - "business_administration": 0.4473684210526316, - "marxism": 0.75, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.4411764705882353, - "teacher_qualification": 0.6326530612244898, - "high_school_politics": 0.875, - "high_school_geography": 0.7083333333333334, - "middle_school_politics": 0.7692307692307693, - "middle_school_geography": 0.5882352941176471, - "modern_chinese_history": 0.5714285714285714, - "ideological_and_moral_cultivation": 0.75, - "logic": 0.48148148148148145, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.5357142857142857, - "art_studies": 0.6052631578947368, - "professional_tour_guide": 0.6176470588235294, - "legal_professional": 0.4642857142857143, - "high_school_chinese": 0.5833333333333334, - "high_school_history": 0.72, - "middle_school_history": 0.6666666666666666, - "civil_servant": 0.46153846153846156, - "sports_science": 0.4583333333333333, - "plant_protection": 0.5925925925925926, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.5294117647058824, - "accountant": 0.4444444444444444, - "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.5555555555555556, - "tax_accountant": 0.37037037037037035, - "physician": 0.5370370370370371 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.5376344086021505 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.5248661716456571, - "category_acc": { - "agronomy": 0.5325443786982249, - "anatomy": 0.3918918918918919, - "ancient_chinese": 0.3475609756097561, - "arts": 0.73125, - "astronomy": 0.38181818181818183, - "business_ethics": 0.5119617224880383, - "chinese_civil_service_exam": 0.5125, - "chinese_driving_rule": 0.7251908396946565, - "chinese_food_culture": 0.5, - "chinese_foreign_policy": 0.5607476635514018, - "chinese_history": 0.6501547987616099, - "chinese_literature": 0.43137254901960786, - "chinese_teacher_qualification": 0.7206703910614525, - "clinical_knowledge": 0.4219409282700422, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.5887850467289719, - "college_engineering_hydrology": 0.4811320754716981, - "college_law": 0.37962962962962965, - "college_mathematics": 0.2857142857142857, - "college_medical_statistics": 0.4716981132075472, - "college_medicine": 0.5164835164835165, - "computer_science": 0.5196078431372549, - "computer_security": 0.5964912280701754, - "conceptual_physics": 0.564625850340136, - "construction_project_management": 0.4316546762589928, - "economics": 0.5408805031446541, - "education": 0.5828220858895705, - "electrical_engineering": 0.5465116279069767, - "elementary_chinese": 0.5476190476190477, - "elementary_commonsense": 0.5959595959595959, - "elementary_information_and_technology": 0.6890756302521008, - "elementary_mathematics": 0.3739130434782609, - "ethnology": 0.5407407407407407, - "food_science": 0.5454545454545454, - "genetics": 0.4375, - "global_facts": 0.5570469798657718, - "high_school_biology": 0.4556213017751479, - "high_school_chemistry": 0.4393939393939394, - "high_school_geography": 0.5508474576271186, - "high_school_mathematics": 0.34146341463414637, - "high_school_physics": 0.39090909090909093, - "high_school_politics": 0.6433566433566433, - "human_sexuality": 0.5079365079365079, - "international_law": 0.40540540540540543, - "journalism": 0.4883720930232558, - "jurisprudence": 0.48661800486618007, - "legal_and_moral_basis": 0.8691588785046729, - "logical": 0.42276422764227645, - "machine_learning": 0.48360655737704916, - "management": 0.6238095238095238, - "marketing": 0.6222222222222222, - "marxist_theory": 0.5925925925925926, - "modern_chinese": 0.4482758620689655, - "nutrition": 0.5241379310344828, - "philosophy": 0.5333333333333333, - "professional_accounting": 0.6114285714285714, - "professional_law": 0.44549763033175355, - "professional_medicine": 0.39361702127659576, - "professional_psychology": 0.5948275862068966, - "public_relations": 0.5747126436781609, - "security_study": 0.6222222222222222, - "sociology": 0.584070796460177, - "sports_science": 0.509090909090909, - "traditional_chinese_medicine": 0.4918918918918919, - "virology": 0.5088757396449705, - "world_history": 0.639751552795031, - "world_religions": 0.575 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.30303030303030304 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.47954545454545455 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.3945762711864407 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.8283470456245325 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.7221950024497795 - } + "prompt_1": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.35700750583113283, - "rouge2": 0.127516217338521, - "rougeL": 0.2753168594089233, - "avg_rouge": 0.2532801941928591 - } + "prompt_1": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.34957034313556845, - "rouge2": 0.11754497364833823, - "rougeL": 0.2693453676332944, - "avg_rouge": 0.24548689480573369 - } + "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.8727064220183486 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.6308724832214765 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.564 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.4425 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.652 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.5774647887323944 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.5740072202166066 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.6102941176470589 - } + "prompt_1": -1 } } }, - "sailor_0_5b": { - "model_size": "0.5B", - "model_link": "https://huggingface.co/sail/Sailor-0.5B", + "sailor_4b_chat": { + "model_size": "4B", + "model_link": "https://huggingface.co/sail/Sailor-4B-Chat", "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.24936974789915967, + "overall_acc": 0.859873949579832, "language_acc": { - "English": 0.24369747899159663, - "Chinese": 0.24201680672268908, - "Spanish": 0.24789915966386555, - "Vietnamese": 0.2638655462184874 - }, - "consistency_score_2": 0.26260504201680673, - "consistency_score_3": 0.07373949579831933, - "consistency_score_4": 0.0226890756302521, + "Spanish": 0.853781512605042, + "English": 0.8823529411764706, + "Chinese": 0.8487394957983193, + "Vietnamese": 0.8546218487394958 + }, + "consistency_score_2": 0.8463585434173669, + "consistency_score_3": 0.7785714285714285, + "consistency_score_4": 0.7344537815126051, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.2596638655462185, - "English,Spanish": 0.26218487394957984, - "English,Vietnamese": 0.2672268907563025, - "Chinese,Spanish": 0.23865546218487396, - "Chinese,Vietnamese": 0.25630252100840334, - "Spanish,Vietnamese": 0.2915966386554622 + "Spanish,English": 0.8689075630252101, + "Spanish,Chinese": 0.8403361344537815, + "Spanish,Vietnamese": 0.846218487394958, + "English,Chinese": 0.8495798319327731, + "English,Vietnamese": 0.846218487394958, + "Chinese,Vietnamese": 0.826890756302521 }, "3_combine": { - "English,Chinese,Spanish": 0.06890756302521009, - "English,Chinese,Vietnamese": 0.07058823529411765, - "English,Spanish,Vietnamese": 0.08487394957983194, - "Chinese,Spanish,Vietnamese": 0.07058823529411765 + "Spanish,English,Chinese": 0.788235294117647, + "Spanish,English,Vietnamese": 0.7890756302521008, + "Spanish,Chinese,Vietnamese": 0.7655462184873949, + "English,Chinese,Vietnamese": 0.7714285714285715 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.0226890756302521 + "Spanish,English,Chinese,Vietnamese": 0.7344537815126051 } }, - "AC3_2": 0.2558163190915596, - "AC3_3": 0.11382156239989012, - "AC3_4": 0.04159371855321998 + "AC3_2": 0.8530627173414949, + "AC3_3": 0.8172055024870784, + "AC3_4": 0.7922306832369328 }, "prompt_2": { - "overall_acc": 0.2621848739495798, + "overall_acc": 0.876890756302521, "language_acc": { - "English": 0.28739495798319326, - "Chinese": 0.2319327731092437, - "Spanish": 0.2638655462184874, - "Vietnamese": 0.26554621848739496 - }, - "consistency_score_2": 0.2661064425770308, - "consistency_score_3": 0.07289915966386555, - "consistency_score_4": 0.0226890756302521, + "Spanish": 0.8747899159663866, + "English": 0.8932773109243698, + "Chinese": 0.8663865546218488, + "Vietnamese": 0.873109243697479 + }, + "consistency_score_2": 0.8628851540616246, + "consistency_score_3": 0.8012605042016806, + "consistency_score_4": 0.7588235294117647, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.24705882352941178, - "English,Spanish": 0.25798319327731095, - "English,Vietnamese": 0.2680672268907563, - "Chinese,Spanish": 0.26218487394957984, - "Chinese,Vietnamese": 0.26218487394957984, - "Spanish,Vietnamese": 0.2991596638655462 + "Spanish,English": 0.8966386554621849, + "Spanish,Chinese": 0.8571428571428571, + "Spanish,Vietnamese": 0.8546218487394958, + "English,Chinese": 0.8697478991596639, + "English,Vietnamese": 0.8579831932773109, + "Chinese,Vietnamese": 0.8411764705882353 }, "3_combine": { - "English,Chinese,Spanish": 0.06134453781512605, - "English,Chinese,Vietnamese": 0.06974789915966387, - "English,Spanish,Vietnamese": 0.08319327731092437, - "Chinese,Spanish,Vietnamese": 0.0773109243697479 + "Spanish,English,Chinese": 0.8159663865546218, + "Spanish,English,Vietnamese": 0.8117647058823529, + "Spanish,Chinese,Vietnamese": 0.7840336134453781, + "English,Chinese,Vietnamese": 0.7932773109243697 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.0226890756302521 + "Spanish,English,Chinese,Vietnamese": 0.7588235294117647 } }, - "AC3_2": 0.26413110308054444, - "AC3_3": 0.11407918649950263, - "AC3_4": 0.041763962207396456 + "AC3_2": 0.869831580948205, + "AC3_3": 0.8373713931743918, + "AC3_4": 0.8135960471541407 }, "prompt_3": { - "overall_acc": 0.26764705882352946, + "overall_acc": 0.8745798319327731, "language_acc": { - "English": 0.280672268907563, - "Chinese": 0.24873949579831933, - "Spanish": 0.2630252100840336, - "Vietnamese": 0.2781512605042017 - }, - "consistency_score_2": 0.2669467787114846, - "consistency_score_3": 0.07668067226890757, - "consistency_score_4": 0.02100840336134454, + "Spanish": 0.8714285714285714, + "English": 0.892436974789916, + "Chinese": 0.8697478991596639, + "Vietnamese": 0.8647058823529412 + }, + "consistency_score_2": 0.8641456582633054, + "consistency_score_3": 0.8033613445378152, + "consistency_score_4": 0.7630252100840336, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.2647058823529412, - "English,Spanish": 0.26974789915966385, - "English,Vietnamese": 0.2722689075630252, - "Chinese,Spanish": 0.2638655462184874, - "Chinese,Vietnamese": 0.24285714285714285, - "Spanish,Vietnamese": 0.28823529411764703 + "Spanish,English": 0.8865546218487395, + "Spanish,Chinese": 0.8579831932773109, + "Spanish,Vietnamese": 0.8596638655462185, + "English,Chinese": 0.8705882352941177, + "English,Vietnamese": 0.8630252100840337, + "Chinese,Vietnamese": 0.8470588235294118 }, "3_combine": { - "English,Chinese,Spanish": 0.07058823529411765, - "English,Chinese,Vietnamese": 0.06974789915966387, - "English,Spanish,Vietnamese": 0.09243697478991597, - "Chinese,Spanish,Vietnamese": 0.07394957983193277 + "Spanish,English,Chinese": 0.8126050420168067, + "Spanish,English,Vietnamese": 0.8117647058823529, + "Spanish,Chinese,Vietnamese": 0.7907563025210084, + "English,Chinese,Vietnamese": 0.7983193277310925 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.02100840336134454 + "Spanish,English,Chinese,Vietnamese": 0.7630252100840336 } }, - "AC3_2": 0.26729646005880126, - "AC3_3": 0.1192082689959935, - "AC3_4": 0.0389588149535382 + "AC3_2": 0.8693314370638944, + "AC3_3": 0.8374591904624037, + "AC3_4": 0.8150029376230561 }, "prompt_4": { - "overall_acc": 0.2588235294117647, + "overall_acc": 0.8657563025210084, "language_acc": { - "English": 0.25546218487394956, - "Chinese": 0.25546218487394956, - "Spanish": 0.2571428571428571, - "Vietnamese": 0.2672268907563025 + "Spanish": 0.8554621848739495, + "English": 0.8890756302521008, + "Chinese": 0.8512605042016806, + "Vietnamese": 0.8672268907563025 }, - "consistency_score_2": 0.2655462184873949, - "consistency_score_3": 0.07247899159663866, - "consistency_score_4": 0.0226890756302521, + "consistency_score_2": 0.8463585434173669, + "consistency_score_3": 0.7800420168067226, + "consistency_score_4": 0.7361344537815127, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.25798319327731095, - "English,Spanish": 0.253781512605042, - "English,Vietnamese": 0.26554621848739496, - "Chinese,Spanish": 0.246218487394958, - "Chinese,Vietnamese": 0.28823529411764703, - "Spanish,Vietnamese": 0.2815126050420168 + "Spanish,English": 0.880672268907563, + "Spanish,Chinese": 0.8302521008403362, + "Spanish,Vietnamese": 0.846218487394958, + "English,Chinese": 0.8512605042016806, + "English,Vietnamese": 0.8512605042016806, + "Chinese,Vietnamese": 0.8184873949579832 }, "3_combine": { - "English,Chinese,Spanish": 0.05966386554621849, - "English,Chinese,Vietnamese": 0.07899159663865546, - "English,Spanish,Vietnamese": 0.07899159663865546, - "Chinese,Spanish,Vietnamese": 0.07226890756302522 + "Spanish,English,Chinese": 0.7924369747899159, + "Spanish,English,Vietnamese": 0.7966386554621848, + "Spanish,Chinese,Vietnamese": 0.7588235294117647, + "English,Chinese,Vietnamese": 0.7722689075630252 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.0226890756302521 + "Spanish,English,Chinese,Vietnamese": 0.7361344537815127 } }, - "AC3_2": 0.2621417797388468, - "AC3_3": 0.11324555183272844, - "AC3_4": 0.041720807711255276 + "AC3_2": 0.8559475374581995, + "AC3_3": 0.8206671307289214, + "AC3_4": 0.7957010055984883 }, "prompt_5": { - "overall_acc": 0.2741596638655462, + "overall_acc": 0.8590336134453782, "language_acc": { - "English": 0.3050420168067227, - "Chinese": 0.2596638655462185, - "Spanish": 0.2672268907563025, - "Vietnamese": 0.2647058823529412 + "Spanish": 0.8369747899159664, + "English": 0.8831932773109243, + "Chinese": 0.8546218487394958, + "Vietnamese": 0.8613445378151261 }, - "consistency_score_2": 0.2612044817927171, - "consistency_score_3": 0.07058823529411765, - "consistency_score_4": 0.019327731092436976, + "consistency_score_2": 0.8570028011204482, + "consistency_score_3": 0.7932773109243697, + "consistency_score_4": 0.7504201680672269, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.2605042016806723, - "English,Spanish": 0.2504201680672269, - "English,Vietnamese": 0.25126050420168067, - "Chinese,Spanish": 0.26218487394957984, - "Chinese,Vietnamese": 0.24369747899159663, - "Spanish,Vietnamese": 0.2991596638655462 + "Spanish,English": 0.8714285714285714, + "Spanish,Chinese": 0.8470588235294118, + "Spanish,Vietnamese": 0.8596638655462185, + "English,Chinese": 0.8714285714285714, + "English,Vietnamese": 0.8638655462184874, + "Chinese,Vietnamese": 0.8285714285714286 }, "3_combine": { - "English,Chinese,Spanish": 0.06722689075630252, - "English,Chinese,Vietnamese": 0.06050420168067227, - "English,Spanish,Vietnamese": 0.07983193277310924, - "Chinese,Spanish,Vietnamese": 0.07478991596638655 + "Spanish,English,Chinese": 0.8016806722689076, + "Spanish,English,Vietnamese": 0.8058823529411765, + "Spanish,Chinese,Vietnamese": 0.7756302521008404, + "English,Chinese,Vietnamese": 0.7899159663865546 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.019327731092436976 + "Spanish,English,Chinese,Vietnamese": 0.7504201680672269 } }, - "AC3_2": 0.2675253227017591, - "AC3_3": 0.11227013654125925, - "AC3_4": 0.036109791076644415 + "AC3_2": 0.8580170055688157, + "AC3_3": 0.8248470245832001, + "AC3_4": 0.8010620198502123 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.2495238095238095, + "overall_acc": 0.48000000000000004, "language_acc": { - "Spanish": 0.25333333333333335, - "English": 0.22, - "Chinese": 0.24, - "Vietnamese": 0.28, - "Filipino": 0.22666666666666666, - "Indonesian": 0.23333333333333334, - "Malay": 0.29333333333333333 - }, - "consistency_score_2": 0.26412698412698415, - "consistency_score_3": 0.07352380952380953, - "consistency_score_4": 0.022857142857142854, - "consistency_score_5": 0.00984126984126984, - "consistency_score_6": 0.006666666666666667, - "consistency_score_7": 0.006666666666666667, + "Filipino": 0.36666666666666664, + "Vietnamese": 0.44666666666666666, + "Chinese": 0.5133333333333333, + "Spanish": 0.4533333333333333, + "Malay": 0.5333333333333333, + "Indonesian": 0.48, + "English": 0.5666666666666667 + }, + "consistency_score_2": 0.6336507936507936, + "consistency_score_3": 0.49371428571428566, + "consistency_score_4": 0.41638095238095235, + "consistency_score_5": 0.36634920634920637, + "consistency_score_6": 0.3314285714285714, + "consistency_score_7": 0.30666666666666664, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.18666666666666668, - "Spanish,Chinese": 0.23333333333333334, - "Spanish,Vietnamese": 0.32666666666666666, - "Spanish,Filipino": 0.29333333333333333, - "Spanish,Indonesian": 0.22666666666666666, - "Spanish,Malay": 0.29333333333333333, - "English,Chinese": 0.2866666666666667, - "English,Vietnamese": 0.2866666666666667, - "English,Filipino": 0.24666666666666667, - "English,Indonesian": 0.20666666666666667, - "English,Malay": 0.24, - "Chinese,Vietnamese": 0.3, - "Chinese,Filipino": 0.29333333333333333, - "Chinese,Indonesian": 0.2733333333333333, - "Chinese,Malay": 0.26, - "Vietnamese,Filipino": 0.26666666666666666, - "Vietnamese,Indonesian": 0.2733333333333333, - "Vietnamese,Malay": 0.29333333333333333, - "Filipino,Indonesian": 0.22666666666666666, - "Filipino,Malay": 0.2866666666666667, - "Indonesian,Malay": 0.24666666666666667 + "Filipino,Vietnamese": 0.54, + "Filipino,Chinese": 0.47333333333333333, + "Filipino,Spanish": 0.5533333333333333, + "Filipino,Malay": 0.5666666666666667, + "Filipino,Indonesian": 0.5, + "Filipino,English": 0.5666666666666667, + "Vietnamese,Chinese": 0.6333333333333333, + "Vietnamese,Spanish": 0.62, + "Vietnamese,Malay": 0.68, + "Vietnamese,Indonesian": 0.66, + "Vietnamese,English": 0.68, + "Chinese,Spanish": 0.6133333333333333, + "Chinese,Malay": 0.6466666666666666, + "Chinese,Indonesian": 0.66, + "Chinese,English": 0.7133333333333334, + "Spanish,Malay": 0.6533333333333333, + "Spanish,Indonesian": 0.6666666666666666, + "Spanish,English": 0.6866666666666666, + "Malay,Indonesian": 0.72, + "Malay,English": 0.76, + "Indonesian,English": 0.7133333333333334 }, "3_combine": { - "Spanish,English,Chinese": 0.03333333333333333, - "Spanish,English,Vietnamese": 0.05333333333333334, - "Spanish,English,Filipino": 0.05333333333333334, - "Spanish,English,Indonesian": 0.02666666666666667, - "Spanish,English,Malay": 0.07333333333333333, - "Spanish,Chinese,Vietnamese": 0.08666666666666667, - "Spanish,Chinese,Filipino": 0.09333333333333334, - "Spanish,Chinese,Indonesian": 0.07333333333333333, - "Spanish,Chinese,Malay": 0.04666666666666667, - "Spanish,Vietnamese,Filipino": 0.09333333333333334, - "Spanish,Vietnamese,Indonesian": 0.09333333333333334, - "Spanish,Vietnamese,Malay": 0.09333333333333334, - "Spanish,Filipino,Indonesian": 0.08666666666666667, - "Spanish,Filipino,Malay": 0.12666666666666668, - "Spanish,Indonesian,Malay": 0.06, - "English,Chinese,Vietnamese": 0.12666666666666668, - "English,Chinese,Filipino": 0.06, - "English,Chinese,Indonesian": 0.07333333333333333, - "English,Chinese,Malay": 0.08, - "English,Vietnamese,Filipino": 0.06666666666666667, - "English,Vietnamese,Indonesian": 0.06, - "English,Vietnamese,Malay": 0.07333333333333333, - "English,Filipino,Indonesian": 0.04, - "English,Filipino,Malay": 0.05333333333333334, - "English,Indonesian,Malay": 0.04666666666666667, - "Chinese,Vietnamese,Filipino": 0.08666666666666667, - "Chinese,Vietnamese,Indonesian": 0.08666666666666667, - "Chinese,Vietnamese,Malay": 0.08666666666666667, - "Chinese,Filipino,Indonesian": 0.05333333333333334, - "Chinese,Filipino,Malay": 0.1, - "Chinese,Indonesian,Malay": 0.07333333333333333, - "Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Vietnamese,Filipino,Malay": 0.08666666666666667, - "Vietnamese,Indonesian,Malay": 0.08666666666666667, - "Filipino,Indonesian,Malay": 0.06666666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.02, - "Spanish,English,Chinese,Filipino": 0.02, - "Spanish,English,Chinese,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Malay": 0.013333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.02, - "Spanish,English,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,English,Vietnamese,Malay": 0.02666666666666667, - "Spanish,English,Filipino,Indonesian": 0.013333333333333334, - "Spanish,English,Filipino,Malay": 0.02666666666666667, - "Spanish,English,Indonesian,Malay": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Filipino": 0.02666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.04, - "Spanish,Chinese,Vietnamese,Malay": 0.006666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.03333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.013333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.04, - "Spanish,Vietnamese,Indonesian,Malay": 0.02666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.02666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.02, - "English,Chinese,Vietnamese,Indonesian": 0.02666666666666667, - "English,Chinese,Vietnamese,Malay": 0.04666666666666667, - "English,Chinese,Filipino,Indonesian": 0.013333333333333334, - "English,Chinese,Filipino,Malay": 0.02, - "English,Chinese,Indonesian,Malay": 0.02666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "English,Vietnamese,Filipino,Malay": 0.02, - "English,Vietnamese,Indonesian,Malay": 0.02, - "English,Filipino,Indonesian,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.03333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.02666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.013333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.02666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.013333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.013333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.02, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.013333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667 - } - }, - "AC3_2": 0.25661781147514484, - "AC3_3": 0.11358041326223542, - "AC3_4": 0.04187812186274703, - "AC3_5": 0.018935711364151085, - "AC3_6": 0.01298636926382812, - "AC3_7": 0.01298636926382812 - }, - "prompt_2": { - "overall_acc": 0.25047619047619046, + "Filipino,Vietnamese,Chinese": 0.3933333333333333, + "Filipino,Vietnamese,Spanish": 0.41333333333333333, + "Filipino,Vietnamese,Malay": 0.4533333333333333, + "Filipino,Vietnamese,Indonesian": 0.41333333333333333, + "Filipino,Vietnamese,English": 0.44666666666666666, + "Filipino,Chinese,Spanish": 0.38666666666666666, + "Filipino,Chinese,Malay": 0.4, + "Filipino,Chinese,Indonesian": 0.37333333333333335, + "Filipino,Chinese,English": 0.4266666666666667, + "Filipino,Spanish,Malay": 0.4533333333333333, + "Filipino,Spanish,Indonesian": 0.4066666666666667, + "Filipino,Spanish,English": 0.4533333333333333, + "Filipino,Malay,Indonesian": 0.44666666666666666, + "Filipino,Malay,English": 0.4866666666666667, + "Filipino,Indonesian,English": 0.4266666666666667, + "Vietnamese,Chinese,Spanish": 0.48, + "Vietnamese,Chinese,Malay": 0.5266666666666666, + "Vietnamese,Chinese,Indonesian": 0.5133333333333333, + "Vietnamese,Chinese,English": 0.5466666666666666, + "Vietnamese,Spanish,Malay": 0.52, + "Vietnamese,Spanish,Indonesian": 0.5133333333333333, + "Vietnamese,Spanish,English": 0.5133333333333333, + "Vietnamese,Malay,Indonesian": 0.58, + "Vietnamese,Malay,English": 0.6, + "Vietnamese,Indonesian,English": 0.5466666666666666, + "Chinese,Spanish,Malay": 0.5, + "Chinese,Spanish,Indonesian": 0.5133333333333333, + "Chinese,Spanish,English": 0.5266666666666666, + "Chinese,Malay,Indonesian": 0.5466666666666666, + "Chinese,Malay,English": 0.6, + "Chinese,Indonesian,English": 0.5733333333333334, + "Spanish,Malay,Indonesian": 0.54, + "Spanish,Malay,English": 0.58, + "Spanish,Indonesian,English": 0.56, + "Malay,Indonesian,English": 0.62 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.3466666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.36, + "Filipino,Vietnamese,Chinese,Indonesian": 0.3466666666666667, + "Filipino,Vietnamese,Chinese,English": 0.37333333333333335, + "Filipino,Vietnamese,Spanish,Malay": 0.38666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.36666666666666664, + "Filipino,Vietnamese,Spanish,English": 0.38, + "Filipino,Vietnamese,Malay,Indonesian": 0.38666666666666666, + "Filipino,Vietnamese,Malay,English": 0.4066666666666667, + "Filipino,Vietnamese,Indonesian,English": 0.36666666666666664, + "Filipino,Chinese,Spanish,Malay": 0.35333333333333333, + "Filipino,Chinese,Spanish,Indonesian": 0.3466666666666667, + "Filipino,Chinese,Spanish,English": 0.36666666666666664, + "Filipino,Chinese,Malay,Indonesian": 0.36, + "Filipino,Chinese,Malay,English": 0.3933333333333333, + "Filipino,Chinese,Indonesian,English": 0.36, + "Filipino,Spanish,Malay,Indonesian": 0.38666666666666666, + "Filipino,Spanish,Malay,English": 0.42, + "Filipino,Spanish,Indonesian,English": 0.38, + "Filipino,Malay,Indonesian,English": 0.4066666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.4266666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.42, + "Vietnamese,Chinese,Spanish,English": 0.43333333333333335, + "Vietnamese,Chinese,Malay,Indonesian": 0.48, + "Vietnamese,Chinese,Malay,English": 0.5066666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.47333333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.4666666666666667, + "Vietnamese,Spanish,Malay,English": 0.48, + "Vietnamese,Spanish,Indonesian,English": 0.44666666666666666, + "Vietnamese,Malay,Indonesian,English": 0.5333333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, + "Chinese,Spanish,Malay,English": 0.47333333333333333, + "Chinese,Spanish,Indonesian,English": 0.4666666666666667, + "Chinese,Malay,Indonesian,English": 0.52, + "Spanish,Malay,Indonesian,English": 0.5066666666666667 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.32666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.32666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.3333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.3333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.35333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.3333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,Spanish,Malay,English": 0.36, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.34, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.36, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.3333333333333333, + "Filipino,Chinese,Spanish,Malay,English": 0.3466666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.3333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.35333333333333333, + "Filipino,Spanish,Malay,Indonesian,English": 0.36666666666666664, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.3933333333333333, + "Vietnamese,Chinese,Spanish,Malay,English": 0.41333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.3933333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.4666666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.44, + "Chinese,Spanish,Malay,Indonesian,English": 0.43333333333333335 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.31333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.32, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.31333333333333335, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.32666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.3333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.38666666666666666 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.30666666666666664 + } + }, + "AC3_2": 0.5462257696202791, + "AC3_3": 0.48676056333029155, + "AC3_4": 0.4459328516289599, + "AC3_5": 0.4155438859223945, + "AC3_6": 0.39211267600801425, + "AC3_7": 0.3742372880880207 + }, + "prompt_2": { + "overall_acc": 0.48, "language_acc": { - "Spanish": 0.18, - "English": 0.32, - "Chinese": 0.28, - "Vietnamese": 0.26666666666666666, - "Filipino": 0.24666666666666667, - "Indonesian": 0.23333333333333334, - "Malay": 0.22666666666666666 - }, - "consistency_score_2": 0.2701587301587302, - "consistency_score_3": 0.08057142857142856, - "consistency_score_4": 0.02742857142857143, - "consistency_score_5": 0.012063492063492063, - "consistency_score_6": 0.00761904761904762, - "consistency_score_7": 0.006666666666666667, + "Filipino": 0.36, + "Vietnamese": 0.4666666666666667, + "Chinese": 0.49333333333333335, + "Spanish": 0.4666666666666667, + "Malay": 0.5333333333333333, + "Indonesian": 0.48, + "English": 0.56 + }, + "consistency_score_2": 0.6298412698412699, + "consistency_score_3": 0.48876190476190484, + "consistency_score_4": 0.41066666666666674, + "consistency_score_5": 0.36000000000000004, + "consistency_score_6": 0.3247619047619047, + "consistency_score_7": 0.3, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.23333333333333334, - "Spanish,Chinese": 0.2733333333333333, - "Spanish,Vietnamese": 0.32, - "Spanish,Filipino": 0.2733333333333333, - "Spanish,Indonesian": 0.30666666666666664, - "Spanish,Malay": 0.3, - "English,Chinese": 0.24, - "English,Vietnamese": 0.24666666666666667, - "English,Filipino": 0.28, - "English,Indonesian": 0.20666666666666667, - "English,Malay": 0.2733333333333333, - "Chinese,Vietnamese": 0.2866666666666667, - "Chinese,Filipino": 0.24, - "Chinese,Indonesian": 0.28, - "Chinese,Malay": 0.21333333333333335, - "Vietnamese,Filipino": 0.34, - "Vietnamese,Indonesian": 0.2866666666666667, - "Vietnamese,Malay": 0.3, - "Filipino,Indonesian": 0.22, - "Filipino,Malay": 0.24, - "Indonesian,Malay": 0.31333333333333335 - }, - "3_combine": { - "Spanish,English,Chinese": 0.08, - "Spanish,English,Vietnamese": 0.11333333333333333, - "Spanish,English,Filipino": 0.06666666666666667, - "Spanish,English,Indonesian": 0.06, - "Spanish,English,Malay": 0.06666666666666667, - "Spanish,Chinese,Vietnamese": 0.14, - "Spanish,Chinese,Filipino": 0.06666666666666667, - "Spanish,Chinese,Indonesian": 0.09333333333333334, - "Spanish,Chinese,Malay": 0.09333333333333334, - "Spanish,Vietnamese,Filipino": 0.09333333333333334, - "Spanish,Vietnamese,Indonesian": 0.12666666666666668, - "Spanish,Vietnamese,Malay": 0.09333333333333334, - "Spanish,Filipino,Indonesian": 0.08, - "Spanish,Filipino,Malay": 0.05333333333333334, - "Spanish,Indonesian,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese": 0.07333333333333333, - "English,Chinese,Filipino": 0.06666666666666667, - "English,Chinese,Indonesian": 0.07333333333333333, - "English,Chinese,Malay": 0.06666666666666667, - "English,Vietnamese,Filipino": 0.10666666666666667, - "English,Vietnamese,Indonesian": 0.06, - "English,Vietnamese,Malay": 0.08666666666666667, - "English,Filipino,Indonesian": 0.05333333333333334, - "English,Filipino,Malay": 0.06666666666666667, - "English,Indonesian,Malay": 0.06, - "Chinese,Vietnamese,Filipino": 0.09333333333333334, - "Chinese,Vietnamese,Indonesian": 0.1, - "Chinese,Vietnamese,Malay": 0.09333333333333334, - "Chinese,Filipino,Indonesian": 0.05333333333333334, - "Chinese,Filipino,Malay": 0.04, - "Chinese,Indonesian,Malay": 0.06, - "Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Vietnamese,Filipino,Malay": 0.10666666666666667, - "Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Filipino,Indonesian,Malay": 0.06 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.05333333333333334, - "Spanish,English,Chinese,Filipino": 0.02, - "Spanish,English,Chinese,Indonesian": 0.02, - "Spanish,English,Chinese,Malay": 0.02666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.04, - "Spanish,English,Vietnamese,Indonesian": 0.04, - "Spanish,English,Vietnamese,Malay": 0.03333333333333333, - "Spanish,English,Filipino,Indonesian": 0.02, - "Spanish,English,Filipino,Malay": 0.013333333333333334, - "Spanish,English,Indonesian,Malay": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Filipino": 0.03333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.04666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.06, - "Spanish,Chinese,Filipino,Indonesian": 0.02666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.02, - "Spanish,Chinese,Indonesian,Malay": 0.03333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.04, - "Spanish,Vietnamese,Filipino,Malay": 0.013333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.03333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.02, - "English,Chinese,Vietnamese,Filipino": 0.02666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.02, - "English,Chinese,Vietnamese,Malay": 0.04, - "English,Chinese,Filipino,Indonesian": 0.02, - "English,Chinese,Filipino,Malay": 0.02, - "English,Chinese,Indonesian,Malay": 0.02, - "English,Vietnamese,Filipino,Indonesian": 0.02666666666666667, - "English,Vietnamese,Filipino,Malay": 0.02, - "English,Vietnamese,Indonesian,Malay": 0.02, - "English,Filipino,Indonesian,Malay": 0.02, - "Chinese,Vietnamese,Filipino,Indonesian": 0.02666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.02, - "Chinese,Vietnamese,Indonesian,Malay": 0.03333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.02, - "Vietnamese,Filipino,Indonesian,Malay": 0.02 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.02, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.02, - "Spanish,English,Chinese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.02, - "Spanish,English,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.02, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.013333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.013333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.013333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667 - } - }, - "AC3_2": 0.25994541226133744, - "AC3_3": 0.12192339302580397, - "AC3_4": 0.04944286691650813, - "AC3_5": 0.02301836816074932, - "AC3_6": 0.0147882621626062, - "AC3_7": 0.012987654315936901 - }, - "prompt_3": { - "overall_acc": 0.2533333333333333, + "Filipino,Vietnamese": 0.5, + "Filipino,Chinese": 0.5066666666666667, + "Filipino,Spanish": 0.52, + "Filipino,Malay": 0.5266666666666666, + "Filipino,Indonesian": 0.52, + "Filipino,English": 0.56, + "Vietnamese,Chinese": 0.6266666666666667, + "Vietnamese,Spanish": 0.5533333333333333, + "Vietnamese,Malay": 0.68, + "Vietnamese,Indonesian": 0.6866666666666666, + "Vietnamese,English": 0.6933333333333334, + "Chinese,Spanish": 0.58, + "Chinese,Malay": 0.6866666666666666, + "Chinese,Indonesian": 0.68, + "Chinese,English": 0.7533333333333333, + "Spanish,Malay": 0.6133333333333333, + "Spanish,Indonesian": 0.64, + "Spanish,English": 0.6666666666666666, + "Malay,Indonesian": 0.7533333333333333, + "Malay,English": 0.72, + "Indonesian,English": 0.76 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.3933333333333333, + "Filipino,Vietnamese,Spanish": 0.36666666666666664, + "Filipino,Vietnamese,Malay": 0.41333333333333333, + "Filipino,Vietnamese,Indonesian": 0.4066666666666667, + "Filipino,Vietnamese,English": 0.41333333333333333, + "Filipino,Chinese,Spanish": 0.37333333333333335, + "Filipino,Chinese,Malay": 0.41333333333333333, + "Filipino,Chinese,Indonesian": 0.3933333333333333, + "Filipino,Chinese,English": 0.44666666666666666, + "Filipino,Spanish,Malay": 0.3933333333333333, + "Filipino,Spanish,Indonesian": 0.4066666666666667, + "Filipino,Spanish,English": 0.43333333333333335, + "Filipino,Malay,Indonesian": 0.44666666666666666, + "Filipino,Malay,English": 0.4533333333333333, + "Filipino,Indonesian,English": 0.44666666666666666, + "Vietnamese,Chinese,Spanish": 0.44, + "Vietnamese,Chinese,Malay": 0.54, + "Vietnamese,Chinese,Indonesian": 0.5266666666666666, + "Vietnamese,Chinese,English": 0.5666666666666667, + "Vietnamese,Spanish,Malay": 0.4866666666666667, + "Vietnamese,Spanish,Indonesian": 0.49333333333333335, + "Vietnamese,Spanish,English": 0.49333333333333335, + "Vietnamese,Malay,Indonesian": 0.6066666666666667, + "Vietnamese,Malay,English": 0.6, + "Vietnamese,Indonesian,English": 0.5933333333333334, + "Chinese,Spanish,Malay": 0.4866666666666667, + "Chinese,Spanish,Indonesian": 0.4866666666666667, + "Chinese,Spanish,English": 0.5266666666666666, + "Chinese,Malay,Indonesian": 0.58, + "Chinese,Malay,English": 0.6066666666666667, + "Chinese,Indonesian,English": 0.62, + "Spanish,Malay,Indonesian": 0.5333333333333333, + "Spanish,Malay,English": 0.5333333333333333, + "Spanish,Indonesian,English": 0.5533333333333333, + "Malay,Indonesian,English": 0.6333333333333333 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.32666666666666666, + "Filipino,Vietnamese,Chinese,Malay": 0.36666666666666664, + "Filipino,Vietnamese,Chinese,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,Chinese,English": 0.37333333333333335, + "Filipino,Vietnamese,Spanish,Malay": 0.3333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian": 0.34, + "Filipino,Vietnamese,Spanish,English": 0.34, + "Filipino,Vietnamese,Malay,Indonesian": 0.38, + "Filipino,Vietnamese,Malay,English": 0.38666666666666666, + "Filipino,Vietnamese,Indonesian,English": 0.37333333333333335, + "Filipino,Chinese,Spanish,Malay": 0.34, + "Filipino,Chinese,Spanish,Indonesian": 0.3466666666666667, + "Filipino,Chinese,Spanish,English": 0.36, + "Filipino,Chinese,Malay,Indonesian": 0.37333333333333335, + "Filipino,Chinese,Malay,English": 0.3933333333333333, + "Filipino,Chinese,Indonesian,English": 0.38, + "Filipino,Spanish,Malay,Indonesian": 0.36666666666666664, + "Filipino,Spanish,Malay,English": 0.36666666666666664, + "Filipino,Spanish,Indonesian,English": 0.38666666666666666, + "Filipino,Malay,Indonesian,English": 0.4066666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.42, + "Vietnamese,Chinese,Spanish,Indonesian": 0.4066666666666667, + "Vietnamese,Chinese,Spanish,English": 0.43333333333333335, + "Vietnamese,Chinese,Malay,Indonesian": 0.49333333333333335, + "Vietnamese,Chinese,Malay,English": 0.5133333333333333, + "Vietnamese,Chinese,Indonesian,English": 0.5, + "Vietnamese,Spanish,Malay,Indonesian": 0.46, + "Vietnamese,Spanish,Malay,English": 0.46, + "Vietnamese,Spanish,Indonesian,English": 0.4533333333333333, + "Vietnamese,Malay,Indonesian,English": 0.5533333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.44, + "Chinese,Spanish,Malay,English": 0.4666666666666667, + "Chinese,Spanish,Indonesian,English": 0.4666666666666667, + "Chinese,Malay,Indonesian,English": 0.5333333333333333, + "Spanish,Malay,Indonesian,English": 0.48 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.31333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.31333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.32, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.3466666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.35333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.34, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.32666666666666666, + "Filipino,Vietnamese,Spanish,Malay,English": 0.32, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.32666666666666666, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.36, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, + "Filipino,Chinese,Spanish,Malay,English": 0.3333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.34, + "Filipino,Chinese,Malay,Indonesian,English": 0.36, + "Filipino,Spanish,Malay,Indonesian,English": 0.3466666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.3933333333333333, + "Vietnamese,Chinese,Spanish,Malay,English": 0.41333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.47333333333333333, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.43333333333333335, + "Chinese,Spanish,Malay,Indonesian,English": 0.42 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.3333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.31333333333333335, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.32, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.38666666666666666 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3 + } + }, + "AC3_2": 0.5448054919417581, + "AC3_3": 0.4843413290888165, + "AC3_4": 0.44263473048922514, + "AC3_5": 0.4114285713795918, + "AC3_6": 0.38740828397552907, + "AC3_7": 0.3692307691834319 + }, + "prompt_3": { + "overall_acc": 0.48761904761904756, "language_acc": { - "Spanish": 0.2, - "English": 0.30666666666666664, - "Chinese": 0.26666666666666666, - "Vietnamese": 0.21333333333333335, - "Filipino": 0.31333333333333335, - "Indonesian": 0.23333333333333334, - "Malay": 0.24 + "Filipino": 0.3333333333333333, + "Vietnamese": 0.49333333333333335, + "Chinese": 0.49333333333333335, + "Spanish": 0.47333333333333333, + "Malay": 0.5066666666666667, + "Indonesian": 0.5466666666666666, + "English": 0.5666666666666667 }, - "consistency_score_2": 0.26285714285714284, - "consistency_score_3": 0.0739047619047619, - "consistency_score_4": 0.021714285714285717, - "consistency_score_5": 0.006031746031746032, - "consistency_score_6": 0.0009523809523809525, - "consistency_score_7": 0.0, + "consistency_score_2": 0.633015873015873, + "consistency_score_3": 0.49161904761904773, + "consistency_score_4": 0.4108571428571428, + "consistency_score_5": 0.3558730158730159, + "consistency_score_6": 0.31428571428571433, + "consistency_score_7": 0.28, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.24, - "Spanish,Chinese": 0.29333333333333333, - "Spanish,Vietnamese": 0.32666666666666666, - "Spanish,Filipino": 0.3, - "Spanish,Indonesian": 0.26666666666666666, - "Spanish,Malay": 0.26, - "English,Chinese": 0.21333333333333335, - "English,Vietnamese": 0.22, - "English,Filipino": 0.20666666666666667, - "English,Indonesian": 0.25333333333333335, - "English,Malay": 0.22666666666666666, - "Chinese,Vietnamese": 0.24, - "Chinese,Filipino": 0.20666666666666667, - "Chinese,Indonesian": 0.26, - "Chinese,Malay": 0.2866666666666667, - "Vietnamese,Filipino": 0.24666666666666667, - "Vietnamese,Indonesian": 0.25333333333333335, - "Vietnamese,Malay": 0.24, - "Filipino,Indonesian": 0.3466666666666667, - "Filipino,Malay": 0.3333333333333333, - "Indonesian,Malay": 0.3 + "Filipino,Vietnamese": 0.47333333333333333, + "Filipino,Chinese": 0.5466666666666666, + "Filipino,Spanish": 0.4666666666666667, + "Filipino,Malay": 0.5333333333333333, + "Filipino,Indonesian": 0.5266666666666666, + "Filipino,English": 0.54, + "Vietnamese,Chinese": 0.6466666666666666, + "Vietnamese,Spanish": 0.64, + "Vietnamese,Malay": 0.6666666666666666, + "Vietnamese,Indonesian": 0.6866666666666666, + "Vietnamese,English": 0.6933333333333334, + "Chinese,Spanish": 0.6333333333333333, + "Chinese,Malay": 0.66, + "Chinese,Indonesian": 0.6733333333333333, + "Chinese,English": 0.7333333333333333, + "Spanish,Malay": 0.6266666666666667, + "Spanish,Indonesian": 0.7, + "Spanish,English": 0.6933333333333334, + "Malay,Indonesian": 0.74, + "Malay,English": 0.6866666666666666, + "Indonesian,English": 0.7266666666666667 }, "3_combine": { - "Spanish,English,Chinese": 0.05333333333333334, - "Spanish,English,Vietnamese": 0.08666666666666667, - "Spanish,English,Filipino": 0.04, - "Spanish,English,Indonesian": 0.05333333333333334, - "Spanish,English,Malay": 0.06, - "Spanish,Chinese,Vietnamese": 0.06666666666666667, - "Spanish,Chinese,Filipino": 0.06, - "Spanish,Chinese,Indonesian": 0.06666666666666667, - "Spanish,Chinese,Malay": 0.11333333333333333, - "Spanish,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,Vietnamese,Indonesian": 0.08, - "Spanish,Vietnamese,Malay": 0.09333333333333334, - "Spanish,Filipino,Indonesian": 0.13333333333333333, - "Spanish,Filipino,Malay": 0.10666666666666667, - "Spanish,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese": 0.05333333333333334, - "English,Chinese,Filipino": 0.013333333333333334, - "English,Chinese,Indonesian": 0.04, - "English,Chinese,Malay": 0.08666666666666667, - "English,Vietnamese,Filipino": 0.02666666666666667, - "English,Vietnamese,Indonesian": 0.06666666666666667, - "English,Vietnamese,Malay": 0.05333333333333334, - "English,Filipino,Indonesian": 0.07333333333333333, - "English,Filipino,Malay": 0.08, - "English,Indonesian,Malay": 0.06666666666666667, - "Chinese,Vietnamese,Filipino": 0.07333333333333333, - "Chinese,Vietnamese,Indonesian": 0.06, - "Chinese,Vietnamese,Malay": 0.06, - "Chinese,Filipino,Indonesian": 0.1, - "Chinese,Filipino,Malay": 0.06, - "Chinese,Indonesian,Malay": 0.07333333333333333, - "Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Vietnamese,Filipino,Malay": 0.09333333333333334, - "Vietnamese,Indonesian,Malay": 0.08, - "Filipino,Indonesian,Malay": 0.14 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.02, - "Spanish,English,Chinese,Filipino": 0.0, - "Spanish,English,Chinese,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Malay": 0.04, - "Spanish,English,Vietnamese,Filipino": 0.013333333333333334, - "Spanish,English,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,English,Vietnamese,Malay": 0.02666666666666667, - "Spanish,English,Filipino,Indonesian": 0.02666666666666667, - "Spanish,English,Filipino,Malay": 0.02, - "Spanish,English,Indonesian,Malay": 0.02, - "Spanish,Chinese,Vietnamese,Filipino": 0.02, - "Spanish,Chinese,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Malay": 0.02666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.02, - "Spanish,Chinese,Indonesian,Malay": 0.02, - "Spanish,Vietnamese,Filipino,Indonesian": 0.04666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.04666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.02, - "Spanish,Filipino,Indonesian,Malay": 0.04, - "English,Chinese,Vietnamese,Filipino": 0.0, - "English,Chinese,Vietnamese,Indonesian": 0.006666666666666667, - "English,Chinese,Vietnamese,Malay": 0.02, - "English,Chinese,Filipino,Indonesian": 0.006666666666666667, - "English,Chinese,Filipino,Malay": 0.006666666666666667, - "English,Chinese,Indonesian,Malay": 0.006666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "English,Vietnamese,Filipino,Malay": 0.02, - "English,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "English,Filipino,Indonesian,Malay": 0.04, - "Chinese,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.02, - "Chinese,Vietnamese,Indonesian,Malay": 0.02, - "Chinese,Filipino,Indonesian,Malay": 0.02666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.04666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.0, - "Spanish,English,Chinese,Vietnamese,Malay": 0.013333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.013333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "English,Chinese,Filipino,Indonesian,Malay": 0.0, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334 + "Filipino,Vietnamese,Chinese": 0.4066666666666667, + "Filipino,Vietnamese,Spanish": 0.3466666666666667, + "Filipino,Vietnamese,Malay": 0.4, + "Filipino,Vietnamese,Indonesian": 0.3933333333333333, + "Filipino,Vietnamese,English": 0.4, + "Filipino,Chinese,Spanish": 0.38666666666666666, + "Filipino,Chinese,Malay": 0.42, + "Filipino,Chinese,Indonesian": 0.42, + "Filipino,Chinese,English": 0.4533333333333333, + "Filipino,Spanish,Malay": 0.38, + "Filipino,Spanish,Indonesian": 0.38666666666666666, + "Filipino,Spanish,English": 0.3933333333333333, + "Filipino,Malay,Indonesian": 0.44, + "Filipino,Malay,English": 0.43333333333333335, + "Filipino,Indonesian,English": 0.44, + "Vietnamese,Chinese,Spanish": 0.5066666666666667, + "Vietnamese,Chinese,Malay": 0.5333333333333333, + "Vietnamese,Chinese,Indonesian": 0.54, + "Vietnamese,Chinese,English": 0.56, + "Vietnamese,Spanish,Malay": 0.5133333333333333, + "Vietnamese,Spanish,Indonesian": 0.5466666666666666, + "Vietnamese,Spanish,English": 0.54, + "Vietnamese,Malay,Indonesian": 0.5866666666666667, + "Vietnamese,Malay,English": 0.5666666666666667, + "Vietnamese,Indonesian,English": 0.58, + "Chinese,Spanish,Malay": 0.52, + "Chinese,Spanish,Indonesian": 0.5466666666666666, + "Chinese,Spanish,English": 0.56, + "Chinese,Malay,Indonesian": 0.56, + "Chinese,Malay,English": 0.5733333333333334, + "Chinese,Indonesian,English": 0.6, + "Spanish,Malay,Indonesian": 0.5533333333333333, + "Spanish,Malay,English": 0.54, + "Spanish,Indonesian,English": 0.58, + "Malay,Indonesian,English": 0.6 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.32, + "Filipino,Vietnamese,Chinese,Malay": 0.35333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,Chinese,English": 0.36666666666666664, + "Filipino,Vietnamese,Spanish,Malay": 0.31333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian": 0.31333333333333335, + "Filipino,Vietnamese,Spanish,English": 0.31333333333333335, + "Filipino,Vietnamese,Malay,Indonesian": 0.36, + "Filipino,Vietnamese,Malay,English": 0.36, + "Filipino,Vietnamese,Indonesian,English": 0.36666666666666664, + "Filipino,Chinese,Spanish,Malay": 0.34, + "Filipino,Chinese,Spanish,Indonesian": 0.34, + "Filipino,Chinese,Spanish,English": 0.3466666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.36666666666666664, + "Filipino,Chinese,Malay,English": 0.38, + "Filipino,Chinese,Indonesian,English": 0.3933333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.35333333333333333, + "Filipino,Spanish,Malay,English": 0.3466666666666667, + "Filipino,Spanish,Indonesian,English": 0.35333333333333333, + "Filipino,Malay,Indonesian,English": 0.38666666666666666, + "Vietnamese,Chinese,Spanish,Malay": 0.46, + "Vietnamese,Chinese,Spanish,Indonesian": 0.46, + "Vietnamese,Chinese,Spanish,English": 0.47333333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.49333333333333335, + "Vietnamese,Chinese,Malay,English": 0.4866666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.5, + "Vietnamese,Spanish,Malay,Indonesian": 0.4866666666666667, + "Vietnamese,Spanish,Malay,English": 0.48, + "Vietnamese,Spanish,Indonesian,English": 0.4866666666666667, + "Vietnamese,Malay,Indonesian,English": 0.5266666666666666, + "Chinese,Spanish,Malay,Indonesian": 0.4866666666666667, + "Chinese,Spanish,Malay,English": 0.4866666666666667, + "Chinese,Spanish,Indonesian,English": 0.5133333333333333, + "Chinese,Malay,Indonesian,English": 0.5066666666666667, + "Spanish,Malay,Indonesian,English": 0.5066666666666667 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.3, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.32666666666666666, + "Filipino,Vietnamese,Chinese,Malay,English": 0.32666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.34, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.3, + "Filipino,Vietnamese,Spanish,Malay,English": 0.3, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.3, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.34, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, + "Filipino,Chinese,Spanish,Malay,English": 0.32, + "Filipino,Chinese,Spanish,Indonesian,English": 0.32666666666666666, + "Filipino,Chinese,Malay,Indonesian,English": 0.3466666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.3333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.44, + "Vietnamese,Chinese,Spanish,Malay,English": 0.44, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.44, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.46, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.46, + "Chinese,Spanish,Malay,Indonesian,English": 0.46 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.31333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.29333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.31333333333333335, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.42 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.28 } }, - "AC3_2": 0.25800738002381773, - "AC3_3": 0.11442762899103152, - "AC3_4": 0.039999999985457065, - "AC3_5": 0.011782945731891111, - "AC3_6": 0.0018976279642974375, - "AC3_7": 0.0 + "AC3_2": 0.5508852016235788, + "AC3_3": 0.48961087795224245, + "AC3_4": 0.44595899325724875, + "AC3_5": 0.41145724675660356, + "AC3_6": 0.3822192059245112, + "AC3_7": 0.35573200987921605 }, "prompt_4": { - "overall_acc": 0.2342857142857143, + "overall_acc": 0.4809523809523809, "language_acc": { - "Spanish": 0.21333333333333335, - "English": 0.26666666666666666, - "Chinese": 0.25333333333333335, - "Vietnamese": 0.20666666666666667, - "Filipino": 0.23333333333333334, - "Indonesian": 0.23333333333333334, - "Malay": 0.23333333333333334 + "Filipino": 0.36, + "Vietnamese": 0.4666666666666667, + "Chinese": 0.47333333333333333, + "Spanish": 0.4866666666666667, + "Malay": 0.5133333333333333, + "Indonesian": 0.5, + "English": 0.5666666666666667 }, - "consistency_score_2": 0.26666666666666666, - "consistency_score_3": 0.06933333333333333, - "consistency_score_4": 0.017904761904761902, - "consistency_score_5": 0.005714285714285715, - "consistency_score_6": 0.001904761904761905, - "consistency_score_7": 0.0, + "consistency_score_2": 0.6301587301587301, + "consistency_score_3": 0.48552380952380947, + "consistency_score_4": 0.40590476190476193, + "consistency_score_5": 0.353968253968254, + "consistency_score_6": 0.31428571428571433, + "consistency_score_7": 0.28, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.23333333333333334, - "Spanish,Chinese": 0.31333333333333335, - "Spanish,Vietnamese": 0.26, - "Spanish,Filipino": 0.2, - "Spanish,Indonesian": 0.29333333333333333, - "Spanish,Malay": 0.23333333333333334, - "English,Chinese": 0.26, - "English,Vietnamese": 0.25333333333333335, - "English,Filipino": 0.24, - "English,Indonesian": 0.30666666666666664, - "English,Malay": 0.26666666666666666, - "Chinese,Vietnamese": 0.23333333333333334, - "Chinese,Filipino": 0.23333333333333334, - "Chinese,Indonesian": 0.24666666666666667, - "Chinese,Malay": 0.2733333333333333, - "Vietnamese,Filipino": 0.32, - "Vietnamese,Indonesian": 0.24666666666666667, - "Vietnamese,Malay": 0.34, - "Filipino,Indonesian": 0.24, - "Filipino,Malay": 0.35333333333333333, - "Indonesian,Malay": 0.25333333333333335 + "Filipino,Vietnamese": 0.5333333333333333, + "Filipino,Chinese": 0.49333333333333335, + "Filipino,Spanish": 0.5333333333333333, + "Filipino,Malay": 0.5466666666666666, + "Filipino,Indonesian": 0.5133333333333333, + "Filipino,English": 0.52, + "Vietnamese,Chinese": 0.62, + "Vietnamese,Spanish": 0.6133333333333333, + "Vietnamese,Malay": 0.6933333333333334, + "Vietnamese,Indonesian": 0.68, + "Vietnamese,English": 0.6466666666666666, + "Chinese,Spanish": 0.5866666666666667, + "Chinese,Malay": 0.68, + "Chinese,Indonesian": 0.6866666666666666, + "Chinese,English": 0.7133333333333334, + "Spanish,Malay": 0.66, + "Spanish,Indonesian": 0.64, + "Spanish,English": 0.6866666666666666, + "Malay,Indonesian": 0.7333333333333333, + "Malay,English": 0.7333333333333333, + "Indonesian,English": 0.72 }, "3_combine": { - "Spanish,English,Chinese": 0.06, - "Spanish,English,Vietnamese": 0.08, - "Spanish,English,Filipino": 0.05333333333333334, - "Spanish,English,Indonesian": 0.06666666666666667, - "Spanish,English,Malay": 0.04, - "Spanish,Chinese,Vietnamese": 0.06666666666666667, - "Spanish,Chinese,Filipino": 0.08, - "Spanish,Chinese,Indonesian": 0.06666666666666667, - "Spanish,Chinese,Malay": 0.08, - "Spanish,Vietnamese,Filipino": 0.07333333333333333, - "Spanish,Vietnamese,Indonesian": 0.06666666666666667, - "Spanish,Vietnamese,Malay": 0.07333333333333333, - "Spanish,Filipino,Indonesian": 0.05333333333333334, - "Spanish,Filipino,Malay": 0.07333333333333333, - "Spanish,Indonesian,Malay": 0.03333333333333333, - "English,Chinese,Vietnamese": 0.04666666666666667, - "English,Chinese,Filipino": 0.04666666666666667, - "English,Chinese,Indonesian": 0.09333333333333334, - "English,Chinese,Malay": 0.06666666666666667, - "English,Vietnamese,Filipino": 0.06666666666666667, - "English,Vietnamese,Indonesian": 0.06, - "English,Vietnamese,Malay": 0.09333333333333334, - "English,Filipino,Indonesian": 0.07333333333333333, - "English,Filipino,Malay": 0.06, - "English,Indonesian,Malay": 0.09333333333333334, - "Chinese,Vietnamese,Filipino": 0.07333333333333333, - "Chinese,Vietnamese,Indonesian": 0.07333333333333333, - "Chinese,Vietnamese,Malay": 0.08, - "Chinese,Filipino,Indonesian": 0.04666666666666667, - "Chinese,Filipino,Malay": 0.07333333333333333, - "Chinese,Indonesian,Malay": 0.06666666666666667, - "Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Vietnamese,Filipino,Malay": 0.12, - "Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Filipino,Indonesian,Malay": 0.08 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.02666666666666667, - "Spanish,English,Chinese,Filipino": 0.013333333333333334, - "Spanish,English,Chinese,Indonesian": 0.02, - "Spanish,English,Chinese,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.02666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.02, - "Spanish,English,Vietnamese,Malay": 0.02, - "Spanish,English,Filipino,Indonesian": 0.013333333333333334, - "Spanish,English,Filipino,Malay": 0.0, - "Spanish,English,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.03333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Malay": 0.02, - "Spanish,Chinese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.02666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.006666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.02, - "Spanish,Vietnamese,Filipino,Malay": 0.03333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.013333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.013333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.02, - "English,Chinese,Vietnamese,Malay": 0.02, - "English,Chinese,Filipino,Indonesian": 0.02, - "English,Chinese,Filipino,Malay": 0.006666666666666667, - "English,Chinese,Indonesian,Malay": 0.02, - "English,Vietnamese,Filipino,Indonesian": 0.02, - "English,Vietnamese,Filipino,Malay": 0.013333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.02666666666666667, - "English,Filipino,Indonesian,Malay": 0.02666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.02, - "Chinese,Vietnamese,Filipino,Malay": 0.013333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.04, - "Chinese,Filipino,Indonesian,Malay": 0.013333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.013333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,English,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.0, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667 + "Filipino,Vietnamese,Chinese": 0.38666666666666666, + "Filipino,Vietnamese,Spanish": 0.4, + "Filipino,Vietnamese,Malay": 0.43333333333333335, + "Filipino,Vietnamese,Indonesian": 0.41333333333333333, + "Filipino,Vietnamese,English": 0.3933333333333333, + "Filipino,Chinese,Spanish": 0.36666666666666664, + "Filipino,Chinese,Malay": 0.4066666666666667, + "Filipino,Chinese,Indonesian": 0.4, + "Filipino,Chinese,English": 0.41333333333333333, + "Filipino,Spanish,Malay": 0.44, + "Filipino,Spanish,Indonesian": 0.3933333333333333, + "Filipino,Spanish,English": 0.41333333333333333, + "Filipino,Malay,Indonesian": 0.44666666666666666, + "Filipino,Malay,English": 0.4533333333333333, + "Filipino,Indonesian,English": 0.41333333333333333, + "Vietnamese,Chinese,Spanish": 0.44666666666666666, + "Vietnamese,Chinese,Malay": 0.54, + "Vietnamese,Chinese,Indonesian": 0.5266666666666666, + "Vietnamese,Chinese,English": 0.52, + "Vietnamese,Spanish,Malay": 0.5266666666666666, + "Vietnamese,Spanish,Indonesian": 0.5, + "Vietnamese,Spanish,English": 0.52, + "Vietnamese,Malay,Indonesian": 0.6, + "Vietnamese,Malay,English": 0.5866666666666667, + "Vietnamese,Indonesian,English": 0.5533333333333333, + "Chinese,Spanish,Malay": 0.49333333333333335, + "Chinese,Spanish,Indonesian": 0.4866666666666667, + "Chinese,Spanish,English": 0.52, + "Chinese,Malay,Indonesian": 0.5666666666666667, + "Chinese,Malay,English": 0.5866666666666667, + "Chinese,Indonesian,English": 0.5733333333333334, + "Spanish,Malay,Indonesian": 0.54, + "Spanish,Malay,English": 0.5733333333333334, + "Spanish,Indonesian,English": 0.5466666666666666, + "Malay,Indonesian,English": 0.6133333333333333 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.32, + "Filipino,Vietnamese,Chinese,Malay": 0.36, + "Filipino,Vietnamese,Chinese,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,Chinese,English": 0.3466666666666667, + "Filipino,Vietnamese,Spanish,Malay": 0.36, + "Filipino,Vietnamese,Spanish,Indonesian": 0.3333333333333333, + "Filipino,Vietnamese,Spanish,English": 0.3333333333333333, + "Filipino,Vietnamese,Malay,Indonesian": 0.38, + "Filipino,Vietnamese,Malay,English": 0.38666666666666666, + "Filipino,Vietnamese,Indonesian,English": 0.36, + "Filipino,Chinese,Spanish,Malay": 0.34, + "Filipino,Chinese,Spanish,Indonesian": 0.32666666666666666, + "Filipino,Chinese,Spanish,English": 0.34, + "Filipino,Chinese,Malay,Indonesian": 0.36666666666666664, + "Filipino,Chinese,Malay,English": 0.38666666666666666, + "Filipino,Chinese,Indonesian,English": 0.36666666666666664, + "Filipino,Spanish,Malay,Indonesian": 0.37333333333333335, + "Filipino,Spanish,Malay,English": 0.37333333333333335, + "Filipino,Spanish,Indonesian,English": 0.3466666666666667, + "Filipino,Malay,Indonesian,English": 0.3933333333333333, + "Vietnamese,Chinese,Spanish,Malay": 0.43333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian": 0.41333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.4266666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.5, + "Vietnamese,Chinese,Malay,English": 0.4866666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.47333333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.46, + "Vietnamese,Spanish,Malay,English": 0.49333333333333335, + "Vietnamese,Spanish,Indonesian,English": 0.4533333333333333, + "Vietnamese,Malay,Indonesian,English": 0.5266666666666666, + "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, + "Chinese,Spanish,Malay,English": 0.47333333333333333, + "Chinese,Spanish,Indonesian,English": 0.46, + "Chinese,Malay,Indonesian,English": 0.5133333333333333, + "Spanish,Malay,Indonesian,English": 0.5 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.3, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.3, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.34, + "Filipino,Vietnamese,Chinese,Malay,English": 0.34, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.3333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.32, + "Filipino,Vietnamese,Spanish,Malay,English": 0.32666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.30666666666666664, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.35333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.32, + "Filipino,Chinese,Spanish,Malay,English": 0.32666666666666666, + "Filipino,Chinese,Spanish,Indonesian,English": 0.31333333333333335, + "Filipino,Chinese,Malay,Indonesian,English": 0.35333333333333333, + "Filipino,Spanish,Malay,Indonesian,English": 0.3333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4066666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.42, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.46, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.44, + "Chinese,Spanish,Malay,Indonesian,English": 0.43333333333333335 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.32666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.3, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.28 } }, - "AC3_2": 0.24942965774488574, - "AC3_3": 0.10700125466990235, - "AC3_4": 0.03326715579172887, - "AC3_5": 0.011156462580385489, - "AC3_6": 0.003778801841718077, - "AC3_7": 0.0 + "AC3_2": 0.545537414916888, + "AC3_3": 0.4832272837685344, + "AC3_4": 0.4402532318273013, + "AC3_5": 0.40780372980811835, + "AC3_6": 0.3801539777109644, + "AC3_7": 0.3539424279885307 }, "prompt_5": { - "overall_acc": 0.2542857142857143, + "overall_acc": 0.4638095238095237, "language_acc": { - "Spanish": 0.26, - "English": 0.20666666666666667, - "Chinese": 0.25333333333333335, - "Vietnamese": 0.24666666666666667, - "Filipino": 0.29333333333333333, - "Indonesian": 0.24, - "Malay": 0.28 + "Filipino": 0.36, + "Vietnamese": 0.44, + "Chinese": 0.4866666666666667, + "Spanish": 0.44666666666666666, + "Malay": 0.49333333333333335, + "Indonesian": 0.47333333333333333, + "English": 0.5466666666666666 }, - "consistency_score_2": 0.2419047619047619, - "consistency_score_3": 0.05561904761904762, - "consistency_score_4": 0.011809523809523806, - "consistency_score_5": 0.0019047619047619048, - "consistency_score_6": 0.0, - "consistency_score_7": 0.0, + "consistency_score_2": 0.6488888888888888, + "consistency_score_3": 0.5167619047619048, + "consistency_score_4": 0.44361904761904764, + "consistency_score_5": 0.3958730158730158, + "consistency_score_6": 0.3609523809523809, + "consistency_score_7": 0.3333333333333333, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.16666666666666666, - "Spanish,Chinese": 0.26, - "Spanish,Vietnamese": 0.24666666666666667, - "Spanish,Filipino": 0.26, - "Spanish,Indonesian": 0.18666666666666668, - "Spanish,Malay": 0.24, - "English,Chinese": 0.23333333333333334, - "English,Vietnamese": 0.25333333333333335, - "English,Filipino": 0.26666666666666666, - "English,Indonesian": 0.2, - "English,Malay": 0.2733333333333333, - "Chinese,Vietnamese": 0.18666666666666668, - "Chinese,Filipino": 0.21333333333333335, - "Chinese,Indonesian": 0.26, - "Chinese,Malay": 0.24, - "Vietnamese,Filipino": 0.26, - "Vietnamese,Indonesian": 0.28, - "Vietnamese,Malay": 0.29333333333333333, - "Filipino,Indonesian": 0.32666666666666666, - "Filipino,Malay": 0.24, - "Indonesian,Malay": 0.19333333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.03333333333333333, - "Spanish,English,Vietnamese": 0.02666666666666667, - "Spanish,English,Filipino": 0.02, - "Spanish,English,Indonesian": 0.02, - "Spanish,English,Malay": 0.06, - "Spanish,Chinese,Vietnamese": 0.03333333333333333, - "Spanish,Chinese,Filipino": 0.04666666666666667, - "Spanish,Chinese,Indonesian": 0.05333333333333334, - "Spanish,Chinese,Malay": 0.06, - "Spanish,Vietnamese,Filipino": 0.04666666666666667, - "Spanish,Vietnamese,Indonesian": 0.02666666666666667, - "Spanish,Vietnamese,Malay": 0.08, - "Spanish,Filipino,Indonesian": 0.05333333333333334, - "Spanish,Filipino,Malay": 0.06666666666666667, - "Spanish,Indonesian,Malay": 0.02666666666666667, - "English,Chinese,Vietnamese": 0.03333333333333333, - "English,Chinese,Filipino": 0.09333333333333334, - "English,Chinese,Indonesian": 0.05333333333333334, - "English,Chinese,Malay": 0.08, - "English,Vietnamese,Filipino": 0.07333333333333333, - "English,Vietnamese,Indonesian": 0.07333333333333333, - "English,Vietnamese,Malay": 0.06666666666666667, - "English,Filipino,Indonesian": 0.08666666666666667, - "English,Filipino,Malay": 0.05333333333333334, - "English,Indonesian,Malay": 0.03333333333333333, - "Chinese,Vietnamese,Filipino": 0.04666666666666667, - "Chinese,Vietnamese,Indonesian": 0.07333333333333333, - "Chinese,Vietnamese,Malay": 0.07333333333333333, - "Chinese,Filipino,Indonesian": 0.07333333333333333, - "Chinese,Filipino,Malay": 0.04666666666666667, - "Chinese,Indonesian,Malay": 0.04, - "Vietnamese,Filipino,Indonesian": 0.1, - "Vietnamese,Filipino,Malay": 0.06666666666666667, - "Vietnamese,Indonesian,Malay": 0.06666666666666667, - "Filipino,Indonesian,Malay": 0.06 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.0, - "Spanish,English,Chinese,Filipino": 0.0, - "Spanish,English,Chinese,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Malay": 0.013333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.006666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.0, - "Spanish,English,Vietnamese,Malay": 0.013333333333333334, - "Spanish,English,Filipino,Indonesian": 0.0, - "Spanish,English,Filipino,Malay": 0.006666666666666667, - "Spanish,English,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Malay": 0.013333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.02, - "Spanish,Chinese,Indonesian,Malay": 0.0, - "Spanish,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.02, - "Spanish,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,Filipino,Indonesian,Malay": 0.013333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.02, - "English,Chinese,Vietnamese,Indonesian": 0.013333333333333334, - "English,Chinese,Vietnamese,Malay": 0.013333333333333334, - "English,Chinese,Filipino,Indonesian": 0.04, - "English,Chinese,Filipino,Malay": 0.02, - "English,Chinese,Indonesian,Malay": 0.013333333333333334, - "English,Vietnamese,Filipino,Indonesian": 0.02, - "English,Vietnamese,Filipino,Malay": 0.006666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "English,Filipino,Indonesian,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.02, - "Chinese,Vietnamese,Filipino,Malay": 0.013333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.02666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.0, - "Spanish,English,Chinese,Vietnamese,Malay": 0.0, - "Spanish,English,Chinese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 - } - }, - "AC3_2": 0.24794077318830915, - "AC3_3": 0.09127403632141258, - "AC3_4": 0.022570815003277653, - "AC3_5": 0.003781200210951045, - "AC3_6": 0.0, - "AC3_7": 0.0 + "Filipino,Vietnamese": 0.54, + "Filipino,Chinese": 0.5466666666666666, + "Filipino,Spanish": 0.56, + "Filipino,Malay": 0.5533333333333333, + "Filipino,Indonesian": 0.5333333333333333, + "Filipino,English": 0.5533333333333333, + "Vietnamese,Chinese": 0.64, + "Vietnamese,Spanish": 0.6466666666666666, + "Vietnamese,Malay": 0.72, + "Vietnamese,Indonesian": 0.6733333333333333, + "Vietnamese,English": 0.6933333333333334, + "Chinese,Spanish": 0.6133333333333333, + "Chinese,Malay": 0.7066666666666667, + "Chinese,Indonesian": 0.7133333333333334, + "Chinese,English": 0.7, + "Spanish,Malay": 0.6333333333333333, + "Spanish,Indonesian": 0.66, + "Spanish,English": 0.66, + "Malay,Indonesian": 0.7733333333333333, + "Malay,English": 0.78, + "Indonesian,English": 0.7266666666666667 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.44, + "Filipino,Vietnamese,Spanish": 0.43333333333333335, + "Filipino,Vietnamese,Malay": 0.46, + "Filipino,Vietnamese,Indonesian": 0.4266666666666667, + "Filipino,Vietnamese,English": 0.43333333333333335, + "Filipino,Chinese,Spanish": 0.43333333333333335, + "Filipino,Chinese,Malay": 0.4666666666666667, + "Filipino,Chinese,Indonesian": 0.44666666666666666, + "Filipino,Chinese,English": 0.4533333333333333, + "Filipino,Spanish,Malay": 0.44, + "Filipino,Spanish,Indonesian": 0.43333333333333335, + "Filipino,Spanish,English": 0.4533333333333333, + "Filipino,Malay,Indonesian": 0.47333333333333333, + "Filipino,Malay,English": 0.47333333333333333, + "Filipino,Indonesian,English": 0.44, + "Vietnamese,Chinese,Spanish": 0.5133333333333333, + "Vietnamese,Chinese,Malay": 0.5733333333333334, + "Vietnamese,Chinese,Indonesian": 0.5466666666666666, + "Vietnamese,Chinese,English": 0.5533333333333333, + "Vietnamese,Spanish,Malay": 0.5466666666666666, + "Vietnamese,Spanish,Indonesian": 0.54, + "Vietnamese,Spanish,English": 0.5466666666666666, + "Vietnamese,Malay,Indonesian": 0.6266666666666667, + "Vietnamese,Malay,English": 0.6266666666666667, + "Vietnamese,Indonesian,English": 0.5666666666666667, + "Chinese,Spanish,Malay": 0.5133333333333333, + "Chinese,Spanish,Indonesian": 0.5266666666666666, + "Chinese,Spanish,English": 0.5133333333333333, + "Chinese,Malay,Indonesian": 0.6133333333333333, + "Chinese,Malay,English": 0.6333333333333333, + "Chinese,Indonesian,English": 0.5933333333333334, + "Spanish,Malay,Indonesian": 0.56, + "Spanish,Malay,English": 0.5733333333333334, + "Spanish,Indonesian,English": 0.56, + "Malay,Indonesian,English": 0.6533333333333333 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.38666666666666666, + "Filipino,Vietnamese,Chinese,Malay": 0.41333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.3933333333333333, + "Filipino,Vietnamese,Chinese,English": 0.4, + "Filipino,Vietnamese,Spanish,Malay": 0.3933333333333333, + "Filipino,Vietnamese,Spanish,Indonesian": 0.38, + "Filipino,Vietnamese,Spanish,English": 0.38666666666666666, + "Filipino,Vietnamese,Malay,Indonesian": 0.41333333333333333, + "Filipino,Vietnamese,Malay,English": 0.41333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.38666666666666666, + "Filipino,Chinese,Spanish,Malay": 0.4, + "Filipino,Chinese,Spanish,Indonesian": 0.38666666666666666, + "Filipino,Chinese,Spanish,English": 0.3933333333333333, + "Filipino,Chinese,Malay,Indonesian": 0.42, + "Filipino,Chinese,Malay,English": 0.43333333333333335, + "Filipino,Chinese,Indonesian,English": 0.4, + "Filipino,Spanish,Malay,Indonesian": 0.4066666666666667, + "Filipino,Spanish,Malay,English": 0.4066666666666667, + "Filipino,Spanish,Indonesian,English": 0.3933333333333333, + "Filipino,Malay,Indonesian,English": 0.42, + "Vietnamese,Chinese,Spanish,Malay": 0.47333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.4666666666666667, + "Vietnamese,Chinese,Spanish,English": 0.4666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.5266666666666666, + "Vietnamese,Chinese,Malay,English": 0.5266666666666666, + "Vietnamese,Chinese,Indonesian,English": 0.49333333333333335, + "Vietnamese,Spanish,Malay,Indonesian": 0.5, + "Vietnamese,Spanish,Malay,English": 0.5066666666666667, + "Vietnamese,Spanish,Indonesian,English": 0.48, + "Vietnamese,Malay,Indonesian,English": 0.5533333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.48, + "Chinese,Spanish,Malay,English": 0.4866666666666667, + "Chinese,Spanish,Indonesian,English": 0.47333333333333333, + "Chinese,Malay,Indonesian,English": 0.5466666666666666, + "Spanish,Malay,Indonesian,English": 0.52 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.36666666666666664, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.36, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.36, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.38666666666666666, + "Filipino,Vietnamese,Chinese,Malay,English": 0.38666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.36666666666666664, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.36666666666666664, + "Filipino,Vietnamese,Spanish,Malay,English": 0.36666666666666664, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.35333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.38, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.38, + "Filipino,Chinese,Spanish,Malay,English": 0.38, + "Filipino,Chinese,Spanish,Indonesian,English": 0.36666666666666664, + "Filipino,Chinese,Malay,Indonesian,English": 0.38666666666666666, + "Filipino,Spanish,Malay,Indonesian,English": 0.38, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, + "Vietnamese,Chinese,Spanish,Malay,English": 0.44666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.43333333333333335, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.48, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.4666666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.4533333333333333 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.3466666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.34, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.36, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.3466666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.36, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.42 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3333333333333333 + } + }, + "AC3_2": 0.5409567284345026, + "AC3_3": 0.48885595880610466, + "AC3_4": 0.45348966449762373, + "AC3_5": 0.42715692475864026, + "AC3_6": 0.40596722748844394, + "AC3_7": 0.3878932695650393 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.23782467532467533, + "overall_acc": 0.3717532467532468, "language_acc": { - "English": 0.26136363636363635, - "Filipino": 0.26704545454545453, - "Spanish": 0.20454545454545456, - "Malay": 0.26136363636363635, - "Chinese": 0.23863636363636365, - "Indonesian": 0.20454545454545456, - "Vietnamese": 0.22727272727272727 + "Indonesian": 0.39204545454545453, + "English": 0.35795454545454547, + "Filipino": 0.30113636363636365, + "Spanish": 0.3693181818181818, + "Chinese": 0.45454545454545453, + "Malay": 0.4034090909090909, + "Vietnamese": 0.32386363636363635 }, - "consistency_score_2": 0.23755411255411255, - "consistency_score_3": 0.05633116883116884, - "consistency_score_4": 0.013798701298701291, - "consistency_score_5": 0.003517316017316017, - "consistency_score_6": 0.0008116883116883117, - "consistency_score_7": 0.0, + "consistency_score_2": 0.5340909090909091, + "consistency_score_3": 0.35259740259740263, + "consistency_score_4": 0.2553571428571428, + "consistency_score_5": 0.19453463203463206, + "consistency_score_6": 0.15178571428571427, + "consistency_score_7": 0.11931818181818182, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.24431818181818182, - "English,Spanish": 0.2784090909090909, - "English,Malay": 0.2784090909090909, - "English,Chinese": 0.2727272727272727, - "English,Indonesian": 0.23295454545454544, - "English,Vietnamese": 0.22727272727272727, - "Filipino,Spanish": 0.26704545454545453, - "Filipino,Malay": 0.23295454545454544, - "Filipino,Chinese": 0.2215909090909091, - "Filipino,Indonesian": 0.22727272727272727, - "Filipino,Vietnamese": 0.26136363636363635, - "Spanish,Malay": 0.22727272727272727, - "Spanish,Chinese": 0.14772727272727273, - "Spanish,Indonesian": 0.22727272727272727, - "Spanish,Vietnamese": 0.16477272727272727, - "Malay,Chinese": 0.23295454545454544, - "Malay,Indonesian": 0.25, - "Malay,Vietnamese": 0.2784090909090909, - "Chinese,Indonesian": 0.19318181818181818, - "Chinese,Vietnamese": 0.2727272727272727, - "Indonesian,Vietnamese": 0.25 + "Indonesian,English": 0.6136363636363636, + "Indonesian,Filipino": 0.4715909090909091, + "Indonesian,Spanish": 0.5795454545454546, + "Indonesian,Chinese": 0.5340909090909091, + "Indonesian,Malay": 0.6988636363636364, + "Indonesian,Vietnamese": 0.5568181818181818, + "English,Filipino": 0.45454545454545453, + "English,Spanish": 0.6193181818181818, + "English,Chinese": 0.5113636363636364, + "English,Malay": 0.5511363636363636, + "English,Vietnamese": 0.5340909090909091, + "Filipino,Spanish": 0.4943181818181818, + "Filipino,Chinese": 0.3806818181818182, + "Filipino,Malay": 0.4431818181818182, + "Filipino,Vietnamese": 0.42045454545454547, + "Spanish,Chinese": 0.5170454545454546, + "Spanish,Malay": 0.6079545454545454, + "Spanish,Vietnamese": 0.625, + "Chinese,Malay": 0.5511363636363636, + "Chinese,Vietnamese": 0.48863636363636365, + "Malay,Vietnamese": 0.5625 }, "3_combine": { - "English,Filipino,Spanish": 0.07954545454545454, - "English,Filipino,Malay": 0.06818181818181818, - "English,Filipino,Chinese": 0.06818181818181818, - "English,Filipino,Indonesian": 0.06818181818181818, - "English,Filipino,Vietnamese": 0.0625, - "English,Spanish,Malay": 0.056818181818181816, - "English,Spanish,Chinese": 0.056818181818181816, - "English,Spanish,Indonesian": 0.05113636363636364, - "English,Spanish,Vietnamese": 0.056818181818181816, - "English,Malay,Chinese": 0.07954545454545454, - "English,Malay,Indonesian": 0.08522727272727272, - "English,Malay,Vietnamese": 0.05113636363636364, - "English,Chinese,Indonesian": 0.056818181818181816, - "English,Chinese,Vietnamese": 0.07954545454545454, - "English,Indonesian,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Malay": 0.06818181818181818, - "Filipino,Spanish,Chinese": 0.045454545454545456, - "Filipino,Spanish,Indonesian": 0.03409090909090909, - "Filipino,Spanish,Vietnamese": 0.045454545454545456, - "Filipino,Malay,Chinese": 0.05113636363636364, - "Filipino,Malay,Indonesian": 0.0625, - "Filipino,Malay,Vietnamese": 0.07386363636363637, - "Filipino,Chinese,Indonesian": 0.03409090909090909, - "Filipino,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Indonesian,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Chinese": 0.022727272727272728, - "Spanish,Malay,Indonesian": 0.056818181818181816, - "Spanish,Malay,Vietnamese": 0.03409090909090909, - "Spanish,Chinese,Indonesian": 0.05113636363636364, - "Spanish,Chinese,Vietnamese": 0.022727272727272728, - "Spanish,Indonesian,Vietnamese": 0.03409090909090909, - "Malay,Chinese,Indonesian": 0.05113636363636364, - "Malay,Chinese,Vietnamese": 0.08522727272727272, - "Malay,Indonesian,Vietnamese": 0.056818181818181816, - "Chinese,Indonesian,Vietnamese": 0.03977272727272727 + "Indonesian,English,Filipino": 0.3409090909090909, + "Indonesian,English,Spanish": 0.4318181818181818, + "Indonesian,English,Chinese": 0.3806818181818182, + "Indonesian,English,Malay": 0.4659090909090909, + "Indonesian,English,Vietnamese": 0.38636363636363635, + "Indonesian,Filipino,Spanish": 0.3465909090909091, + "Indonesian,Filipino,Chinese": 0.26704545454545453, + "Indonesian,Filipino,Malay": 0.3522727272727273, + "Indonesian,Filipino,Vietnamese": 0.30113636363636365, + "Indonesian,Spanish,Chinese": 0.36363636363636365, + "Indonesian,Spanish,Malay": 0.48295454545454547, + "Indonesian,Spanish,Vietnamese": 0.4147727272727273, + "Indonesian,Chinese,Malay": 0.4318181818181818, + "Indonesian,Chinese,Vietnamese": 0.32954545454545453, + "Indonesian,Malay,Vietnamese": 0.4431818181818182, + "English,Filipino,Spanish": 0.3409090909090909, + "English,Filipino,Chinese": 0.23863636363636365, + "English,Filipino,Malay": 0.2897727272727273, + "English,Filipino,Vietnamese": 0.2727272727272727, + "English,Spanish,Chinese": 0.3806818181818182, + "English,Spanish,Malay": 0.4147727272727273, + "English,Spanish,Vietnamese": 0.42045454545454547, + "English,Chinese,Malay": 0.35795454545454547, + "English,Chinese,Vietnamese": 0.3125, + "English,Malay,Vietnamese": 0.35795454545454547, + "Filipino,Spanish,Chinese": 0.2727272727272727, + "Filipino,Spanish,Malay": 0.32954545454545453, + "Filipino,Spanish,Vietnamese": 0.32386363636363635, + "Filipino,Chinese,Malay": 0.2556818181818182, + "Filipino,Chinese,Vietnamese": 0.2159090909090909, + "Filipino,Malay,Vietnamese": 0.2840909090909091, + "Spanish,Chinese,Malay": 0.39204545454545453, + "Spanish,Chinese,Vietnamese": 0.35795454545454547, + "Spanish,Malay,Vietnamese": 0.4375, + "Chinese,Malay,Vietnamese": 0.3465909090909091 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.017045454545454544, - "English,Filipino,Spanish,Chinese": 0.028409090909090908, - "English,Filipino,Spanish,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian": 0.028409090909090908, - "English,Filipino,Malay,Vietnamese": 0.011363636363636364, - "English,Filipino,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Indonesian,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Chinese": 0.005681818181818182, - "English,Spanish,Malay,Indonesian": 0.017045454545454544, - "English,Spanish,Malay,Vietnamese": 0.005681818181818182, - "English,Spanish,Chinese,Indonesian": 0.011363636363636364, - "English,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Spanish,Indonesian,Vietnamese": 0.011363636363636364, - "English,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "English,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese": 0.005681818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.011363636363636364, - "Filipino,Spanish,Malay,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Chinese,Indonesian": 0.0, - "Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.005681818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Malay,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Chinese,Indonesian,Vietnamese": 0.0, - "Spanish,Malay,Chinese,Indonesian": 0.017045454545454544, - "Spanish,Malay,Chinese,Vietnamese": 0.0, - "Spanish,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 + "Indonesian,English,Filipino,Spanish": 0.2727272727272727, + "Indonesian,English,Filipino,Chinese": 0.21022727272727273, + "Indonesian,English,Filipino,Malay": 0.26704545454545453, + "Indonesian,English,Filipino,Vietnamese": 0.22727272727272727, + "Indonesian,English,Spanish,Chinese": 0.30113636363636365, + "Indonesian,English,Spanish,Malay": 0.36363636363636365, + "Indonesian,English,Spanish,Vietnamese": 0.32386363636363635, + "Indonesian,English,Chinese,Malay": 0.32386363636363635, + "Indonesian,English,Chinese,Vietnamese": 0.25, + "Indonesian,English,Malay,Vietnamese": 0.3125, + "Indonesian,Filipino,Spanish,Chinese": 0.19886363636363635, + "Indonesian,Filipino,Spanish,Malay": 0.2897727272727273, + "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.2159090909090909, + "Indonesian,Filipino,Chinese,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,Spanish,Chinese,Malay": 0.32954545454545453, + "Indonesian,Spanish,Chinese,Vietnamese": 0.2784090909090909, + "Indonesian,Spanish,Malay,Vietnamese": 0.35795454545454547, + "Indonesian,Chinese,Malay,Vietnamese": 0.2897727272727273, + "English,Filipino,Spanish,Chinese": 0.19318181818181818, + "English,Filipino,Spanish,Malay": 0.23863636363636365, + "English,Filipino,Spanish,Vietnamese": 0.23863636363636365, + "English,Filipino,Chinese,Malay": 0.18181818181818182, + "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, + "English,Filipino,Malay,Vietnamese": 0.20454545454545456, + "English,Spanish,Chinese,Malay": 0.2897727272727273, + "English,Spanish,Chinese,Vietnamese": 0.2784090909090909, + "English,Spanish,Malay,Vietnamese": 0.32386363636363635, + "English,Chinese,Malay,Vietnamese": 0.24431818181818182, + "Filipino,Spanish,Chinese,Malay": 0.21022727272727273, + "Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, + "Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, + "Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, + "Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.0, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.0, - "English,Filipino,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian": 0.0, - "English,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.0, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 - }, + "Indonesian,English,Filipino,Spanish,Chinese": 0.17045454545454544, + "Indonesian,English,Filipino,Spanish,Malay": 0.22727272727272727, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.21022727272727273, + "Indonesian,English,Filipino,Chinese,Malay": 0.18181818181818182, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.14204545454545456, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.19886363636363635, + "Indonesian,English,Spanish,Chinese,Malay": 0.26704545454545453, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.23295454545454544, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2840909090909091, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, + "English,Filipino,Spanish,Chinese,Malay": 0.1534090909090909, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, + "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, + "English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, + "English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 + }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.1534090909090909, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 } }, - "AC3_2": 0.23768931689372788, - "AC3_3": 0.09108737561766826, - "AC3_4": 0.026083996638146475, - "AC3_5": 0.0069321093675778584, - "AC3_6": 0.0016178549335035064, - "AC3_7": 0.0 + "AC3_2": 0.4383756923595563, + "AC3_3": 0.36192203130281486, + "AC3_4": 0.3027532265335796, + "AC3_5": 0.2554138407724643, + "AC3_6": 0.21555924691342232, + "AC3_7": 0.18065364383999674 }, "prompt_2": { - "overall_acc": 0.23295454545454547, + "overall_acc": 0.37581168831168826, "language_acc": { - "English": 0.18181818181818182, - "Filipino": 0.22727272727272727, - "Spanish": 0.20454545454545456, - "Malay": 0.20454545454545456, - "Chinese": 0.25, - "Indonesian": 0.2556818181818182, - "Vietnamese": 0.3068181818181818 + "Indonesian": 0.4034090909090909, + "English": 0.375, + "Filipino": 0.29545454545454547, + "Spanish": 0.32386363636363635, + "Chinese": 0.42045454545454547, + "Malay": 0.4375, + "Vietnamese": 0.375 }, - "consistency_score_2": 0.2678571428571429, - "consistency_score_3": 0.0771103896103896, - "consistency_score_4": 0.024350649350649352, - "consistency_score_5": 0.008928571428571426, - "consistency_score_6": 0.003246753246753247, - "consistency_score_7": 0.0, + "consistency_score_2": 0.5284090909090909, + "consistency_score_3": 0.341396103896104, + "consistency_score_4": 0.24025974025974023, + "consistency_score_5": 0.17775974025974026, + "consistency_score_6": 0.13636363636363638, + "consistency_score_7": 0.10795454545454546, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.2897727272727273, - "English,Spanish": 0.3352272727272727, - "English,Malay": 0.3181818181818182, - "English,Chinese": 0.26136363636363635, - "English,Indonesian": 0.2215909090909091, - "English,Vietnamese": 0.24431818181818182, - "Filipino,Spanish": 0.3068181818181818, - "Filipino,Malay": 0.3409090909090909, - "Filipino,Chinese": 0.2840909090909091, - "Filipino,Indonesian": 0.22727272727272727, - "Filipino,Vietnamese": 0.25, - "Spanish,Malay": 0.2727272727272727, - "Spanish,Chinese": 0.23295454545454544, - "Spanish,Indonesian": 0.22727272727272727, - "Spanish,Vietnamese": 0.26136363636363635, - "Malay,Chinese": 0.26704545454545453, - "Malay,Indonesian": 0.22727272727272727, - "Malay,Vietnamese": 0.2840909090909091, - "Chinese,Indonesian": 0.1875, - "Chinese,Vietnamese": 0.2897727272727273, - "Indonesian,Vietnamese": 0.29545454545454547 + "Indonesian,English": 0.5909090909090909, + "Indonesian,Filipino": 0.4375, + "Indonesian,Spanish": 0.5568181818181818, + "Indonesian,Chinese": 0.5681818181818182, + "Indonesian,Malay": 0.6931818181818182, + "Indonesian,Vietnamese": 0.5568181818181818, + "English,Filipino": 0.42613636363636365, + "English,Spanish": 0.5852272727272727, + "English,Chinese": 0.5227272727272727, + "English,Malay": 0.5625, + "English,Vietnamese": 0.5170454545454546, + "Filipino,Spanish": 0.44886363636363635, + "Filipino,Chinese": 0.38636363636363635, + "Filipino,Malay": 0.45454545454545453, + "Filipino,Vietnamese": 0.4034090909090909, + "Spanish,Chinese": 0.5170454545454546, + "Spanish,Malay": 0.5795454545454546, + "Spanish,Vietnamese": 0.5965909090909091, + "Chinese,Malay": 0.5738636363636364, + "Chinese,Vietnamese": 0.5568181818181818, + "Malay,Vietnamese": 0.5625 }, "3_combine": { - "English,Filipino,Spanish": 0.10795454545454546, - "English,Filipino,Malay": 0.10227272727272728, - "English,Filipino,Chinese": 0.09659090909090909, - "English,Filipino,Indonesian": 0.0625, - "English,Filipino,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay": 0.10795454545454546, - "English,Spanish,Chinese": 0.09659090909090909, - "English,Spanish,Indonesian": 0.08522727272727272, - "English,Spanish,Vietnamese": 0.09090909090909091, - "English,Malay,Chinese": 0.11931818181818182, - "English,Malay,Indonesian": 0.0625, - "English,Malay,Vietnamese": 0.07954545454545454, - "English,Chinese,Indonesian": 0.03977272727272727, - "English,Chinese,Vietnamese": 0.0625, - "English,Indonesian,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Malay": 0.11931818181818182, - "Filipino,Spanish,Chinese": 0.09090909090909091, - "Filipino,Spanish,Indonesian": 0.0625, - "Filipino,Spanish,Vietnamese": 0.07386363636363637, - "Filipino,Malay,Chinese": 0.10795454545454546, - "Filipino,Malay,Indonesian": 0.09090909090909091, - "Filipino,Malay,Vietnamese": 0.09090909090909091, - "Filipino,Chinese,Indonesian": 0.0625, - "Filipino,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Indonesian,Vietnamese": 0.07954545454545454, - "Spanish,Malay,Chinese": 0.08522727272727272, - "Spanish,Malay,Indonesian": 0.056818181818181816, - "Spanish,Malay,Vietnamese": 0.08522727272727272, - "Spanish,Chinese,Indonesian": 0.03977272727272727, - "Spanish,Chinese,Vietnamese": 0.05113636363636364, - "Spanish,Indonesian,Vietnamese": 0.045454545454545456, - "Malay,Chinese,Indonesian": 0.045454545454545456, - "Malay,Chinese,Vietnamese": 0.0625, - "Malay,Indonesian,Vietnamese": 0.05113636363636364, - "Chinese,Indonesian,Vietnamese": 0.07954545454545454 + "Indonesian,English,Filipino": 0.29545454545454547, + "Indonesian,English,Spanish": 0.3977272727272727, + "Indonesian,English,Chinese": 0.375, + "Indonesian,English,Malay": 0.44886363636363635, + "Indonesian,English,Vietnamese": 0.3693181818181818, + "Indonesian,Filipino,Spanish": 0.29545454545454547, + "Indonesian,Filipino,Chinese": 0.26136363636363635, + "Indonesian,Filipino,Malay": 0.32954545454545453, + "Indonesian,Filipino,Vietnamese": 0.26704545454545453, + "Indonesian,Spanish,Chinese": 0.375, + "Indonesian,Spanish,Malay": 0.44886363636363635, + "Indonesian,Spanish,Vietnamese": 0.39204545454545453, + "Indonesian,Chinese,Malay": 0.45454545454545453, + "Indonesian,Chinese,Vietnamese": 0.38636363636363635, + "Indonesian,Malay,Vietnamese": 0.4375, + "English,Filipino,Spanish": 0.2840909090909091, + "English,Filipino,Chinese": 0.23295454545454544, + "English,Filipino,Malay": 0.2897727272727273, + "English,Filipino,Vietnamese": 0.24431818181818182, + "English,Spanish,Chinese": 0.35795454545454547, + "English,Spanish,Malay": 0.39204545454545453, + "English,Spanish,Vietnamese": 0.3977272727272727, + "English,Chinese,Malay": 0.36363636363636365, + "English,Chinese,Vietnamese": 0.3352272727272727, + "English,Malay,Vietnamese": 0.35795454545454547, + "Filipino,Spanish,Chinese": 0.25, + "Filipino,Spanish,Malay": 0.29545454545454547, + "Filipino,Spanish,Vietnamese": 0.2784090909090909, + "Filipino,Chinese,Malay": 0.2727272727272727, + "Filipino,Chinese,Vietnamese": 0.23295454545454544, + "Filipino,Malay,Vietnamese": 0.2727272727272727, + "Spanish,Chinese,Malay": 0.38636363636363635, + "Spanish,Chinese,Vietnamese": 0.3693181818181818, + "Spanish,Malay,Vietnamese": 0.42045454545454547, + "Chinese,Malay,Vietnamese": 0.3806818181818182 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.03409090909090909, - "English,Filipino,Spanish,Chinese": 0.045454545454545456, - "English,Filipino,Spanish,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese": 0.045454545454545456, - "English,Filipino,Malay,Indonesian": 0.028409090909090908, - "English,Filipino,Malay,Vietnamese": 0.017045454545454544, - "English,Filipino,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese": 0.045454545454545456, - "English,Spanish,Malay,Indonesian": 0.022727272727272728, - "English,Spanish,Malay,Vietnamese": 0.03409090909090909, - "English,Spanish,Chinese,Indonesian": 0.022727272727272728, - "English,Spanish,Chinese,Vietnamese": 0.011363636363636364, - "English,Spanish,Indonesian,Vietnamese": 0.022727272727272728, - "English,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese": 0.045454545454545456, - "Filipino,Spanish,Malay,Indonesian": 0.03409090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Chinese,Indonesian": 0.022727272727272728, - "Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Malay,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Malay,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "Spanish,Malay,Indonesian,Vietnamese": 0.0, - "Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 + "Indonesian,English,Filipino,Spanish": 0.2159090909090909, + "Indonesian,English,Filipino,Chinese": 0.17613636363636365, + "Indonesian,English,Filipino,Malay": 0.23295454545454544, + "Indonesian,English,Filipino,Vietnamese": 0.19318181818181818, + "Indonesian,English,Spanish,Chinese": 0.2784090909090909, + "Indonesian,English,Spanish,Malay": 0.32386363636363635, + "Indonesian,English,Spanish,Vietnamese": 0.2784090909090909, + "Indonesian,English,Chinese,Malay": 0.3181818181818182, + "Indonesian,English,Chinese,Vietnamese": 0.26704545454545453, + "Indonesian,English,Malay,Vietnamese": 0.3068181818181818, + "Indonesian,Filipino,Spanish,Chinese": 0.19318181818181818, + "Indonesian,Filipino,Spanish,Malay": 0.24431818181818182, + "Indonesian,Filipino,Spanish,Vietnamese": 0.19318181818181818, + "Indonesian,Filipino,Chinese,Malay": 0.2215909090909091, + "Indonesian,Filipino,Chinese,Vietnamese": 0.18181818181818182, + "Indonesian,Filipino,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,Spanish,Chinese,Malay": 0.3465909090909091, + "Indonesian,Spanish,Chinese,Vietnamese": 0.29545454545454547, + "Indonesian,Spanish,Malay,Vietnamese": 0.3522727272727273, + "Indonesian,Chinese,Malay,Vietnamese": 0.3181818181818182, + "English,Filipino,Spanish,Chinese": 0.17045454545454544, + "English,Filipino,Spanish,Malay": 0.19886363636363635, + "English,Filipino,Spanish,Vietnamese": 0.18181818181818182, + "English,Filipino,Chinese,Malay": 0.1875, + "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, + "English,Filipino,Malay,Vietnamese": 0.19318181818181818, + "English,Spanish,Chinese,Malay": 0.26704545454545453, + "English,Spanish,Chinese,Vietnamese": 0.2727272727272727, + "English,Spanish,Malay,Vietnamese": 0.30113636363636365, + "English,Chinese,Malay,Vietnamese": 0.24431818181818182, + "Filipino,Spanish,Chinese,Malay": 0.20454545454545456, + "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, + "Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, + "Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, + "Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Spanish,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese": 0.14204545454545456, + "Indonesian,English,Filipino,Spanish,Malay": 0.17613636363636365, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "Indonesian,English,Filipino,Chinese,Malay": 0.1590909090909091, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,English,Spanish,Chinese,Malay": 0.25, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2215909090909091, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.22727272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909, + "English,Filipino,Spanish,Chinese,Malay": 0.13636363636363635, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.125, + "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, + "English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, + "English,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.125, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 } }, - "AC3_2": 0.24918962717876797, - "AC3_3": 0.1158674440368816, - "AC3_4": 0.044092342971105834, - "AC3_5": 0.017197986570071166, - "AC3_6": 0.006404248669578558, - "AC3_7": 0.0 + "AC3_2": 0.43923412758321767, + "AC3_3": 0.35777817130343365, + "AC3_4": 0.29312321402425123, + "AC3_5": 0.24135706664337872, + "AC3_6": 0.20011525712848346, + "AC3_7": 0.16772803535276004 }, "prompt_3": { - "overall_acc": 0.22321428571428573, + "overall_acc": 0.38961038961038963, "language_acc": { - "English": 0.2556818181818182, - "Filipino": 0.19318181818181818, - "Spanish": 0.25, - "Malay": 0.19318181818181818, - "Chinese": 0.22727272727272727, - "Indonesian": 0.20454545454545456, - "Vietnamese": 0.23863636363636365 + "Indonesian": 0.45454545454545453, + "English": 0.4034090909090909, + "Filipino": 0.2840909090909091, + "Spanish": 0.3352272727272727, + "Chinese": 0.4318181818181818, + "Malay": 0.42045454545454547, + "Vietnamese": 0.3977272727272727 }, - "consistency_score_2": 0.25027056277056275, - "consistency_score_3": 0.06509740259740261, - "consistency_score_4": 0.01801948051948052, - "consistency_score_5": 0.00541125541125541, - "consistency_score_6": 0.0016233766233766235, - "consistency_score_7": 0.0, + "consistency_score_2": 0.549512987012987, + "consistency_score_3": 0.3704545454545455, + "consistency_score_4": 0.26931818181818185, + "consistency_score_5": 0.20454545454545456, + "consistency_score_6": 0.1590909090909091, + "consistency_score_7": 0.125, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.25, - "English,Spanish": 0.29545454545454547, - "English,Malay": 0.2784090909090909, - "English,Chinese": 0.26704545454545453, - "English,Indonesian": 0.24431818181818182, - "English,Vietnamese": 0.26136363636363635, - "Filipino,Spanish": 0.23295454545454544, - "Filipino,Malay": 0.23295454545454544, - "Filipino,Chinese": 0.2159090909090909, - "Filipino,Indonesian": 0.2727272727272727, - "Filipino,Vietnamese": 0.2727272727272727, - "Spanish,Malay": 0.30113636363636365, - "Spanish,Chinese": 0.19318181818181818, - "Spanish,Indonesian": 0.24431818181818182, - "Spanish,Vietnamese": 0.18181818181818182, - "Malay,Chinese": 0.2556818181818182, - "Malay,Indonesian": 0.26704545454545453, - "Malay,Vietnamese": 0.2556818181818182, - "Chinese,Indonesian": 0.21022727272727273, - "Chinese,Vietnamese": 0.29545454545454547, - "Indonesian,Vietnamese": 0.22727272727272727 + "Indonesian,English": 0.6363636363636364, + "Indonesian,Filipino": 0.4659090909090909, + "Indonesian,Spanish": 0.5738636363636364, + "Indonesian,Chinese": 0.5397727272727273, + "Indonesian,Malay": 0.7329545454545454, + "Indonesian,Vietnamese": 0.5909090909090909, + "English,Filipino": 0.4090909090909091, + "English,Spanish": 0.5795454545454546, + "English,Chinese": 0.5738636363636364, + "English,Malay": 0.6022727272727273, + "English,Vietnamese": 0.5795454545454546, + "Filipino,Spanish": 0.4375, + "Filipino,Chinese": 0.375, + "Filipino,Malay": 0.4715909090909091, + "Filipino,Vietnamese": 0.4318181818181818, + "Spanish,Chinese": 0.5454545454545454, + "Spanish,Malay": 0.5965909090909091, + "Spanish,Vietnamese": 0.6704545454545454, + "Chinese,Malay": 0.5397727272727273, + "Chinese,Vietnamese": 0.5738636363636364, + "Malay,Vietnamese": 0.6136363636363636 }, "3_combine": { - "English,Filipino,Spanish": 0.056818181818181816, - "English,Filipino,Malay": 0.07386363636363637, - "English,Filipino,Chinese": 0.06818181818181818, - "English,Filipino,Indonesian": 0.09090909090909091, - "English,Filipino,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay": 0.11363636363636363, - "English,Spanish,Chinese": 0.07386363636363637, - "English,Spanish,Indonesian": 0.07386363636363637, - "English,Spanish,Vietnamese": 0.07386363636363637, - "English,Malay,Chinese": 0.07386363636363637, - "English,Malay,Indonesian": 0.08522727272727272, - "English,Malay,Vietnamese": 0.07386363636363637, - "English,Chinese,Indonesian": 0.0625, - "English,Chinese,Vietnamese": 0.06818181818181818, - "English,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay": 0.056818181818181816, - "Filipino,Spanish,Chinese": 0.045454545454545456, - "Filipino,Spanish,Indonesian": 0.0625, - "Filipino,Spanish,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Chinese": 0.05113636363636364, - "Filipino,Malay,Indonesian": 0.06818181818181818, - "Filipino,Malay,Vietnamese": 0.056818181818181816, - "Filipino,Chinese,Indonesian": 0.045454545454545456, - "Filipino,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Indonesian,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Chinese": 0.03977272727272727, - "Spanish,Malay,Indonesian": 0.08522727272727272, - "Spanish,Malay,Vietnamese": 0.056818181818181816, - "Spanish,Chinese,Indonesian": 0.03409090909090909, - "Spanish,Chinese,Vietnamese": 0.03409090909090909, - "Spanish,Indonesian,Vietnamese": 0.03977272727272727, - "Malay,Chinese,Indonesian": 0.06818181818181818, - "Malay,Chinese,Vietnamese": 0.08522727272727272, - "Malay,Indonesian,Vietnamese": 0.056818181818181816, - "Chinese,Indonesian,Vietnamese": 0.0625 + "Indonesian,English,Filipino": 0.32954545454545453, + "Indonesian,English,Spanish": 0.42613636363636365, + "Indonesian,English,Chinese": 0.4034090909090909, + "Indonesian,English,Malay": 0.5284090909090909, + "Indonesian,English,Vietnamese": 0.42613636363636365, + "Indonesian,Filipino,Spanish": 0.3181818181818182, + "Indonesian,Filipino,Chinese": 0.26136363636363635, + "Indonesian,Filipino,Malay": 0.38636363636363635, + "Indonesian,Filipino,Vietnamese": 0.3125, + "Indonesian,Spanish,Chinese": 0.35795454545454547, + "Indonesian,Spanish,Malay": 0.48295454545454547, + "Indonesian,Spanish,Vietnamese": 0.4431818181818182, + "Indonesian,Chinese,Malay": 0.44886363636363635, + "Indonesian,Chinese,Vietnamese": 0.3806818181818182, + "Indonesian,Malay,Vietnamese": 0.5113636363636364, + "English,Filipino,Spanish": 0.2840909090909091, + "English,Filipino,Chinese": 0.24431818181818182, + "English,Filipino,Malay": 0.30113636363636365, + "English,Filipino,Vietnamese": 0.2784090909090909, + "English,Spanish,Chinese": 0.39204545454545453, + "English,Spanish,Malay": 0.4147727272727273, + "English,Spanish,Vietnamese": 0.44886363636363635, + "English,Chinese,Malay": 0.39204545454545453, + "English,Chinese,Vietnamese": 0.4034090909090909, + "English,Malay,Vietnamese": 0.42613636363636365, + "Filipino,Spanish,Chinese": 0.23863636363636365, + "Filipino,Spanish,Malay": 0.3068181818181818, + "Filipino,Spanish,Vietnamese": 0.3125, + "Filipino,Chinese,Malay": 0.2556818181818182, + "Filipino,Chinese,Vietnamese": 0.25, + "Filipino,Malay,Vietnamese": 0.32386363636363635, + "Spanish,Chinese,Malay": 0.3806818181818182, + "Spanish,Chinese,Vietnamese": 0.42045454545454547, + "Spanish,Malay,Vietnamese": 0.4715909090909091, + "Chinese,Malay,Vietnamese": 0.4034090909090909 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.022727272727272728, - "English,Filipino,Spanish,Chinese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian": 0.022727272727272728, - "English,Filipino,Spanish,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian": 0.03409090909090909, - "English,Filipino,Malay,Vietnamese": 0.017045454545454544, - "English,Filipino,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Chinese": 0.022727272727272728, - "English,Spanish,Malay,Indonesian": 0.03409090909090909, - "English,Spanish,Malay,Vietnamese": 0.03409090909090909, - "English,Spanish,Chinese,Indonesian": 0.005681818181818182, - "English,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Spanish,Indonesian,Vietnamese": 0.028409090909090908, - "English,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Malay,Indonesian,Vietnamese": 0.017045454545454544, - "English,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese": 0.005681818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.028409090909090908, - "Filipino,Spanish,Malay,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Chinese,Indonesian": 0.011363636363636364, - "Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Malay,Chinese,Indonesian": 0.005681818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "Spanish,Malay,Chinese,Vietnamese": 0.005681818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544 + "Indonesian,English,Filipino,Spanish": 0.24431818181818182, + "Indonesian,English,Filipino,Chinese": 0.19886363636363635, + "Indonesian,English,Filipino,Malay": 0.29545454545454547, + "Indonesian,English,Filipino,Vietnamese": 0.23863636363636365, + "Indonesian,English,Spanish,Chinese": 0.2897727272727273, + "Indonesian,English,Spanish,Malay": 0.36363636363636365, + "Indonesian,English,Spanish,Vietnamese": 0.3352272727272727, + "Indonesian,English,Chinese,Malay": 0.35795454545454547, + "Indonesian,English,Chinese,Vietnamese": 0.3068181818181818, + "Indonesian,English,Malay,Vietnamese": 0.38636363636363635, + "Indonesian,Filipino,Spanish,Chinese": 0.1875, + "Indonesian,Filipino,Spanish,Malay": 0.2784090909090909, + "Indonesian,Filipino,Spanish,Vietnamese": 0.23863636363636365, + "Indonesian,Filipino,Chinese,Malay": 0.23863636363636365, + "Indonesian,Filipino,Chinese,Vietnamese": 0.19318181818181818, + "Indonesian,Filipino,Malay,Vietnamese": 0.30113636363636365, + "Indonesian,Spanish,Chinese,Malay": 0.32386363636363635, + "Indonesian,Spanish,Chinese,Vietnamese": 0.29545454545454547, + "Indonesian,Spanish,Malay,Vietnamese": 0.3977272727272727, + "Indonesian,Chinese,Malay,Vietnamese": 0.3352272727272727, + "English,Filipino,Spanish,Chinese": 0.17613636363636365, + "English,Filipino,Spanish,Malay": 0.2215909090909091, + "English,Filipino,Spanish,Vietnamese": 0.2215909090909091, + "English,Filipino,Chinese,Malay": 0.19318181818181818, + "English,Filipino,Chinese,Vietnamese": 0.18181818181818182, + "English,Filipino,Malay,Vietnamese": 0.23295454545454544, + "English,Spanish,Chinese,Malay": 0.2897727272727273, + "English,Spanish,Chinese,Vietnamese": 0.32386363636363635, + "English,Spanish,Malay,Vietnamese": 0.3409090909090909, + "English,Chinese,Malay,Vietnamese": 0.3068181818181818, + "Filipino,Spanish,Chinese,Malay": 0.19318181818181818, + "Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, + "Filipino,Spanish,Malay,Vietnamese": 0.24431818181818182, + "Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, + "Spanish,Chinese,Malay,Vietnamese": 0.3125 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese": 0.1534090909090909, + "Indonesian,English,Filipino,Spanish,Malay": 0.2215909090909091, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1875, + "Indonesian,English,Filipino,Chinese,Malay": 0.19318181818181818, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1590909090909091, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,English,Spanish,Chinese,Malay": 0.26136363636363635, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.24431818181818182, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.3068181818181818, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453, + "English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, + "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, + "English,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, + "English,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.125, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.1875, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 } }, - "AC3_2": 0.2359693877052653, - "AC3_3": 0.10079834295088141, - "AC3_4": 0.033346952495309355, - "AC3_5": 0.010566356715581184, - "AC3_6": 0.0032233109836102202, - "AC3_7": 0.0 + "AC3_2": 0.45594854578763583, + "AC3_3": 0.3797910760247473, + "AC3_4": 0.3184841765289693, + "AC3_5": 0.26825633378495517, + "AC3_6": 0.22592791819443553, + "AC3_7": 0.18927444791274667 }, "prompt_4": { - "overall_acc": 0.25405844155844154, + "overall_acc": 0.3766233766233766, "language_acc": { - "English": 0.2556818181818182, - "Filipino": 0.25, - "Spanish": 0.25, - "Malay": 0.2840909090909091, - "Chinese": 0.2159090909090909, - "Indonesian": 0.2897727272727273, - "Vietnamese": 0.23295454545454544 + "Indonesian": 0.3806818181818182, + "English": 0.4034090909090909, + "Filipino": 0.3068181818181818, + "Spanish": 0.3522727272727273, + "Chinese": 0.42045454545454547, + "Malay": 0.4090909090909091, + "Vietnamese": 0.36363636363636365 }, - "consistency_score_2": 0.2597402597402597, - "consistency_score_3": 0.06785714285714285, - "consistency_score_4": 0.018668831168831168, - "consistency_score_5": 0.005952380952380952, - "consistency_score_6": 0.0016233766233766235, - "consistency_score_7": 0.0, + "consistency_score_2": 0.5484307359307358, + "consistency_score_3": 0.3693181818181818, + "consistency_score_4": 0.26948051948051943, + "consistency_score_5": 0.20589826839826836, + "consistency_score_6": 0.16233766233766236, + "consistency_score_7": 0.13068181818181818, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.1875, - "English,Spanish": 0.32954545454545453, - "English,Malay": 0.24431818181818182, - "English,Chinese": 0.23295454545454544, - "English,Indonesian": 0.25, - "English,Vietnamese": 0.23295454545454544, - "Filipino,Spanish": 0.23295454545454544, - "Filipino,Malay": 0.26704545454545453, - "Filipino,Chinese": 0.3125, - "Filipino,Indonesian": 0.2159090909090909, - "Filipino,Vietnamese": 0.2727272727272727, - "Spanish,Malay": 0.3125, - "Spanish,Chinese": 0.25, - "Spanish,Indonesian": 0.3125, - "Spanish,Vietnamese": 0.22727272727272727, - "Malay,Chinese": 0.3068181818181818, - "Malay,Indonesian": 0.23863636363636365, - "Malay,Vietnamese": 0.26136363636363635, - "Chinese,Indonesian": 0.26704545454545453, - "Chinese,Vietnamese": 0.26136363636363635, - "Indonesian,Vietnamese": 0.23863636363636365 + "Indonesian,English": 0.6477272727272727, + "Indonesian,Filipino": 0.48295454545454547, + "Indonesian,Spanish": 0.5965909090909091, + "Indonesian,Chinese": 0.5568181818181818, + "Indonesian,Malay": 0.6931818181818182, + "Indonesian,Vietnamese": 0.5738636363636364, + "English,Filipino": 0.48863636363636365, + "English,Spanish": 0.6363636363636364, + "English,Chinese": 0.6022727272727273, + "English,Malay": 0.5909090909090909, + "English,Vietnamese": 0.5625, + "Filipino,Spanish": 0.48863636363636365, + "Filipino,Chinese": 0.36363636363636365, + "Filipino,Malay": 0.4602272727272727, + "Filipino,Vietnamese": 0.39204545454545453, + "Spanish,Chinese": 0.5113636363636364, + "Spanish,Malay": 0.6079545454545454, + "Spanish,Vietnamese": 0.5852272727272727, + "Chinese,Malay": 0.5397727272727273, + "Chinese,Vietnamese": 0.5340909090909091, + "Malay,Vietnamese": 0.6022727272727273 }, "3_combine": { - "English,Filipino,Spanish": 0.05113636363636364, - "English,Filipino,Malay": 0.056818181818181816, - "English,Filipino,Chinese": 0.07386363636363637, - "English,Filipino,Indonesian": 0.05113636363636364, - "English,Filipino,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay": 0.09090909090909091, - "English,Spanish,Chinese": 0.09090909090909091, - "English,Spanish,Indonesian": 0.10227272727272728, - "English,Spanish,Vietnamese": 0.0625, - "English,Malay,Chinese": 0.0625, - "English,Malay,Indonesian": 0.05113636363636364, - "English,Malay,Vietnamese": 0.05113636363636364, - "English,Chinese,Indonesian": 0.07386363636363637, - "English,Chinese,Vietnamese": 0.05113636363636364, - "English,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay": 0.05113636363636364, - "Filipino,Spanish,Chinese": 0.07386363636363637, - "Filipino,Spanish,Indonesian": 0.06818181818181818, - "Filipino,Spanish,Vietnamese": 0.0625, - "Filipino,Malay,Chinese": 0.10795454545454546, - "Filipino,Malay,Indonesian": 0.05113636363636364, - "Filipino,Malay,Vietnamese": 0.056818181818181816, - "Filipino,Chinese,Indonesian": 0.06818181818181818, - "Filipino,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Indonesian,Vietnamese": 0.06818181818181818, - "Spanish,Malay,Chinese": 0.10227272727272728, - "Spanish,Malay,Indonesian": 0.08522727272727272, - "Spanish,Malay,Vietnamese": 0.07386363636363637, - "Spanish,Chinese,Indonesian": 0.07386363636363637, - "Spanish,Chinese,Vietnamese": 0.056818181818181816, - "Spanish,Indonesian,Vietnamese": 0.056818181818181816, - "Malay,Chinese,Indonesian": 0.08522727272727272, - "Malay,Chinese,Vietnamese": 0.07954545454545454, - "Malay,Indonesian,Vietnamese": 0.05113636363636364, - "Chinese,Indonesian,Vietnamese": 0.07386363636363637 + "Indonesian,English,Filipino": 0.3522727272727273, + "Indonesian,English,Spanish": 0.4602272727272727, + "Indonesian,English,Chinese": 0.4431818181818182, + "Indonesian,English,Malay": 0.4943181818181818, + "Indonesian,English,Vietnamese": 0.42613636363636365, + "Indonesian,Filipino,Spanish": 0.3352272727272727, + "Indonesian,Filipino,Chinese": 0.26704545454545453, + "Indonesian,Filipino,Malay": 0.3522727272727273, + "Indonesian,Filipino,Vietnamese": 0.30113636363636365, + "Indonesian,Spanish,Chinese": 0.375, + "Indonesian,Spanish,Malay": 0.4659090909090909, + "Indonesian,Spanish,Vietnamese": 0.4034090909090909, + "Indonesian,Chinese,Malay": 0.4375, + "Indonesian,Chinese,Vietnamese": 0.375, + "Indonesian,Malay,Vietnamese": 0.4659090909090909, + "English,Filipino,Spanish": 0.3522727272727273, + "English,Filipino,Chinese": 0.2897727272727273, + "English,Filipino,Malay": 0.32954545454545453, + "English,Filipino,Vietnamese": 0.2897727272727273, + "English,Spanish,Chinese": 0.42045454545454547, + "English,Spanish,Malay": 0.4375, + "English,Spanish,Vietnamese": 0.42045454545454547, + "English,Chinese,Malay": 0.42045454545454547, + "English,Chinese,Vietnamese": 0.3977272727272727, + "English,Malay,Vietnamese": 0.3977272727272727, + "Filipino,Spanish,Chinese": 0.2556818181818182, + "Filipino,Spanish,Malay": 0.3352272727272727, + "Filipino,Spanish,Vietnamese": 0.29545454545454547, + "Filipino,Chinese,Malay": 0.26136363636363635, + "Filipino,Chinese,Vietnamese": 0.22727272727272727, + "Filipino,Malay,Vietnamese": 0.29545454545454547, + "Spanish,Chinese,Malay": 0.375, + "Spanish,Chinese,Vietnamese": 0.3693181818181818, + "Spanish,Malay,Vietnamese": 0.42045454545454547, + "Chinese,Malay,Vietnamese": 0.3806818181818182 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.005681818181818182, - "English,Filipino,Spanish,Chinese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Vietnamese": 0.017045454545454544, - "English,Filipino,Malay,Chinese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian": 0.011363636363636364, - "English,Filipino,Malay,Vietnamese": 0.005681818181818182, - "English,Filipino,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese": 0.028409090909090908, - "English,Spanish,Malay,Indonesian": 0.028409090909090908, - "English,Spanish,Malay,Vietnamese": 0.017045454545454544, - "English,Spanish,Chinese,Indonesian": 0.03977272727272727, - "English,Spanish,Chinese,Vietnamese": 0.028409090909090908, - "English,Spanish,Indonesian,Vietnamese": 0.017045454545454544, - "English,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese": 0.017045454545454544, - "Filipino,Spanish,Malay,Indonesian": 0.005681818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Chinese,Indonesian": 0.022727272727272728, - "Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Malay,Chinese,Indonesian": 0.022727272727272728, - "Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Spanish,Malay,Chinese,Indonesian": 0.03409090909090909, - "Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Spanish,Malay,Indonesian,Vietnamese": 0.017045454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 + "Indonesian,English,Filipino,Spanish": 0.26704545454545453, + "Indonesian,English,Filipino,Chinese": 0.2215909090909091, + "Indonesian,English,Filipino,Malay": 0.2784090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.25, + "Indonesian,English,Spanish,Chinese": 0.3352272727272727, + "Indonesian,English,Spanish,Malay": 0.35795454545454547, + "Indonesian,English,Spanish,Vietnamese": 0.32954545454545453, + "Indonesian,English,Chinese,Malay": 0.3693181818181818, + "Indonesian,English,Chinese,Vietnamese": 0.3125, + "Indonesian,English,Malay,Vietnamese": 0.36363636363636365, + "Indonesian,Filipino,Spanish,Chinese": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Malay": 0.26704545454545453, + "Indonesian,Filipino,Spanish,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Chinese,Malay": 0.2215909090909091, + "Indonesian,Filipino,Chinese,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Malay,Vietnamese": 0.2727272727272727, + "Indonesian,Spanish,Chinese,Malay": 0.3068181818181818, + "Indonesian,Spanish,Chinese,Vietnamese": 0.2784090909090909, + "Indonesian,Spanish,Malay,Vietnamese": 0.3522727272727273, + "Indonesian,Chinese,Malay,Vietnamese": 0.32954545454545453, + "English,Filipino,Spanish,Chinese": 0.2215909090909091, + "English,Filipino,Spanish,Malay": 0.2556818181818182, + "English,Filipino,Spanish,Vietnamese": 0.22727272727272727, + "English,Filipino,Chinese,Malay": 0.2215909090909091, + "English,Filipino,Chinese,Vietnamese": 0.19318181818181818, + "English,Filipino,Malay,Vietnamese": 0.2215909090909091, + "English,Spanish,Chinese,Malay": 0.32386363636363635, + "English,Spanish,Chinese,Vietnamese": 0.3068181818181818, + "English,Spanish,Malay,Vietnamese": 0.32386363636363635, + "English,Chinese,Malay,Vietnamese": 0.30113636363636365, + "Filipino,Spanish,Chinese,Malay": 0.21022727272727273, + "Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, + "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, + "Filipino,Chinese,Malay,Vietnamese": 0.1875, + "Spanish,Chinese,Malay,Vietnamese": 0.2897727272727273 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.0, - "English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 + "Indonesian,English,Filipino,Spanish,Chinese": 0.18181818181818182, + "Indonesian,English,Filipino,Spanish,Malay": 0.2159090909090909, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.19886363636363635, + "Indonesian,English,Filipino,Chinese,Malay": 0.19318181818181818, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.17045454545454544, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,English,Spanish,Chinese,Malay": 0.2784090909090909, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.25, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2897727272727273, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17613636363636365, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.25, + "English,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, + "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, + "English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, + "English,Spanish,Chinese,Malay,Vietnamese": 0.25, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 } }, - "AC3_2": 0.2568679345430592, - "AC3_3": 0.10710683664996434, - "AC3_4": 0.034781810438699975, - "AC3_5": 0.011632228329107343, - "AC3_6": 0.003226138939162876, - "AC3_7": 0.0 + "AC3_2": 0.4465724389196009, + "AC3_3": 0.37293500835835375, + "AC3_4": 0.3141682437678599, + "AC3_5": 0.2662428142695661, + "AC3_6": 0.2268815521406609, + "AC3_7": 0.19403636359811519 }, "prompt_5": { - "overall_acc": 0.24431818181818185, + "overall_acc": 0.38636363636363635, "language_acc": { - "English": 0.2556818181818182, - "Filipino": 0.2556818181818182, - "Spanish": 0.26704545454545453, - "Malay": 0.23863636363636365, - "Chinese": 0.23863636363636365, - "Indonesian": 0.1875, - "Vietnamese": 0.26704545454545453 + "Indonesian": 0.42613636363636365, + "English": 0.4147727272727273, + "Filipino": 0.26136363636363635, + "Spanish": 0.36363636363636365, + "Chinese": 0.4375, + "Malay": 0.42613636363636365, + "Vietnamese": 0.375 }, - "consistency_score_2": 0.24594155844155846, - "consistency_score_3": 0.056818181818181816, - "consistency_score_4": 0.011038961038961034, - "consistency_score_5": 0.001352813852813853, - "consistency_score_6": 0.0, - "consistency_score_7": 0.0, + "consistency_score_2": 0.5533008658008659, + "consistency_score_3": 0.375487012987013, + "consistency_score_4": 0.2788961038961038, + "consistency_score_5": 0.21780303030303033, + "consistency_score_6": 0.1728896103896104, + "consistency_score_7": 0.13636363636363635, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.26136363636363635, - "English,Malay": 0.24431818181818182, - "English,Chinese": 0.26136363636363635, - "English,Indonesian": 0.22727272727272727, - "English,Vietnamese": 0.26136363636363635, - "Filipino,Spanish": 0.2556818181818182, - "Filipino,Malay": 0.2727272727272727, - "Filipino,Chinese": 0.1875, - "Filipino,Indonesian": 0.22727272727272727, - "Filipino,Vietnamese": 0.2840909090909091, - "Spanish,Malay": 0.2556818181818182, - "Spanish,Chinese": 0.2556818181818182, - "Spanish,Indonesian": 0.21022727272727273, - "Spanish,Vietnamese": 0.2159090909090909, - "Malay,Chinese": 0.2556818181818182, - "Malay,Indonesian": 0.21022727272727273, - "Malay,Vietnamese": 0.25, - "Chinese,Indonesian": 0.25, - "Chinese,Vietnamese": 0.2215909090909091, - "Indonesian,Vietnamese": 0.2215909090909091 - }, - "3_combine": { - "English,Filipino,Spanish": 0.07386363636363637, - "English,Filipino,Malay": 0.09659090909090909, - "English,Filipino,Chinese": 0.07386363636363637, - "English,Filipino,Indonesian": 0.03977272727272727, - "English,Filipino,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay": 0.06818181818181818, - "English,Spanish,Chinese": 0.07954545454545454, - "English,Spanish,Indonesian": 0.06818181818181818, - "English,Spanish,Vietnamese": 0.03977272727272727, - "English,Malay,Chinese": 0.056818181818181816, - "English,Malay,Indonesian": 0.045454545454545456, - "English,Malay,Vietnamese": 0.056818181818181816, - "English,Chinese,Indonesian": 0.06818181818181818, - "English,Chinese,Vietnamese": 0.03977272727272727, - "English,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay": 0.09090909090909091, - "Filipino,Spanish,Chinese": 0.045454545454545456, - "Filipino,Spanish,Indonesian": 0.06818181818181818, - "Filipino,Spanish,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Chinese": 0.028409090909090908, - "Filipino,Malay,Indonesian": 0.028409090909090908, - "Filipino,Malay,Vietnamese": 0.09090909090909091, - "Filipino,Chinese,Indonesian": 0.03409090909090909, - "Filipino,Chinese,Vietnamese": 0.03977272727272727, - "Filipino,Indonesian,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Chinese": 0.0625, - "Spanish,Malay,Indonesian": 0.03977272727272727, - "Spanish,Malay,Vietnamese": 0.056818181818181816, - "Spanish,Chinese,Indonesian": 0.03977272727272727, - "Spanish,Chinese,Vietnamese": 0.05113636363636364, - "Spanish,Indonesian,Vietnamese": 0.03409090909090909, - "Malay,Chinese,Indonesian": 0.03977272727272727, - "Malay,Chinese,Vietnamese": 0.056818181818181816, - "Malay,Indonesian,Vietnamese": 0.0625, - "Chinese,Indonesian,Vietnamese": 0.022727272727272728 + "Indonesian,English": 0.6193181818181818, + "Indonesian,Filipino": 0.4375, + "Indonesian,Spanish": 0.5738636363636364, + "Indonesian,Chinese": 0.5965909090909091, + "Indonesian,Malay": 0.7045454545454546, + "Indonesian,Vietnamese": 0.6079545454545454, + "English,Filipino": 0.4147727272727273, + "English,Spanish": 0.5681818181818182, + "English,Chinese": 0.5965909090909091, + "English,Malay": 0.5965909090909091, + "English,Vietnamese": 0.6193181818181818, + "Filipino,Spanish": 0.42613636363636365, + "Filipino,Chinese": 0.38636363636363635, + "Filipino,Malay": 0.4318181818181818, + "Filipino,Vietnamese": 0.44886363636363635, + "Spanish,Chinese": 0.5625, + "Spanish,Malay": 0.5852272727272727, + "Spanish,Vietnamese": 0.6306818181818182, + "Chinese,Malay": 0.5909090909090909, + "Chinese,Vietnamese": 0.5909090909090909, + "Malay,Vietnamese": 0.6306818181818182 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.30113636363636365, + "Indonesian,English,Spanish": 0.4090909090909091, + "Indonesian,English,Chinese": 0.4375, + "Indonesian,English,Malay": 0.48295454545454547, + "Indonesian,English,Vietnamese": 0.45454545454545453, + "Indonesian,Filipino,Spanish": 0.29545454545454547, + "Indonesian,Filipino,Chinese": 0.2784090909090909, + "Indonesian,Filipino,Malay": 0.3352272727272727, + "Indonesian,Filipino,Vietnamese": 0.3068181818181818, + "Indonesian,Spanish,Chinese": 0.4034090909090909, + "Indonesian,Spanish,Malay": 0.44886363636363635, + "Indonesian,Spanish,Vietnamese": 0.44886363636363635, + "Indonesian,Chinese,Malay": 0.4772727272727273, + "Indonesian,Chinese,Vietnamese": 0.42613636363636365, + "Indonesian,Malay,Vietnamese": 0.5, + "English,Filipino,Spanish": 0.2727272727272727, + "English,Filipino,Chinese": 0.2556818181818182, + "English,Filipino,Malay": 0.2897727272727273, + "English,Filipino,Vietnamese": 0.2897727272727273, + "English,Spanish,Chinese": 0.4090909090909091, + "English,Spanish,Malay": 0.39204545454545453, + "English,Spanish,Vietnamese": 0.44886363636363635, + "English,Chinese,Malay": 0.4318181818181818, + "English,Chinese,Vietnamese": 0.4375, + "English,Malay,Vietnamese": 0.4602272727272727, + "Filipino,Spanish,Chinese": 0.26136363636363635, + "Filipino,Spanish,Malay": 0.29545454545454547, + "Filipino,Spanish,Vietnamese": 0.3068181818181818, + "Filipino,Chinese,Malay": 0.26136363636363635, + "Filipino,Chinese,Vietnamese": 0.2784090909090909, + "Filipino,Malay,Vietnamese": 0.32954545454545453, + "Spanish,Chinese,Malay": 0.4090909090909091, + "Spanish,Chinese,Vietnamese": 0.42045454545454547, + "Spanish,Malay,Vietnamese": 0.45454545454545453, + "Chinese,Malay,Vietnamese": 0.4318181818181818 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.03409090909090909, - "English,Filipino,Spanish,Chinese": 0.017045454545454544, - "English,Filipino,Spanish,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Vietnamese": 0.017045454545454544, - "English,Filipino,Malay,Chinese": 0.011363636363636364, - "English,Filipino,Malay,Indonesian": 0.005681818181818182, - "English,Filipino,Malay,Vietnamese": 0.022727272727272728, - "English,Filipino,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Indonesian,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Chinese": 0.011363636363636364, - "English,Spanish,Malay,Indonesian": 0.011363636363636364, - "English,Spanish,Malay,Vietnamese": 0.011363636363636364, - "English,Spanish,Chinese,Indonesian": 0.028409090909090908, - "English,Spanish,Chinese,Vietnamese": 0.005681818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.0, - "English,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese": 0.005681818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.011363636363636364, - "Filipino,Spanish,Malay,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Chinese,Indonesian": 0.005681818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Malay,Chinese,Indonesian": 0.0, - "Filipino,Malay,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Chinese,Indonesian,Vietnamese": 0.0, - "Spanish,Malay,Chinese,Indonesian": 0.0, - "Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "Spanish,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 + "Indonesian,English,Filipino,Spanish": 0.2215909090909091, + "Indonesian,English,Filipino,Chinese": 0.20454545454545456, + "Indonesian,English,Filipino,Malay": 0.24431818181818182, + "Indonesian,English,Filipino,Vietnamese": 0.24431818181818182, + "Indonesian,English,Spanish,Chinese": 0.32386363636363635, + "Indonesian,English,Spanish,Malay": 0.32954545454545453, + "Indonesian,English,Spanish,Vietnamese": 0.35795454545454547, + "Indonesian,English,Chinese,Malay": 0.3693181818181818, + "Indonesian,English,Chinese,Vietnamese": 0.3465909090909091, + "Indonesian,English,Malay,Vietnamese": 0.39204545454545453, + "Indonesian,Filipino,Spanish,Chinese": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Malay": 0.2556818181818182, + "Indonesian,Filipino,Spanish,Vietnamese": 0.25, + "Indonesian,Filipino,Chinese,Malay": 0.22727272727272727, + "Indonesian,Filipino,Chinese,Vietnamese": 0.2215909090909091, + "Indonesian,Filipino,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,Spanish,Chinese,Malay": 0.3409090909090909, + "Indonesian,Spanish,Chinese,Vietnamese": 0.3352272727272727, + "Indonesian,Spanish,Malay,Vietnamese": 0.3806818181818182, + "Indonesian,Chinese,Malay,Vietnamese": 0.3693181818181818, + "English,Filipino,Spanish,Chinese": 0.19318181818181818, + "English,Filipino,Spanish,Malay": 0.20454545454545456, + "English,Filipino,Spanish,Vietnamese": 0.22727272727272727, + "English,Filipino,Chinese,Malay": 0.21022727272727273, + "English,Filipino,Chinese,Vietnamese": 0.20454545454545456, + "English,Filipino,Malay,Vietnamese": 0.24431818181818182, + "English,Spanish,Chinese,Malay": 0.3125, + "English,Spanish,Chinese,Vietnamese": 0.3409090909090909, + "English,Spanish,Malay,Vietnamese": 0.35795454545454547, + "English,Chinese,Malay,Vietnamese": 0.3465909090909091, + "Filipino,Spanish,Chinese,Malay": 0.20454545454545456, + "Filipino,Spanish,Chinese,Vietnamese": 0.2215909090909091, + "Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, + "Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Spanish,Chinese,Malay,Vietnamese": 0.3352272727272727 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.0, - "English,Filipino,Spanish,Malay,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.0, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian": 0.0, - "English,Filipino,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian": 0.0, - "English,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.0, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese": 0.1590909090909091, + "Indonesian,English,Filipino,Spanish,Malay": 0.1875, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.20454545454545456, + "Indonesian,English,Filipino,Chinese,Malay": 0.18181818181818182, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.18181818181818182, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,English,Spanish,Chinese,Malay": 0.2727272727272727, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2897727272727273, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.3125, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.3068181818181818, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1875, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.19886363636363635, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547, + "English,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, + "English,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, + "English,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, + "English,Spanish,Chinese,Malay,Vietnamese": 0.2840909090909091, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 } }, - "AC3_2": 0.24512718237022935, - "AC3_3": 0.09219554027813198, - "AC3_4": 0.021123504586306824, - "AC3_5": 0.0026907288735543225, - "AC3_6": 0.0, - "AC3_7": 0.0 + "AC3_2": 0.45500353370547175, + "AC3_3": 0.38084768412179404, + "AC3_4": 0.3239495984250613, + "AC3_5": 0.27856939294334765, + "AC3_6": 0.23888375770896753, + "AC3_7": 0.20158102762942087 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.2815533980582524 + "accuracy": 0.4563106796116505 }, "prompt_2": { - "accuracy": 0.32038834951456313 + "accuracy": 0.5436893203883495 }, "prompt_3": { - "accuracy": 0.30097087378640774 + "accuracy": 0.5048543689320388 }, "prompt_4": { - "accuracy": 0.2621359223300971 + "accuracy": 0.4368932038834951 }, "prompt_5": { - "accuracy": 0.23300970873786409 + "accuracy": 0.4854368932038835 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.24761904761904763 + "accuracy": 0.5238095238095238 }, "prompt_2": { - "accuracy": 0.3619047619047619 + "accuracy": 0.49523809523809526 }, "prompt_3": { - "accuracy": 0.2761904761904762 + "accuracy": 0.5142857142857142 }, "prompt_4": { - "accuracy": 0.21904761904761905 + "accuracy": 0.4857142857142857 }, "prompt_5": { - "accuracy": 0.3333333333333333 + "accuracy": 0.47619047619047616 } }, "us_eval": { "prompt_1": { - "accuracy": 0.2897196261682243 + "accuracy": 0.48598130841121495 }, "prompt_2": { - "accuracy": 0.2523364485981308 + "accuracy": 0.4672897196261682 }, "prompt_3": { - "accuracy": 0.2803738317757009 + "accuracy": 0.48598130841121495 }, "prompt_4": { - "accuracy": 0.205607476635514 + "accuracy": 0.48598130841121495 }, "prompt_5": { - "accuracy": 0.27102803738317754 + "accuracy": 0.48598130841121495 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.25, + "accuracy": 0.33, "category_acc": { "brand": 0.3, - "demographics": 0.2, - "biology": 0.3, - "history": 0.2, - "literature": 0.2, - "politics": 0.5, - "culture": 0.1, - "film": 0.2, - "law": 0.2, - "geography": 0.3 + "demographics": 0.4, + "biology": 0.4, + "history": 0.26666666666666666, + "literature": 0.1, + "politics": 0.2, + "culture": 0.4, + "film": 0.4, + "law": 0.3, + "geography": 0.6 } }, "prompt_2": { - "accuracy": 0.21, + "accuracy": 0.32, "category_acc": { - "brand": 0.2, + "brand": 0.4, "demographics": 0.0, - "biology": 0.1, + "biology": 0.4, "history": 0.26666666666666666, - "literature": 0.0, - "politics": 0.4, + "literature": 0.1, + "politics": 0.2, "culture": 0.3, - "film": 0.2, + "film": 0.3, "law": 0.3, - "geography": 0.2 + "geography": 0.8 } }, "prompt_3": { - "accuracy": 0.21, + "accuracy": 0.38, "category_acc": { - "brand": 0.0, + "brand": 0.5, "demographics": 0.2, - "biology": 0.3, - "history": 0.13333333333333333, + "biology": 0.4, + "history": 0.26666666666666666, "literature": 0.2, - "politics": 0.4, - "culture": 0.3, - "film": 0.1, + "politics": 0.2, + "culture": 0.4, + "film": 0.4, "law": 0.3, - "geography": 0.2 + "geography": 0.9 } }, "prompt_4": { - "accuracy": 0.33, + "accuracy": 0.31, "category_acc": { - "brand": 0.4, - "demographics": 0.2, + "brand": 0.2, + "demographics": 0.0, "biology": 0.4, - "history": 0.3333333333333333, + "history": 0.2, "literature": 0.1, - "politics": 0.6, - "culture": 0.2, + "politics": 0.4, + "culture": 0.3, "film": 0.4, "law": 0.3, - "geography": 0.3 + "geography": 0.7 } }, "prompt_5": { - "accuracy": 0.32, + "accuracy": 0.36, "category_acc": { - "brand": 0.4, + "brand": 0.3, "demographics": 0.4, - "biology": 0.2, + "biology": 0.5, "history": 0.26666666666666666, - "literature": 0.4, - "politics": 0.3, - "culture": 0.1, + "literature": 0.2, + "politics": 0.4, + "culture": 0.3, "film": 0.4, "law": 0.3, - "geography": 0.5 + "geography": 0.6 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.043001602599385534 + "bleu_score": 0.1866125409947737 }, "prompt_2": { - "bleu_score": 0.026464425346842502 + "bleu_score": 0.18638187736641829 }, "prompt_3": { - "bleu_score": 0.040085889752202446 + "bleu_score": 0.18540130650550757 }, "prompt_4": { - "bleu_score": 0.05080641488782834 + "bleu_score": 0.1880559410252317 }, "prompt_5": { - "bleu_score": 0.04599537851212927 + "bleu_score": 0.15033083280606302 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "prompt_1": { + "accuracy": 0.4552373322651712, + "category_acc": { + "History": 0.43172690763052207, + "Geography": 0.37755102040816324, + "Lampungic": 0.4013605442176871, + "Social science": 0.6978297161936561, + "Balinese": 0.3078556263269639, + "Makassarese": 0.34408602150537637, + "Banjarese": 0.3958333333333333, + "Chemistry": 0.2656934306569343, + "Biology": 0.40828402366863903, + "Science": 0.5366357069143447, + "Christian religion": 0.5572139303482587, + "Art": 0.5590682196339434, + "Islam religion": 0.5689900426742532, + "Hindu religion": 0.3933333333333333, + "Madurese": 0.2983050847457627, + "Sport": 0.4864864864864865, + "Indonesian language": 0.5077833125778332, + "Physics": 0.36363636363636365, + "Minangkabau culture": 0.37185929648241206, + "Dayak language": 0.26605504587155965, + "Sociology": 0.46169354838709675, + "Economy": 0.4180327868852459, + "Sundanese": 0.4044943820224719, + "Javanese": 0.3659274193548387, + "Civic education": 0.5493562231759657 + } + }, + "prompt_2": { + "accuracy": 0.46404966953735227, + "category_acc": { + "History": 0.44377510040160645, + "Geography": 0.3877551020408163, + "Lampungic": 0.4013605442176871, + "Social science": 0.6961602671118531, + "Balinese": 0.3036093418259023, + "Makassarese": 0.3602150537634409, + "Banjarese": 0.4097222222222222, + "Chemistry": 0.2671532846715328, + "Biology": 0.4059171597633136, + "Science": 0.5655314757481941, + "Christian religion": 0.582089552238806, + "Art": 0.5723793677204659, + "Islam religion": 0.5846372688477952, + "Hindu religion": 0.42, + "Madurese": 0.3220338983050847, + "Sport": 0.5067567567567568, + "Indonesian language": 0.5121419676214197, + "Physics": 0.3575757575757576, + "Minangkabau culture": 0.4271356783919598, + "Dayak language": 0.30275229357798167, + "Sociology": 0.4939516129032258, + "Economy": 0.42213114754098363, + "Sundanese": 0.39671564390665515, + "Javanese": 0.3790322580645161, + "Civic education": 0.5579399141630901 + } + }, + "prompt_3": { + "accuracy": 0.4582415381534148, + "category_acc": { + "History": 0.41164658634538154, + "Geography": 0.38979591836734695, + "Lampungic": 0.4217687074829932, + "Social science": 0.6844741235392321, + "Balinese": 0.3354564755838641, + "Makassarese": 0.40860215053763443, + "Banjarese": 0.375, + "Chemistry": 0.24525547445255474, + "Biology": 0.42248520710059173, + "Science": 0.5614035087719298, + "Christian religion": 0.5671641791044776, + "Art": 0.5773710482529119, + "Islam religion": 0.5633001422475107, + "Hindu religion": 0.46, + "Madurese": 0.2677966101694915, + "Sport": 0.47297297297297297, + "Indonesian language": 0.5161892901618929, + "Physics": 0.36363636363636365, + "Minangkabau culture": 0.39195979899497485, + "Dayak language": 0.3119266055045872, + "Sociology": 0.49193548387096775, + "Economy": 0.41598360655737704, + "Sundanese": 0.3863439930855661, + "Javanese": 0.35080645161290325, + "Civic education": 0.5321888412017167 + } + }, + "prompt_4": { + "accuracy": 0.4557714133119701, + "category_acc": { + "History": 0.4397590361445783, + "Geography": 0.37142857142857144, + "Lampungic": 0.32653061224489793, + "Social science": 0.679465776293823, + "Balinese": 0.3099787685774947, + "Makassarese": 0.3763440860215054, + "Banjarese": 0.3541666666666667, + "Chemistry": 0.272992700729927, + "Biology": 0.40946745562130177, + "Science": 0.5490196078431373, + "Christian religion": 0.5771144278606966, + "Art": 0.5490848585690515, + "Islam religion": 0.5761024182076814, + "Hindu religion": 0.43333333333333335, + "Madurese": 0.3050847457627119, + "Sport": 0.43243243243243246, + "Indonesian language": 0.5158779576587795, + "Physics": 0.3878787878787879, + "Minangkabau culture": 0.3869346733668342, + "Dayak language": 0.29357798165137616, + "Sociology": 0.4737903225806452, + "Economy": 0.430327868852459, + "Sundanese": 0.38807260155574763, + "Javanese": 0.3467741935483871, + "Civic education": 0.5336194563662375 + } + }, + "prompt_5": { + "accuracy": 0.4355430936644636, + "category_acc": { + "History": 0.41566265060240964, + "Geography": 0.35306122448979593, + "Lampungic": 0.3673469387755102, + "Social science": 0.6477462437395659, + "Balinese": 0.2802547770700637, + "Makassarese": 0.34408602150537637, + "Banjarese": 0.3680555555555556, + "Chemistry": 0.2671532846715328, + "Biology": 0.3857988165680473, + "Science": 0.5211558307533539, + "Christian religion": 0.5422885572139303, + "Art": 0.5291181364392679, + "Islam religion": 0.5405405405405406, + "Hindu religion": 0.41333333333333333, + "Madurese": 0.3254237288135593, + "Sport": 0.4527027027027027, + "Indonesian language": 0.4953300124533001, + "Physics": 0.3898989898989899, + "Minangkabau culture": 0.34673366834170855, + "Dayak language": 0.24770642201834864, + "Sociology": 0.4435483870967742, + "Economy": 0.4077868852459016, + "Sundanese": 0.3500432152117545, + "Javanese": 0.3336693548387097, + "Civic education": 0.5321888412017167 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.013285014209555142 + "bleu_score": 0.3236011529924488 }, "prompt_2": { - "bleu_score": 0.006157642089722674 + "bleu_score": 0.32440891006073647 }, "prompt_3": { - "bleu_score": 0.006701263103034912 + "bleu_score": 0.3204429258830498 }, "prompt_4": { - "bleu_score": 0.014981019260253739 + "bleu_score": 0.32704458117805263 }, "prompt_5": { - "bleu_score": 0.009118472573818275 + "bleu_score": 0.31802407870582927 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.031094937080711402 + "bleu_score": 0.27250852620501337 }, "prompt_2": { - "bleu_score": 0.00981561403840906 + "bleu_score": 0.26977172917246206 }, "prompt_3": { - "bleu_score": 0.010961263356457206 + "bleu_score": 0.26954366575043015 }, "prompt_4": { - "bleu_score": 0.05232878701423688 + "bleu_score": 0.2723245835557172 }, "prompt_5": { - "bleu_score": 0.025224189183119416 + "bleu_score": 0.2629241654872022 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.009540597961044034 + "bleu_score": 0.18710585898067542 }, "prompt_2": { - "bleu_score": 0.004498303439778538 + "bleu_score": 0.18518476878559328 }, "prompt_3": { - "bleu_score": 0.0020757596860400173 + "bleu_score": 0.1814602853117574 }, "prompt_4": { - "bleu_score": 0.013612574682903687 + "bleu_score": 0.18619098922321092 }, "prompt_5": { - "bleu_score": 0.004572799974216072 + "bleu_score": 0.18341704036900028 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.00939526555021961 + "bleu_score": 0.33312415897753844 }, "prompt_2": { - "bleu_score": 0.005493095723226531 + "bleu_score": 0.33441784160936017 }, "prompt_3": { - "bleu_score": 0.005849900179344782 + "bleu_score": 0.3288102573880272 }, "prompt_4": { - "bleu_score": 0.017291761334341137 + "bleu_score": 0.33525485700633156 }, "prompt_5": { - "bleu_score": 0.007292092374995376 + "bleu_score": 0.32441550434919486 } }, "mmlu": { "prompt_1": { - "accuracy": 0.2555425904317386 + "accuracy": 0.4515752625437573 }, "prompt_2": { - "accuracy": 0.24504084014002334 + "accuracy": 0.4690781796966161 }, "prompt_3": { - "accuracy": 0.2660443407234539 + "accuracy": 0.4690781796966161 }, "prompt_4": { - "accuracy": 0.24387397899649943 + "accuracy": 0.46557759626604434 }, "prompt_5": { - "accuracy": 0.26254375729288215 + "accuracy": 0.4469078179696616 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.24962459778333929, + "accuracy": 0.42452627815516625, "category_acc": { - "high_school_european_history": 0.20121951219512196, - "business_ethics": 0.30303030303030304, - "clinical_knowledge": 0.21212121212121213, - "medical_genetics": 0.26262626262626265, - "high_school_us_history": 0.2315270935960591, - "high_school_physics": 0.3, - "high_school_world_history": 0.24152542372881355, - "virology": 0.30303030303030304, - "high_school_microeconomics": 0.25738396624472576, - "econometrics": 0.23008849557522124, - "college_computer_science": 0.23232323232323232, - "high_school_biology": 0.255663430420712, + "high_school_european_history": 0.5670731707317073, + "business_ethics": 0.45454545454545453, + "clinical_knowledge": 0.4356060606060606, + "medical_genetics": 0.5050505050505051, + "high_school_us_history": 0.5024630541871922, + "high_school_physics": 0.25333333333333335, + "high_school_world_history": 0.6059322033898306, + "virology": 0.4121212121212121, + "high_school_microeconomics": 0.3881856540084388, + "econometrics": 0.26548672566371684, + "college_computer_science": 0.29292929292929293, + "high_school_biology": 0.44336569579288027, "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.23843416370106763, - "philosophy": 0.2645161290322581, - "professional_medicine": 0.2988929889298893, - "nutrition": 0.23934426229508196, - "global_facts": 0.2727272727272727, - "machine_learning": 0.3063063063063063, - "security_studies": 0.20491803278688525, - "public_relations": 0.22018348623853212, - "professional_psychology": 0.24549918166939444, - "prehistory": 0.2693498452012384, - "anatomy": 0.208955223880597, - "human_sexuality": 0.25384615384615383, - "college_medicine": 0.26744186046511625, - "high_school_government_and_politics": 0.3020833333333333, - "college_chemistry": 0.26262626262626265, - "logical_fallacies": 0.2345679012345679, - "high_school_geography": 0.27411167512690354, - "elementary_mathematics": 0.246684350132626, - "human_aging": 0.25225225225225223, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.22794117647058823, - "formal_logic": 0.24, - "high_school_statistics": 0.2372093023255814, - "international_law": 0.2916666666666667, - "high_school_mathematics": 0.22304832713754646, - "high_school_computer_science": 0.2222222222222222, - "conceptual_physics": 0.2222222222222222, - "miscellaneous": 0.2544757033248082, - "high_school_chemistry": 0.22277227722772278, - "marketing": 0.2446351931330472, - "professional_law": 0.24592302674494454, - "management": 0.29411764705882354, - "college_physics": 0.26732673267326734, - "jurisprudence": 0.21495327102803738, - "world_religions": 0.27647058823529413, - "sociology": 0.225, - "us_foreign_policy": 0.24242424242424243, - "high_school_macroeconomics": 0.2262210796915167, - "computer_security": 0.26262626262626265, - "moral_scenarios": 0.2684563758389262, - "moral_disputes": 0.24347826086956523, - "electrical_engineering": 0.2222222222222222, - "astronomy": 0.2847682119205298, - "college_biology": 0.24475524475524477 + "professional_accounting": 0.3487544483985765, + "philosophy": 0.49032258064516127, + "professional_medicine": 0.3284132841328413, + "nutrition": 0.4131147540983607, + "global_facts": 0.3434343434343434, + "machine_learning": 0.3783783783783784, + "security_studies": 0.4016393442622951, + "public_relations": 0.5137614678899083, + "professional_psychology": 0.40098199672667756, + "prehistory": 0.47678018575851394, + "anatomy": 0.4626865671641791, + "human_sexuality": 0.5, + "college_medicine": 0.36627906976744184, + "high_school_government_and_politics": 0.609375, + "college_chemistry": 0.3434343434343434, + "logical_fallacies": 0.5370370370370371, + "high_school_geography": 0.5532994923857868, + "elementary_mathematics": 0.3103448275862069, + "human_aging": 0.5045045045045045, + "college_mathematics": 0.3333333333333333, + "high_school_psychology": 0.5753676470588235, + "formal_logic": 0.256, + "high_school_statistics": 0.2651162790697674, + "international_law": 0.575, + "high_school_mathematics": 0.2862453531598513, + "high_school_computer_science": 0.45454545454545453, + "conceptual_physics": 0.34615384615384615, + "miscellaneous": 0.6214833759590793, + "high_school_chemistry": 0.31683168316831684, + "marketing": 0.6738197424892703, + "professional_law": 0.31833007175472927, + "management": 0.6274509803921569, + "college_physics": 0.21782178217821782, + "jurisprudence": 0.48598130841121495, + "world_religions": 0.5882352941176471, + "sociology": 0.565, + "us_foreign_policy": 0.6565656565656566, + "high_school_macroeconomics": 0.38817480719794345, + "computer_security": 0.5656565656565656, + "moral_scenarios": 0.2516778523489933, + "moral_disputes": 0.4753623188405797, + "electrical_engineering": 0.4166666666666667, + "astronomy": 0.47019867549668876, + "college_biology": 0.46153846153846156 } }, "prompt_2": { - "accuracy": 0.24790847336431893, + "accuracy": 0.4466928852341795, "category_acc": { - "high_school_european_history": 0.2621951219512195, - "business_ethics": 0.2222222222222222, - "clinical_knowledge": 0.22348484848484848, - "medical_genetics": 0.2727272727272727, - "high_school_us_history": 0.2413793103448276, - "high_school_physics": 0.22, - "high_school_world_history": 0.2711864406779661, - "virology": 0.2606060606060606, - "high_school_microeconomics": 0.2869198312236287, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.1919191919191919, - "high_school_biology": 0.2168284789644013, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.27402135231316727, - "philosophy": 0.2645161290322581, - "professional_medicine": 0.2029520295202952, - "nutrition": 0.24262295081967214, - "global_facts": 0.18181818181818182, - "machine_learning": 0.18018018018018017, - "security_studies": 0.22131147540983606, - "public_relations": 0.1926605504587156, - "professional_psychology": 0.22913256955810146, - "prehistory": 0.25077399380804954, - "anatomy": 0.2462686567164179, - "human_sexuality": 0.3076923076923077, - "college_medicine": 0.2441860465116279, - "high_school_government_and_politics": 0.22395833333333334, - "college_chemistry": 0.1919191919191919, - "logical_fallacies": 0.2839506172839506, - "high_school_geography": 0.25380710659898476, - "elementary_mathematics": 0.2625994694960212, - "human_aging": 0.22972972972972974, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.2536764705882353, - "formal_logic": 0.344, - "high_school_statistics": 0.21395348837209302, - "international_law": 0.175, - "high_school_mathematics": 0.22304832713754646, - "high_school_computer_science": 0.2727272727272727, - "conceptual_physics": 0.25213675213675213, - "miscellaneous": 0.25191815856777494, - "high_school_chemistry": 0.21782178217821782, - "marketing": 0.30472103004291845, - "professional_law": 0.24592302674494454, - "management": 0.3235294117647059, - "college_physics": 0.1485148514851485, - "jurisprudence": 0.2803738317757009, - "world_religions": 0.27058823529411763, - "sociology": 0.28, - "us_foreign_policy": 0.23232323232323232, - "high_school_macroeconomics": 0.2519280205655527, - "computer_security": 0.29292929292929293, - "moral_scenarios": 0.2550335570469799, - "moral_disputes": 0.2318840579710145, - "electrical_engineering": 0.3263888888888889, - "astronomy": 0.2251655629139073, - "college_biology": 0.3006993006993007 + "high_school_european_history": 0.5548780487804879, + "business_ethics": 0.45454545454545453, + "clinical_knowledge": 0.45454545454545453, + "medical_genetics": 0.494949494949495, + "high_school_us_history": 0.5369458128078818, + "high_school_physics": 0.28, + "high_school_world_history": 0.597457627118644, + "virology": 0.41818181818181815, + "high_school_microeconomics": 0.4430379746835443, + "econometrics": 0.2831858407079646, + "college_computer_science": 0.36363636363636365, + "high_school_biology": 0.4919093851132686, + "abstract_algebra": 0.20202020202020202, + "professional_accounting": 0.3665480427046263, + "philosophy": 0.49032258064516127, + "professional_medicine": 0.33948339483394835, + "nutrition": 0.4721311475409836, + "global_facts": 0.30303030303030304, + "machine_learning": 0.3063063063063063, + "security_studies": 0.4385245901639344, + "public_relations": 0.5137614678899083, + "professional_psychology": 0.4386252045826514, + "prehistory": 0.5139318885448917, + "anatomy": 0.5, + "human_sexuality": 0.5076923076923077, + "college_medicine": 0.4069767441860465, + "high_school_government_and_politics": 0.5989583333333334, + "college_chemistry": 0.3333333333333333, + "logical_fallacies": 0.5802469135802469, + "high_school_geography": 0.6091370558375635, + "elementary_mathematics": 0.32625994694960214, + "human_aging": 0.5180180180180181, + "college_mathematics": 0.31313131313131315, + "high_school_psychology": 0.6102941176470589, + "formal_logic": 0.256, + "high_school_statistics": 0.29767441860465116, + "international_law": 0.625, + "high_school_mathematics": 0.29739776951672864, + "high_school_computer_science": 0.48484848484848486, + "conceptual_physics": 0.3547008547008547, + "miscellaneous": 0.649616368286445, + "high_school_chemistry": 0.36633663366336633, + "marketing": 0.7167381974248928, + "professional_law": 0.3424657534246575, + "management": 0.6078431372549019, + "college_physics": 0.24752475247524752, + "jurisprudence": 0.5327102803738317, + "world_religions": 0.5764705882352941, + "sociology": 0.615, + "us_foreign_policy": 0.6767676767676768, + "high_school_macroeconomics": 0.41645244215938304, + "computer_security": 0.5555555555555556, + "moral_scenarios": 0.25727069351230425, + "moral_disputes": 0.4927536231884058, + "electrical_engineering": 0.4583333333333333, + "astronomy": 0.5231788079470199, + "college_biology": 0.4755244755244755 } }, "prompt_3": { - "accuracy": 0.2554165176975331, + "accuracy": 0.4493385770468359, "category_acc": { - "high_school_european_history": 0.22560975609756098, - "business_ethics": 0.1919191919191919, - "clinical_knowledge": 0.2727272727272727, - "medical_genetics": 0.21212121212121213, - "high_school_us_history": 0.22660098522167488, - "high_school_physics": 0.25333333333333335, - "high_school_world_history": 0.2966101694915254, - "virology": 0.30303030303030304, - "high_school_microeconomics": 0.3333333333333333, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.2727272727272727, - "high_school_biology": 0.18770226537216828, - "abstract_algebra": 0.20202020202020202, - "professional_accounting": 0.28113879003558717, - "philosophy": 0.24838709677419354, - "professional_medicine": 0.25092250922509224, - "nutrition": 0.24262295081967214, - "global_facts": 0.21212121212121213, - "machine_learning": 0.2882882882882883, - "security_studies": 0.23770491803278687, - "public_relations": 0.28440366972477066, - "professional_psychology": 0.25368248772504093, - "prehistory": 0.25696594427244585, - "anatomy": 0.23134328358208955, - "human_sexuality": 0.3153846153846154, - "college_medicine": 0.28488372093023256, - "high_school_government_and_politics": 0.2604166666666667, - "college_chemistry": 0.1717171717171717, - "logical_fallacies": 0.345679012345679, - "high_school_geography": 0.233502538071066, - "elementary_mathematics": 0.26525198938992045, - "human_aging": 0.27927927927927926, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.25735294117647056, - "formal_logic": 0.328, - "high_school_statistics": 0.2372093023255814, - "international_law": 0.225, - "high_school_mathematics": 0.19330855018587362, - "high_school_computer_science": 0.26262626262626265, - "conceptual_physics": 0.23076923076923078, - "miscellaneous": 0.2442455242966752, - "high_school_chemistry": 0.26732673267326734, - "marketing": 0.22746781115879827, - "professional_law": 0.2648401826484018, - "management": 0.27450980392156865, - "college_physics": 0.19801980198019803, - "jurisprudence": 0.2616822429906542, - "world_religions": 0.2647058823529412, - "sociology": 0.285, - "us_foreign_policy": 0.29292929292929293, - "high_school_macroeconomics": 0.2262210796915167, - "computer_security": 0.24242424242424243, - "moral_scenarios": 0.26286353467561524, - "moral_disputes": 0.26956521739130435, - "electrical_engineering": 0.2569444444444444, - "astronomy": 0.18543046357615894, - "college_biology": 0.3076923076923077 + "high_school_european_history": 0.5853658536585366, + "business_ethics": 0.43434343434343436, + "clinical_knowledge": 0.45075757575757575, + "medical_genetics": 0.5252525252525253, + "high_school_us_history": 0.5763546798029556, + "high_school_physics": 0.24666666666666667, + "high_school_world_history": 0.5932203389830508, + "virology": 0.3878787878787879, + "high_school_microeconomics": 0.4472573839662447, + "econometrics": 0.2831858407079646, + "college_computer_science": 0.37373737373737376, + "high_school_biology": 0.5048543689320388, + "abstract_algebra": 0.23232323232323232, + "professional_accounting": 0.3309608540925267, + "philosophy": 0.46774193548387094, + "professional_medicine": 0.33210332103321033, + "nutrition": 0.46229508196721314, + "global_facts": 0.37373737373737376, + "machine_learning": 0.3333333333333333, + "security_studies": 0.45491803278688525, + "public_relations": 0.5412844036697247, + "professional_psychology": 0.43044189852700493, + "prehistory": 0.5325077399380805, + "anatomy": 0.5, + "human_sexuality": 0.5153846153846153, + "college_medicine": 0.38372093023255816, + "high_school_government_and_politics": 0.6145833333333334, + "college_chemistry": 0.3838383838383838, + "logical_fallacies": 0.5740740740740741, + "high_school_geography": 0.5786802030456852, + "elementary_mathematics": 0.3129973474801061, + "human_aging": 0.5, + "college_mathematics": 0.3434343434343434, + "high_school_psychology": 0.6158088235294118, + "formal_logic": 0.28, + "high_school_statistics": 0.27906976744186046, + "international_law": 0.625, + "high_school_mathematics": 0.30855018587360594, + "high_school_computer_science": 0.47474747474747475, + "conceptual_physics": 0.358974358974359, + "miscellaneous": 0.6470588235294118, + "high_school_chemistry": 0.32673267326732675, + "marketing": 0.7467811158798283, + "professional_law": 0.34833659491193736, + "management": 0.6666666666666666, + "college_physics": 0.2871287128712871, + "jurisprudence": 0.5046728971962616, + "world_religions": 0.5647058823529412, + "sociology": 0.665, + "us_foreign_policy": 0.6565656565656566, + "high_school_macroeconomics": 0.4087403598971722, + "computer_security": 0.5757575757575758, + "moral_scenarios": 0.28523489932885904, + "moral_disputes": 0.4811594202898551, + "electrical_engineering": 0.4305555555555556, + "astronomy": 0.5165562913907285, + "college_biology": 0.46853146853146854 } }, "prompt_4": { - "accuracy": 0.25084018591347873, + "accuracy": 0.443618162316768, "category_acc": { - "high_school_european_history": 0.25, - "business_ethics": 0.29292929292929293, - "clinical_knowledge": 0.20075757575757575, - "medical_genetics": 0.29292929292929293, - "high_school_us_history": 0.26108374384236455, - "high_school_physics": 0.25333333333333335, - "high_school_world_history": 0.2711864406779661, - "virology": 0.28484848484848485, - "high_school_microeconomics": 0.2320675105485232, - "econometrics": 0.22123893805309736, - "college_computer_science": 0.24242424242424243, - "high_school_biology": 0.2815533980582524, + "high_school_european_history": 0.5670731707317073, + "business_ethics": 0.494949494949495, + "clinical_knowledge": 0.45454545454545453, + "medical_genetics": 0.48484848484848486, + "high_school_us_history": 0.5763546798029556, + "high_school_physics": 0.28, + "high_school_world_history": 0.6101694915254238, + "virology": 0.4, + "high_school_microeconomics": 0.42616033755274263, + "econometrics": 0.23008849557522124, + "college_computer_science": 0.35353535353535354, + "high_school_biology": 0.4919093851132686, "abstract_algebra": 0.25252525252525254, - "professional_accounting": 0.23843416370106763, - "philosophy": 0.24193548387096775, - "professional_medicine": 0.22878228782287824, - "nutrition": 0.2786885245901639, - "global_facts": 0.18181818181818182, - "machine_learning": 0.36036036036036034, - "security_studies": 0.26229508196721313, - "public_relations": 0.26605504587155965, - "professional_psychology": 0.22749590834697217, - "prehistory": 0.23839009287925697, - "anatomy": 0.22388059701492538, - "human_sexuality": 0.2692307692307692, - "college_medicine": 0.28488372093023256, - "high_school_government_and_politics": 0.2604166666666667, - "college_chemistry": 0.24242424242424243, - "logical_fallacies": 0.24691358024691357, - "high_school_geography": 0.2182741116751269, - "elementary_mathematics": 0.23872679045092837, - "human_aging": 0.2927927927927928, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.25, - "formal_logic": 0.248, - "high_school_statistics": 0.24651162790697675, - "international_law": 0.3, - "high_school_mathematics": 0.20817843866171004, - "high_school_computer_science": 0.21212121212121213, - "conceptual_physics": 0.2777777777777778, - "miscellaneous": 0.23529411764705882, - "high_school_chemistry": 0.22772277227722773, - "marketing": 0.26180257510729615, - "professional_law": 0.2465753424657534, - "management": 0.3431372549019608, - "college_physics": 0.25742574257425743, - "jurisprudence": 0.2336448598130841, - "world_religions": 0.2, - "sociology": 0.27, - "us_foreign_policy": 0.15151515151515152, - "high_school_macroeconomics": 0.2236503856041131, - "computer_security": 0.30303030303030304, - "moral_scenarios": 0.2684563758389262, - "moral_disputes": 0.263768115942029, - "electrical_engineering": 0.3333333333333333, - "astronomy": 0.26490066225165565, - "college_biology": 0.26573426573426573 + "professional_accounting": 0.3594306049822064, + "philosophy": 0.4838709677419355, + "professional_medicine": 0.3505535055350554, + "nutrition": 0.4459016393442623, + "global_facts": 0.36363636363636365, + "machine_learning": 0.32432432432432434, + "security_studies": 0.45491803278688525, + "public_relations": 0.5045871559633027, + "professional_psychology": 0.4402618657937807, + "prehistory": 0.5170278637770898, + "anatomy": 0.5149253731343284, + "human_sexuality": 0.4846153846153846, + "college_medicine": 0.36046511627906974, + "high_school_government_and_politics": 0.6197916666666666, + "college_chemistry": 0.30303030303030304, + "logical_fallacies": 0.5740740740740741, + "high_school_geography": 0.5939086294416244, + "elementary_mathematics": 0.32360742705570295, + "human_aging": 0.5180180180180181, + "college_mathematics": 0.3434343434343434, + "high_school_psychology": 0.6047794117647058, + "formal_logic": 0.184, + "high_school_statistics": 0.2837209302325581, + "international_law": 0.6166666666666667, + "high_school_mathematics": 0.2788104089219331, + "high_school_computer_science": 0.48484848484848486, + "conceptual_physics": 0.36752136752136755, + "miscellaneous": 0.6406649616368286, + "high_school_chemistry": 0.3465346534653465, + "marketing": 0.703862660944206, + "professional_law": 0.34637964774951074, + "management": 0.6372549019607843, + "college_physics": 0.2079207920792079, + "jurisprudence": 0.514018691588785, + "world_religions": 0.5588235294117647, + "sociology": 0.635, + "us_foreign_policy": 0.6767676767676768, + "high_school_macroeconomics": 0.390745501285347, + "computer_security": 0.5858585858585859, + "moral_scenarios": 0.2505592841163311, + "moral_disputes": 0.5101449275362319, + "electrical_engineering": 0.4583333333333333, + "astronomy": 0.4768211920529801, + "college_biology": 0.46153846153846156 } }, "prompt_5": { - "accuracy": 0.25084018591347873, + "accuracy": 0.43575259206292455, "category_acc": { - "high_school_european_history": 0.21951219512195122, - "business_ethics": 0.25252525252525254, - "clinical_knowledge": 0.25, - "medical_genetics": 0.2828282828282828, - "high_school_us_history": 0.29064039408866993, - "high_school_physics": 0.2733333333333333, - "high_school_world_history": 0.2796610169491525, - "virology": 0.23636363636363636, - "high_school_microeconomics": 0.2742616033755274, - "econometrics": 0.13274336283185842, - "college_computer_science": 0.23232323232323232, - "high_school_biology": 0.26537216828478966, - "abstract_algebra": 0.2727272727272727, - "professional_accounting": 0.2313167259786477, - "philosophy": 0.2161290322580645, - "professional_medicine": 0.25461254612546125, - "nutrition": 0.2262295081967213, - "global_facts": 0.30303030303030304, - "machine_learning": 0.23423423423423423, - "security_studies": 0.24180327868852458, - "public_relations": 0.28440366972477066, - "professional_psychology": 0.2553191489361702, - "prehistory": 0.2724458204334365, - "anatomy": 0.2537313432835821, - "human_sexuality": 0.26153846153846155, - "college_medicine": 0.23255813953488372, - "high_school_government_and_politics": 0.1875, - "college_chemistry": 0.2727272727272727, - "logical_fallacies": 0.29012345679012347, - "high_school_geography": 0.29949238578680204, - "elementary_mathematics": 0.2413793103448276, - "human_aging": 0.25675675675675674, - "college_mathematics": 0.21212121212121213, - "high_school_psychology": 0.24080882352941177, - "formal_logic": 0.28, - "high_school_statistics": 0.22325581395348837, - "international_law": 0.3333333333333333, - "high_school_mathematics": 0.241635687732342, - "high_school_computer_science": 0.20202020202020202, - "conceptual_physics": 0.21367521367521367, - "miscellaneous": 0.26982097186700765, - "high_school_chemistry": 0.21287128712871287, - "marketing": 0.24892703862660945, - "professional_law": 0.2511415525114155, - "management": 0.29411764705882354, - "college_physics": 0.26732673267326734, - "jurisprudence": 0.32710280373831774, - "world_religions": 0.27058823529411763, - "sociology": 0.245, - "us_foreign_policy": 0.23232323232323232, - "high_school_macroeconomics": 0.23393316195372751, - "computer_security": 0.25252525252525254, - "moral_scenarios": 0.24720357941834453, - "moral_disputes": 0.22318840579710145, - "electrical_engineering": 0.2708333333333333, - "astronomy": 0.2781456953642384, - "college_biology": 0.2727272727272727 + "high_school_european_history": 0.5792682926829268, + "business_ethics": 0.48484848484848486, + "clinical_knowledge": 0.4583333333333333, + "medical_genetics": 0.46464646464646464, + "high_school_us_history": 0.5615763546798029, + "high_school_physics": 0.26666666666666666, + "high_school_world_history": 0.6228813559322034, + "virology": 0.41818181818181815, + "high_school_microeconomics": 0.39662447257383965, + "econometrics": 0.23893805309734514, + "college_computer_science": 0.32323232323232326, + "high_school_biology": 0.4854368932038835, + "abstract_algebra": 0.24242424242424243, + "professional_accounting": 0.3487544483985765, + "philosophy": 0.4838709677419355, + "professional_medicine": 0.3505535055350554, + "nutrition": 0.4426229508196721, + "global_facts": 0.32323232323232326, + "machine_learning": 0.3063063063063063, + "security_studies": 0.44672131147540983, + "public_relations": 0.5321100917431193, + "professional_psychology": 0.4238952536824877, + "prehistory": 0.5046439628482973, + "anatomy": 0.4701492537313433, + "human_sexuality": 0.5, + "college_medicine": 0.37790697674418605, + "high_school_government_and_politics": 0.6041666666666666, + "college_chemistry": 0.3434343434343434, + "logical_fallacies": 0.5493827160493827, + "high_school_geography": 0.5736040609137056, + "elementary_mathematics": 0.3156498673740053, + "human_aging": 0.509009009009009, + "college_mathematics": 0.31313131313131315, + "high_school_psychology": 0.5900735294117647, + "formal_logic": 0.256, + "high_school_statistics": 0.24651162790697675, + "international_law": 0.5666666666666667, + "high_school_mathematics": 0.275092936802974, + "high_school_computer_science": 0.45454545454545453, + "conceptual_physics": 0.3547008547008547, + "miscellaneous": 0.6355498721227621, + "high_school_chemistry": 0.3415841584158416, + "marketing": 0.703862660944206, + "professional_law": 0.33529028049575993, + "management": 0.6274509803921569, + "college_physics": 0.21782178217821782, + "jurisprudence": 0.4953271028037383, + "world_religions": 0.5882352941176471, + "sociology": 0.605, + "us_foreign_policy": 0.6565656565656566, + "high_school_macroeconomics": 0.37789203084832906, + "computer_security": 0.5757575757575758, + "moral_scenarios": 0.2494407158836689, + "moral_disputes": 0.48405797101449277, + "electrical_engineering": 0.4791666666666667, + "astronomy": 0.47019867549668876, + "college_biology": 0.46853146853146854 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.25705794947994054 + "accuracy": 0.437592867756315 }, "prompt_2": { - "accuracy": 0.2399702823179792 + "accuracy": 0.4309063893016345 }, "prompt_3": { - "accuracy": 0.2704309063893016 + "accuracy": 0.43536404160475484 }, "prompt_4": { - "accuracy": 0.2511144130757801 + "accuracy": 0.4175334323922734 }, "prompt_5": { - "accuracy": 0.2362555720653789 + "accuracy": 0.42570579494799404 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.23163138231631383, + "accuracy": 0.43586550435865506, "category_acc": { - "computer_network": 0.125, - "operating_system": 0.16666666666666666, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.2619047619047619, - "college_physics": 0.25, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.2916666666666667, + "computer_network": 0.20833333333333334, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.34615384615384615, + "college_programming": 0.2857142857142857, + "college_physics": 0.4166666666666667, + "college_chemistry": 0.3793103448275862, + "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.10344827586206896, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.125, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.16666666666666666, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.16666666666666666, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.25, - "college_economics": 0.25, - "business_administration": 0.21052631578947367, - "marxism": 0.25, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.11764705882352941, - "teacher_qualification": 0.24489795918367346, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.375, - "logic": 0.2222222222222222, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.2631578947368421, - "professional_tour_guide": 0.20588235294117646, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.24, - "middle_school_history": 0.07407407407407407, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.3333333333333333, + "metrology_engineer": 0.4482758620689655, + "high_school_mathematics": 0.13043478260869565, + "high_school_physics": 0.375, + "high_school_chemistry": 0.4583333333333333, + "high_school_biology": 0.4583333333333333, + "middle_school_mathematics": 0.3333333333333333, + "middle_school_biology": 0.5, + "middle_school_physics": 0.5416666666666666, + "middle_school_chemistry": 0.36, + "veterinary_medicine": 0.35714285714285715, + "college_economics": 0.36666666666666664, + "business_administration": 0.4473684210526316, + "marxism": 0.5833333333333334, + "mao_zedong_thought": 0.5862068965517241, + "education_science": 0.6176470588235294, + "teacher_qualification": 0.6530612244897959, + "high_school_politics": 0.5833333333333334, + "high_school_geography": 0.5, + "middle_school_politics": 0.6923076923076923, + "middle_school_geography": 0.47058823529411764, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.5833333333333334, + "logic": 0.18518518518518517, + "law": 0.27586206896551724, + "chinese_language_and_literature": 0.35714285714285715, + "art_studies": 0.5, + "professional_tour_guide": 0.4411764705882353, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.5833333333333334, + "high_school_history": 0.64, + "middle_school_history": 0.5555555555555556, "civil_servant": 0.3076923076923077, - "sports_science": 0.125, - "plant_protection": 0.1111111111111111, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.43137254901960786, - "accountant": 0.2777777777777778, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.25925925925925924, - "physician": 0.16666666666666666 + "sports_science": 0.375, + "plant_protection": 0.5185185185185185, + "basic_medicine": 0.375, + "clinical_medicine": 0.4074074074074074, + "urban_and_rural_planner": 0.6078431372549019, + "accountant": 0.37037037037037035, + "fire_engineer": 0.4722222222222222, + "environmental_impact_assessment_engineer": 0.5277777777777778, + "tax_accountant": 0.42592592592592593, + "physician": 0.5555555555555556 } }, "prompt_2": { - "accuracy": 0.25840597758405975, + "accuracy": 0.4414694894146949, "category_acc": { - "computer_network": 0.125, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.23809523809523808, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.5, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.1724137931034483, - "high_school_mathematics": 0.5652173913043478, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.125, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.14285714285714285, - "college_economics": 0.31666666666666665, - "business_administration": 0.18421052631578946, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.42857142857142855, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.08333333333333333, - "middle_school_politics": 0.19230769230769232, - "middle_school_geography": 0.0, - "modern_chinese_history": 0.32142857142857145, - "ideological_and_moral_cultivation": 0.125, - "logic": 0.2962962962962963, + "computer_network": 0.2916666666666667, + "operating_system": 0.375, + "computer_architecture": 0.34615384615384615, + "college_programming": 0.3333333333333333, + "college_physics": 0.4583333333333333, + "college_chemistry": 0.3448275862068966, + "advanced_mathematics": 0.375, + "probability_and_statistics": 0.13043478260869565, + "discrete_mathematics": 0.14285714285714285, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.5172413793103449, + "high_school_mathematics": 0.17391304347826086, + "high_school_physics": 0.4583333333333333, + "high_school_chemistry": 0.4583333333333333, + "high_school_biology": 0.375, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.5384615384615384, + "middle_school_physics": 0.5416666666666666, + "middle_school_chemistry": 0.44, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.38333333333333336, + "business_administration": 0.42105263157894735, + "marxism": 0.5416666666666666, + "mao_zedong_thought": 0.6551724137931034, + "education_science": 0.6470588235294118, + "teacher_qualification": 0.6326530612244898, + "high_school_politics": 0.5416666666666666, + "high_school_geography": 0.4583333333333333, + "middle_school_politics": 0.6538461538461539, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.5357142857142857, + "ideological_and_moral_cultivation": 0.5833333333333334, + "logic": 0.3333333333333333, "law": 0.3448275862068966, - "chinese_language_and_literature": 0.25, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.23529411764705882, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.25, - "high_school_history": 0.32, - "middle_school_history": 0.14814814814814814, - "civil_servant": 0.1346153846153846, - "sports_science": 0.2916666666666667, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.18518518518518517, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.2037037037037037, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.25925925925925924, - "physician": 0.18518518518518517 + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.5263157894736842, + "professional_tour_guide": 0.47058823529411764, + "legal_professional": 0.39285714285714285, + "high_school_chinese": 0.5833333333333334, + "high_school_history": 0.64, + "middle_school_history": 0.6296296296296297, + "civil_servant": 0.36538461538461536, + "sports_science": 0.5416666666666666, + "plant_protection": 0.5555555555555556, + "basic_medicine": 0.5, + "clinical_medicine": 0.3333333333333333, + "urban_and_rural_planner": 0.5294117647058824, + "accountant": 0.3333333333333333, + "fire_engineer": 0.4166666666666667, + "environmental_impact_assessment_engineer": 0.5, + "tax_accountant": 0.42592592592592593, + "physician": 0.5 } }, "prompt_3": { - "accuracy": 0.2465753424657534, + "accuracy": 0.4564134495641345, "category_acc": { - "computer_network": 0.125, + "computer_network": 0.3333333333333333, "operating_system": 0.375, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.35714285714285715, - "college_physics": 0.25, - "college_chemistry": 0.1724137931034483, + "computer_architecture": 0.4230769230769231, + "college_programming": 0.47619047619047616, + "college_physics": 0.4583333333333333, + "college_chemistry": 0.3793103448275862, "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.09523809523809523, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.375, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.125, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.21666666666666667, - "business_administration": 0.23684210526315788, - "marxism": 0.25, - "mao_zedong_thought": 0.41379310344827586, - "education_science": 0.058823529411764705, - "teacher_qualification": 0.32653061224489793, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.25, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.18518518518518517, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.047619047619047616, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.5172413793103449, + "high_school_mathematics": 0.08695652173913043, + "high_school_physics": 0.5416666666666666, + "high_school_chemistry": 0.4583333333333333, + "high_school_biology": 0.375, + "middle_school_mathematics": 0.16666666666666666, + "middle_school_biology": 0.5, + "middle_school_physics": 0.5416666666666666, + "middle_school_chemistry": 0.4, + "veterinary_medicine": 0.35714285714285715, + "college_economics": 0.35, + "business_administration": 0.3684210526315789, + "marxism": 0.5416666666666666, + "mao_zedong_thought": 0.6551724137931034, + "education_science": 0.6176470588235294, + "teacher_qualification": 0.6938775510204082, + "high_school_politics": 0.5416666666666666, + "high_school_geography": 0.5, + "middle_school_politics": 0.6538461538461539, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.5714285714285714, + "ideological_and_moral_cultivation": 0.5833333333333334, + "logic": 0.4444444444444444, "law": 0.2413793103448276, - "chinese_language_and_literature": 0.17857142857142858, - "art_studies": 0.23684210526315788, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.28, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.3076923076923077, - "sports_science": 0.20833333333333334, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.1568627450980392, - "accountant": 0.12962962962962962, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.24074074074074073, - "physician": 0.18518518518518517 + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.5, + "professional_tour_guide": 0.5, + "legal_professional": 0.42857142857142855, + "high_school_chinese": 0.5833333333333334, + "high_school_history": 0.64, + "middle_school_history": 0.5925925925925926, + "civil_servant": 0.40384615384615385, + "sports_science": 0.4583333333333333, + "plant_protection": 0.5925925925925926, + "basic_medicine": 0.5, + "clinical_medicine": 0.3333333333333333, + "urban_and_rural_planner": 0.6078431372549019, + "accountant": 0.3888888888888889, + "fire_engineer": 0.3888888888888889, + "environmental_impact_assessment_engineer": 0.5555555555555556, + "tax_accountant": 0.46296296296296297, + "physician": 0.5555555555555556 } }, "prompt_4": { - "accuracy": 0.2590286425902864, + "accuracy": 0.4315068493150685, "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.125, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.30952380952380953, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.16666666666666666, - "metrology_engineer": 0.06896551724137931, - "high_school_mathematics": 0.21739130434782608, + "computer_network": 0.375, + "operating_system": 0.375, + "computer_architecture": 0.38461538461538464, + "college_programming": 0.3333333333333333, + "college_physics": 0.5, + "college_chemistry": 0.41379310344827586, + "advanced_mathematics": 0.125, + "probability_and_statistics": 0.30434782608695654, + "discrete_mathematics": 0.19047619047619047, + "electrical_engineer": 0.2857142857142857, + "metrology_engineer": 0.5172413793103449, + "high_school_mathematics": 0.17391304347826086, "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.5, + "high_school_chemistry": 0.375, "high_school_biology": 0.4166666666666667, - "middle_school_mathematics": 0.08333333333333333, - "middle_school_biology": 0.19230769230769232, - "middle_school_physics": 0.20833333333333334, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.5769230769230769, + "middle_school_physics": 0.4583333333333333, "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.10714285714285714, - "college_economics": 0.31666666666666665, - "business_administration": 0.3157894736842105, - "marxism": 0.25, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.2857142857142857, - "high_school_politics": 0.25, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.32142857142857145, - "ideological_and_moral_cultivation": 0.25, + "veterinary_medicine": 0.35714285714285715, + "college_economics": 0.3333333333333333, + "business_administration": 0.4473684210526316, + "marxism": 0.5416666666666666, + "mao_zedong_thought": 0.6551724137931034, + "education_science": 0.6176470588235294, + "teacher_qualification": 0.5918367346938775, + "high_school_politics": 0.6666666666666666, + "high_school_geography": 0.4583333333333333, + "middle_school_politics": 0.6923076923076923, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.5833333333333334, "logic": 0.25925925925925924, "law": 0.27586206896551724, - "chinese_language_and_literature": 0.14285714285714285, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.10714285714285714, - "high_school_chinese": 0.125, - "high_school_history": 0.28, - "middle_school_history": 0.2222222222222222, - "civil_servant": 0.17307692307692307, + "chinese_language_and_literature": 0.35714285714285715, + "art_studies": 0.47368421052631576, + "professional_tour_guide": 0.5294117647058824, + "legal_professional": 0.35714285714285715, + "high_school_chinese": 0.4166666666666667, + "high_school_history": 0.64, + "middle_school_history": 0.6296296296296297, + "civil_servant": 0.4423076923076923, "sports_science": 0.4166666666666667, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.20833333333333334, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.25925925925925924, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.25925925925925924, - "physician": 0.2222222222222222 + "plant_protection": 0.4444444444444444, + "basic_medicine": 0.2916666666666667, + "clinical_medicine": 0.37037037037037035, + "urban_and_rural_planner": 0.5686274509803921, + "accountant": 0.3148148148148148, + "fire_engineer": 0.5277777777777778, + "environmental_impact_assessment_engineer": 0.5, + "tax_accountant": 0.4074074074074074, + "physician": 0.48148148148148145 } }, "prompt_5": { - "accuracy": 0.26089663760896636, + "accuracy": 0.43586550435865506, "category_acc": { - "computer_network": 0.16666666666666666, - "operating_system": 0.16666666666666666, + "computer_network": 0.3333333333333333, + "operating_system": 0.375, "computer_architecture": 0.34615384615384615, "college_programming": 0.30952380952380953, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.4782608695652174, - "discrete_mathematics": 0.09523809523809523, - "electrical_engineer": 0.19047619047619047, - "metrology_engineer": 0.1724137931034483, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.2916666666666667, + "college_physics": 0.5, + "college_chemistry": 0.3793103448275862, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.17391304347826086, + "discrete_mathematics": 0.19047619047619047, + "electrical_engineer": 0.3333333333333333, + "metrology_engineer": 0.5172413793103449, + "high_school_mathematics": 0.13043478260869565, + "high_school_physics": 0.375, + "high_school_chemistry": 0.5, + "high_school_biology": 0.4166666666666667, "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.3, - "business_administration": 0.13157894736842105, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.24489795918367346, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.15384615384615385, - "middle_school_geography": 0.11764705882352941, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.125, - "logic": 0.37037037037037035, - "law": 0.3448275862068966, + "middle_school_biology": 0.5, + "middle_school_physics": 0.5833333333333334, + "middle_school_chemistry": 0.4, + "veterinary_medicine": 0.39285714285714285, + "college_economics": 0.36666666666666664, + "business_administration": 0.42105263157894735, + "marxism": 0.5416666666666666, + "mao_zedong_thought": 0.6551724137931034, + "education_science": 0.5882352941176471, + "teacher_qualification": 0.6530612244897959, + "high_school_politics": 0.6666666666666666, + "high_school_geography": 0.4583333333333333, + "middle_school_politics": 0.6923076923076923, + "middle_school_geography": 0.35294117647058826, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.625, + "logic": 0.3333333333333333, + "law": 0.2413793103448276, "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.20588235294117646, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.32, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.2692307692307692, - "sports_science": 0.16666666666666666, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.20833333333333334, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.19607843137254902, - "accountant": 0.2777777777777778, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.2222222222222222, - "physician": 0.3333333333333333 + "art_studies": 0.5, + "professional_tour_guide": 0.4411764705882353, + "legal_professional": 0.4642857142857143, + "high_school_chinese": 0.5, + "high_school_history": 0.68, + "middle_school_history": 0.5555555555555556, + "civil_servant": 0.3269230769230769, + "sports_science": 0.4583333333333333, + "plant_protection": 0.4074074074074074, + "basic_medicine": 0.2916666666666667, + "clinical_medicine": 0.37037037037037035, + "urban_and_rural_planner": 0.5882352941176471, + "accountant": 0.35185185185185186, + "fire_engineer": 0.4444444444444444, + "environmental_impact_assessment_engineer": 0.5833333333333334, + "tax_accountant": 0.42592592592592593, + "physician": 0.46296296296296297 } } }, "cmmlu": { "prompt_1": { - "accuracy": 0.25089605734767023 + "accuracy": 0.48028673835125446 }, "prompt_2": { - "accuracy": 0.25089605734767023 + "accuracy": 0.4659498207885305 }, "prompt_3": { - "accuracy": 0.2974910394265233 + "accuracy": 0.46236559139784944 }, "prompt_4": { - "accuracy": 0.25806451612903225 + "accuracy": 0.44802867383512546 }, "prompt_5": { - "accuracy": 0.23297491039426524 + "accuracy": 0.46236559139784944 } }, "cmmlu_full": { "prompt_1": { - "accuracy": 0.2514246244171991, + "accuracy": 0.43645311690554306, "category_acc": { - "agronomy": 0.22485207100591717, - "anatomy": 0.2905405405405405, - "ancient_chinese": 0.18292682926829268, - "arts": 0.1875, - "astronomy": 0.2545454545454545, - "business_ethics": 0.2679425837320574, - "chinese_civil_service_exam": 0.2625, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.2647058823529412, - "chinese_foreign_policy": 0.2897196261682243, - "chinese_history": 0.24458204334365324, - "chinese_literature": 0.24019607843137256, - "chinese_teacher_qualification": 0.2737430167597765, - "clinical_knowledge": 0.24472573839662448, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.3177570093457944, - "college_engineering_hydrology": 0.2641509433962264, - "college_law": 0.25925925925925924, - "college_mathematics": 0.2, - "college_medical_statistics": 0.3113207547169811, - "college_medicine": 0.28205128205128205, - "computer_science": 0.23529411764705882, - "computer_security": 0.21637426900584794, - "conceptual_physics": 0.24489795918367346, - "construction_project_management": 0.2446043165467626, - "economics": 0.1761006289308176, - "education": 0.24539877300613497, - "electrical_engineering": 0.26744186046511625, - "elementary_chinese": 0.17857142857142858, - "elementary_commonsense": 0.31313131313131315, - "elementary_information_and_technology": 0.24789915966386555, - "elementary_mathematics": 0.27391304347826084, - "ethnology": 0.28888888888888886, - "food_science": 0.23776223776223776, - "genetics": 0.2556818181818182, - "global_facts": 0.2348993288590604, - "high_school_biology": 0.31952662721893493, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.2288135593220339, + "agronomy": 0.39644970414201186, + "anatomy": 0.35135135135135137, + "ancient_chinese": 0.21951219512195122, + "arts": 0.60625, + "astronomy": 0.296969696969697, + "business_ethics": 0.5119617224880383, + "chinese_civil_service_exam": 0.41875, + "chinese_driving_rule": 0.6793893129770993, + "chinese_food_culture": 0.41911764705882354, + "chinese_foreign_policy": 0.4953271028037383, + "chinese_history": 0.47678018575851394, + "chinese_literature": 0.3480392156862745, + "chinese_teacher_qualification": 0.5363128491620112, + "clinical_knowledge": 0.35864978902953587, + "college_actuarial_science": 0.20754716981132076, + "college_education": 0.616822429906542, + "college_engineering_hydrology": 0.41509433962264153, + "college_law": 0.3888888888888889, + "college_mathematics": 0.3333333333333333, + "college_medical_statistics": 0.37735849056603776, + "college_medicine": 0.3992673992673993, + "computer_science": 0.45098039215686275, + "computer_security": 0.45614035087719296, + "conceptual_physics": 0.47619047619047616, + "construction_project_management": 0.34532374100719426, + "economics": 0.44025157232704404, + "education": 0.50920245398773, + "electrical_engineering": 0.42441860465116277, + "elementary_chinese": 0.42063492063492064, + "elementary_commonsense": 0.494949494949495, + "elementary_information_and_technology": 0.6596638655462185, + "elementary_mathematics": 0.2826086956521739, + "ethnology": 0.43703703703703706, + "food_science": 0.46153846153846156, + "genetics": 0.3693181818181818, + "global_facts": 0.4228187919463087, + "high_school_biology": 0.33727810650887574, + "high_school_chemistry": 0.30303030303030304, + "high_school_geography": 0.4661016949152542, "high_school_mathematics": 0.2865853658536585, - "high_school_physics": 0.2, - "high_school_politics": 0.3006993006993007, - "human_sexuality": 0.23015873015873015, - "international_law": 0.1783783783783784, - "journalism": 0.22674418604651161, - "jurisprudence": 0.26520681265206814, - "legal_and_moral_basis": 0.2850467289719626, - "logical": 0.18699186991869918, - "machine_learning": 0.28688524590163933, - "management": 0.24761904761904763, - "marketing": 0.25, - "marxist_theory": 0.20634920634920634, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.22758620689655173, - "philosophy": 0.2857142857142857, - "professional_accounting": 0.21714285714285714, - "professional_law": 0.26066350710900477, - "professional_medicine": 0.25, - "professional_psychology": 0.23275862068965517, - "public_relations": 0.25862068965517243, - "security_study": 0.24444444444444444, - "sociology": 0.30973451327433627, - "sports_science": 0.2909090909090909, - "traditional_chinese_medicine": 0.23243243243243245, - "virology": 0.26627218934911245, - "world_history": 0.2484472049689441, - "world_religions": 0.25625 + "high_school_physics": 0.37272727272727274, + "high_school_politics": 0.40559440559440557, + "human_sexuality": 0.4603174603174603, + "international_law": 0.3891891891891892, + "journalism": 0.5174418604651163, + "jurisprudence": 0.45012165450121655, + "legal_and_moral_basis": 0.7757009345794392, + "logical": 0.4065040650406504, + "machine_learning": 0.38524590163934425, + "management": 0.48095238095238096, + "marketing": 0.5444444444444444, + "marxist_theory": 0.5079365079365079, + "modern_chinese": 0.33620689655172414, + "nutrition": 0.4482758620689655, + "philosophy": 0.5142857142857142, + "professional_accounting": 0.45714285714285713, + "professional_law": 0.3744075829383886, + "professional_medicine": 0.3271276595744681, + "professional_psychology": 0.5043103448275862, + "public_relations": 0.4540229885057471, + "security_study": 0.5111111111111111, + "sociology": 0.4469026548672566, + "sports_science": 0.44242424242424244, + "traditional_chinese_medicine": 0.3837837837837838, + "virology": 0.40236686390532544, + "world_history": 0.45962732919254656, + "world_religions": 0.45 } }, "prompt_2": { - "accuracy": 0.25453289587290623, + "accuracy": 0.4392160248661717, "category_acc": { - "agronomy": 0.2485207100591716, - "anatomy": 0.27702702702702703, - "ancient_chinese": 0.2621951219512195, - "arts": 0.29375, - "astronomy": 0.2606060606060606, - "business_ethics": 0.3157894736842105, - "chinese_civil_service_exam": 0.2375, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.19852941176470587, - "chinese_foreign_policy": 0.205607476635514, - "chinese_history": 0.26006191950464397, - "chinese_literature": 0.24019607843137256, - "chinese_teacher_qualification": 0.22346368715083798, - "clinical_knowledge": 0.2911392405063291, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.29906542056074764, - "college_engineering_hydrology": 0.2358490566037736, - "college_law": 0.26851851851851855, - "college_mathematics": 0.29523809523809524, - "college_medical_statistics": 0.3113207547169811, - "college_medicine": 0.23076923076923078, - "computer_science": 0.20098039215686275, - "computer_security": 0.27485380116959063, - "conceptual_physics": 0.25170068027210885, - "construction_project_management": 0.2446043165467626, - "economics": 0.2138364779874214, - "education": 0.2331288343558282, - "electrical_engineering": 0.27325581395348836, - "elementary_chinese": 0.2896825396825397, - "elementary_commonsense": 0.22727272727272727, - "elementary_information_and_technology": 0.23949579831932774, - "elementary_mathematics": 0.2956521739130435, - "ethnology": 0.3037037037037037, - "food_science": 0.26573426573426573, - "genetics": 0.23295454545454544, - "global_facts": 0.24161073825503357, - "high_school_biology": 0.21301775147928995, - "high_school_chemistry": 0.20454545454545456, - "high_school_geography": 0.22033898305084745, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.23636363636363636, - "high_school_politics": 0.23076923076923078, - "human_sexuality": 0.2857142857142857, - "international_law": 0.2648648648648649, - "journalism": 0.23837209302325582, - "jurisprudence": 0.2725060827250608, - "legal_and_moral_basis": 0.2383177570093458, - "logical": 0.23577235772357724, - "machine_learning": 0.26229508196721313, - "management": 0.2523809523809524, - "marketing": 0.2833333333333333, - "marxist_theory": 0.2857142857142857, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.2206896551724138, - "philosophy": 0.21904761904761905, - "professional_accounting": 0.19428571428571428, - "professional_law": 0.26540284360189575, - "professional_medicine": 0.26595744680851063, - "professional_psychology": 0.24568965517241378, - "public_relations": 0.22988505747126436, - "security_study": 0.3037037037037037, - "sociology": 0.2610619469026549, - "sports_science": 0.30303030303030304, - "traditional_chinese_medicine": 0.23783783783783785, - "virology": 0.27218934911242604, - "world_history": 0.2236024844720497, - "world_religions": 0.2375 + "agronomy": 0.39644970414201186, + "anatomy": 0.32432432432432434, + "ancient_chinese": 0.2926829268292683, + "arts": 0.65, + "astronomy": 0.2909090909090909, + "business_ethics": 0.4880382775119617, + "chinese_civil_service_exam": 0.41875, + "chinese_driving_rule": 0.7099236641221374, + "chinese_food_culture": 0.4411764705882353, + "chinese_foreign_policy": 0.48598130841121495, + "chinese_history": 0.49226006191950467, + "chinese_literature": 0.35784313725490197, + "chinese_teacher_qualification": 0.553072625698324, + "clinical_knowledge": 0.3670886075949367, + "college_actuarial_science": 0.22641509433962265, + "college_education": 0.6261682242990654, + "college_engineering_hydrology": 0.4056603773584906, + "college_law": 0.39814814814814814, + "college_mathematics": 0.3238095238095238, + "college_medical_statistics": 0.39622641509433965, + "college_medicine": 0.4139194139194139, + "computer_science": 0.44607843137254904, + "computer_security": 0.4327485380116959, + "conceptual_physics": 0.48299319727891155, + "construction_project_management": 0.37410071942446044, + "economics": 0.46540880503144655, + "education": 0.50920245398773, + "electrical_engineering": 0.4418604651162791, + "elementary_chinese": 0.4166666666666667, + "elementary_commonsense": 0.48484848484848486, + "elementary_information_and_technology": 0.6764705882352942, + "elementary_mathematics": 0.2782608695652174, + "ethnology": 0.4222222222222222, + "food_science": 0.45454545454545453, + "genetics": 0.3409090909090909, + "global_facts": 0.42953020134228187, + "high_school_biology": 0.3431952662721893, + "high_school_chemistry": 0.29545454545454547, + "high_school_geography": 0.4491525423728814, + "high_school_mathematics": 0.2621951219512195, + "high_school_physics": 0.38181818181818183, + "high_school_politics": 0.3986013986013986, + "human_sexuality": 0.47619047619047616, + "international_law": 0.372972972972973, + "journalism": 0.5116279069767442, + "jurisprudence": 0.4257907542579075, + "legal_and_moral_basis": 0.780373831775701, + "logical": 0.42276422764227645, + "machine_learning": 0.39344262295081966, + "management": 0.5333333333333333, + "marketing": 0.5277777777777778, + "marxist_theory": 0.5132275132275133, + "modern_chinese": 0.3706896551724138, + "nutrition": 0.46206896551724136, + "philosophy": 0.45714285714285713, + "professional_accounting": 0.44, + "professional_law": 0.41232227488151657, + "professional_medicine": 0.300531914893617, + "professional_psychology": 0.5129310344827587, + "public_relations": 0.47126436781609193, + "security_study": 0.5333333333333333, + "sociology": 0.4469026548672566, + "sports_science": 0.43636363636363634, + "traditional_chinese_medicine": 0.3621621621621622, + "virology": 0.40828402366863903, + "world_history": 0.45962732919254656, + "world_religions": 0.46875 } }, "prompt_3": { - "accuracy": 0.24658953548609913, + "accuracy": 0.4429286824382663, "category_acc": { - "agronomy": 0.25443786982248523, - "anatomy": 0.20945945945945946, - "ancient_chinese": 0.2621951219512195, - "arts": 0.2625, - "astronomy": 0.24242424242424243, - "business_ethics": 0.24880382775119617, - "chinese_civil_service_exam": 0.225, - "chinese_driving_rule": 0.22137404580152673, - "chinese_food_culture": 0.29411764705882354, - "chinese_foreign_policy": 0.18691588785046728, - "chinese_history": 0.24148606811145512, - "chinese_literature": 0.24019607843137256, - "chinese_teacher_qualification": 0.2905027932960894, - "clinical_knowledge": 0.2109704641350211, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.22429906542056074, - "college_engineering_hydrology": 0.33962264150943394, - "college_law": 0.21296296296296297, + "agronomy": 0.4260355029585799, + "anatomy": 0.34459459459459457, + "ancient_chinese": 0.23780487804878048, + "arts": 0.63125, + "astronomy": 0.3151515151515151, + "business_ethics": 0.48325358851674644, + "chinese_civil_service_exam": 0.4125, + "chinese_driving_rule": 0.7175572519083969, + "chinese_food_culture": 0.4338235294117647, + "chinese_foreign_policy": 0.4766355140186916, + "chinese_history": 0.48606811145510836, + "chinese_literature": 0.3872549019607843, + "chinese_teacher_qualification": 0.547486033519553, + "clinical_knowledge": 0.3670886075949367, + "college_actuarial_science": 0.25471698113207547, + "college_education": 0.6074766355140186, + "college_engineering_hydrology": 0.39622641509433965, + "college_law": 0.37962962962962965, "college_mathematics": 0.2857142857142857, - "college_medical_statistics": 0.3018867924528302, - "college_medicine": 0.1978021978021978, - "computer_science": 0.25, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.2653061224489796, - "construction_project_management": 0.23741007194244604, - "economics": 0.34591194968553457, - "education": 0.20245398773006135, - "electrical_engineering": 0.2441860465116279, - "elementary_chinese": 0.25793650793650796, - "elementary_commonsense": 0.2828282828282828, - "elementary_information_and_technology": 0.23109243697478993, - "elementary_mathematics": 0.25217391304347825, - "ethnology": 0.2814814814814815, - "food_science": 0.3006993006993007, - "genetics": 0.19318181818181818, - "global_facts": 0.2550335570469799, - "high_school_biology": 0.22485207100591717, - "high_school_chemistry": 0.2878787878787879, - "high_school_geography": 0.2966101694915254, - "high_school_mathematics": 0.2073170731707317, - "high_school_physics": 0.2909090909090909, - "high_school_politics": 0.24475524475524477, - "human_sexuality": 0.25396825396825395, - "international_law": 0.22162162162162163, - "journalism": 0.27906976744186046, - "jurisprudence": 0.27007299270072993, - "legal_and_moral_basis": 0.2523364485981308, - "logical": 0.22764227642276422, - "machine_learning": 0.20491803278688525, - "management": 0.1761904761904762, - "marketing": 0.25555555555555554, - "marxist_theory": 0.2222222222222222, - "modern_chinese": 0.31896551724137934, - "nutrition": 0.2827586206896552, - "philosophy": 0.24761904761904763, - "professional_accounting": 0.2342857142857143, - "professional_law": 0.2559241706161137, - "professional_medicine": 0.24468085106382978, - "professional_psychology": 0.21120689655172414, - "public_relations": 0.2988505747126437, - "security_study": 0.25925925925925924, - "sociology": 0.23893805309734514, - "sports_science": 0.20606060606060606, - "traditional_chinese_medicine": 0.2756756756756757, - "virology": 0.20710059171597633, - "world_history": 0.20496894409937888, - "world_religions": 0.21875 + "college_medical_statistics": 0.4056603773584906, + "college_medicine": 0.4175824175824176, + "computer_science": 0.46568627450980393, + "computer_security": 0.4502923976608187, + "conceptual_physics": 0.4897959183673469, + "construction_project_management": 0.4028776978417266, + "economics": 0.4528301886792453, + "education": 0.5153374233128835, + "electrical_engineering": 0.42441860465116277, + "elementary_chinese": 0.42857142857142855, + "elementary_commonsense": 0.4595959595959596, + "elementary_information_and_technology": 0.680672268907563, + "elementary_mathematics": 0.2956521739130435, + "ethnology": 0.43703703703703706, + "food_science": 0.4405594405594406, + "genetics": 0.3806818181818182, + "global_facts": 0.4228187919463087, + "high_school_biology": 0.3431952662721893, + "high_school_chemistry": 0.3106060606060606, + "high_school_geography": 0.4830508474576271, + "high_school_mathematics": 0.25609756097560976, + "high_school_physics": 0.4, + "high_school_politics": 0.4195804195804196, + "human_sexuality": 0.46825396825396826, + "international_law": 0.3891891891891892, + "journalism": 0.5290697674418605, + "jurisprudence": 0.44038929440389296, + "legal_and_moral_basis": 0.7850467289719626, + "logical": 0.4146341463414634, + "machine_learning": 0.39344262295081966, + "management": 0.5333333333333333, + "marketing": 0.5388888888888889, + "marxist_theory": 0.5238095238095238, + "modern_chinese": 0.3620689655172414, + "nutrition": 0.496551724137931, + "philosophy": 0.47619047619047616, + "professional_accounting": 0.4342857142857143, + "professional_law": 0.3933649289099526, + "professional_medicine": 0.32180851063829785, + "professional_psychology": 0.5043103448275862, + "public_relations": 0.47701149425287354, + "security_study": 0.5185185185185185, + "sociology": 0.4247787610619469, + "sports_science": 0.44242424242424244, + "traditional_chinese_medicine": 0.3783783783783784, + "virology": 0.4260355029585799, + "world_history": 0.4472049689440994, + "world_religions": 0.4625 } }, "prompt_4": { - "accuracy": 0.24589880849594198, + "accuracy": 0.42764634778103955, "category_acc": { - "agronomy": 0.21893491124260356, - "anatomy": 0.25, - "ancient_chinese": 0.2621951219512195, - "arts": 0.24375, - "astronomy": 0.2909090909090909, - "business_ethics": 0.19138755980861244, - "chinese_civil_service_exam": 0.2375, - "chinese_driving_rule": 0.2748091603053435, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.22429906542056074, - "chinese_history": 0.2260061919504644, - "chinese_literature": 0.21568627450980393, - "chinese_teacher_qualification": 0.24022346368715083, - "clinical_knowledge": 0.27848101265822783, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.2616822429906542, - "college_engineering_hydrology": 0.3113207547169811, - "college_law": 0.26851851851851855, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.2358490566037736, - "college_medicine": 0.26373626373626374, - "computer_science": 0.23039215686274508, - "computer_security": 0.23391812865497075, - "conceptual_physics": 0.2585034013605442, - "construction_project_management": 0.23741007194244604, - "economics": 0.24528301886792453, - "education": 0.32515337423312884, - "electrical_engineering": 0.29651162790697677, - "elementary_chinese": 0.23809523809523808, - "elementary_commonsense": 0.24242424242424243, - "elementary_information_and_technology": 0.2647058823529412, + "agronomy": 0.38461538461538464, + "anatomy": 0.33783783783783783, + "ancient_chinese": 0.25, + "arts": 0.59375, + "astronomy": 0.2727272727272727, + "business_ethics": 0.4784688995215311, + "chinese_civil_service_exam": 0.4125, + "chinese_driving_rule": 0.6641221374045801, + "chinese_food_culture": 0.47058823529411764, + "chinese_foreign_policy": 0.48598130841121495, + "chinese_history": 0.4551083591331269, + "chinese_literature": 0.37254901960784315, + "chinese_teacher_qualification": 0.547486033519553, + "clinical_knowledge": 0.350210970464135, + "college_actuarial_science": 0.22641509433962265, + "college_education": 0.5887850467289719, + "college_engineering_hydrology": 0.3867924528301887, + "college_law": 0.3888888888888889, + "college_mathematics": 0.26666666666666666, + "college_medical_statistics": 0.4056603773584906, + "college_medicine": 0.3882783882783883, + "computer_science": 0.45588235294117646, + "computer_security": 0.4502923976608187, + "conceptual_physics": 0.4421768707482993, + "construction_project_management": 0.39568345323741005, + "economics": 0.42138364779874216, + "education": 0.5214723926380368, + "electrical_engineering": 0.43023255813953487, + "elementary_chinese": 0.42857142857142855, + "elementary_commonsense": 0.45454545454545453, + "elementary_information_and_technology": 0.6554621848739496, "elementary_mathematics": 0.2782608695652174, - "ethnology": 0.2222222222222222, - "food_science": 0.21678321678321677, - "genetics": 0.29545454545454547, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.2958579881656805, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.23728813559322035, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.17272727272727273, - "high_school_politics": 0.16783216783216784, - "human_sexuality": 0.2222222222222222, - "international_law": 0.2702702702702703, - "journalism": 0.22674418604651161, - "jurisprudence": 0.2141119221411192, - "legal_and_moral_basis": 0.22897196261682243, - "logical": 0.25203252032520324, - "machine_learning": 0.2459016393442623, - "management": 0.2714285714285714, - "marketing": 0.2833333333333333, - "marxist_theory": 0.21693121693121692, - "modern_chinese": 0.25, - "nutrition": 0.21379310344827587, - "philosophy": 0.2761904761904762, - "professional_accounting": 0.24571428571428572, - "professional_law": 0.23222748815165878, - "professional_medicine": 0.2074468085106383, - "professional_psychology": 0.22844827586206898, - "public_relations": 0.22413793103448276, - "security_study": 0.24444444444444444, - "sociology": 0.25663716814159293, - "sports_science": 0.2909090909090909, - "traditional_chinese_medicine": 0.2648648648648649, - "virology": 0.21893491124260356, - "world_history": 0.2422360248447205, - "world_religions": 0.29375 + "ethnology": 0.4, + "food_science": 0.45454545454545453, + "genetics": 0.3522727272727273, + "global_facts": 0.4228187919463087, + "high_school_biology": 0.3254437869822485, + "high_school_chemistry": 0.3484848484848485, + "high_school_geography": 0.4576271186440678, + "high_school_mathematics": 0.25609756097560976, + "high_school_physics": 0.34545454545454546, + "high_school_politics": 0.4125874125874126, + "human_sexuality": 0.42857142857142855, + "international_law": 0.3783783783783784, + "journalism": 0.4941860465116279, + "jurisprudence": 0.43795620437956206, + "legal_and_moral_basis": 0.7663551401869159, + "logical": 0.43902439024390244, + "machine_learning": 0.36885245901639346, + "management": 0.5142857142857142, + "marketing": 0.5333333333333333, + "marxist_theory": 0.4603174603174603, + "modern_chinese": 0.31896551724137934, + "nutrition": 0.42758620689655175, + "philosophy": 0.4666666666666667, + "professional_accounting": 0.45714285714285713, + "professional_law": 0.3459715639810427, + "professional_medicine": 0.31382978723404253, + "professional_psychology": 0.4827586206896552, + "public_relations": 0.4425287356321839, + "security_study": 0.48148148148148145, + "sociology": 0.4336283185840708, + "sports_science": 0.42424242424242425, + "traditional_chinese_medicine": 0.372972972972973, + "virology": 0.38461538461538464, + "world_history": 0.484472049689441, + "world_religions": 0.43125 } }, "prompt_5": { - "accuracy": 0.2509929200483509, + "accuracy": 0.43653945777931275, "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.2905405405405405, + "agronomy": 0.40828402366863903, + "anatomy": 0.36486486486486486, "ancient_chinese": 0.27439024390243905, - "arts": 0.1875, - "astronomy": 0.24848484848484848, - "business_ethics": 0.23444976076555024, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.2824427480916031, - "chinese_food_culture": 0.3235294117647059, - "chinese_foreign_policy": 0.205607476635514, - "chinese_history": 0.25077399380804954, - "chinese_literature": 0.25980392156862747, - "chinese_teacher_qualification": 0.2905027932960894, - "clinical_knowledge": 0.29535864978902954, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.2336448598130841, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.23148148148148148, - "college_mathematics": 0.3238095238095238, - "college_medical_statistics": 0.22641509433962265, - "college_medicine": 0.27472527472527475, - "computer_science": 0.23529411764705882, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.21768707482993196, - "construction_project_management": 0.20863309352517986, - "economics": 0.22641509433962265, - "education": 0.19631901840490798, - "electrical_engineering": 0.27325581395348836, - "elementary_chinese": 0.23015873015873015, - "elementary_commonsense": 0.24242424242424243, - "elementary_information_and_technology": 0.27310924369747897, - "elementary_mathematics": 0.26956521739130435, - "ethnology": 0.21481481481481482, - "food_science": 0.2517482517482518, - "genetics": 0.1590909090909091, - "global_facts": 0.2550335570469799, - "high_school_biology": 0.1893491124260355, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.1694915254237288, - "high_school_mathematics": 0.22560975609756098, - "high_school_physics": 0.21818181818181817, - "high_school_politics": 0.23076923076923078, - "human_sexuality": 0.20634920634920634, - "international_law": 0.22162162162162163, - "journalism": 0.25, - "jurisprudence": 0.26277372262773724, - "legal_and_moral_basis": 0.21495327102803738, - "logical": 0.2032520325203252, - "machine_learning": 0.28688524590163933, - "management": 0.2857142857142857, - "marketing": 0.31666666666666665, - "marxist_theory": 0.2275132275132275, - "modern_chinese": 0.23275862068965517, - "nutrition": 0.27586206896551724, - "philosophy": 0.26666666666666666, - "professional_accounting": 0.2857142857142857, - "professional_law": 0.24170616113744076, - "professional_medicine": 0.23404255319148937, - "professional_psychology": 0.2672413793103448, - "public_relations": 0.22413793103448276, - "security_study": 0.2814814814814815, - "sociology": 0.2743362831858407, - "sports_science": 0.3090909090909091, - "traditional_chinese_medicine": 0.2864864864864865, - "virology": 0.25443786982248523, - "world_history": 0.2795031055900621, - "world_religions": 0.3 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.21212121212121213 - }, - "prompt_2": { - "accuracy": 0.42424242424242425 - }, - "prompt_3": { - "accuracy": 0.3939393939393939 - }, - "prompt_4": { - "accuracy": 0.30303030303030304 - }, - "prompt_5": { - "accuracy": 0.36363636363636365 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.18863636363636363 - }, - "prompt_2": { - "accuracy": 0.15454545454545454 - }, - "prompt_3": { - "accuracy": 0.15454545454545454 - }, - "prompt_4": { - "accuracy": 0.17272727272727273 - }, - "prompt_5": { - "accuracy": 0.17045454545454544 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3440677966101695 - }, - "prompt_2": { - "accuracy": 0.32305084745762713 - }, + "arts": 0.59375, + "astronomy": 0.296969696969697, + "business_ethics": 0.48325358851674644, + "chinese_civil_service_exam": 0.43125, + "chinese_driving_rule": 0.6793893129770993, + "chinese_food_culture": 0.45588235294117646, + "chinese_foreign_policy": 0.4953271028037383, + "chinese_history": 0.4674922600619195, + "chinese_literature": 0.39215686274509803, + "chinese_teacher_qualification": 0.5307262569832403, + "clinical_knowledge": 0.35443037974683544, + "college_actuarial_science": 0.24528301886792453, + "college_education": 0.5887850467289719, + "college_engineering_hydrology": 0.3867924528301887, + "college_law": 0.4074074074074074, + "college_mathematics": 0.2761904761904762, + "college_medical_statistics": 0.44339622641509435, + "college_medicine": 0.3992673992673993, + "computer_science": 0.46568627450980393, + "computer_security": 0.4853801169590643, + "conceptual_physics": 0.48299319727891155, + "construction_project_management": 0.39568345323741005, + "economics": 0.4528301886792453, + "education": 0.5337423312883436, + "electrical_engineering": 0.4127906976744186, + "elementary_chinese": 0.4246031746031746, + "elementary_commonsense": 0.4898989898989899, + "elementary_information_and_technology": 0.6596638655462185, + "elementary_mathematics": 0.28695652173913044, + "ethnology": 0.45185185185185184, + "food_science": 0.46853146853146854, + "genetics": 0.36363636363636365, + "global_facts": 0.4161073825503356, + "high_school_biology": 0.33136094674556216, + "high_school_chemistry": 0.30303030303030304, + "high_school_geography": 0.4576271186440678, + "high_school_mathematics": 0.2804878048780488, + "high_school_physics": 0.34545454545454546, + "high_school_politics": 0.42657342657342656, + "human_sexuality": 0.4365079365079365, + "international_law": 0.35135135135135137, + "journalism": 0.5174418604651163, + "jurisprudence": 0.4306569343065693, + "legal_and_moral_basis": 0.7710280373831776, + "logical": 0.42276422764227645, + "machine_learning": 0.36885245901639346, + "management": 0.5047619047619047, + "marketing": 0.5333333333333333, + "marxist_theory": 0.5132275132275133, + "modern_chinese": 0.35344827586206895, + "nutrition": 0.4413793103448276, + "philosophy": 0.4380952380952381, + "professional_accounting": 0.46285714285714286, + "professional_law": 0.3696682464454976, + "professional_medicine": 0.32978723404255317, + "professional_psychology": 0.4956896551724138, + "public_relations": 0.4540229885057471, + "security_study": 0.5037037037037037, + "sociology": 0.43805309734513276, + "sports_science": 0.43636363636363634, + "traditional_chinese_medicine": 0.35135135135135137, + "virology": 0.39644970414201186, + "world_history": 0.4472049689440994, + "world_religions": 0.45625 + } + } + }, + "zbench": { + "prompt_1": { + "accuracy": 0.18181818181818182 + }, + "prompt_2": { + "accuracy": 0.2727272727272727 + }, "prompt_3": { - "accuracy": 0.3176271186440678 + "accuracy": 0.21212121212121213 + }, + "prompt_4": { + "accuracy": 0.24242424242424243 + }, + "prompt_5": { + "accuracy": 0.24242424242424243 + } + }, + "ind_emotion": { + "prompt_1": { + "accuracy": 0.6295454545454545 + }, + "prompt_2": { + "accuracy": 0.6613636363636364 + }, + "prompt_3": { + "accuracy": 0.6772727272727272 + }, + "prompt_4": { + "accuracy": 0.5727272727272728 + }, + "prompt_5": { + "accuracy": 0.6068181818181818 + } + }, + "ocnli": { + "prompt_1": { + "accuracy": 0.39796610169491525 + }, + "prompt_2": { + "accuracy": 0.39084745762711864 + }, + "prompt_3": { + "accuracy": 0.4040677966101695 }, "prompt_4": { - "accuracy": 0.3369491525423729 + "accuracy": 0.3827118644067797 }, "prompt_5": { - "accuracy": 0.3467796610169491 + "accuracy": 0.36135593220338985 } }, "c3": { "prompt_1": { - "accuracy": 0.2629020194465221 + "accuracy": 0.7774869109947644 }, "prompt_2": { - "accuracy": 0.29020194465220644 + "accuracy": 0.7729992520568437 }, "prompt_3": { - "accuracy": 0.2801047120418848 + "accuracy": 0.7778608825729244 }, "prompt_4": { - "accuracy": 0.2677636499626028 + "accuracy": 0.7741211667913238 }, "prompt_5": { - "accuracy": 0.27711293941660436 + "accuracy": 0.7696335078534031 } }, "dream": { "prompt_1": { - "accuracy": 0.35080842724154826 + "accuracy": 0.7143557079862812 }, "prompt_2": { - "accuracy": 0.33610975012248895 + "accuracy": 0.7261146496815286 }, "prompt_3": { - "accuracy": 0.32533072023517884 + "accuracy": 0.7275845173934345 }, "prompt_4": { - "accuracy": 0.3547280744732974 + "accuracy": 0.7035766780989711 }, "prompt_5": { - "accuracy": 0.3395394414502695 + "accuracy": 0.698187163155316 } }, "samsum": { "prompt_1": { - "rouge1": 0.012377281576480267, - "rouge2": 0.0023322492433952573, - "rougeL": 0.009611773766619633, - "avg_rouge": 0.008107101528831718 + "rouge1": 0.3313739994617144, + "rouge2": 0.10653091519534055, + "rougeL": 0.24154018848608264, + "avg_rouge": 0.2264817010477125 }, "prompt_2": { - "rouge1": 0.01597823516968608, - "rouge2": 0.0030294744191448728, - "rougeL": 0.01215827742374666, - "avg_rouge": 0.01038866233752587 + "rouge1": 0.33083257013215717, + "rouge2": 0.10462933943338155, + "rougeL": 0.24150892295839743, + "avg_rouge": 0.22565694417464535 }, "prompt_3": { - "rouge1": 0.010943760341091367, - "rouge2": 0.002195574862102358, - "rougeL": 0.008608337626769446, - "avg_rouge": 0.00724922427665439 + "rouge1": 0.34835996028886995, + "rouge2": 0.10982634542283559, + "rougeL": 0.2575315647612755, + "avg_rouge": 0.23857262349099365 }, "prompt_4": { - "rouge1": 0.014893974859736696, - "rouge2": 0.002978348139378036, - "rougeL": 0.011541536922048646, - "avg_rouge": 0.009804619973721125 + "rouge1": 0.3349310528982967, + "rouge2": 0.10951537129251994, + "rougeL": 0.24659014274854002, + "avg_rouge": 0.2303455223131189 }, "prompt_5": { - "rouge1": 0.012217159420843697, - "rouge2": 0.002467765630826918, - "rougeL": 0.009507628176067678, - "avg_rouge": 0.008064184409246097 + "rouge1": 0.3428935036440538, + "rouge2": 0.11041972210880038, + "rougeL": 0.25412988565195127, + "avg_rouge": 0.23581437046826847 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.028895630179341263, - "rouge2": 0.0065963896033818135, - "rougeL": 0.019966356620012747, - "avg_rouge": 0.018486125467578607 + "rouge1": 0.21956463999285744, + "rouge2": 0.05747048037363373, + "rougeL": 0.1596940858570499, + "avg_rouge": 0.1455764020745137 }, "prompt_2": { - "rouge1": 0.02781831017354505, - "rouge2": 0.005894644901671599, - "rougeL": 0.019824454177794996, - "avg_rouge": 0.017845803084337215 + "rouge1": 0.2174604871339299, + "rouge2": 0.05475645193832106, + "rougeL": 0.15851239276061685, + "avg_rouge": 0.14357644394428928 }, "prompt_3": { - "rouge1": 0.025789912879256485, - "rouge2": 0.00552802083781569, - "rougeL": 0.01868739285568793, - "avg_rouge": 0.016668442190920035 + "rouge1": 0.217054414110199, + "rouge2": 0.055400616357914836, + "rougeL": 0.15783238159600757, + "avg_rouge": 0.1434291373547071 }, "prompt_4": { - "rouge1": 0.029556250929129935, - "rouge2": 0.006610810657440766, - "rougeL": 0.020687220868247126, - "avg_rouge": 0.018951427484939276 + "rouge1": 0.2188521463516572, + "rouge2": 0.05698676971087247, + "rougeL": 0.1595901861437309, + "avg_rouge": 0.14514303406875353 }, "prompt_5": { - "rouge1": 0.025641618760452407, - "rouge2": 0.006237021442129534, - "rougeL": 0.018278749880372716, - "avg_rouge": 0.016719130027651553 + "rouge1": 0.2217691129634524, + "rouge2": 0.05864590992084784, + "rougeL": 0.16271839979241123, + "avg_rouge": 0.14771114089223714 } }, "sst2": { "prompt_1": { - "accuracy": 0.5068807339449541 + "accuracy": 0.9139908256880734 }, "prompt_2": { - "accuracy": 0.5275229357798165 + "accuracy": 0.9013761467889908 }, "prompt_3": { - "accuracy": 0.49770642201834864 + "accuracy": 0.9025229357798165 }, "prompt_4": { - "accuracy": 0.5252293577981652 + "accuracy": 0.9185779816513762 }, "prompt_5": { - "accuracy": 0.5080275229357798 + "accuracy": 0.9036697247706422 } }, "cola": { "prompt_1": { - "accuracy": 0.5196548418024928 + "accuracy": 0.5589645254074784 }, "prompt_2": { - "accuracy": 0.5129434324065196 + "accuracy": 0.6049856184084372 }, "prompt_3": { - "accuracy": 0.5292425695110259 + "accuracy": 0.6519654841802492 }, "prompt_4": { - "accuracy": 0.5234899328859061 + "accuracy": 0.6097794822627037 }, "prompt_5": { - "accuracy": 0.5723873441994247 + "accuracy": 0.6538830297219559 } }, "qqp": { "prompt_1": { - "accuracy": 0.538 + "accuracy": 0.5695 }, "prompt_2": { - "accuracy": 0.544 + "accuracy": 0.7025 }, "prompt_3": { - "accuracy": 0.5475 + "accuracy": 0.603 }, "prompt_4": { - "accuracy": 0.548 + "accuracy": 0.623 }, "prompt_5": { - "accuracy": 0.563 + "accuracy": 0.71 } }, "mnli": { "prompt_1": { - "accuracy": 0.3345 + "accuracy": 0.6 }, "prompt_2": { - "accuracy": 0.333 + "accuracy": 0.7 }, "prompt_3": { - "accuracy": 0.3195 + "accuracy": 0.7 }, "prompt_4": { - "accuracy": 0.338 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.333 + "accuracy": 0.5 } }, "qnli": { "prompt_1": { - "accuracy": 0.495 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.4925 + "accuracy": 0.7 }, "prompt_3": { - "accuracy": 0.4915 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.507 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.49 + "accuracy": 0.8 } }, "wnli": { "prompt_1": { - "accuracy": 0.38028169014084506 + "accuracy": 0.3 }, "prompt_2": { - "accuracy": 0.49295774647887325 + "accuracy": 0.5 }, "prompt_3": { - "accuracy": 0.5774647887323944 + "accuracy": 0.3 }, "prompt_4": { - "accuracy": 0.5211267605633803 + "accuracy": 0.3 }, "prompt_5": { - "accuracy": 0.5492957746478874 + "accuracy": 0.4 } }, "rte": { "prompt_1": { - "accuracy": 0.5306859205776173 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.45126353790613716 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.5451263537906137 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.4332129963898917 + "accuracy": 0.7 }, "prompt_5": { - "accuracy": 0.4729241877256318 + "accuracy": 1.0 } }, "mrpc": { "prompt_1": { - "accuracy": 0.5098039215686274 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.37745098039215685 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.40931372549019607 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.47794117647058826 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.4166666666666667 + "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.24012605042016805, - "language_acc": { - "English": 0.23949579831932774, - "Chinese": 0.25546218487394956, - "Spanish": 0.22857142857142856, - "Vietnamese": 0.23697478991596638 - }, - "consistency_score_2": 0.38431372549019605, - "consistency_score_3": 0.19327731092436973, - "consistency_score_4": 0.11008403361344538, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.35714285714285715, - "English,Spanish": 0.36722689075630255, - "English,Vietnamese": 0.39747899159663863, - "Chinese,Spanish": 0.3831932773109244, - "Chinese,Vietnamese": 0.37478991596638656, - "Spanish,Vietnamese": 0.4260504201680672 - }, - "3_combine": { - "English,Chinese,Spanish": 0.17899159663865546, - "English,Chinese,Vietnamese": 0.18823529411764706, - "English,Spanish,Vietnamese": 0.2042016806722689, - "Chinese,Spanish,Vietnamese": 0.20168067226890757 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.11008403361344538 - } - }, - "AC3_2": 0.29557289772229295, - "AC3_3": 0.21416962320463617, - "AC3_4": 0.1509610682474377 - } + "prompt_1": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.21428571428571427, - "language_acc": { - "Spanish": 0.22666666666666666, - "English": 0.22, - "Chinese": 0.23333333333333334, - "Vietnamese": 0.19333333333333333, - "Filipino": 0.19333333333333333, - "Indonesian": 0.2, - "Malay": 0.23333333333333334 - }, - "consistency_score_2": 0.40952380952380957, - "consistency_score_3": 0.21580952380952378, - "consistency_score_4": 0.12476190476190475, - "consistency_score_5": 0.07555555555555554, - "consistency_score_6": 0.04857142857142858, - "consistency_score_7": 0.03333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.38, - "Spanish,Chinese": 0.4, - "Spanish,Vietnamese": 0.42, - "Spanish,Filipino": 0.4, - "Spanish,Indonesian": 0.48, - "Spanish,Malay": 0.38666666666666666, - "English,Chinese": 0.36666666666666664, - "English,Vietnamese": 0.3933333333333333, - "English,Filipino": 0.38, - "English,Indonesian": 0.38, - "English,Malay": 0.3466666666666667, - "Chinese,Vietnamese": 0.42, - "Chinese,Filipino": 0.44, - "Chinese,Indonesian": 0.36, - "Chinese,Malay": 0.4, - "Vietnamese,Filipino": 0.46, - "Vietnamese,Indonesian": 0.36, - "Vietnamese,Malay": 0.4266666666666667, - "Filipino,Indonesian": 0.43333333333333335, - "Filipino,Malay": 0.49333333333333335, - "Indonesian,Malay": 0.47333333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.16, - "Spanish,English,Vietnamese": 0.22, - "Spanish,English,Filipino": 0.18666666666666668, - "Spanish,English,Indonesian": 0.20666666666666667, - "Spanish,English,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese": 0.20666666666666667, - "Spanish,Chinese,Filipino": 0.21333333333333335, - "Spanish,Chinese,Indonesian": 0.22, - "Spanish,Chinese,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino": 0.22, - "Spanish,Vietnamese,Indonesian": 0.22, - "Spanish,Vietnamese,Malay": 0.2, - "Spanish,Filipino,Indonesian": 0.24, - "Spanish,Filipino,Malay": 0.23333333333333334, - "Spanish,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese": 0.18666666666666668, - "English,Chinese,Filipino": 0.2, - "English,Chinese,Indonesian": 0.17333333333333334, - "English,Chinese,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino": 0.21333333333333335, - "English,Vietnamese,Indonesian": 0.18, - "English,Vietnamese,Malay": 0.2, - "English,Filipino,Indonesian": 0.22, - "English,Filipino,Malay": 0.22666666666666666, - "English,Indonesian,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "Chinese,Vietnamese,Malay": 0.23333333333333334, - "Chinese,Filipino,Indonesian": 0.24666666666666667, - "Chinese,Filipino,Malay": 0.29333333333333333, - "Chinese,Indonesian,Malay": 0.22666666666666666, - "Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Vietnamese,Filipino,Malay": 0.29333333333333333, - "Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Filipino,Indonesian,Malay": 0.2733333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.12, - "Spanish,English,Chinese,Filipino": 0.09333333333333334, - "Spanish,English,Chinese,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.12, - "Spanish,English,Vietnamese,Malay": 0.11333333333333333, - "Spanish,English,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.1, - "Spanish,Chinese,Vietnamese,Malay": 0.10666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.14, - "Spanish,Chinese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Indonesian,Malay": 0.14, - "Spanish,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.12, - "Spanish,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.08666666666666667, - "English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Filipino,Malay": 0.14, - "English,Chinese,Indonesian,Malay": 0.12, - "English,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "English,Vietnamese,Filipino,Malay": 0.14, - "English,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "English,Filipino,Indonesian,Malay": 0.14, - "Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.06, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.06, - "Spanish,English,Chinese,Vietnamese,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.08, - "Spanish,English,Chinese,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.06666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.06666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.06666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.04, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.05333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.06, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.04, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.03333333333333333 - } - }, - "AC3_2": 0.2813522355056066, - "AC3_3": 0.2150449196007661, - "AC3_4": 0.1577046548491521, - "AC3_5": 0.11171960565696419, - "AC3_6": 0.07919254655372335, - "AC3_7": 0.057692307669008874 - } + "prompt_1": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.26785714285714285, - "language_acc": { - "English": 0.21022727272727273, - "Filipino": 0.26704545454545453, - "Spanish": 0.2727272727272727, - "Malay": 0.26136363636363635, - "Chinese": 0.3352272727272727, - "Indonesian": 0.2556818181818182, - "Vietnamese": 0.2727272727272727 - }, - "consistency_score_2": 0.354978354978355, - "consistency_score_3": 0.15892857142857142, - "consistency_score_4": 0.07873376623376624, - "consistency_score_5": 0.041396103896103896, - "consistency_score_6": 0.022727272727272728, - "consistency_score_7": 0.011363636363636364, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.2727272727272727, - "English,Spanish": 0.3181818181818182, - "English,Malay": 0.3465909090909091, - "English,Chinese": 0.3125, - "English,Indonesian": 0.32954545454545453, - "English,Vietnamese": 0.3465909090909091, - "Filipino,Spanish": 0.4375, - "Filipino,Malay": 0.3806818181818182, - "Filipino,Chinese": 0.3181818181818182, - "Filipino,Indonesian": 0.35795454545454547, - "Filipino,Vietnamese": 0.4375, - "Spanish,Malay": 0.38636363636363635, - "Spanish,Chinese": 0.3409090909090909, - "Spanish,Indonesian": 0.3352272727272727, - "Spanish,Vietnamese": 0.4034090909090909, - "Malay,Chinese": 0.3181818181818182, - "Malay,Indonesian": 0.3693181818181818, - "Malay,Vietnamese": 0.4090909090909091, - "Chinese,Indonesian": 0.32386363636363635, - "Chinese,Vietnamese": 0.3465909090909091, - "Indonesian,Vietnamese": 0.36363636363636365 - }, - "3_combine": { - "English,Filipino,Spanish": 0.14772727272727273, - "English,Filipino,Malay": 0.125, - "English,Filipino,Chinese": 0.09090909090909091, - "English,Filipino,Indonesian": 0.11931818181818182, - "English,Filipino,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay": 0.16477272727272727, - "English,Spanish,Chinese": 0.125, - "English,Spanish,Indonesian": 0.125, - "English,Spanish,Vietnamese": 0.1534090909090909, - "English,Malay,Chinese": 0.13068181818181818, - "English,Malay,Indonesian": 0.14204545454545456, - "English,Malay,Vietnamese": 0.17045454545454544, - "English,Chinese,Indonesian": 0.13636363636363635, - "English,Chinese,Vietnamese": 0.14772727272727273, - "English,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay": 0.21022727272727273, - "Filipino,Spanish,Chinese": 0.1534090909090909, - "Filipino,Spanish,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese": 0.13068181818181818, - "Filipino,Malay,Indonesian": 0.17613636363636365, - "Filipino,Malay,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Indonesian,Vietnamese": 0.19886363636363635, - "Spanish,Malay,Chinese": 0.17045454545454544, - "Spanish,Malay,Indonesian": 0.17613636363636365, - "Spanish,Malay,Vietnamese": 0.2159090909090909, - "Spanish,Chinese,Indonesian": 0.13636363636363635, - "Spanish,Chinese,Vietnamese": 0.18181818181818182, - "Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "Malay,Chinese,Indonesian": 0.1590909090909091, - "Malay,Chinese,Vietnamese": 0.1534090909090909, - "Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Chinese,Indonesian,Vietnamese": 0.14772727272727273 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.07954545454545454, - "English,Filipino,Spanish,Chinese": 0.0625, - "English,Filipino,Spanish,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese": 0.03977272727272727, - "English,Filipino,Malay,Indonesian": 0.045454545454545456, - "English,Filipino,Malay,Vietnamese": 0.09659090909090909, - "English,Filipino,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.0625, - "English,Filipino,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Chinese": 0.06818181818181818, - "English,Spanish,Malay,Indonesian": 0.06818181818181818, - "English,Spanish,Malay,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian": 0.05113636363636364, - "English,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "English,Spanish,Indonesian,Vietnamese": 0.06818181818181818, - "English,Malay,Chinese,Indonesian": 0.0625, - "English,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Chinese": 0.08522727272727272, - "Filipino,Spanish,Malay,Indonesian": 0.10795454545454546, - "Filipino,Spanish,Malay,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Malay,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian": 0.03409090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.03409090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.045454545454545456, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.05113636363636364, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 - } - }, - "AC3_2": 0.3053245624432211, - "AC3_3": 0.19949193061673587, - "AC3_4": 0.12169621943628278, - "AC3_5": 0.07170978625352736, - "AC3_6": 0.0418994413263631, - "AC3_7": 0.021802325573587075 - } + "prompt_1": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.23300970873786409 - } + "prompt_1": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.2761904761904762 - } + "prompt_1": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.22429906542056074 - } + "prompt_1": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.26, - "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.2, - "history": 0.13333333333333333, - "literature": 0.3, - "politics": 0.6, - "culture": 0.1, - "film": 0.3, - "law": 0.2, - "geography": 0.2 - } - } + "prompt_1": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.03761924026752369 - } + "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.12991865854591933 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.1254239847711969 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.03939020873093385 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.13194803209514458 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.2543757292882147 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.24254558455488023, - "category_acc": { - "high_school_european_history": 0.21341463414634146, - "business_ethics": 0.26262626262626265, - "clinical_knowledge": 0.2196969696969697, - "medical_genetics": 0.25252525252525254, - "high_school_us_history": 0.2413793103448276, - "high_school_physics": 0.25333333333333335, - "high_school_world_history": 0.25, - "virology": 0.28484848484848485, - "high_school_microeconomics": 0.22784810126582278, - "econometrics": 0.30973451327433627, - "college_computer_science": 0.25252525252525254, - "high_school_biology": 0.22006472491909385, - "abstract_algebra": 0.20202020202020202, - "professional_accounting": 0.23487544483985764, - "philosophy": 0.23225806451612904, - "professional_medicine": 0.2029520295202952, - "nutrition": 0.21639344262295082, - "global_facts": 0.20202020202020202, - "machine_learning": 0.21621621621621623, - "security_studies": 0.2459016393442623, - "public_relations": 0.1926605504587156, - "professional_psychology": 0.22913256955810146, - "prehistory": 0.22291021671826625, - "anatomy": 0.26119402985074625, - "human_sexuality": 0.25384615384615383, - "college_medicine": 0.21511627906976744, - "high_school_government_and_politics": 0.24479166666666666, - "college_chemistry": 0.26262626262626265, - "logical_fallacies": 0.22839506172839505, - "high_school_geography": 0.2182741116751269, - "elementary_mathematics": 0.2493368700265252, - "human_aging": 0.23423423423423423, - "college_mathematics": 0.1919191919191919, - "high_school_psychology": 0.21691176470588236, - "formal_logic": 0.304, - "high_school_statistics": 0.22790697674418606, - "international_law": 0.30833333333333335, - "high_school_mathematics": 0.22304832713754646, - "high_school_computer_science": 0.24242424242424243, - "conceptual_physics": 0.24358974358974358, - "miscellaneous": 0.24296675191815856, - "high_school_chemistry": 0.21782178217821782, - "marketing": 0.30042918454935624, - "professional_law": 0.24592302674494454, - "management": 0.22549019607843138, - "college_physics": 0.22772277227722773, - "jurisprudence": 0.2803738317757009, - "world_religions": 0.3, - "sociology": 0.27, - "us_foreign_policy": 0.2222222222222222, - "high_school_macroeconomics": 0.27249357326478146, - "computer_security": 0.3333333333333333, - "moral_scenarios": 0.24161073825503357, - "moral_disputes": 0.2927536231884058, - "electrical_engineering": 0.2777777777777778, - "astronomy": 0.2913907284768212, - "college_biology": 0.1888111888111888 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.237741456166419 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.23474470734744707, - "category_acc": { - "computer_network": 0.041666666666666664, - "operating_system": 0.16666666666666666, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.3333333333333333, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.21428571428571427, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.125, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.19230769230769232, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.3, - "business_administration": 0.2894736842105263, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.20588235294117646, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.25, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.14285714285714285, - "ideological_and_moral_cultivation": 0.125, - "logic": 0.14814814814814814, - "law": 0.1724137931034483, - "chinese_language_and_literature": 0.17857142857142858, - "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.23529411764705882, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.36, - "middle_school_history": 0.2222222222222222, - "civil_servant": 0.25, - "sports_science": 0.125, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.25, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.24074074074074073, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.24074074074074073, - "physician": 0.2777777777777778 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.3154121863799283 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.256518735969608, - "category_acc": { - "agronomy": 0.2485207100591716, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.24390243902439024, - "arts": 0.26875, - "astronomy": 0.2606060606060606, - "business_ethics": 0.24401913875598086, - "chinese_civil_service_exam": 0.26875, - "chinese_driving_rule": 0.2366412213740458, - "chinese_food_culture": 0.3088235294117647, - "chinese_foreign_policy": 0.205607476635514, - "chinese_history": 0.2631578947368421, - "chinese_literature": 0.24509803921568626, - "chinese_teacher_qualification": 0.2849162011173184, - "clinical_knowledge": 0.2320675105485232, - "college_actuarial_science": 0.20754716981132076, - "college_education": 0.2897196261682243, - "college_engineering_hydrology": 0.3490566037735849, - "college_law": 0.23148148148148148, - "college_mathematics": 0.2, - "college_medical_statistics": 0.19811320754716982, - "college_medicine": 0.24175824175824176, - "computer_science": 0.3235294117647059, - "computer_security": 0.27485380116959063, - "conceptual_physics": 0.2789115646258503, - "construction_project_management": 0.2446043165467626, - "economics": 0.20754716981132076, - "education": 0.26380368098159507, - "electrical_engineering": 0.27906976744186046, - "elementary_chinese": 0.25793650793650796, - "elementary_commonsense": 0.26262626262626265, - "elementary_information_and_technology": 0.3067226890756303, - "elementary_mathematics": 0.23043478260869565, - "ethnology": 0.23703703703703705, - "food_science": 0.23076923076923078, - "genetics": 0.2159090909090909, - "global_facts": 0.2550335570469799, - "high_school_biology": 0.2958579881656805, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.3135593220338983, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.3181818181818182, - "high_school_politics": 0.22377622377622378, - "human_sexuality": 0.29365079365079366, - "international_law": 0.31891891891891894, - "journalism": 0.2558139534883721, - "jurisprudence": 0.25304136253041365, - "legal_and_moral_basis": 0.2102803738317757, - "logical": 0.2601626016260163, - "machine_learning": 0.20491803278688525, - "management": 0.2761904761904762, - "marketing": 0.2722222222222222, - "marxist_theory": 0.30158730158730157, - "modern_chinese": 0.22413793103448276, - "nutrition": 0.2206896551724138, - "philosophy": 0.22857142857142856, - "professional_accounting": 0.26285714285714284, - "professional_law": 0.27014218009478674, - "professional_medicine": 0.22606382978723405, - "professional_psychology": 0.2629310344827586, - "public_relations": 0.25862068965517243, - "security_study": 0.25925925925925924, - "sociology": 0.25663716814159293, - "sports_science": 0.26666666666666666, - "traditional_chinese_medicine": 0.24324324324324326, - "virology": 0.21893491124260356, - "world_history": 0.2919254658385093, - "world_religions": 0.21875 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.12121212121212122 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.15681818181818183 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.31796610169491524 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.24981301421091998 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.3341499265066144 - } + "prompt_1": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.10204994918596774, - "rouge2": 0.03236347095261923, - "rougeL": 0.08077443541061587, - "avg_rouge": 0.0717292851830676 - } + "prompt_1": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.11382635212790682, - "rouge2": 0.03023667946317905, - "rougeL": 0.08920693350348612, - "avg_rouge": 0.077756655031524 - } + "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.4713302752293578 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.37775647171620325 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.5265 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.336 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.505 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.5774647887323944 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.51985559566787 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.45588235294117646 - } + "prompt_1": -1 } } }, - "sailor_1_8b": { - "model_size": "1.8B", - "model_link": "https://huggingface.co/sail/Sailor-1.8B", + "sailor_7b_chat": { + "model_size": "7B", + "model_link": "https://huggingface.co/sail/Sailor-7B-Chat", "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.26953781512605046, + "overall_acc": 0.9147058823529411, "language_acc": { - "English": 0.3042016806722689, - "Chinese": 0.24705882352941178, - "Spanish": 0.25210084033613445, - "Vietnamese": 0.2747899159663866 - }, - "consistency_score_2": 0.5693277310924371, - "consistency_score_3": 0.39327731092436974, - "consistency_score_4": 0.2865546218487395, + "Spanish": 0.9042016806722689, + "English": 0.9512605042016806, + "Chinese": 0.9042016806722689, + "Vietnamese": 0.8991596638655462 + }, + "consistency_score_2": 0.8918767507002802, + "consistency_score_3": 0.8445378151260504, + "consistency_score_4": 0.8100840336134454, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.3949579831932773, - "English,Spanish": 0.5235294117647059, - "English,Vietnamese": 0.5453781512605042, - "Chinese,Spanish": 0.6352941176470588, - "Chinese,Vietnamese": 0.5630252100840336, - "Spanish,Vietnamese": 0.753781512605042 + "Spanish,English": 0.915126050420168, + "Spanish,Chinese": 0.8798319327731092, + "Spanish,Vietnamese": 0.8764705882352941, + "English,Chinese": 0.8974789915966387, + "English,Vietnamese": 0.9193277310924369, + "Chinese,Vietnamese": 0.8630252100840337 }, "3_combine": { - "English,Chinese,Spanish": 0.32605042016806723, - "English,Chinese,Vietnamese": 0.30840336134453783, - "English,Spanish,Vietnamese": 0.4336134453781513, - "Chinese,Spanish,Vietnamese": 0.5050420168067227 + "Spanish,English,Chinese": 0.8529411764705882, + "Spanish,English,Vietnamese": 0.8596638655462185, + "Spanish,Chinese,Vietnamese": 0.8201680672268907, + "English,Chinese,Vietnamese": 0.8453781512605042 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.2865546218487395 + "Spanish,English,Chinese,Vietnamese": 0.8100840336134454 } }, - "AC3_2": 0.36586400145480646, - "AC3_3": 0.31985723603159455, - "AC3_4": 0.2777858555087418 + "AC3_2": 0.903147074734647, + "AC3_3": 0.8782225093980909, + "AC3_4": 0.8592218957730685 }, "prompt_2": { - "overall_acc": 0.25819327731092434, + "overall_acc": 0.9201680672268907, "language_acc": { - "English": 0.28739495798319326, - "Chinese": 0.23781512605042016, - "Spanish": 0.24789915966386555, - "Vietnamese": 0.2596638655462185 + "Spanish": 0.9100840336134454, + "English": 0.9495798319327731, + "Chinese": 0.9084033613445378, + "Vietnamese": 0.9126050420168067 }, - "consistency_score_2": 0.584733893557423, - "consistency_score_3": 0.4191176470588235, - "consistency_score_4": 0.3176470588235294, + "consistency_score_2": 0.8988795518207283, + "consistency_score_3": 0.8535714285714286, + "consistency_score_4": 0.819327731092437, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.49747899159663866, - "English,Spanish": 0.7361344537815127, - "English,Vietnamese": 0.5579831932773109, - "Chinese,Spanish": 0.5596638655462185, - "Chinese,Vietnamese": 0.5050420168067227, - "Spanish,Vietnamese": 0.6521008403361345 + "Spanish,English": 0.9159663865546218, + "Spanish,Chinese": 0.8798319327731092, + "Spanish,Vietnamese": 0.8840336134453781, + "English,Chinese": 0.9092436974789916, + "English,Vietnamese": 0.9218487394957983, + "Chinese,Vietnamese": 0.8823529411764706 }, "3_combine": { - "English,Chinese,Spanish": 0.4327731092436975, - "English,Chinese,Vietnamese": 0.3369747899159664, - "English,Spanish,Vietnamese": 0.5058823529411764, - "Chinese,Spanish,Vietnamese": 0.4008403361344538 + "Spanish,English,Chinese": 0.8588235294117647, + "Spanish,English,Vietnamese": 0.865546218487395, + "Spanish,Chinese,Vietnamese": 0.8294117647058824, + "English,Chinese,Vietnamese": 0.8605042016806723 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3176470588235294 + "Spanish,English,Chinese,Vietnamese": 0.819327731092437 } }, - "AC3_2": 0.3582144828928227, - "AC3_3": 0.31953820605412153, - "AC3_4": 0.28485095602066623 + "AC3_2": 0.9093992385490498, + "AC3_3": 0.8856195326144926, + "AC3_4": 0.8668249908160796 }, "prompt_3": { - "overall_acc": 0.26134453781512607, + "overall_acc": 0.9262605042016806, "language_acc": { - "English": 0.292436974789916, - "Chinese": 0.2361344537815126, - "Spanish": 0.25210084033613445, - "Vietnamese": 0.2647058823529412 + "Spanish": 0.9218487394957983, + "English": 0.9478991596638655, + "Chinese": 0.9176470588235294, + "Vietnamese": 0.9176470588235294 }, - "consistency_score_2": 0.5822128851540617, - "consistency_score_3": 0.41869747899159665, - "consistency_score_4": 0.31932773109243695, + "consistency_score_2": 0.9099439775910364, + "consistency_score_3": 0.8682773109243698, + "consistency_score_4": 0.8369747899159664, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.4647058823529412, - "English,Spanish": 0.6588235294117647, - "English,Vietnamese": 0.5949579831932773, - "Chinese,Spanish": 0.5546218487394958, - "Chinese,Vietnamese": 0.5084033613445378, - "Spanish,Vietnamese": 0.711764705882353 + "Spanish,English": 0.9310924369747899, + "Spanish,Chinese": 0.8966386554621849, + "Spanish,Vietnamese": 0.9, + "English,Chinese": 0.9134453781512605, + "English,Vietnamese": 0.9260504201680673, + "Chinese,Vietnamese": 0.892436974789916 }, "3_combine": { - "English,Chinese,Spanish": 0.38823529411764707, - "English,Chinese,Vietnamese": 0.3420168067226891, - "English,Spanish,Vietnamese": 0.5134453781512605, - "Chinese,Spanish,Vietnamese": 0.43109243697478994 + "Spanish,English,Chinese": 0.8739495798319328, + "Spanish,English,Vietnamese": 0.8815126050420168, + "Spanish,Chinese,Vietnamese": 0.8487394957983193, + "English,Chinese,Vietnamese": 0.8689075630252101 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.31932773109243695 + "Spanish,English,Chinese,Vietnamese": 0.8369747899159664 } }, - "AC3_2": 0.36075352600653826, - "AC3_3": 0.32181628902623044, - "AC3_4": 0.287441170340507 + "AC3_2": 0.9180297464582713, + "AC3_3": 0.896332161941517, + "AC3_4": 0.8793570471893741 }, "prompt_4": { - "overall_acc": 0.2842436974789916, + "overall_acc": 0.9159663865546218, "language_acc": { - "English": 0.31092436974789917, - "Chinese": 0.253781512605042, - "Spanish": 0.2722689075630252, - "Vietnamese": 0.3 - }, - "consistency_score_2": 0.5813725490196079, - "consistency_score_3": 0.4046218487394958, - "consistency_score_4": 0.3033613445378151, + "Spanish": 0.9092436974789916, + "English": 0.9504201680672268, + "Chinese": 0.9033613445378151, + "Vietnamese": 0.9008403361344538 + }, + "consistency_score_2": 0.892296918767507, + "consistency_score_3": 0.8453781512605042, + "consistency_score_4": 0.8117647058823529, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.43781512605042017, - "English,Spanish": 0.5117647058823529, - "English,Vietnamese": 0.5386554621848739, - "Chinese,Spanish": 0.7075630252100841, - "Chinese,Vietnamese": 0.5907563025210084, - "Spanish,Vietnamese": 0.7016806722689075 + "Spanish,English": 0.915126050420168, + "Spanish,Chinese": 0.880672268907563, + "Spanish,Vietnamese": 0.8764705882352941, + "English,Chinese": 0.9, + "English,Vietnamese": 0.9142857142857143, + "Chinese,Vietnamese": 0.8672268907563025 }, "3_combine": { - "English,Chinese,Spanish": 0.36050420168067226, - "English,Chinese,Vietnamese": 0.3302521008403361, - "English,Spanish,Vietnamese": 0.4042016806722689, - "Chinese,Spanish,Vietnamese": 0.5235294117647059 + "Spanish,English,Chinese": 0.853781512605042, + "Spanish,English,Vietnamese": 0.8579831932773109, + "Spanish,Chinese,Vietnamese": 0.8226890756302521, + "English,Chinese,Vietnamese": 0.8470588235294118 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3033613445378151 + "Spanish,English,Chinese,Vietnamese": 0.8117647058823529 } }, - "AC3_2": 0.38181234142835924, - "AC3_3": 0.3339148284629863, - "AC3_4": 0.2934915259848501 + "AC3_2": 0.9039767405184379, + "AC3_3": 0.879257809944307, + "AC3_4": 0.8607232775880852 }, "prompt_5": { - "overall_acc": 0.2907563025210084, + "overall_acc": 0.9197478991596638, "language_acc": { - "English": 0.31596638655462184, - "Chinese": 0.24789915966386555, - "Spanish": 0.29747899159663865, - "Vietnamese": 0.30168067226890755 - }, - "consistency_score_2": 0.48487394957983193, - "consistency_score_3": 0.2827731092436975, - "consistency_score_4": 0.1773109243697479, + "Spanish": 0.9084033613445378, + "English": 0.9394957983193277, + "Chinese": 0.9159663865546218, + "Vietnamese": 0.915126050420168 + }, + "consistency_score_2": 0.900140056022409, + "consistency_score_3": 0.8546218487394959, + "consistency_score_4": 0.8210084033613445, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.3403361344537815, - "English,Spanish": 0.4050420168067227, - "English,Vietnamese": 0.4008403361344538, - "Chinese,Spanish": 0.6319327731092437, - "Chinese,Vietnamese": 0.5277310924369748, - "Spanish,Vietnamese": 0.6033613445378151 + "Spanish,English": 0.9126050420168067, + "Spanish,Chinese": 0.8831932773109243, + "Spanish,Vietnamese": 0.8907563025210085, + "English,Chinese": 0.9109243697478991, + "English,Vietnamese": 0.9176470588235294, + "Chinese,Vietnamese": 0.8857142857142857 }, "3_combine": { - "English,Chinese,Spanish": 0.23445378151260504, - "English,Chinese,Vietnamese": 0.21176470588235294, - "English,Spanish,Vietnamese": 0.25798319327731095, - "Chinese,Spanish,Vietnamese": 0.426890756302521 + "Spanish,English,Chinese": 0.8579831932773109, + "Spanish,English,Vietnamese": 0.8638655462184874, + "Spanish,Chinese,Vietnamese": 0.8352941176470589, + "English,Chinese,Vietnamese": 0.8613445378151261 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.1773109243697479 + "Spanish,English,Chinese,Vietnamese": 0.8210084033613445 } }, - "AC3_2": 0.36352413116574683, - "AC3_3": 0.2867091451465499, - "AC3_4": 0.2202857444123008 + "AC3_2": 0.9098383480951743, + "AC3_3": 0.8859896882715244, + "AC3_4": 0.8675777913853445 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.2761904761904762, + "overall_acc": 0.5161904761904762, "language_acc": { - "Spanish": 0.28, - "English": 0.3, - "Chinese": 0.25333333333333335, - "Vietnamese": 0.26666666666666666, - "Filipino": 0.26, - "Indonesian": 0.30666666666666664, - "Malay": 0.26666666666666666 + "Filipino": 0.3466666666666667, + "Vietnamese": 0.5266666666666666, + "Chinese": 0.5666666666666667, + "Spanish": 0.52, + "Malay": 0.5, + "Indonesian": 0.5266666666666666, + "English": 0.6266666666666667 }, - "consistency_score_2": 0.5120634920634921, - "consistency_score_3": 0.3190476190476191, - "consistency_score_4": 0.21657142857142864, - "consistency_score_5": 0.15396825396825395, - "consistency_score_6": 0.11333333333333333, - "consistency_score_7": 0.08666666666666667, + "consistency_score_2": 0.5923809523809523, + "consistency_score_3": 0.43085714285714294, + "consistency_score_4": 0.336, + "consistency_score_5": 0.27142857142857146, + "consistency_score_6": 0.22380952380952382, + "consistency_score_7": 0.18666666666666668, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5133333333333333, - "Spanish,Chinese": 0.54, - "Spanish,Vietnamese": 0.6133333333333333, - "Spanish,Filipino": 0.47333333333333333, - "Spanish,Indonesian": 0.5533333333333333, - "Spanish,Malay": 0.6333333333333333, - "English,Chinese": 0.38, - "English,Vietnamese": 0.4266666666666667, - "English,Filipino": 0.4866666666666667, - "English,Indonesian": 0.5066666666666667, - "English,Malay": 0.5133333333333333, - "Chinese,Vietnamese": 0.54, - "Chinese,Filipino": 0.4066666666666667, - "Chinese,Indonesian": 0.46, - "Chinese,Malay": 0.54, - "Vietnamese,Filipino": 0.4533333333333333, - "Vietnamese,Indonesian": 0.5, - "Vietnamese,Malay": 0.66, - "Filipino,Indonesian": 0.44666666666666666, - "Filipino,Malay": 0.4533333333333333, - "Indonesian,Malay": 0.6533333333333333 + "Filipino,Vietnamese": 0.42, + "Filipino,Chinese": 0.4266666666666667, + "Filipino,Spanish": 0.43333333333333335, + "Filipino,Malay": 0.4066666666666667, + "Filipino,Indonesian": 0.4066666666666667, + "Filipino,English": 0.43333333333333335, + "Vietnamese,Chinese": 0.6666666666666666, + "Vietnamese,Spanish": 0.6533333333333333, + "Vietnamese,Malay": 0.5866666666666667, + "Vietnamese,Indonesian": 0.68, + "Vietnamese,English": 0.6866666666666666, + "Chinese,Spanish": 0.6466666666666666, + "Chinese,Malay": 0.5733333333333334, + "Chinese,Indonesian": 0.6266666666666667, + "Chinese,English": 0.66, + "Spanish,Malay": 0.64, + "Spanish,Indonesian": 0.6733333333333333, + "Spanish,English": 0.7, + "Malay,Indonesian": 0.7066666666666667, + "Malay,English": 0.6933333333333334, + "Indonesian,English": 0.72 }, "3_combine": { - "Spanish,English,Chinese": 0.28, - "Spanish,English,Vietnamese": 0.34, - "Spanish,English,Filipino": 0.29333333333333333, - "Spanish,English,Indonesian": 0.34, - "Spanish,English,Malay": 0.38, - "Spanish,Chinese,Vietnamese": 0.38666666666666666, - "Spanish,Chinese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Indonesian": 0.3333333333333333, - "Spanish,Chinese,Malay": 0.4, - "Spanish,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,Vietnamese,Indonesian": 0.36666666666666664, - "Spanish,Vietnamese,Malay": 0.49333333333333335, - "Spanish,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Filipino,Malay": 0.32, - "Spanish,Indonesian,Malay": 0.4533333333333333, - "English,Chinese,Vietnamese": 0.26, - "English,Chinese,Filipino": 0.22, - "English,Chinese,Indonesian": 0.22666666666666666, - "English,Chinese,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino": 0.23333333333333334, - "English,Vietnamese,Indonesian": 0.28, - "English,Vietnamese,Malay": 0.36, - "English,Filipino,Indonesian": 0.2733333333333333, - "English,Filipino,Malay": 0.2733333333333333, - "English,Indonesian,Malay": 0.38, - "Chinese,Vietnamese,Filipino": 0.26, - "Chinese,Vietnamese,Indonesian": 0.30666666666666664, - "Chinese,Vietnamese,Malay": 0.41333333333333333, - "Chinese,Filipino,Indonesian": 0.22, - "Chinese,Filipino,Malay": 0.24666666666666667, - "Chinese,Indonesian,Malay": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Vietnamese,Filipino,Malay": 0.32666666666666666, - "Vietnamese,Indonesian,Malay": 0.44, - "Filipino,Indonesian,Malay": 0.32 + "Filipino,Vietnamese,Chinese": 0.30666666666666664, + "Filipino,Vietnamese,Spanish": 0.30666666666666664, + "Filipino,Vietnamese,Malay": 0.2866666666666667, + "Filipino,Vietnamese,Indonesian": 0.32, + "Filipino,Vietnamese,English": 0.32666666666666666, + "Filipino,Chinese,Spanish": 0.3, + "Filipino,Chinese,Malay": 0.26666666666666666, + "Filipino,Chinese,Indonesian": 0.2866666666666667, + "Filipino,Chinese,English": 0.32, + "Filipino,Spanish,Malay": 0.30666666666666664, + "Filipino,Spanish,Indonesian": 0.31333333333333335, + "Filipino,Spanish,English": 0.3333333333333333, + "Filipino,Malay,Indonesian": 0.31333333333333335, + "Filipino,Malay,English": 0.3, + "Filipino,Indonesian,English": 0.3333333333333333, + "Vietnamese,Chinese,Spanish": 0.5066666666666667, + "Vietnamese,Chinese,Malay": 0.4533333333333333, + "Vietnamese,Chinese,Indonesian": 0.5066666666666667, + "Vietnamese,Chinese,English": 0.5333333333333333, + "Vietnamese,Spanish,Malay": 0.48, + "Vietnamese,Spanish,Indonesian": 0.5466666666666666, + "Vietnamese,Spanish,English": 0.5466666666666666, + "Vietnamese,Malay,Indonesian": 0.52, + "Vietnamese,Malay,English": 0.49333333333333335, + "Vietnamese,Indonesian,English": 0.5666666666666667, + "Chinese,Spanish,Malay": 0.48, + "Chinese,Spanish,Indonesian": 0.5133333333333333, + "Chinese,Spanish,English": 0.54, + "Chinese,Malay,Indonesian": 0.49333333333333335, + "Chinese,Malay,English": 0.49333333333333335, + "Chinese,Indonesian,English": 0.5266666666666666, + "Spanish,Malay,Indonesian": 0.5466666666666666, + "Spanish,Malay,English": 0.54, + "Spanish,Indonesian,English": 0.5733333333333334, + "Malay,Indonesian,English": 0.6 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,English,Chinese,Filipino": 0.16666666666666666, - "Spanish,English,Chinese,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.23333333333333334, - "Spanish,English,Vietnamese,Malay": 0.29333333333333333, - "Spanish,English,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Filipino,Malay": 0.2, - "Spanish,English,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.2, - "Spanish,Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.32, - "Spanish,Chinese,Filipino,Indonesian": 0.18, - "Spanish,Chinese,Filipino,Malay": 0.2, - "Spanish,Chinese,Indonesian,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.34, - "Spanish,Filipino,Indonesian,Malay": 0.26, - "English,Chinese,Vietnamese,Filipino": 0.16666666666666666, - "English,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "English,Chinese,Vietnamese,Malay": 0.21333333333333335, - "English,Chinese,Filipino,Indonesian": 0.12666666666666668, - "English,Chinese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Indonesian,Malay": 0.2, - "English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 + "Filipino,Vietnamese,Chinese,Spanish": 0.24, + "Filipino,Vietnamese,Chinese,Malay": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay": 0.24, + "Filipino,Vietnamese,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Spanish,English": 0.2733333333333333, + "Filipino,Vietnamese,Malay,Indonesian": 0.26, + "Filipino,Vietnamese,Malay,English": 0.24666666666666667, + "Filipino,Vietnamese,Indonesian,English": 0.2866666666666667, + "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, + "Filipino,Chinese,Spanish,Indonesian": 0.24666666666666667, + "Filipino,Chinese,Spanish,English": 0.2733333333333333, + "Filipino,Chinese,Malay,Indonesian": 0.23333333333333334, + "Filipino,Chinese,Malay,English": 0.23333333333333334, + "Filipino,Chinese,Indonesian,English": 0.26, + "Filipino,Spanish,Malay,Indonesian": 0.26, + "Filipino,Spanish,Malay,English": 0.26, + "Filipino,Spanish,Indonesian,English": 0.2866666666666667, + "Filipino,Malay,Indonesian,English": 0.28, + "Vietnamese,Chinese,Spanish,Malay": 0.4, + "Vietnamese,Chinese,Spanish,Indonesian": 0.44666666666666666, + "Vietnamese,Chinese,Spanish,English": 0.4533333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.4066666666666667, + "Vietnamese,Chinese,Malay,English": 0.4, + "Vietnamese,Chinese,Indonesian,English": 0.44666666666666666, + "Vietnamese,Spanish,Malay,Indonesian": 0.44666666666666666, + "Vietnamese,Spanish,Malay,English": 0.43333333333333335, + "Vietnamese,Spanish,Indonesian,English": 0.4866666666666667, + "Vietnamese,Malay,Indonesian,English": 0.46, + "Chinese,Spanish,Malay,Indonesian": 0.43333333333333335, + "Chinese,Spanish,Malay,English": 0.44, + "Chinese,Spanish,Indonesian,English": 0.4666666666666667, + "Chinese,Malay,Indonesian,English": 0.44666666666666666, + "Spanish,Malay,Indonesian,English": 0.48 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,English,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "English,Chinese,Filipino,Indonesian,Malay": 0.12, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.22, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.2, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Malay,English": 0.22, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.25333333333333335, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.24, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, + "Filipino,Chinese,Spanish,Malay,English": 0.22, + "Filipino,Chinese,Spanish,Indonesian,English": 0.24, + "Filipino,Chinese,Malay,Indonesian,English": 0.22, + "Filipino,Spanish,Malay,Indonesian,English": 0.24, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.37333333333333335, + "Vietnamese,Chinese,Spanish,Malay,English": 0.37333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.41333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.37333333333333335, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.4066666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.4 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.19333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.35333333333333333 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668 } }, - "AC3_2": 0.3588362771153616, - "AC3_3": 0.29607619042644967, - "AC3_4": 0.24277431106675162, - "AC3_5": 0.19771569139666526, - "AC3_6": 0.160717196372758, - "AC3_7": 0.13193350827510145 + "AC3_2": 0.5516674848135973, + "AC3_3": 0.4696793471518771, + "AC3_4": 0.40704514970637046, + "AC3_5": 0.35577819998937693, + "AC3_6": 0.3122387693394321, + "AC3_7": 0.27418247511907523 }, "prompt_2": { - "overall_acc": 0.26666666666666666, + "overall_acc": 0.5342857142857144, "language_acc": { - "Spanish": 0.26666666666666666, - "English": 0.25333333333333335, - "Chinese": 0.31333333333333335, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.24, - "Indonesian": 0.3, - "Malay": 0.20666666666666667 - }, - "consistency_score_2": 0.5279365079365079, - "consistency_score_3": 0.34933333333333344, - "consistency_score_4": 0.2502857142857142, - "consistency_score_5": 0.1866666666666667, - "consistency_score_6": 0.14380952380952383, - "consistency_score_7": 0.11333333333333333, + "Filipino": 0.36, + "Vietnamese": 0.54, + "Chinese": 0.5733333333333334, + "Spanish": 0.5266666666666666, + "Malay": 0.5466666666666666, + "Indonesian": 0.5466666666666666, + "English": 0.6466666666666666 + }, + "consistency_score_2": 0.5955555555555556, + "consistency_score_3": 0.4342857142857143, + "consistency_score_4": 0.34247619047619055, + "consistency_score_5": 0.2812698412698412, + "consistency_score_6": 0.2361904761904762, + "consistency_score_7": 0.2, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5933333333333334, - "Spanish,Chinese": 0.52, - "Spanish,Vietnamese": 0.56, - "Spanish,Filipino": 0.58, - "Spanish,Indonesian": 0.58, - "Spanish,Malay": 0.62, - "English,Chinese": 0.5, - "English,Vietnamese": 0.46, - "English,Filipino": 0.5733333333333334, - "English,Indonesian": 0.5066666666666667, - "English,Malay": 0.5466666666666666, - "Chinese,Vietnamese": 0.49333333333333335, - "Chinese,Filipino": 0.4866666666666667, - "Chinese,Indonesian": 0.4266666666666667, - "Chinese,Malay": 0.4866666666666667, - "Vietnamese,Filipino": 0.4866666666666667, - "Vietnamese,Indonesian": 0.5066666666666667, - "Vietnamese,Malay": 0.5266666666666666, - "Filipino,Indonesian": 0.5266666666666666, - "Filipino,Malay": 0.5066666666666667, - "Indonesian,Malay": 0.6 + "Filipino,Vietnamese": 0.4, + "Filipino,Chinese": 0.44, + "Filipino,Spanish": 0.4066666666666667, + "Filipino,Malay": 0.3933333333333333, + "Filipino,Indonesian": 0.4266666666666667, + "Filipino,English": 0.44666666666666666, + "Vietnamese,Chinese": 0.6333333333333333, + "Vietnamese,Spanish": 0.6866666666666666, + "Vietnamese,Malay": 0.6266666666666667, + "Vietnamese,Indonesian": 0.6733333333333333, + "Vietnamese,English": 0.7066666666666667, + "Chinese,Spanish": 0.62, + "Chinese,Malay": 0.6533333333333333, + "Chinese,Indonesian": 0.6066666666666667, + "Chinese,English": 0.6733333333333333, + "Spanish,Malay": 0.6733333333333333, + "Spanish,Indonesian": 0.68, + "Spanish,English": 0.68, + "Malay,Indonesian": 0.6866666666666666, + "Malay,English": 0.7066666666666667, + "Indonesian,English": 0.6866666666666666 }, "3_combine": { - "Spanish,English,Chinese": 0.36666666666666664, - "Spanish,English,Vietnamese": 0.36, - "Spanish,English,Filipino": 0.4266666666666667, - "Spanish,English,Indonesian": 0.3933333333333333, - "Spanish,English,Malay": 0.41333333333333333, - "Spanish,Chinese,Vietnamese": 0.36666666666666664, - "Spanish,Chinese,Filipino": 0.34, - "Spanish,Chinese,Indonesian": 0.3333333333333333, - "Spanish,Chinese,Malay": 0.36, - "Spanish,Vietnamese,Filipino": 0.37333333333333335, - "Spanish,Vietnamese,Indonesian": 0.38, - "Spanish,Vietnamese,Malay": 0.4, - "Spanish,Filipino,Indonesian": 0.38666666666666666, - "Spanish,Filipino,Malay": 0.4, - "Spanish,Indonesian,Malay": 0.46, - "English,Chinese,Vietnamese": 0.30666666666666664, - "English,Chinese,Filipino": 0.3333333333333333, - "English,Chinese,Indonesian": 0.2866666666666667, - "English,Chinese,Malay": 0.32666666666666666, - "English,Vietnamese,Filipino": 0.3, - "English,Vietnamese,Indonesian": 0.29333333333333333, - "English,Vietnamese,Malay": 0.32666666666666666, - "English,Filipino,Indonesian": 0.36, - "English,Filipino,Malay": 0.3466666666666667, - "English,Indonesian,Malay": 0.36666666666666664, - "Chinese,Vietnamese,Filipino": 0.29333333333333333, - "Chinese,Vietnamese,Indonesian": 0.32, - "Chinese,Vietnamese,Malay": 0.3333333333333333, - "Chinese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Filipino,Malay": 0.2866666666666667, - "Chinese,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Vietnamese,Filipino,Malay": 0.31333333333333335, - "Vietnamese,Indonesian,Malay": 0.38, - "Filipino,Indonesian,Malay": 0.38 + "Filipino,Vietnamese,Chinese": 0.2866666666666667, + "Filipino,Vietnamese,Spanish": 0.3, + "Filipino,Vietnamese,Malay": 0.26666666666666666, + "Filipino,Vietnamese,Indonesian": 0.3, + "Filipino,Vietnamese,English": 0.31333333333333335, + "Filipino,Chinese,Spanish": 0.28, + "Filipino,Chinese,Malay": 0.31333333333333335, + "Filipino,Chinese,Indonesian": 0.29333333333333333, + "Filipino,Chinese,English": 0.32666666666666666, + "Filipino,Spanish,Malay": 0.26666666666666666, + "Filipino,Spanish,Indonesian": 0.30666666666666664, + "Filipino,Spanish,English": 0.30666666666666664, + "Filipino,Malay,Indonesian": 0.29333333333333333, + "Filipino,Malay,English": 0.32, + "Filipino,Indonesian,English": 0.32666666666666666, + "Vietnamese,Chinese,Spanish": 0.5133333333333333, + "Vietnamese,Chinese,Malay": 0.5133333333333333, + "Vietnamese,Chinese,Indonesian": 0.49333333333333335, + "Vietnamese,Chinese,English": 0.5333333333333333, + "Vietnamese,Spanish,Malay": 0.5333333333333333, + "Vietnamese,Spanish,Indonesian": 0.5533333333333333, + "Vietnamese,Spanish,English": 0.5733333333333334, + "Vietnamese,Malay,Indonesian": 0.54, + "Vietnamese,Malay,English": 0.5533333333333333, + "Vietnamese,Indonesian,English": 0.5666666666666667, + "Chinese,Spanish,Malay": 0.52, + "Chinese,Spanish,Indonesian": 0.5, + "Chinese,Spanish,English": 0.5133333333333333, + "Chinese,Malay,Indonesian": 0.5133333333333333, + "Chinese,Malay,English": 0.5466666666666666, + "Chinese,Indonesian,English": 0.5266666666666666, + "Spanish,Malay,Indonesian": 0.5533333333333333, + "Spanish,Malay,English": 0.54, + "Spanish,Indonesian,English": 0.5466666666666666, + "Malay,Indonesian,English": 0.5666666666666667 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.26, - "Spanish,English,Chinese,Filipino": 0.26, - "Spanish,English,Chinese,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Malay": 0.26666666666666666, - "Spanish,English,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,English,Vietnamese,Malay": 0.26666666666666666, - "Spanish,English,Filipino,Indonesian": 0.29333333333333333, - "Spanish,English,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.28, - "Spanish,Chinese,Vietnamese,Malay": 0.28, - "Spanish,Chinese,Filipino,Indonesian": 0.22, - "Spanish,Chinese,Filipino,Malay": 0.24, - "Spanish,Chinese,Indonesian,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,Vietnamese,Filipino,Malay": 0.26, - "Spanish,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "English,Chinese,Vietnamese,Indonesian": 0.22, - "English,Chinese,Vietnamese,Malay": 0.22666666666666666, - "English,Chinese,Filipino,Indonesian": 0.20666666666666667, - "English,Chinese,Filipino,Malay": 0.21333333333333335, - "English,Chinese,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "English,Vietnamese,Filipino,Malay": 0.2, - "English,Vietnamese,Indonesian,Malay": 0.24, - "English,Filipino,Indonesian,Malay": 0.26, - "Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Chinese,Vietnamese,Filipino,Malay": 0.2, - "Chinese,Vietnamese,Indonesian,Malay": 0.26, - "Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.26 + "Filipino,Vietnamese,Chinese,Spanish": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Malay": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,English": 0.24666666666666667, + "Filipino,Vietnamese,Spanish,Malay": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,English": 0.2733333333333333, + "Filipino,Vietnamese,Malay,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Malay,English": 0.24666666666666667, + "Filipino,Vietnamese,Indonesian,English": 0.2733333333333333, + "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, + "Filipino,Chinese,Spanish,Indonesian": 0.24666666666666667, + "Filipino,Chinese,Spanish,English": 0.24666666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.25333333333333335, + "Filipino,Chinese,Malay,English": 0.28, + "Filipino,Chinese,Indonesian,English": 0.2733333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.24666666666666667, + "Filipino,Spanish,Malay,English": 0.24666666666666667, + "Filipino,Spanish,Indonesian,English": 0.26666666666666666, + "Filipino,Malay,Indonesian,English": 0.26, + "Vietnamese,Chinese,Spanish,Malay": 0.4666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.44, + "Vietnamese,Chinese,Spanish,English": 0.4666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.4533333333333333, + "Vietnamese,Chinese,Malay,English": 0.4666666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.4533333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.4666666666666667, + "Vietnamese,Spanish,Malay,English": 0.48, + "Vietnamese,Spanish,Indonesian,English": 0.4866666666666667, + "Vietnamese,Malay,Indonesian,English": 0.4866666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, + "Chinese,Spanish,Malay,English": 0.46, + "Chinese,Spanish,Indonesian,English": 0.44666666666666666, + "Chinese,Malay,Indonesian,English": 0.46, + "Spanish,Malay,Indonesian,English": 0.47333333333333333 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.20666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Filipino,Malay": 0.18666666666666668, - "Spanish,English,Chinese,Indonesian,Malay": 0.2, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Malay": 0.18, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,English,Filipino,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.18, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "English,Chinese,Filipino,Indonesian,Malay": 0.16, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.22, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Malay,English": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22, + "Filipino,Vietnamese,Spanish,Malay,English": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.24, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.23333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.22, + "Filipino,Chinese,Spanish,Malay,English": 0.22666666666666666, + "Filipino,Chinese,Spanish,Indonesian,English": 0.23333333333333334, + "Filipino,Chinese,Malay,Indonesian,English": 0.24, + "Filipino,Spanish,Malay,Indonesian,English": 0.22666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, + "Vietnamese,Chinese,Spanish,Malay,English": 0.43333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.41333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.42, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.4266666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.4066666666666667 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.22, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.38666666666666666 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2 } }, - "AC3_2": 0.3543481155500255, - "AC3_3": 0.30245310240400297, - "AC3_4": 0.2582166543347223, - "AC3_5": 0.21960784308881204, - "AC3_6": 0.18685228147033375, - "AC3_7": 0.15906432744352106 + "AC3_2": 0.5632593424620886, + "AC3_3": 0.4791234723483417, + "AC3_4": 0.4173998323598293, + "AC3_5": 0.3685302785512646, + "AC3_6": 0.32757195828346014, + "AC3_7": 0.2910505836179503 }, "prompt_3": { - "overall_acc": 0.2695238095238095, + "overall_acc": 0.5361904761904762, "language_acc": { - "Spanish": 0.25333333333333335, - "English": 0.2733333333333333, - "Chinese": 0.21333333333333335, - "Vietnamese": 0.2733333333333333, - "Filipino": 0.26, - "Indonesian": 0.3333333333333333, - "Malay": 0.28 + "Filipino": 0.36666666666666664, + "Vietnamese": 0.52, + "Chinese": 0.5666666666666667, + "Spanish": 0.5533333333333333, + "Malay": 0.5533333333333333, + "Indonesian": 0.5333333333333333, + "English": 0.66 }, - "consistency_score_2": 0.44444444444444436, - "consistency_score_3": 0.2548571428571428, - "consistency_score_4": 0.16533333333333333, - "consistency_score_5": 0.11142857142857142, - "consistency_score_6": 0.07428571428571429, - "consistency_score_7": 0.04666666666666667, + "consistency_score_2": 0.5984126984126984, + "consistency_score_3": 0.43866666666666665, + "consistency_score_4": 0.34514285714285725, + "consistency_score_5": 0.280952380952381, + "consistency_score_6": 0.2323809523809524, + "consistency_score_7": 0.19333333333333333, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.43333333333333335, - "Spanish,Chinese": 0.49333333333333335, - "Spanish,Vietnamese": 0.46, - "Spanish,Filipino": 0.4866666666666667, - "Spanish,Indonesian": 0.52, - "Spanish,Malay": 0.47333333333333333, - "English,Chinese": 0.38666666666666666, - "English,Vietnamese": 0.43333333333333335, - "English,Filipino": 0.4, - "English,Indonesian": 0.3933333333333333, - "English,Malay": 0.49333333333333335, - "Chinese,Vietnamese": 0.42, - "Chinese,Filipino": 0.4866666666666667, - "Chinese,Indonesian": 0.44666666666666666, - "Chinese,Malay": 0.4, - "Vietnamese,Filipino": 0.4066666666666667, - "Vietnamese,Indonesian": 0.48, - "Vietnamese,Malay": 0.4066666666666667, - "Filipino,Indonesian": 0.42, - "Filipino,Malay": 0.38666666666666666, - "Indonesian,Malay": 0.5066666666666667 + "Filipino,Vietnamese": 0.4066666666666667, + "Filipino,Chinese": 0.44, + "Filipino,Spanish": 0.46, + "Filipino,Malay": 0.42, + "Filipino,Indonesian": 0.4066666666666667, + "Filipino,English": 0.4266666666666667, + "Vietnamese,Chinese": 0.68, + "Vietnamese,Spanish": 0.6666666666666666, + "Vietnamese,Malay": 0.68, + "Vietnamese,Indonesian": 0.6666666666666666, + "Vietnamese,English": 0.6733333333333333, + "Chinese,Spanish": 0.66, + "Chinese,Malay": 0.6133333333333333, + "Chinese,Indonesian": 0.6533333333333333, + "Chinese,English": 0.66, + "Spanish,Malay": 0.6866666666666666, + "Spanish,Indonesian": 0.6666666666666666, + "Spanish,English": 0.6866666666666666, + "Malay,Indonesian": 0.7, + "Malay,English": 0.6666666666666666, + "Indonesian,English": 0.6466666666666666 }, "3_combine": { - "Spanish,English,Chinese": 0.24666666666666667, - "Spanish,English,Vietnamese": 0.2733333333333333, - "Spanish,English,Filipino": 0.26, - "Spanish,English,Indonesian": 0.2733333333333333, - "Spanish,English,Malay": 0.3, - "Spanish,Chinese,Vietnamese": 0.2733333333333333, - "Spanish,Chinese,Filipino": 0.3, - "Spanish,Chinese,Indonesian": 0.3, - "Spanish,Chinese,Malay": 0.2866666666666667, - "Spanish,Vietnamese,Filipino": 0.26, - "Spanish,Vietnamese,Indonesian": 0.30666666666666664, - "Spanish,Vietnamese,Malay": 0.2733333333333333, - "Spanish,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Filipino,Malay": 0.24666666666666667, - "Spanish,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese": 0.21333333333333335, - "English,Chinese,Filipino": 0.22, - "English,Chinese,Indonesian": 0.22666666666666666, - "English,Chinese,Malay": 0.24666666666666667, - "English,Vietnamese,Filipino": 0.20666666666666667, - "English,Vietnamese,Indonesian": 0.26666666666666666, - "English,Vietnamese,Malay": 0.26, - "English,Filipino,Indonesian": 0.20666666666666667, - "English,Filipino,Malay": 0.23333333333333334, - "English,Indonesian,Malay": 0.28, - "Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Chinese,Vietnamese,Indonesian": 0.24, - "Chinese,Vietnamese,Malay": 0.21333333333333335, - "Chinese,Filipino,Indonesian": 0.26, - "Chinese,Filipino,Malay": 0.23333333333333334, - "Chinese,Indonesian,Malay": 0.26666666666666666, - "Vietnamese,Filipino,Indonesian": 0.22, - "Vietnamese,Filipino,Malay": 0.19333333333333333, - "Vietnamese,Indonesian,Malay": 0.28, - "Filipino,Indonesian,Malay": 0.23333333333333334 + "Filipino,Vietnamese,Chinese": 0.31333333333333335, + "Filipino,Vietnamese,Spanish": 0.32666666666666666, + "Filipino,Vietnamese,Malay": 0.32, + "Filipino,Vietnamese,Indonesian": 0.29333333333333333, + "Filipino,Vietnamese,English": 0.31333333333333335, + "Filipino,Chinese,Spanish": 0.34, + "Filipino,Chinese,Malay": 0.32, + "Filipino,Chinese,Indonesian": 0.31333333333333335, + "Filipino,Chinese,English": 0.32, + "Filipino,Spanish,Malay": 0.3333333333333333, + "Filipino,Spanish,Indonesian": 0.32, + "Filipino,Spanish,English": 0.34, + "Filipino,Malay,Indonesian": 0.31333333333333335, + "Filipino,Malay,English": 0.31333333333333335, + "Filipino,Indonesian,English": 0.3, + "Vietnamese,Chinese,Spanish": 0.5266666666666666, + "Vietnamese,Chinese,Malay": 0.52, + "Vietnamese,Chinese,Indonesian": 0.5266666666666666, + "Vietnamese,Chinese,English": 0.5333333333333333, + "Vietnamese,Spanish,Malay": 0.5333333333333333, + "Vietnamese,Spanish,Indonesian": 0.5266666666666666, + "Vietnamese,Spanish,English": 0.5266666666666666, + "Vietnamese,Malay,Indonesian": 0.5533333333333333, + "Vietnamese,Malay,English": 0.5266666666666666, + "Vietnamese,Indonesian,English": 0.5266666666666666, + "Chinese,Spanish,Malay": 0.5133333333333333, + "Chinese,Spanish,Indonesian": 0.52, + "Chinese,Spanish,English": 0.5333333333333333, + "Chinese,Malay,Indonesian": 0.5133333333333333, + "Chinese,Malay,English": 0.5133333333333333, + "Chinese,Indonesian,English": 0.5266666666666666, + "Spanish,Malay,Indonesian": 0.5533333333333333, + "Spanish,Malay,English": 0.5333333333333333, + "Spanish,Indonesian,English": 0.52, + "Malay,Indonesian,English": 0.5466666666666666 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.16, - "Spanish,English,Chinese,Filipino": 0.16, - "Spanish,English,Chinese,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Malay": 0.2, - "Spanish,English,Vietnamese,Filipino": 0.16, - "Spanish,English,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Filipino,Malay": 0.16, - "Spanish,English,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian": 0.2, - "Spanish,Chinese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Indonesian,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.2, - "Spanish,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "English,Chinese,Vietnamese,Indonesian": 0.14, - "English,Chinese,Vietnamese,Malay": 0.14, - "English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Indonesian,Malay": 0.16, - "English,Vietnamese,Filipino,Indonesian": 0.12, - "English,Vietnamese,Filipino,Malay": 0.12, - "English,Vietnamese,Indonesian,Malay": 0.18, - "English,Filipino,Indonesian,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 + "Filipino,Vietnamese,Chinese,Spanish": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,English": 0.26, + "Filipino,Vietnamese,Spanish,Malay": 0.2733333333333333, + "Filipino,Vietnamese,Spanish,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Spanish,English": 0.26, + "Filipino,Vietnamese,Malay,Indonesian": 0.26, + "Filipino,Vietnamese,Malay,English": 0.25333333333333335, + "Filipino,Vietnamese,Indonesian,English": 0.24666666666666667, + "Filipino,Chinese,Spanish,Malay": 0.28, + "Filipino,Chinese,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Chinese,Spanish,English": 0.28, + "Filipino,Chinese,Malay,Indonesian": 0.2733333333333333, + "Filipino,Chinese,Malay,English": 0.2733333333333333, + "Filipino,Chinese,Indonesian,English": 0.2733333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.2733333333333333, + "Filipino,Spanish,Malay,English": 0.26666666666666666, + "Filipino,Spanish,Indonesian,English": 0.26, + "Filipino,Malay,Indonesian,English": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,Malay": 0.44, + "Vietnamese,Chinese,Spanish,Indonesian": 0.44, + "Vietnamese,Chinese,Spanish,English": 0.44666666666666666, + "Vietnamese,Chinese,Malay,Indonesian": 0.44, + "Vietnamese,Chinese,Malay,English": 0.44, + "Vietnamese,Chinese,Indonesian,English": 0.4533333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.4666666666666667, + "Vietnamese,Spanish,Malay,English": 0.4533333333333333, + "Vietnamese,Spanish,Indonesian,English": 0.46, + "Vietnamese,Malay,Indonesian,English": 0.46, + "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, + "Chinese,Spanish,Malay,English": 0.44, + "Chinese,Spanish,Indonesian,English": 0.44666666666666666, + "Chinese,Malay,Indonesian,English": 0.44666666666666666, + "Spanish,Malay,Indonesian,English": 0.47333333333333333 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.12, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.1, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.24, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Malay,English": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Malay,English": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.22, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.22666666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.24, + "Filipino,Chinese,Spanish,Malay,English": 0.24, + "Filipino,Chinese,Spanish,Indonesian,English": 0.24, + "Filipino,Chinese,Malay,Indonesian,English": 0.24666666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.24, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.38666666666666666, + "Vietnamese,Chinese,Spanish,Malay,English": 0.38666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.3933333333333333, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.42, + "Chinese,Spanish,Malay,Indonesian,English": 0.4 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.08, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.36 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 } }, - "AC3_2": 0.3355565436020303, - "AC3_3": 0.2619853665630845, - "AC3_4": 0.20494670750141758, - "AC3_5": 0.1576714285300398, - "AC3_6": 0.11647012264123247, - "AC3_7": 0.07955823290656541 + "AC3_2": 0.565595437901492, + "AC3_3": 0.4825504753966382, + "AC3_4": 0.4199598628687718, + "AC3_5": 0.3687090686639471, + "AC3_6": 0.32423909832330516, + "AC3_7": 0.28419495209333273 }, "prompt_4": { - "overall_acc": 0.2533333333333333, + "overall_acc": 0.5333333333333333, "language_acc": { - "Spanish": 0.24, - "English": 0.26666666666666666, - "Chinese": 0.2866666666666667, - "Vietnamese": 0.26666666666666666, - "Filipino": 0.21333333333333335, - "Indonesian": 0.24666666666666667, - "Malay": 0.25333333333333335 + "Filipino": 0.3933333333333333, + "Vietnamese": 0.5133333333333333, + "Chinese": 0.58, + "Spanish": 0.5333333333333333, + "Malay": 0.5533333333333333, + "Indonesian": 0.5133333333333333, + "English": 0.6466666666666666 }, - "consistency_score_2": 0.5057142857142858, - "consistency_score_3": 0.31809523809523804, - "consistency_score_4": 0.22209523809523812, - "consistency_score_5": 0.16634920634920633, - "consistency_score_6": 0.13142857142857142, - "consistency_score_7": 0.10666666666666667, + "consistency_score_2": 0.6044444444444443, + "consistency_score_3": 0.44876190476190475, + "consistency_score_4": 0.36095238095238097, + "consistency_score_5": 0.3028571428571429, + "consistency_score_6": 0.26, + "consistency_score_7": 0.22666666666666666, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.41333333333333333, - "Spanish,Chinese": 0.6266666666666667, - "Spanish,Vietnamese": 0.6933333333333334, - "Spanish,Filipino": 0.49333333333333335, - "Spanish,Indonesian": 0.5533333333333333, - "Spanish,Malay": 0.5933333333333334, - "English,Chinese": 0.35333333333333333, - "English,Vietnamese": 0.4266666666666667, - "English,Filipino": 0.48, - "English,Indonesian": 0.46, - "English,Malay": 0.44666666666666666, - "Chinese,Vietnamese": 0.52, - "Chinese,Filipino": 0.43333333333333335, - "Chinese,Indonesian": 0.43333333333333335, - "Chinese,Malay": 0.4533333333333333, - "Vietnamese,Filipino": 0.5066666666666667, - "Vietnamese,Indonesian": 0.5533333333333333, - "Vietnamese,Malay": 0.64, - "Filipino,Indonesian": 0.49333333333333335, - "Filipino,Malay": 0.44666666666666666, - "Indonesian,Malay": 0.6 + "Filipino,Vietnamese": 0.4266666666666667, + "Filipino,Chinese": 0.46, + "Filipino,Spanish": 0.48, + "Filipino,Malay": 0.44, + "Filipino,Indonesian": 0.44, + "Filipino,English": 0.46, + "Vietnamese,Chinese": 0.7066666666666667, + "Vietnamese,Spanish": 0.64, + "Vietnamese,Malay": 0.6533333333333333, + "Vietnamese,Indonesian": 0.6533333333333333, + "Vietnamese,English": 0.6666666666666666, + "Chinese,Spanish": 0.64, + "Chinese,Malay": 0.66, + "Chinese,Indonesian": 0.6533333333333333, + "Chinese,English": 0.6733333333333333, + "Spanish,Malay": 0.66, + "Spanish,Indonesian": 0.6666666666666666, + "Spanish,English": 0.68, + "Malay,Indonesian": 0.7066666666666667, + "Malay,English": 0.6733333333333333, + "Indonesian,English": 0.6533333333333333 }, "3_combine": { - "Spanish,English,Chinese": 0.25333333333333335, - "Spanish,English,Vietnamese": 0.30666666666666664, - "Spanish,English,Filipino": 0.28, - "Spanish,English,Indonesian": 0.29333333333333333, - "Spanish,English,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese": 0.4533333333333333, - "Spanish,Chinese,Filipino": 0.34, - "Spanish,Chinese,Indonesian": 0.37333333333333335, - "Spanish,Chinese,Malay": 0.3933333333333333, - "Spanish,Vietnamese,Filipino": 0.38, - "Spanish,Vietnamese,Indonesian": 0.43333333333333335, - "Spanish,Vietnamese,Malay": 0.4866666666666667, - "Spanish,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Filipino,Malay": 0.32666666666666666, - "Spanish,Indonesian,Malay": 0.42, - "English,Chinese,Vietnamese": 0.2, - "English,Chinese,Filipino": 0.22, - "English,Chinese,Indonesian": 0.21333333333333335, - "English,Chinese,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino": 0.26666666666666666, - "English,Vietnamese,Indonesian": 0.29333333333333333, - "English,Vietnamese,Malay": 0.32, - "English,Filipino,Indonesian": 0.28, - "English,Filipino,Malay": 0.25333333333333335, - "English,Indonesian,Malay": 0.34, - "Chinese,Vietnamese,Filipino": 0.2733333333333333, - "Chinese,Vietnamese,Indonesian": 0.30666666666666664, - "Chinese,Vietnamese,Malay": 0.3466666666666667, - "Chinese,Filipino,Indonesian": 0.25333333333333335, - "Chinese,Filipino,Malay": 0.24666666666666667, - "Chinese,Indonesian,Malay": 0.30666666666666664, - "Vietnamese,Filipino,Indonesian": 0.32, - "Vietnamese,Filipino,Malay": 0.3466666666666667, - "Vietnamese,Indonesian,Malay": 0.42, - "Filipino,Indonesian,Malay": 0.31333333333333335 + "Filipino,Vietnamese,Chinese": 0.3333333333333333, + "Filipino,Vietnamese,Spanish": 0.34, + "Filipino,Vietnamese,Malay": 0.32, + "Filipino,Vietnamese,Indonesian": 0.32, + "Filipino,Vietnamese,English": 0.34, + "Filipino,Chinese,Spanish": 0.34, + "Filipino,Chinese,Malay": 0.3333333333333333, + "Filipino,Chinese,Indonesian": 0.31333333333333335, + "Filipino,Chinese,English": 0.35333333333333333, + "Filipino,Spanish,Malay": 0.34, + "Filipino,Spanish,Indonesian": 0.3466666666666667, + "Filipino,Spanish,English": 0.37333333333333335, + "Filipino,Malay,Indonesian": 0.35333333333333333, + "Filipino,Malay,English": 0.3333333333333333, + "Filipino,Indonesian,English": 0.3333333333333333, + "Vietnamese,Chinese,Spanish": 0.5333333333333333, + "Vietnamese,Chinese,Malay": 0.54, + "Vietnamese,Chinese,Indonesian": 0.54, + "Vietnamese,Chinese,English": 0.54, + "Vietnamese,Spanish,Malay": 0.5133333333333333, + "Vietnamese,Spanish,Indonesian": 0.52, + "Vietnamese,Spanish,English": 0.52, + "Vietnamese,Malay,Indonesian": 0.5333333333333333, + "Vietnamese,Malay,English": 0.5133333333333333, + "Vietnamese,Indonesian,English": 0.52, + "Chinese,Spanish,Malay": 0.5266666666666666, + "Chinese,Spanish,Indonesian": 0.5266666666666666, + "Chinese,Spanish,English": 0.5266666666666666, + "Chinese,Malay,Indonesian": 0.5533333333333333, + "Chinese,Malay,English": 0.5333333333333333, + "Chinese,Indonesian,English": 0.5266666666666666, + "Spanish,Malay,Indonesian": 0.5466666666666666, + "Spanish,Malay,English": 0.5333333333333333, + "Spanish,Indonesian,English": 0.54, + "Malay,Indonesian,English": 0.5466666666666666 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.18, - "Spanish,English,Chinese,Filipino": 0.19333333333333333, - "Spanish,English,Chinese,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.22, - "Spanish,English,Vietnamese,Indonesian": 0.22, - "Spanish,English,Vietnamese,Malay": 0.24666666666666667, - "Spanish,English,Filipino,Indonesian": 0.2, - "Spanish,English,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.26, - "Spanish,Chinese,Vietnamese,Indonesian": 0.28, - "Spanish,Chinese,Vietnamese,Malay": 0.30666666666666664, - "Spanish,Chinese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.23333333333333334, - "Spanish,Chinese,Indonesian,Malay": 0.28, - "Spanish,Vietnamese,Filipino,Indonesian": 0.28, - "Spanish,Vietnamese,Filipino,Malay": 0.2866666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino": 0.15333333333333332, - "English,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "English,Chinese,Vietnamese,Malay": 0.18, - "English,Chinese,Filipino,Indonesian": 0.14, - "English,Chinese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Indonesian,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "English,Vietnamese,Filipino,Malay": 0.2, - "English,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "English,Filipino,Indonesian,Malay": 0.2, - "Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Chinese,Filipino,Indonesian,Malay": 0.2, - "Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 + "Filipino,Vietnamese,Chinese,Spanish": 0.28, + "Filipino,Vietnamese,Chinese,Malay": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,English": 0.2866666666666667, + "Filipino,Vietnamese,Spanish,Malay": 0.28, + "Filipino,Vietnamese,Spanish,Indonesian": 0.28, + "Filipino,Vietnamese,Spanish,English": 0.30666666666666664, + "Filipino,Vietnamese,Malay,Indonesian": 0.28, + "Filipino,Vietnamese,Malay,English": 0.2733333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.2866666666666667, + "Filipino,Chinese,Spanish,Malay": 0.29333333333333333, + "Filipino,Chinese,Spanish,Indonesian": 0.2866666666666667, + "Filipino,Chinese,Spanish,English": 0.3, + "Filipino,Chinese,Malay,Indonesian": 0.29333333333333333, + "Filipino,Chinese,Malay,English": 0.2866666666666667, + "Filipino,Chinese,Indonesian,English": 0.29333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.29333333333333333, + "Filipino,Spanish,Malay,English": 0.30666666666666664, + "Filipino,Spanish,Indonesian,English": 0.30666666666666664, + "Filipino,Malay,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish,Malay": 0.4533333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.4533333333333333, + "Vietnamese,Chinese,Spanish,English": 0.44666666666666666, + "Vietnamese,Chinese,Malay,Indonesian": 0.4666666666666667, + "Vietnamese,Chinese,Malay,English": 0.44, + "Vietnamese,Chinese,Indonesian,English": 0.4533333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.4533333333333333, + "Vietnamese,Spanish,Malay,English": 0.43333333333333335, + "Vietnamese,Spanish,Indonesian,English": 0.44666666666666666, + "Vietnamese,Malay,Indonesian,English": 0.44666666666666666, + "Chinese,Spanish,Malay,Indonesian": 0.4666666666666667, + "Chinese,Spanish,Malay,English": 0.4533333333333333, + "Chinese,Spanish,Indonesian,English": 0.4666666666666667, + "Chinese,Malay,Indonesian,English": 0.48, + "Spanish,Malay,Indonesian,English": 0.48 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.18, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.2, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.26, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Malay,English": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay,English": 0.26, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.26666666666666666, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.26, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Filipino,Chinese,Spanish,Malay,English": 0.26666666666666666, + "Filipino,Chinese,Spanish,Indonesian,English": 0.2733333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.28, + "Filipino,Spanish,Malay,Indonesian,English": 0.2866666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4066666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.38666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4066666666666667, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.4066666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.4, + "Chinese,Spanish,Malay,Indonesian,English": 0.4266666666666667 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.24, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.24, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.24666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.36666666666666664 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666 } }, - "AC3_2": 0.33756587197560295, - "AC3_3": 0.28204444439508664, - "AC3_4": 0.23668803413825004, - "AC3_5": 0.20082702970504748, - "AC3_6": 0.17306930688571215, - "AC3_7": 0.1501234567484225 + "AC3_2": 0.5666666666168619, + "AC3_3": 0.4874062580315706, + "AC3_4": 0.4305289314392557, + "AC3_5": 0.3863325739856892, + "AC3_6": 0.3495798318887084, + "AC3_7": 0.3181286549289012 }, "prompt_5": { - "overall_acc": 0.26761904761904765, + "overall_acc": 0.5361904761904762, "language_acc": { - "Spanish": 0.24666666666666667, - "English": 0.25333333333333335, - "Chinese": 0.23333333333333334, - "Vietnamese": 0.2733333333333333, - "Filipino": 0.2733333333333333, - "Indonesian": 0.3, - "Malay": 0.29333333333333333 - }, - "consistency_score_2": 0.4158730158730159, - "consistency_score_3": 0.21676190476190477, - "consistency_score_4": 0.12533333333333335, - "consistency_score_5": 0.07714285714285714, - "consistency_score_6": 0.049523809523809526, - "consistency_score_7": 0.03333333333333333, + "Filipino": 0.3933333333333333, + "Vietnamese": 0.52, + "Chinese": 0.6, + "Spanish": 0.5266666666666666, + "Malay": 0.5333333333333333, + "Indonesian": 0.54, + "English": 0.64 + }, + "consistency_score_2": 0.6215873015873016, + "consistency_score_3": 0.4666666666666666, + "consistency_score_4": 0.37428571428571433, + "consistency_score_5": 0.31015873015873013, + "consistency_score_6": 0.26095238095238094, + "consistency_score_7": 0.22, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.3, - "Spanish,Chinese": 0.5, - "Spanish,Vietnamese": 0.4266666666666667, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.29333333333333333, - "Spanish,Malay": 0.35333333333333333, - "English,Chinese": 0.32666666666666666, - "English,Vietnamese": 0.4533333333333333, - "English,Filipino": 0.28, - "English,Indonesian": 0.4266666666666667, - "English,Malay": 0.4666666666666667, - "Chinese,Vietnamese": 0.54, - "Chinese,Filipino": 0.43333333333333335, - "Chinese,Indonesian": 0.37333333333333335, - "Chinese,Malay": 0.44666666666666666, - "Vietnamese,Filipino": 0.35333333333333333, - "Vietnamese,Indonesian": 0.5533333333333333, - "Vietnamese,Malay": 0.6133333333333333, - "Filipino,Indonesian": 0.26666666666666666, - "Filipino,Malay": 0.24666666666666667, - "Indonesian,Malay": 0.64 + "Filipino,Vietnamese": 0.4, + "Filipino,Chinese": 0.49333333333333335, + "Filipino,Spanish": 0.4666666666666667, + "Filipino,Malay": 0.4533333333333333, + "Filipino,Indonesian": 0.4533333333333333, + "Filipino,English": 0.5133333333333333, + "Vietnamese,Chinese": 0.6666666666666666, + "Vietnamese,Spanish": 0.66, + "Vietnamese,Malay": 0.64, + "Vietnamese,Indonesian": 0.6666666666666666, + "Vietnamese,English": 0.68, + "Chinese,Spanish": 0.6466666666666666, + "Chinese,Malay": 0.6733333333333333, + "Chinese,Indonesian": 0.66, + "Chinese,English": 0.6666666666666666, + "Spanish,Malay": 0.6733333333333333, + "Spanish,Indonesian": 0.7133333333333334, + "Spanish,English": 0.6866666666666666, + "Malay,Indonesian": 0.7533333333333333, + "Malay,English": 0.7266666666666667, + "Indonesian,English": 0.76 }, "3_combine": { - "Spanish,English,Chinese": 0.16, - "Spanish,English,Vietnamese": 0.17333333333333334, - "Spanish,English,Filipino": 0.15333333333333332, - "Spanish,English,Indonesian": 0.13333333333333333, - "Spanish,English,Malay": 0.16666666666666666, - "Spanish,Chinese,Vietnamese": 0.30666666666666664, - "Spanish,Chinese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,Vietnamese,Indonesian": 0.21333333333333335, - "Spanish,Vietnamese,Malay": 0.26, - "Spanish,Filipino,Indonesian": 0.14, - "Spanish,Filipino,Malay": 0.16666666666666666, - "Spanish,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese": 0.24666666666666667, - "English,Chinese,Filipino": 0.16, - "English,Chinese,Indonesian": 0.2, - "English,Chinese,Malay": 0.22, - "English,Vietnamese,Filipino": 0.16666666666666666, - "English,Vietnamese,Indonesian": 0.30666666666666664, - "English,Vietnamese,Malay": 0.34, - "English,Filipino,Indonesian": 0.12, - "English,Filipino,Malay": 0.12, - "English,Indonesian,Malay": 0.34, - "Chinese,Vietnamese,Filipino": 0.26, - "Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Malay": 0.3333333333333333, - "Chinese,Filipino,Indonesian": 0.14666666666666667, - "Chinese,Filipino,Malay": 0.18, - "Chinese,Indonesian,Malay": 0.28, - "Vietnamese,Filipino,Indonesian": 0.16, - "Vietnamese,Filipino,Malay": 0.18666666666666668, - "Vietnamese,Indonesian,Malay": 0.44, - "Filipino,Indonesian,Malay": 0.12666666666666668 + "Filipino,Vietnamese,Chinese": 0.31333333333333335, + "Filipino,Vietnamese,Spanish": 0.31333333333333335, + "Filipino,Vietnamese,Malay": 0.31333333333333335, + "Filipino,Vietnamese,Indonesian": 0.32, + "Filipino,Vietnamese,English": 0.34, + "Filipino,Chinese,Spanish": 0.35333333333333333, + "Filipino,Chinese,Malay": 0.36666666666666664, + "Filipino,Chinese,Indonesian": 0.3466666666666667, + "Filipino,Chinese,English": 0.38, + "Filipino,Spanish,Malay": 0.3466666666666667, + "Filipino,Spanish,Indonesian": 0.35333333333333333, + "Filipino,Spanish,English": 0.36666666666666664, + "Filipino,Malay,Indonesian": 0.36, + "Filipino,Malay,English": 0.36666666666666664, + "Filipino,Indonesian,English": 0.3933333333333333, + "Vietnamese,Chinese,Spanish": 0.52, + "Vietnamese,Chinese,Malay": 0.52, + "Vietnamese,Chinese,Indonesian": 0.5333333333333333, + "Vietnamese,Chinese,English": 0.5266666666666666, + "Vietnamese,Spanish,Malay": 0.52, + "Vietnamese,Spanish,Indonesian": 0.5533333333333333, + "Vietnamese,Spanish,English": 0.5333333333333333, + "Vietnamese,Malay,Indonesian": 0.5533333333333333, + "Vietnamese,Malay,English": 0.5533333333333333, + "Vietnamese,Indonesian,English": 0.5866666666666667, + "Chinese,Spanish,Malay": 0.5266666666666666, + "Chinese,Spanish,Indonesian": 0.5466666666666666, + "Chinese,Spanish,English": 0.5266666666666666, + "Chinese,Malay,Indonesian": 0.5666666666666667, + "Chinese,Malay,English": 0.56, + "Chinese,Indonesian,English": 0.5733333333333334, + "Spanish,Malay,Indonesian": 0.6, + "Spanish,Malay,English": 0.5666666666666667, + "Spanish,Indonesian,English": 0.5933333333333334, + "Malay,Indonesian,English": 0.64 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.12666666666666668, - "Spanish,English,Chinese,Filipino": 0.1, - "Spanish,English,Chinese,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.1, - "Spanish,English,Vietnamese,Indonesian": 0.10666666666666667, - "Spanish,English,Vietnamese,Malay": 0.12666666666666668, - "Spanish,English,Filipino,Indonesian": 0.07333333333333333, - "Spanish,English,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino": 0.16, - "Spanish,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.12666666666666668, - "Spanish,Chinese,Indonesian,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,Vietnamese,Filipino,Malay": 0.12, - "Spanish,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.08666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.12, - "English,Chinese,Vietnamese,Indonesian": 0.18, - "English,Chinese,Vietnamese,Malay": 0.18, - "English,Chinese,Filipino,Indonesian": 0.08, - "English,Chinese,Filipino,Malay": 0.1, - "English,Chinese,Indonesian,Malay": 0.17333333333333334, - "English,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "English,Vietnamese,Filipino,Malay": 0.09333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "English,Filipino,Indonesian,Malay": 0.06666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 + "Filipino,Vietnamese,Chinese,Spanish": 0.26, + "Filipino,Vietnamese,Chinese,Malay": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,English": 0.28, + "Filipino,Vietnamese,Spanish,Malay": 0.26666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Spanish,English": 0.2866666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Malay,English": 0.2866666666666667, + "Filipino,Vietnamese,Indonesian,English": 0.31333333333333335, + "Filipino,Chinese,Spanish,Malay": 0.29333333333333333, + "Filipino,Chinese,Spanish,Indonesian": 0.3, + "Filipino,Chinese,Spanish,English": 0.32, + "Filipino,Chinese,Malay,Indonesian": 0.30666666666666664, + "Filipino,Chinese,Malay,English": 0.32666666666666666, + "Filipino,Chinese,Indonesian,English": 0.3333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.3, + "Filipino,Spanish,Malay,English": 0.31333333333333335, + "Filipino,Spanish,Indonesian,English": 0.32666666666666666, + "Filipino,Malay,Indonesian,English": 0.3333333333333333, + "Vietnamese,Chinese,Spanish,Malay": 0.44, + "Vietnamese,Chinese,Spanish,Indonesian": 0.47333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.44, + "Vietnamese,Chinese,Malay,Indonesian": 0.4666666666666667, + "Vietnamese,Chinese,Malay,English": 0.44666666666666666, + "Vietnamese,Chinese,Indonesian,English": 0.48, + "Vietnamese,Spanish,Malay,Indonesian": 0.4866666666666667, + "Vietnamese,Spanish,Malay,English": 0.46, + "Vietnamese,Spanish,Indonesian,English": 0.49333333333333335, + "Vietnamese,Malay,Indonesian,English": 0.5066666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.4866666666666667, + "Chinese,Spanish,Malay,English": 0.4666666666666667, + "Chinese,Spanish,Indonesian,English": 0.4866666666666667, + "Chinese,Malay,Indonesian,English": 0.5066666666666667, + "Spanish,Malay,Indonesian,English": 0.5333333333333333 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.05333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.05333333333333334, - "Spanish,English,Vietnamese,Filipino,Malay": 0.06666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.04666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.06, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian,Malay": 0.06, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.06, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.24, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.24, + "Filipino,Vietnamese,Chinese,Malay,English": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.26666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2733333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.2733333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Filipino,Chinese,Spanish,Malay,English": 0.28, + "Filipino,Chinese,Spanish,Indonesian,English": 0.29333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.3, + "Filipino,Spanish,Malay,Indonesian,English": 0.29333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4266666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.3933333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4266666666666667, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.4266666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.44666666666666666, + "Chinese,Spanish,Malay,Indonesian,English": 0.44666666666666666 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.04666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.04, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.04, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.22, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.24, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.24, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.24666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.03333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22 } }, - "AC3_2": 0.3256673964975938, - "AC3_3": 0.2395206261326009, - "AC3_4": 0.17071578602861895, - "AC3_5": 0.11976322017047121, - "AC3_6": 0.0835807235543693, - "AC3_7": 0.05928270040224256 + "AC3_2": 0.5757394857637913, + "AC3_3": 0.49901867674669487, + "AC3_4": 0.4408427973215751, + "AC3_5": 0.3929918193369835, + "AC3_6": 0.3510542185370612, + "AC3_7": 0.31198992439199125 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.2702922077922078, + "overall_acc": 0.44724025974025977, "language_acc": { - "English": 0.24431818181818182, - "Filipino": 0.2215909090909091, - "Spanish": 0.29545454545454547, - "Malay": 0.26136363636363635, - "Chinese": 0.25, - "Indonesian": 0.2840909090909091, - "Vietnamese": 0.3352272727272727 + "Indonesian": 0.42613636363636365, + "English": 0.5056818181818182, + "Filipino": 0.3409090909090909, + "Spanish": 0.3977272727272727, + "Chinese": 0.5738636363636364, + "Malay": 0.4772727272727273, + "Vietnamese": 0.4090909090909091 }, - "consistency_score_2": 0.4253246753246753, - "consistency_score_3": 0.22045454545454546, - "consistency_score_4": 0.13084415584415587, - "consistency_score_5": 0.08685064935064936, - "consistency_score_6": 0.06168831168831169, - "consistency_score_7": 0.045454545454545456, + "consistency_score_2": 0.518939393939394, + "consistency_score_3": 0.34480519480519484, + "consistency_score_4": 0.2512987012987013, + "consistency_score_5": 0.191017316017316, + "consistency_score_6": 0.1477272727272727, + "consistency_score_7": 0.11363636363636363, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4715909090909091, - "English,Spanish": 0.3181818181818182, - "English,Malay": 0.3693181818181818, - "English,Chinese": 0.36363636363636365, - "English,Indonesian": 0.42613636363636365, - "English,Vietnamese": 0.3465909090909091, - "Filipino,Spanish": 0.5284090909090909, - "Filipino,Malay": 0.4375, - "Filipino,Chinese": 0.4772727272727273, - "Filipino,Indonesian": 0.3352272727272727, - "Filipino,Vietnamese": 0.375, - "Spanish,Malay": 0.5340909090909091, + "Indonesian,English": 0.5852272727272727, + "Indonesian,Filipino": 0.38636363636363635, + "Indonesian,Spanish": 0.6363636363636364, + "Indonesian,Chinese": 0.5681818181818182, + "Indonesian,Malay": 0.6761363636363636, + "Indonesian,Vietnamese": 0.5965909090909091, + "English,Filipino": 0.375, + "English,Spanish": 0.5113636363636364, + "English,Chinese": 0.6079545454545454, + "English,Malay": 0.5568181818181818, + "English,Vietnamese": 0.5284090909090909, + "Filipino,Spanish": 0.4318181818181818, + "Filipino,Chinese": 0.3522727272727273, + "Filipino,Malay": 0.42045454545454547, + "Filipino,Vietnamese": 0.3465909090909091, "Spanish,Chinese": 0.4659090909090909, - "Spanish,Indonesian": 0.3409090909090909, - "Spanish,Vietnamese": 0.5511363636363636, - "Malay,Chinese": 0.4318181818181818, - "Malay,Indonesian": 0.4772727272727273, - "Malay,Vietnamese": 0.5, - "Chinese,Indonesian": 0.3693181818181818, - "Chinese,Vietnamese": 0.4090909090909091, - "Indonesian,Vietnamese": 0.4034090909090909 + "Spanish,Malay": 0.6193181818181818, + "Spanish,Vietnamese": 0.5227272727272727, + "Chinese,Malay": 0.5852272727272727, + "Chinese,Vietnamese": 0.5454545454545454, + "Malay,Vietnamese": 0.5795454545454546 }, "3_combine": { - "English,Filipino,Spanish": 0.21022727272727273, - "English,Filipino,Malay": 0.22727272727272727, - "English,Filipino,Chinese": 0.22727272727272727, - "English,Filipino,Indonesian": 0.20454545454545456, - "English,Filipino,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay": 0.19318181818181818, - "English,Spanish,Chinese": 0.1590909090909091, - "English,Spanish,Indonesian": 0.1590909090909091, - "English,Spanish,Vietnamese": 0.2159090909090909, - "English,Malay,Chinese": 0.1590909090909091, - "English,Malay,Indonesian": 0.22727272727272727, - "English,Malay,Vietnamese": 0.19886363636363635, - "English,Chinese,Indonesian": 0.19318181818181818, - "English,Chinese,Vietnamese": 0.17045454545454544, - "English,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay": 0.29545454545454547, - "Filipino,Spanish,Chinese": 0.2897727272727273, - "Filipino,Spanish,Indonesian": 0.1875, - "Filipino,Spanish,Vietnamese": 0.30113636363636365, - "Filipino,Malay,Chinese": 0.22727272727272727, - "Filipino,Malay,Indonesian": 0.20454545454545456, + "Indonesian,English,Filipino": 0.26704545454545453, + "Indonesian,English,Spanish": 0.42045454545454547, + "Indonesian,English,Chinese": 0.4431818181818182, + "Indonesian,English,Malay": 0.44886363636363635, + "Indonesian,English,Vietnamese": 0.4090909090909091, + "Indonesian,Filipino,Spanish": 0.30113636363636365, + "Indonesian,Filipino,Chinese": 0.23295454545454544, + "Indonesian,Filipino,Malay": 0.29545454545454547, + "Indonesian,Filipino,Vietnamese": 0.25, + "Indonesian,Spanish,Chinese": 0.38636363636363635, + "Indonesian,Spanish,Malay": 0.5113636363636364, + "Indonesian,Spanish,Vietnamese": 0.4147727272727273, + "Indonesian,Chinese,Malay": 0.44886363636363635, + "Indonesian,Chinese,Vietnamese": 0.4090909090909091, + "Indonesian,Malay,Vietnamese": 0.48295454545454547, + "English,Filipino,Spanish": 0.25, + "English,Filipino,Chinese": 0.26136363636363635, + "English,Filipino,Malay": 0.2784090909090909, + "English,Filipino,Vietnamese": 0.22727272727272727, + "English,Spanish,Chinese": 0.35795454545454547, + "English,Spanish,Malay": 0.38636363636363635, + "English,Spanish,Vietnamese": 0.3522727272727273, + "English,Chinese,Malay": 0.4318181818181818, + "English,Chinese,Vietnamese": 0.3977272727272727, + "English,Malay,Vietnamese": 0.38636363636363635, + "Filipino,Spanish,Chinese": 0.2159090909090909, + "Filipino,Spanish,Malay": 0.3068181818181818, + "Filipino,Spanish,Vietnamese": 0.24431818181818182, + "Filipino,Chinese,Malay": 0.25, + "Filipino,Chinese,Vietnamese": 0.2215909090909091, "Filipino,Malay,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Chinese,Vietnamese": 0.23863636363636365, - "Filipino,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese": 0.2784090909090909, - "Spanish,Malay,Indonesian": 0.23863636363636365, - "Spanish,Malay,Vietnamese": 0.35795454545454547, - "Spanish,Chinese,Indonesian": 0.17045454545454544, - "Spanish,Chinese,Vietnamese": 0.2897727272727273, - "Spanish,Indonesian,Vietnamese": 0.22727272727272727, - "Malay,Chinese,Indonesian": 0.2159090909090909, - "Malay,Chinese,Vietnamese": 0.25, - "Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Chinese,Indonesian,Vietnamese": 0.19886363636363635 + "Spanish,Chinese,Malay": 0.3806818181818182, + "Spanish,Chinese,Vietnamese": 0.32386363636363635, + "Spanish,Malay,Vietnamese": 0.4147727272727273, + "Chinese,Malay,Vietnamese": 0.4090909090909091 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.13068181818181818, - "English,Filipino,Spanish,Chinese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese": 0.11931818181818182, - "English,Filipino,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Vietnamese": 0.11931818181818182, - "English,Filipino,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese": 0.11931818181818182, - "English,Spanish,Malay,Indonesian": 0.10795454545454546, - "English,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Spanish,Chinese,Indonesian": 0.10227272727272728, - "English,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.11363636363636363, - "English,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Malay,Indonesian,Vietnamese": 0.125, - "English,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese": 0.17613636363636365, - "Filipino,Spanish,Malay,Indonesian": 0.13068181818181818, - "Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Indonesian,Vietnamese": 0.125, - "Filipino,Malay,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "Spanish,Malay,Chinese,Vietnamese": 0.19886363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 + "Indonesian,English,Filipino,Spanish": 0.2159090909090909, + "Indonesian,English,Filipino,Chinese": 0.19886363636363635, + "Indonesian,English,Filipino,Malay": 0.23295454545454544, + "Indonesian,English,Filipino,Vietnamese": 0.19886363636363635, + "Indonesian,English,Spanish,Chinese": 0.32954545454545453, + "Indonesian,English,Spanish,Malay": 0.36363636363636365, + "Indonesian,English,Spanish,Vietnamese": 0.3125, + "Indonesian,English,Chinese,Malay": 0.36363636363636365, + "Indonesian,English,Chinese,Vietnamese": 0.3352272727272727, + "Indonesian,English,Malay,Vietnamese": 0.3409090909090909, + "Indonesian,Filipino,Spanish,Chinese": 0.18181818181818182, + "Indonesian,Filipino,Spanish,Malay": 0.25, + "Indonesian,Filipino,Spanish,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Chinese,Malay": 0.19318181818181818, + "Indonesian,Filipino,Chinese,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,Spanish,Chinese,Malay": 0.3352272727272727, + "Indonesian,Spanish,Chinese,Vietnamese": 0.2840909090909091, + "Indonesian,Spanish,Malay,Vietnamese": 0.3806818181818182, + "Indonesian,Chinese,Malay,Vietnamese": 0.3409090909090909, + "English,Filipino,Spanish,Chinese": 0.17045454545454544, + "English,Filipino,Spanish,Malay": 0.21022727272727273, + "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, + "English,Filipino,Chinese,Malay": 0.21022727272727273, + "English,Filipino,Chinese,Vietnamese": 0.17045454545454544, + "English,Filipino,Malay,Vietnamese": 0.1875, + "English,Spanish,Chinese,Malay": 0.3125, + "English,Spanish,Chinese,Vietnamese": 0.2727272727272727, + "English,Spanish,Malay,Vietnamese": 0.29545454545454547, + "English,Chinese,Malay,Vietnamese": 0.3181818181818182, + "Filipino,Spanish,Chinese,Malay": 0.1875, + "Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, + "Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, + "Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, + "Spanish,Chinese,Malay,Vietnamese": 0.2897727272727273 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 + "Indonesian,English,Filipino,Spanish,Chinese": 0.1590909090909091, + "Indonesian,English,Filipino,Spanish,Malay": 0.19318181818181818, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1590909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.17613636363636365, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1590909090909091, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,English,Spanish,Chinese,Malay": 0.29545454545454547, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2556818181818182, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2840909090909091, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.1875, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635, + "English,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, + "English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, + "English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, + "English,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.125, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 } }, - "AC3_2": 0.3305323614066653, - "AC3_3": 0.24284275098049793, - "AC3_4": 0.17632984170474297, - "AC3_5": 0.13146030102576492, - "AC3_6": 0.10045089381916879, - "AC3_7": 0.07782192098493473 + "AC3_2": 0.4804294696684598, + "AC3_3": 0.38939877498191305, + "AC3_4": 0.3217884833738374, + "AC3_5": 0.26769955347787405, + "AC3_6": 0.22209475377638804, + "AC3_7": 0.18122615442105017 }, "prompt_2": { - "overall_acc": 0.262987012987013, + "overall_acc": 0.461038961038961, "language_acc": { - "English": 0.2897727272727273, - "Filipino": 0.2897727272727273, - "Spanish": 0.24431818181818182, - "Malay": 0.25, - "Chinese": 0.2556818181818182, - "Indonesian": 0.21022727272727273, - "Vietnamese": 0.30113636363636365 + "Indonesian": 0.44886363636363635, + "English": 0.5113636363636364, + "Filipino": 0.3409090909090909, + "Spanish": 0.4034090909090909, + "Chinese": 0.5852272727272727, + "Malay": 0.4943181818181818, + "Vietnamese": 0.4431818181818182 }, - "consistency_score_2": 0.4334415584415584, - "consistency_score_3": 0.22987012987012992, - "consistency_score_4": 0.13652597402597402, - "consistency_score_5": 0.08901515151515153, - "consistency_score_6": 0.06412337662337662, - "consistency_score_7": 0.05113636363636364, + "consistency_score_2": 0.5192099567099567, + "consistency_score_3": 0.3487012987012987, + "consistency_score_4": 0.2581168831168831, + "consistency_score_5": 0.2015692640692641, + "consistency_score_6": 0.16152597402597405, + "consistency_score_7": 0.13068181818181818, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.5, - "English,Spanish": 0.4943181818181818, - "English,Malay": 0.5056818181818182, - "English,Chinese": 0.48295454545454547, - "English,Indonesian": 0.3522727272727273, - "English,Vietnamese": 0.3977272727272727, - "Filipino,Spanish": 0.42613636363636365, - "Filipino,Malay": 0.48295454545454547, - "Filipino,Chinese": 0.4943181818181818, - "Filipino,Indonesian": 0.3806818181818182, - "Filipino,Vietnamese": 0.4431818181818182, - "Spanish,Malay": 0.4659090909090909, - "Spanish,Chinese": 0.4375, - "Spanish,Indonesian": 0.39204545454545453, - "Spanish,Vietnamese": 0.38636363636363635, - "Malay,Chinese": 0.4375, - "Malay,Indonesian": 0.4431818181818182, - "Malay,Vietnamese": 0.42045454545454547, - "Chinese,Indonesian": 0.3522727272727273, - "Chinese,Vietnamese": 0.3977272727272727, - "Indonesian,Vietnamese": 0.4090909090909091 + "Indonesian,English": 0.6420454545454546, + "Indonesian,Filipino": 0.3465909090909091, + "Indonesian,Spanish": 0.5909090909090909, + "Indonesian,Chinese": 0.5454545454545454, + "Indonesian,Malay": 0.6420454545454546, + "Indonesian,Vietnamese": 0.5738636363636364, + "English,Filipino": 0.3977272727272727, + "English,Spanish": 0.5852272727272727, + "English,Chinese": 0.5909090909090909, + "English,Malay": 0.6363636363636364, + "English,Vietnamese": 0.5625, + "Filipino,Spanish": 0.4090909090909091, + "Filipino,Chinese": 0.3465909090909091, + "Filipino,Malay": 0.375, + "Filipino,Vietnamese": 0.36363636363636365, + "Spanish,Chinese": 0.5284090909090909, + "Spanish,Malay": 0.6079545454545454, + "Spanish,Vietnamese": 0.5340909090909091, + "Chinese,Malay": 0.5909090909090909, + "Chinese,Vietnamese": 0.5170454545454546, + "Malay,Vietnamese": 0.5170454545454546 }, "3_combine": { - "English,Filipino,Spanish": 0.2784090909090909, - "English,Filipino,Malay": 0.3125, - "English,Filipino,Chinese": 0.3068181818181818, - "English,Filipino,Indonesian": 0.1875, - "English,Filipino,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay": 0.29545454545454547, - "English,Spanish,Chinese": 0.2840909090909091, - "English,Spanish,Indonesian": 0.20454545454545456, - "English,Spanish,Vietnamese": 0.23863636363636365, - "English,Malay,Chinese": 0.3068181818181818, - "English,Malay,Indonesian": 0.24431818181818182, - "English,Malay,Vietnamese": 0.24431818181818182, - "English,Chinese,Indonesian": 0.21022727272727273, - "English,Chinese,Vietnamese": 0.22727272727272727, - "English,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay": 0.2556818181818182, - "Filipino,Spanish,Chinese": 0.23863636363636365, - "Filipino,Spanish,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Chinese": 0.26136363636363635, - "Filipino,Malay,Indonesian": 0.22727272727272727, + "Indonesian,English,Filipino": 0.26704545454545453, + "Indonesian,English,Spanish": 0.45454545454545453, + "Indonesian,English,Chinese": 0.44886363636363635, + "Indonesian,English,Malay": 0.5113636363636364, + "Indonesian,English,Vietnamese": 0.4431818181818182, + "Indonesian,Filipino,Spanish": 0.2840909090909091, + "Indonesian,Filipino,Chinese": 0.22727272727272727, + "Indonesian,Filipino,Malay": 0.26136363636363635, + "Indonesian,Filipino,Vietnamese": 0.23295454545454544, + "Indonesian,Spanish,Chinese": 0.38636363636363635, + "Indonesian,Spanish,Malay": 0.48295454545454547, + "Indonesian,Spanish,Vietnamese": 0.4034090909090909, + "Indonesian,Chinese,Malay": 0.42613636363636365, + "Indonesian,Chinese,Vietnamese": 0.4034090909090909, + "Indonesian,Malay,Vietnamese": 0.42613636363636365, + "English,Filipino,Spanish": 0.29545454545454547, + "English,Filipino,Chinese": 0.23863636363636365, + "English,Filipino,Malay": 0.2784090909090909, + "English,Filipino,Vietnamese": 0.23295454545454544, + "English,Spanish,Chinese": 0.3977272727272727, + "English,Spanish,Malay": 0.4375, + "English,Spanish,Vietnamese": 0.3977272727272727, + "English,Chinese,Malay": 0.45454545454545453, + "English,Chinese,Vietnamese": 0.4034090909090909, + "English,Malay,Vietnamese": 0.4147727272727273, + "Filipino,Spanish,Chinese": 0.24431818181818182, + "Filipino,Spanish,Malay": 0.2784090909090909, + "Filipino,Spanish,Vietnamese": 0.25, + "Filipino,Chinese,Malay": 0.24431818181818182, + "Filipino,Chinese,Vietnamese": 0.2159090909090909, "Filipino,Malay,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian": 0.1875, - "Filipino,Chinese,Vietnamese": 0.23863636363636365, - "Filipino,Indonesian,Vietnamese": 0.19886363636363635, - "Spanish,Malay,Chinese": 0.24431818181818182, - "Spanish,Malay,Indonesian": 0.23295454545454544, - "Spanish,Malay,Vietnamese": 0.2159090909090909, - "Spanish,Chinese,Indonesian": 0.17045454545454544, - "Spanish,Chinese,Vietnamese": 0.20454545454545456, - "Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "Malay,Chinese,Indonesian": 0.2159090909090909, - "Malay,Chinese,Vietnamese": 0.19886363636363635, - "Malay,Indonesian,Vietnamese": 0.22727272727272727, - "Chinese,Indonesian,Vietnamese": 0.17045454545454544 + "Spanish,Chinese,Malay": 0.42045454545454547, + "Spanish,Chinese,Vietnamese": 0.3522727272727273, + "Spanish,Malay,Vietnamese": 0.39204545454545453, + "Chinese,Malay,Vietnamese": 0.375 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.18181818181818182, - "English,Filipino,Spanish,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Chinese": 0.19318181818181818, - "English,Filipino,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Malay,Vietnamese": 0.16477272727272727, - "English,Filipino,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Chinese": 0.19318181818181818, - "English,Spanish,Malay,Indonesian": 0.14772727272727273, - "English,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Spanish,Chinese,Indonesian": 0.125, - "English,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "English,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "English,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Malay,Chinese,Vietnamese": 0.1590909090909091, - "English,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.09090909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.125, - "Filipino,Malay,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Spanish,Malay,Chinese,Indonesian": 0.125, - "Spanish,Malay,Chinese,Vietnamese": 0.125, - "Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.125 + "Indonesian,English,Filipino,Spanish": 0.23863636363636365, + "Indonesian,English,Filipino,Chinese": 0.1875, + "Indonesian,English,Filipino,Malay": 0.22727272727272727, + "Indonesian,English,Filipino,Vietnamese": 0.19318181818181818, + "Indonesian,English,Spanish,Chinese": 0.3465909090909091, + "Indonesian,English,Spanish,Malay": 0.3977272727272727, + "Indonesian,English,Spanish,Vietnamese": 0.3352272727272727, + "Indonesian,English,Chinese,Malay": 0.38636363636363635, + "Indonesian,English,Chinese,Vietnamese": 0.3522727272727273, + "Indonesian,English,Malay,Vietnamese": 0.36363636363636365, + "Indonesian,Filipino,Spanish,Chinese": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Malay": 0.24431818181818182, + "Indonesian,Filipino,Spanish,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Chinese,Malay": 0.19886363636363635, + "Indonesian,Filipino,Chinese,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Malay,Vietnamese": 0.19886363636363635, + "Indonesian,Spanish,Chinese,Malay": 0.3409090909090909, + "Indonesian,Spanish,Chinese,Vietnamese": 0.30113636363636365, + "Indonesian,Spanish,Malay,Vietnamese": 0.3465909090909091, + "Indonesian,Chinese,Malay,Vietnamese": 0.32954545454545453, + "English,Filipino,Spanish,Chinese": 0.19318181818181818, + "English,Filipino,Spanish,Malay": 0.2215909090909091, + "English,Filipino,Spanish,Vietnamese": 0.19886363636363635, + "English,Filipino,Chinese,Malay": 0.19886363636363635, + "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, + "English,Filipino,Malay,Vietnamese": 0.17613636363636365, + "English,Spanish,Chinese,Malay": 0.3352272727272727, + "English,Spanish,Chinese,Vietnamese": 0.29545454545454547, + "English,Spanish,Malay,Vietnamese": 0.3181818181818182, + "English,Chinese,Malay,Vietnamese": 0.32954545454545453, + "Filipino,Spanish,Chinese,Malay": 0.20454545454545456, + "Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, + "Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, + "Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, + "Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Spanish,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 + "Indonesian,English,Filipino,Spanish,Chinese": 0.17613636363636365, + "Indonesian,English,Filipino,Spanish,Malay": 0.21022727272727273, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.17045454545454544, + "Indonesian,English,Filipino,Chinese,Malay": 0.17613636363636365, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.14772727272727273, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.17045454545454544, + "Indonesian,English,Spanish,Chinese,Malay": 0.3125, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.30113636363636365, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.30113636363636365, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.1875, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727, + "English,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, + "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, + "English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, + "English,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.25, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 } }, - "AC3_2": 0.32735446367109533, - "AC3_3": 0.2453159488234869, - "AC3_4": 0.179741631748653, - "AC3_5": 0.13300957301790486, - "AC3_6": 0.10310657081698592, - "AC3_7": 0.08562367861967649 + "AC3_2": 0.4883984356298704, + "AC3_3": 0.39707766166374836, + "AC3_4": 0.33094896069978136, + "AC3_5": 0.2805014503243686, + "AC3_6": 0.23923534089182885, + "AC3_7": 0.2036413517550848 }, "prompt_3": { - "overall_acc": 0.2694805194805195, + "overall_acc": 0.45698051948051954, "language_acc": { - "English": 0.3465909090909091, - "Filipino": 0.26136363636363635, - "Spanish": 0.26704545454545453, - "Malay": 0.23863636363636365, - "Chinese": 0.23863636363636365, - "Indonesian": 0.19886363636363635, - "Vietnamese": 0.3352272727272727 - }, - "consistency_score_2": 0.39042207792207795, - "consistency_score_3": 0.19074675324675325, - "consistency_score_4": 0.10405844155844157, - "consistency_score_5": 0.060335497835497826, - "consistency_score_6": 0.036525974025974024, - "consistency_score_7": 0.022727272727272728, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4659090909090909, - "English,Spanish": 0.4602272727272727, - "English,Malay": 0.42045454545454547, - "English,Chinese": 0.3693181818181818, - "English,Indonesian": 0.32386363636363635, - "English,Vietnamese": 0.4090909090909091, - "Filipino,Spanish": 0.42613636363636365, - "Filipino,Malay": 0.4602272727272727, - "Filipino,Chinese": 0.4659090909090909, - "Filipino,Indonesian": 0.3522727272727273, - "Filipino,Vietnamese": 0.3465909090909091, - "Spanish,Malay": 0.4147727272727273, - "Spanish,Chinese": 0.4147727272727273, - "Spanish,Indonesian": 0.3125, - "Spanish,Vietnamese": 0.3806818181818182, - "Malay,Chinese": 0.32954545454545453, - "Malay,Indonesian": 0.4943181818181818, - "Malay,Vietnamese": 0.3977272727272727, - "Chinese,Indonesian": 0.2784090909090909, - "Chinese,Vietnamese": 0.32386363636363635, - "Indonesian,Vietnamese": 0.3522727272727273 + "Indonesian": 0.4375, + "English": 0.48863636363636365, + "Filipino": 0.3409090909090909, + "Spanish": 0.42613636363636365, + "Chinese": 0.5795454545454546, + "Malay": 0.5056818181818182, + "Vietnamese": 0.42045454545454547 + }, + "consistency_score_2": 0.5211038961038961, + "consistency_score_3": 0.34512987012987006, + "consistency_score_4": 0.2506493506493506, + "consistency_score_5": 0.18966450216450217, + "consistency_score_6": 0.1461038961038961, + "consistency_score_7": 0.11363636363636363, + "detailed_consistency_score": { + "2_combine": { + "Indonesian,English": 0.5795454545454546, + "Indonesian,Filipino": 0.36363636363636365, + "Indonesian,Spanish": 0.6079545454545454, + "Indonesian,Chinese": 0.5625, + "Indonesian,Malay": 0.6590909090909091, + "Indonesian,Vietnamese": 0.6022727272727273, + "English,Filipino": 0.3977272727272727, + "English,Spanish": 0.5965909090909091, + "English,Chinese": 0.5852272727272727, + "English,Malay": 0.6022727272727273, + "English,Vietnamese": 0.5397727272727273, + "Filipino,Spanish": 0.4375, + "Filipino,Chinese": 0.36363636363636365, + "Filipino,Malay": 0.38636363636363635, + "Filipino,Vietnamese": 0.3806818181818182, + "Spanish,Chinese": 0.5511363636363636, + "Spanish,Malay": 0.6079545454545454, + "Spanish,Vietnamese": 0.4772727272727273, + "Chinese,Malay": 0.5738636363636364, + "Chinese,Vietnamese": 0.5284090909090909, + "Malay,Vietnamese": 0.5397727272727273 }, "3_combine": { + "Indonesian,English,Filipino": 0.24431818181818182, + "Indonesian,English,Spanish": 0.4431818181818182, + "Indonesian,English,Chinese": 0.4090909090909091, + "Indonesian,English,Malay": 0.4659090909090909, + "Indonesian,English,Vietnamese": 0.4147727272727273, + "Indonesian,Filipino,Spanish": 0.2840909090909091, + "Indonesian,Filipino,Chinese": 0.22727272727272727, + "Indonesian,Filipino,Malay": 0.2727272727272727, + "Indonesian,Filipino,Vietnamese": 0.2556818181818182, + "Indonesian,Spanish,Chinese": 0.42045454545454547, + "Indonesian,Spanish,Malay": 0.48863636363636365, + "Indonesian,Spanish,Vietnamese": 0.4090909090909091, + "Indonesian,Chinese,Malay": 0.42613636363636365, + "Indonesian,Chinese,Vietnamese": 0.42045454545454547, + "Indonesian,Malay,Vietnamese": 0.45454545454545453, "English,Filipino,Spanish": 0.2727272727272727, - "English,Filipino,Malay": 0.26704545454545453, "English,Filipino,Chinese": 0.23295454545454544, - "English,Filipino,Indonesian": 0.1875, - "English,Filipino,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay": 0.24431818181818182, - "English,Spanish,Chinese": 0.20454545454545456, - "English,Spanish,Indonesian": 0.1875, - "English,Spanish,Vietnamese": 0.23295454545454544, - "English,Malay,Chinese": 0.19318181818181818, - "English,Malay,Indonesian": 0.22727272727272727, - "English,Malay,Vietnamese": 0.2215909090909091, - "English,Chinese,Indonesian": 0.11363636363636363, - "English,Chinese,Vietnamese": 0.1590909090909091, - "English,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Malay": 0.24431818181818182, - "Filipino,Spanish,Chinese": 0.2159090909090909, - "Filipino,Spanish,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese": 0.2159090909090909, - "Filipino,Malay,Indonesian": 0.24431818181818182, - "Filipino,Malay,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese": 0.19886363636363635, - "Spanish,Malay,Indonesian": 0.19318181818181818, - "Spanish,Malay,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Indonesian": 0.13068181818181818, - "Spanish,Chinese,Vietnamese": 0.18181818181818182, - "Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "Malay,Chinese,Indonesian": 0.14204545454545456, - "Malay,Chinese,Vietnamese": 0.13636363636363635, - "Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Chinese,Indonesian,Vietnamese": 0.09659090909090909 + "English,Filipino,Malay": 0.2784090909090909, + "English,Filipino,Vietnamese": 0.23295454545454544, + "English,Spanish,Chinese": 0.42045454545454547, + "English,Spanish,Malay": 0.4318181818181818, + "English,Spanish,Vietnamese": 0.36363636363636365, + "English,Chinese,Malay": 0.42613636363636365, + "English,Chinese,Vietnamese": 0.3806818181818182, + "English,Malay,Vietnamese": 0.39204545454545453, + "Filipino,Spanish,Chinese": 0.26136363636363635, + "Filipino,Spanish,Malay": 0.2897727272727273, + "Filipino,Spanish,Vietnamese": 0.23295454545454544, + "Filipino,Chinese,Malay": 0.23863636363636365, + "Filipino,Chinese,Vietnamese": 0.23863636363636365, + "Filipino,Malay,Vietnamese": 0.23863636363636365, + "Spanish,Chinese,Malay": 0.42613636363636365, + "Spanish,Chinese,Vietnamese": 0.3465909090909091, + "Spanish,Malay,Vietnamese": 0.3693181818181818, + "Chinese,Malay,Vietnamese": 0.3693181818181818 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.17045454545454544, - "English,Filipino,Spanish,Chinese": 0.14204545454545456, - "English,Filipino,Spanish,Indonesian": 0.125, - "English,Filipino,Spanish,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Malay,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Chinese": 0.125, - "English,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian": 0.07386363636363637, - "English,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "English,Spanish,Indonesian,Vietnamese": 0.09659090909090909, - "English,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese": 0.13068181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.125, - "Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian": 0.0625, - "Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Filipino,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Spanish,Malay,Chinese,Indonesian": 0.07386363636363637, - "Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 + "Indonesian,English,Filipino,Spanish": 0.20454545454545456, + "Indonesian,English,Filipino,Chinese": 0.1590909090909091, + "Indonesian,English,Filipino,Malay": 0.21022727272727273, + "Indonesian,English,Filipino,Vietnamese": 0.18181818181818182, + "Indonesian,English,Spanish,Chinese": 0.3409090909090909, + "Indonesian,English,Spanish,Malay": 0.36363636363636365, + "Indonesian,English,Spanish,Vietnamese": 0.32954545454545453, + "Indonesian,English,Chinese,Malay": 0.3465909090909091, + "Indonesian,English,Chinese,Vietnamese": 0.3125, + "Indonesian,English,Malay,Vietnamese": 0.3409090909090909, + "Indonesian,Filipino,Spanish,Chinese": 0.19886363636363635, + "Indonesian,Filipino,Spanish,Malay": 0.22727272727272727, + "Indonesian,Filipino,Spanish,Vietnamese": 0.20454545454545456, + "Indonesian,Filipino,Chinese,Malay": 0.1875, + "Indonesian,Filipino,Chinese,Vietnamese": 0.1875, + "Indonesian,Filipino,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Spanish,Chinese,Malay": 0.3522727272727273, + "Indonesian,Spanish,Chinese,Vietnamese": 0.3181818181818182, + "Indonesian,Spanish,Malay,Vietnamese": 0.3409090909090909, + "Indonesian,Chinese,Malay,Vietnamese": 0.3352272727272727, + "English,Filipino,Spanish,Chinese": 0.19318181818181818, + "English,Filipino,Spanish,Malay": 0.2159090909090909, + "English,Filipino,Spanish,Vietnamese": 0.18181818181818182, + "English,Filipino,Chinese,Malay": 0.18181818181818182, + "English,Filipino,Chinese,Vietnamese": 0.17045454545454544, + "English,Filipino,Malay,Vietnamese": 0.18181818181818182, + "English,Spanish,Chinese,Malay": 0.3465909090909091, + "English,Spanish,Chinese,Vietnamese": 0.2840909090909091, + "English,Spanish,Malay,Vietnamese": 0.29545454545454547, + "English,Chinese,Malay,Vietnamese": 0.3068181818181818, + "Filipino,Spanish,Chinese,Malay": 0.21022727272727273, + "Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, + "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, + "Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, + "Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727 + "Indonesian,English,Filipino,Spanish,Chinese": 0.14772727272727273, + "Indonesian,English,Filipino,Spanish,Malay": 0.17613636363636365, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1590909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.14204545454545456, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,English,Spanish,Chinese,Malay": 0.29545454545454547, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.26136363636363635, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2727272727272727, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.2727272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909, + "English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, + "English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, + "English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, + "English,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.045454545454545456, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.125, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.125, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 } }, - "AC3_2": 0.31886870811928064, - "AC3_3": 0.22337891380656769, - "AC3_4": 0.15014081959215092, - "AC3_5": 0.09859582580137052, - "AC3_6": 0.06433221948361013, - "AC3_7": 0.04191919190484625 + "AC3_2": 0.4869402381352664, + "AC3_3": 0.3932566623333177, + "AC3_4": 0.3237338481203083, + "AC3_5": 0.2680697440220125, + "AC3_6": 0.2214172099403882, + "AC3_7": 0.18201215566446877 }, "prompt_4": { - "overall_acc": 0.2767857142857143, + "overall_acc": 0.4488636363636364, "language_acc": { - "English": 0.3352272727272727, - "Filipino": 0.2556818181818182, - "Spanish": 0.26704545454545453, - "Malay": 0.3068181818181818, - "Chinese": 0.23295454545454544, - "Indonesian": 0.23863636363636365, - "Vietnamese": 0.30113636363636365 + "Indonesian": 0.44886363636363635, + "English": 0.5, + "Filipino": 0.32386363636363635, + "Spanish": 0.42045454545454547, + "Chinese": 0.5625, + "Malay": 0.48863636363636365, + "Vietnamese": 0.3977272727272727 }, - "consistency_score_2": 0.46861471861471865, - "consistency_score_3": 0.2764610389610389, - "consistency_score_4": 0.18425324675324675, - "consistency_score_5": 0.13203463203463206, - "consistency_score_6": 0.09983766233766232, - "consistency_score_7": 0.07954545454545454, + "consistency_score_2": 0.5186688311688312, + "consistency_score_3": 0.3362012987012988, + "consistency_score_4": 0.23522727272727276, + "consistency_score_5": 0.16937229437229437, + "consistency_score_6": 0.12337662337662338, + "consistency_score_7": 0.09090909090909091, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.48863636363636365, - "English,Spanish": 0.48295454545454547, - "English,Malay": 0.4318181818181818, - "English,Chinese": 0.4090909090909091, - "English,Indonesian": 0.3806818181818182, - "English,Vietnamese": 0.4034090909090909, - "Filipino,Spanish": 0.5852272727272727, - "Filipino,Malay": 0.4602272727272727, - "Filipino,Chinese": 0.48863636363636365, - "Filipino,Indonesian": 0.3806818181818182, - "Filipino,Vietnamese": 0.45454545454545453, - "Spanish,Malay": 0.6022727272727273, - "Spanish,Chinese": 0.5397727272727273, - "Spanish,Indonesian": 0.375, - "Spanish,Vietnamese": 0.5454545454545454, - "Malay,Chinese": 0.44886363636363635, - "Malay,Indonesian": 0.48863636363636365, - "Malay,Vietnamese": 0.5795454545454546, - "Chinese,Indonesian": 0.375, - "Chinese,Vietnamese": 0.4602272727272727, - "Indonesian,Vietnamese": 0.4602272727272727 + "Indonesian,English": 0.6136363636363636, + "Indonesian,Filipino": 0.375, + "Indonesian,Spanish": 0.6363636363636364, + "Indonesian,Chinese": 0.5340909090909091, + "Indonesian,Malay": 0.6931818181818182, + "Indonesian,Vietnamese": 0.5568181818181818, + "English,Filipino": 0.38636363636363635, + "English,Spanish": 0.5681818181818182, + "English,Chinese": 0.5568181818181818, + "English,Malay": 0.6420454545454546, + "English,Vietnamese": 0.5113636363636364, + "Filipino,Spanish": 0.3465909090909091, + "Filipino,Chinese": 0.3806818181818182, + "Filipino,Malay": 0.3977272727272727, + "Filipino,Vietnamese": 0.3352272727272727, + "Spanish,Chinese": 0.48295454545454547, + "Spanish,Malay": 0.6534090909090909, + "Spanish,Vietnamese": 0.5681818181818182, + "Chinese,Malay": 0.5795454545454546, + "Chinese,Vietnamese": 0.5170454545454546, + "Malay,Vietnamese": 0.5568181818181818 }, "3_combine": { - "English,Filipino,Spanish": 0.32386363636363635, + "Indonesian,English,Filipino": 0.2727272727272727, + "Indonesian,English,Spanish": 0.4375, + "Indonesian,English,Chinese": 0.3977272727272727, + "Indonesian,English,Malay": 0.4943181818181818, + "Indonesian,English,Vietnamese": 0.375, + "Indonesian,Filipino,Spanish": 0.26704545454545453, + "Indonesian,Filipino,Chinese": 0.24431818181818182, + "Indonesian,Filipino,Malay": 0.2897727272727273, + "Indonesian,Filipino,Vietnamese": 0.23295454545454544, + "Indonesian,Spanish,Chinese": 0.36363636363636365, + "Indonesian,Spanish,Malay": 0.5227272727272727, + "Indonesian,Spanish,Vietnamese": 0.42613636363636365, + "Indonesian,Chinese,Malay": 0.42613636363636365, + "Indonesian,Chinese,Vietnamese": 0.35795454545454547, + "Indonesian,Malay,Vietnamese": 0.4318181818181818, + "English,Filipino,Spanish": 0.23295454545454544, + "English,Filipino,Chinese": 0.23295454545454544, "English,Filipino,Malay": 0.2784090909090909, - "English,Filipino,Chinese": 0.2556818181818182, - "English,Filipino,Indonesian": 0.2215909090909091, - "English,Filipino,Vietnamese": 0.26704545454545453, - "English,Spanish,Malay": 0.32386363636363635, - "English,Spanish,Chinese": 0.26704545454545453, - "English,Spanish,Indonesian": 0.20454545454545456, - "English,Spanish,Vietnamese": 0.2897727272727273, - "English,Malay,Chinese": 0.23295454545454544, - "English,Malay,Indonesian": 0.22727272727272727, - "English,Malay,Vietnamese": 0.2784090909090909, - "English,Chinese,Indonesian": 0.17045454545454544, - "English,Chinese,Vietnamese": 0.2215909090909091, - "English,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay": 0.39204545454545453, - "Filipino,Spanish,Chinese": 0.36363636363636365, - "Filipino,Spanish,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Vietnamese": 0.3693181818181818, - "Filipino,Malay,Chinese": 0.2727272727272727, - "Filipino,Malay,Indonesian": 0.2215909090909091, - "Filipino,Malay,Vietnamese": 0.32386363636363635, - "Filipino,Chinese,Indonesian": 0.20454545454545456, - "Filipino,Chinese,Vietnamese": 0.29545454545454547, - "Filipino,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Chinese": 0.35795454545454547, - "Spanish,Malay,Indonesian": 0.2897727272727273, - "Spanish,Malay,Vietnamese": 0.4147727272727273, - "Spanish,Chinese,Indonesian": 0.2215909090909091, - "Spanish,Chinese,Vietnamese": 0.3522727272727273, - "Spanish,Indonesian,Vietnamese": 0.24431818181818182, - "Malay,Chinese,Indonesian": 0.22727272727272727, - "Malay,Chinese,Vietnamese": 0.3181818181818182, - "Malay,Indonesian,Vietnamese": 0.3181818181818182, - "Chinese,Indonesian,Vietnamese": 0.22727272727272727 + "English,Filipino,Vietnamese": 0.19886363636363635, + "English,Spanish,Chinese": 0.36363636363636365, + "English,Spanish,Malay": 0.45454545454545453, + "English,Spanish,Vietnamese": 0.3806818181818182, + "English,Chinese,Malay": 0.42613636363636365, + "English,Chinese,Vietnamese": 0.3522727272727273, + "English,Malay,Vietnamese": 0.38636363636363635, + "Filipino,Spanish,Chinese": 0.21022727272727273, + "Filipino,Spanish,Malay": 0.2784090909090909, + "Filipino,Spanish,Vietnamese": 0.21022727272727273, + "Filipino,Chinese,Malay": 0.25, + "Filipino,Chinese,Vietnamese": 0.2159090909090909, + "Filipino,Malay,Vietnamese": 0.2215909090909091, + "Spanish,Chinese,Malay": 0.3977272727272727, + "Spanish,Chinese,Vietnamese": 0.3352272727272727, + "Spanish,Malay,Vietnamese": 0.4375, + "Chinese,Malay,Vietnamese": 0.36363636363636365 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.24431818181818182, - "English,Filipino,Spanish,Chinese": 0.19886363636363635, - "English,Filipino,Spanish,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Vietnamese": 0.2215909090909091, - "English,Filipino,Malay,Chinese": 0.17613636363636365, - "English,Filipino,Malay,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Vietnamese": 0.19886363636363635, - "English,Filipino,Chinese,Indonesian": 0.125, - "English,Filipino,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese": 0.19318181818181818, - "English,Spanish,Malay,Indonesian": 0.17045454545454544, - "English,Spanish,Malay,Vietnamese": 0.22727272727272727, - "English,Spanish,Chinese,Indonesian": 0.125, - "English,Spanish,Chinese,Vietnamese": 0.1875, - "English,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "English,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Malay,Chinese,Vietnamese": 0.17613636363636365, - "English,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese": 0.23863636363636365, - "Filipino,Spanish,Malay,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Malay,Vietnamese": 0.29545454545454547, - "Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.25, - "Filipino,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.18181818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365 + "Indonesian,English,Filipino,Spanish": 0.19318181818181818, + "Indonesian,English,Filipino,Chinese": 0.19318181818181818, + "Indonesian,English,Filipino,Malay": 0.2215909090909091, + "Indonesian,English,Filipino,Vietnamese": 0.17045454545454544, + "Indonesian,English,Spanish,Chinese": 0.29545454545454547, + "Indonesian,English,Spanish,Malay": 0.3806818181818182, + "Indonesian,English,Spanish,Vietnamese": 0.30113636363636365, + "Indonesian,English,Chinese,Malay": 0.3352272727272727, + "Indonesian,English,Chinese,Vietnamese": 0.26704545454545453, + "Indonesian,English,Malay,Vietnamese": 0.3181818181818182, + "Indonesian,Filipino,Spanish,Chinese": 0.18181818181818182, + "Indonesian,Filipino,Spanish,Malay": 0.23295454545454544, + "Indonesian,Filipino,Spanish,Vietnamese": 0.18181818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.21022727272727273, + "Indonesian,Filipino,Chinese,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Malay,Vietnamese": 0.19318181818181818, + "Indonesian,Spanish,Chinese,Malay": 0.3181818181818182, + "Indonesian,Spanish,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,Spanish,Malay,Vietnamese": 0.3693181818181818, + "Indonesian,Chinese,Malay,Vietnamese": 0.2840909090909091, + "English,Filipino,Spanish,Chinese": 0.1534090909090909, + "English,Filipino,Spanish,Malay": 0.19886363636363635, + "English,Filipino,Spanish,Vietnamese": 0.14772727272727273, + "English,Filipino,Chinese,Malay": 0.19318181818181818, + "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, + "English,Filipino,Malay,Vietnamese": 0.1590909090909091, + "English,Spanish,Chinese,Malay": 0.3181818181818182, + "English,Spanish,Chinese,Vietnamese": 0.26136363636363635, + "English,Spanish,Malay,Vietnamese": 0.3125, + "English,Chinese,Malay,Vietnamese": 0.2840909090909091, + "Filipino,Spanish,Chinese,Malay": 0.19318181818181818, + "Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, + "Filipino,Spanish,Malay,Vietnamese": 0.1875, + "Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, + "Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 + "Indonesian,English,Filipino,Spanish,Chinese": 0.13636363636363635, + "Indonesian,English,Filipino,Spanish,Malay": 0.17045454545454544, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.125, + "Indonesian,English,Filipino,Chinese,Malay": 0.17045454545454544, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,English,Spanish,Chinese,Malay": 0.26704545454545453, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.21022727272727273, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.22727272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, + "English,Filipino,Spanish,Chinese,Malay": 0.14772727272727273, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, + "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, + "English,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, + "English,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1875, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 } }, - "AC3_2": 0.3480165931604673, - "AC3_3": 0.2766232813047954, - "AC3_4": 0.22123365186348293, - "AC3_5": 0.17878415425332797, - "AC3_6": 0.1467441502073423, - "AC3_7": 0.12357630976030842 + "AC3_2": 0.48124809329986373, + "AC3_3": 0.3844485487140946, + "AC3_4": 0.30868695253717776, + "AC3_5": 0.24594191362642126, + "AC3_6": 0.19355254671021244, + "AC3_7": 0.15119617222079276 }, "prompt_5": { - "overall_acc": 0.2865259740259741, + "overall_acc": 0.44724025974025977, "language_acc": { - "English": 0.30113636363636365, - "Filipino": 0.26704545454545453, - "Spanish": 0.3352272727272727, - "Malay": 0.2727272727272727, - "Chinese": 0.26704545454545453, - "Indonesian": 0.25, - "Vietnamese": 0.3125 + "Indonesian": 0.42613636363636365, + "English": 0.4715909090909091, + "Filipino": 0.3409090909090909, + "Spanish": 0.4318181818181818, + "Chinese": 0.5738636363636364, + "Malay": 0.4772727272727273, + "Vietnamese": 0.4090909090909091 }, - "consistency_score_2": 0.36958874458874463, - "consistency_score_3": 0.17094155844155845, - "consistency_score_4": 0.09545454545454546, - "consistency_score_5": 0.06439393939393938, - "consistency_score_6": 0.05113636363636364, - "consistency_score_7": 0.045454545454545456, + "consistency_score_2": 0.533279220779221, + "consistency_score_3": 0.3573051948051947, + "consistency_score_4": 0.26022727272727275, + "consistency_score_5": 0.1966991341991342, + "consistency_score_6": 0.15178571428571427, + "consistency_score_7": 0.11931818181818182, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.25, - "English,Spanish": 0.2784090909090909, - "English,Malay": 0.4375, - "English,Chinese": 0.35795454545454547, - "English,Indonesian": 0.48863636363636365, - "English,Vietnamese": 0.4090909090909091, - "Filipino,Spanish": 0.4034090909090909, - "Filipino,Malay": 0.2897727272727273, - "Filipino,Chinese": 0.48295454545454547, - "Filipino,Indonesian": 0.20454545454545456, - "Filipino,Vietnamese": 0.30113636363636365, - "Spanish,Malay": 0.2784090909090909, - "Spanish,Chinese": 0.2727272727272727, - "Spanish,Indonesian": 0.19886363636363635, - "Spanish,Vietnamese": 0.2840909090909091, - "Malay,Chinese": 0.3693181818181818, - "Malay,Indonesian": 0.625, - "Malay,Vietnamese": 0.5738636363636364, - "Chinese,Indonesian": 0.3068181818181818, - "Chinese,Vietnamese": 0.38636363636363635, - "Indonesian,Vietnamese": 0.5625 + "Indonesian,English": 0.6079545454545454, + "Indonesian,Filipino": 0.3465909090909091, + "Indonesian,Spanish": 0.6477272727272727, + "Indonesian,Chinese": 0.5397727272727273, + "Indonesian,Malay": 0.6818181818181818, + "Indonesian,Vietnamese": 0.6306818181818182, + "English,Filipino": 0.38636363636363635, + "English,Spanish": 0.6306818181818182, + "English,Chinese": 0.5625, + "English,Malay": 0.6306818181818182, + "English,Vietnamese": 0.5284090909090909, + "Filipino,Spanish": 0.38636363636363635, + "Filipino,Chinese": 0.35795454545454547, + "Filipino,Malay": 0.4034090909090909, + "Filipino,Vietnamese": 0.3409090909090909, + "Spanish,Chinese": 0.5568181818181818, + "Spanish,Malay": 0.6420454545454546, + "Spanish,Vietnamese": 0.5795454545454546, + "Chinese,Malay": 0.5909090909090909, + "Chinese,Vietnamese": 0.5681818181818182, + "Malay,Vietnamese": 0.5795454545454546 }, "3_combine": { - "English,Filipino,Spanish": 0.14204545454545456, - "English,Filipino,Malay": 0.10795454545454546, - "English,Filipino,Chinese": 0.13636363636363635, - "English,Filipino,Indonesian": 0.09090909090909091, - "English,Filipino,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay": 0.11363636363636363, - "English,Spanish,Chinese": 0.11931818181818182, - "English,Spanish,Indonesian": 0.125, - "English,Spanish,Vietnamese": 0.1534090909090909, - "English,Malay,Chinese": 0.20454545454545456, - "English,Malay,Indonesian": 0.3352272727272727, - "English,Malay,Vietnamese": 0.2897727272727273, - "English,Chinese,Indonesian": 0.17613636363636365, - "English,Chinese,Vietnamese": 0.18181818181818182, - "English,Indonesian,Vietnamese": 0.3068181818181818, - "Filipino,Spanish,Malay": 0.18181818181818182, - "Filipino,Spanish,Chinese": 0.20454545454545456, - "Filipino,Spanish,Indonesian": 0.11931818181818182, - "Filipino,Spanish,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Chinese": 0.1590909090909091, - "Filipino,Malay,Indonesian": 0.125, - "Filipino,Malay,Vietnamese": 0.1590909090909091, - "Filipino,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Indonesian,Vietnamese": 0.11363636363636363, - "Spanish,Malay,Chinese": 0.10795454545454546, - "Spanish,Malay,Indonesian": 0.13068181818181818, - "Spanish,Malay,Vietnamese": 0.1534090909090909, - "Spanish,Chinese,Indonesian": 0.08522727272727272, - "Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "Malay,Chinese,Indonesian": 0.2159090909090909, - "Malay,Chinese,Vietnamese": 0.2215909090909091, - "Malay,Indonesian,Vietnamese": 0.4375, - "Chinese,Indonesian,Vietnamese": 0.19886363636363635 + "Indonesian,English,Filipino": 0.24431818181818182, + "Indonesian,English,Spanish": 0.4659090909090909, + "Indonesian,English,Chinese": 0.4147727272727273, + "Indonesian,English,Malay": 0.48863636363636365, + "Indonesian,English,Vietnamese": 0.4318181818181818, + "Indonesian,Filipino,Spanish": 0.2556818181818182, + "Indonesian,Filipino,Chinese": 0.21022727272727273, + "Indonesian,Filipino,Malay": 0.2784090909090909, + "Indonesian,Filipino,Vietnamese": 0.25, + "Indonesian,Spanish,Chinese": 0.4090909090909091, + "Indonesian,Spanish,Malay": 0.5056818181818182, + "Indonesian,Spanish,Vietnamese": 0.4715909090909091, + "Indonesian,Chinese,Malay": 0.4431818181818182, + "Indonesian,Chinese,Vietnamese": 0.42045454545454547, + "Indonesian,Malay,Vietnamese": 0.48295454545454547, + "English,Filipino,Spanish": 0.2556818181818182, + "English,Filipino,Chinese": 0.24431818181818182, + "English,Filipino,Malay": 0.2727272727272727, + "English,Filipino,Vietnamese": 0.2215909090909091, + "English,Spanish,Chinese": 0.4375, + "English,Spanish,Malay": 0.48295454545454547, + "English,Spanish,Vietnamese": 0.4147727272727273, + "English,Chinese,Malay": 0.44886363636363635, + "English,Chinese,Vietnamese": 0.38636363636363635, + "English,Malay,Vietnamese": 0.4147727272727273, + "Filipino,Spanish,Chinese": 0.23863636363636365, + "Filipino,Spanish,Malay": 0.2840909090909091, + "Filipino,Spanish,Vietnamese": 0.22727272727272727, + "Filipino,Chinese,Malay": 0.24431818181818182, + "Filipino,Chinese,Vietnamese": 0.2215909090909091, + "Filipino,Malay,Vietnamese": 0.23863636363636365, + "Spanish,Chinese,Malay": 0.4375, + "Spanish,Chinese,Vietnamese": 0.4090909090909091, + "Spanish,Malay,Vietnamese": 0.4375, + "Chinese,Malay,Vietnamese": 0.4147727272727273 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.08522727272727272, - "English,Filipino,Spanish,Chinese": 0.09659090909090909, - "English,Filipino,Spanish,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese": 0.06818181818181818, - "English,Filipino,Malay,Indonesian": 0.06818181818181818, - "English,Filipino,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese": 0.056818181818181816, - "English,Spanish,Malay,Indonesian": 0.08522727272727272, - "English,Spanish,Malay,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian": 0.056818181818181816, - "English,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "English,Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Malay,Chinese,Vietnamese": 0.125, - "English,Malay,Indonesian,Vietnamese": 0.25, - "English,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Chinese": 0.08522727272727272, - "Filipino,Spanish,Malay,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian": 0.06818181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Malay,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Spanish,Malay,Chinese,Indonesian": 0.0625, - "Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091 + "Indonesian,English,Filipino,Spanish": 0.1875, + "Indonesian,English,Filipino,Chinese": 0.18181818181818182, + "Indonesian,English,Filipino,Malay": 0.20454545454545456, + "Indonesian,English,Filipino,Vietnamese": 0.1875, + "Indonesian,English,Spanish,Chinese": 0.3465909090909091, + "Indonesian,English,Spanish,Malay": 0.39204545454545453, + "Indonesian,English,Spanish,Vietnamese": 0.35795454545454547, + "Indonesian,English,Chinese,Malay": 0.36363636363636365, + "Indonesian,English,Chinese,Vietnamese": 0.32954545454545453, + "Indonesian,English,Malay,Vietnamese": 0.375, + "Indonesian,Filipino,Spanish,Chinese": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Malay": 0.2215909090909091, + "Indonesian,Filipino,Spanish,Vietnamese": 0.1875, + "Indonesian,Filipino,Chinese,Malay": 0.1875, + "Indonesian,Filipino,Chinese,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Malay,Vietnamese": 0.20454545454545456, + "Indonesian,Spanish,Chinese,Malay": 0.3522727272727273, + "Indonesian,Spanish,Chinese,Vietnamese": 0.3352272727272727, + "Indonesian,Spanish,Malay,Vietnamese": 0.39204545454545453, + "Indonesian,Chinese,Malay,Vietnamese": 0.3522727272727273, + "English,Filipino,Spanish,Chinese": 0.19886363636363635, + "English,Filipino,Spanish,Malay": 0.21022727272727273, + "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, + "English,Filipino,Chinese,Malay": 0.20454545454545456, + "English,Filipino,Chinese,Vietnamese": 0.17045454545454544, + "English,Filipino,Malay,Vietnamese": 0.1875, + "English,Spanish,Chinese,Malay": 0.3806818181818182, + "English,Spanish,Chinese,Vietnamese": 0.3181818181818182, + "English,Spanish,Malay,Vietnamese": 0.3352272727272727, + "English,Chinese,Malay,Vietnamese": 0.32954545454545453, + "Filipino,Spanish,Chinese,Malay": 0.20454545454545456, + "Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, + "Filipino,Spanish,Malay,Vietnamese": 0.1875, + "Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, + "Spanish,Chinese,Malay,Vietnamese": 0.3352272727272727 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Indonesian": 0.0625, - "English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0625, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 + "Indonesian,English,Filipino,Spanish,Chinese": 0.14772727272727273, + "Indonesian,English,Filipino,Spanish,Malay": 0.16477272727272727, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.14772727272727273, + "Indonesian,English,Filipino,Chinese,Malay": 0.16477272727272727, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.14772727272727273, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,English,Spanish,Chinese,Malay": 0.3125, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2784090909090909, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.3125, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.29545454545454547, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547, + "English,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, + "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, + "English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, + "English,Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.125, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 } }, - "AC3_2": 0.32279957151325955, - "AC3_3": 0.21413190247224842, - "AC3_4": 0.14320210172814832, - "AC3_5": 0.1051552533519647, - "AC3_6": 0.08678430941485793, - "AC3_7": 0.07846188039423616 + "AC3_2": 0.4864848520188464, + "AC3_3": 0.39724609022032276, + "AC3_4": 0.32901612494827875, + "AC3_5": 0.2732299737652739, + "AC3_6": 0.2266502128928643, + "AC3_7": 0.1883791351582064 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.24271844660194175 + "accuracy": 0.5631067961165048 }, "prompt_2": { - "accuracy": 0.24271844660194175 + "accuracy": 0.5922330097087378 }, "prompt_3": { - "accuracy": 0.21359223300970873 + "accuracy": 0.5922330097087378 }, "prompt_4": { - "accuracy": 0.2815533980582524 + "accuracy": 0.5436893203883495 }, "prompt_5": { - "accuracy": 0.30097087378640774 + "accuracy": 0.5922330097087378 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.22857142857142856 + "accuracy": 0.5428571428571428 }, "prompt_2": { - "accuracy": 0.2761904761904762 + "accuracy": 0.6095238095238096 }, "prompt_3": { - "accuracy": 0.3142857142857143 + "accuracy": 0.580952380952381 }, "prompt_4": { - "accuracy": 0.24761904761904763 + "accuracy": 0.5428571428571428 }, "prompt_5": { - "accuracy": 0.22857142857142856 + "accuracy": 0.6095238095238096 } }, "us_eval": { "prompt_1": { - "accuracy": 0.22429906542056074 + "accuracy": 0.5794392523364486 }, "prompt_2": { - "accuracy": 0.29906542056074764 + "accuracy": 0.5420560747663551 }, "prompt_3": { - "accuracy": 0.2897196261682243 + "accuracy": 0.514018691588785 }, "prompt_4": { - "accuracy": 0.2803738317757009 + "accuracy": 0.5700934579439252 }, "prompt_5": { - "accuracy": 0.2803738317757009 + "accuracy": 0.5607476635514018 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.28, + "accuracy": 0.34, "category_acc": { "brand": 0.3, - "demographics": 0.2, - "biology": 0.3, - "history": 0.2, + "demographics": 0.0, + "biology": 0.2, + "history": 0.13333333333333333, "literature": 0.4, - "politics": 0.1, - "culture": 0.4, - "film": 0.6, - "law": 0.1, - "geography": 0.2 + "politics": 0.6, + "culture": 0.2, + "film": 0.4, + "law": 0.4, + "geography": 0.7 } }, "prompt_2": { - "accuracy": 0.28, + "accuracy": 0.36, "category_acc": { - "brand": 0.3, - "demographics": 0.6, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.2, - "politics": 0.2, - "culture": 0.4, + "brand": 0.4, + "demographics": 0.0, + "biology": 0.3, + "history": 0.13333333333333333, + "literature": 0.4, + "politics": 0.6, + "culture": 0.2, "film": 0.3, - "law": 0.1, - "geography": 0.2 + "law": 0.5, + "geography": 0.7 } }, "prompt_3": { - "accuracy": 0.31, + "accuracy": 0.4, "category_acc": { - "brand": 0.2, - "demographics": 0.6, - "biology": 0.6, - "history": 0.06666666666666667, - "literature": 0.4, - "politics": 0.3, + "brand": 0.6, + "demographics": 0.2, + "biology": 0.3, + "history": 0.13333333333333333, + "literature": 0.5, + "politics": 0.6, "culture": 0.3, - "film": 0.4, - "law": 0.2, - "geography": 0.3 + "film": 0.3, + "law": 0.5, + "geography": 0.6 } }, "prompt_4": { - "accuracy": 0.32, + "accuracy": 0.35, "category_acc": { "brand": 0.4, - "demographics": 0.6, + "demographics": 0.0, "biology": 0.2, - "history": 0.26666666666666666, - "literature": 0.5, + "history": 0.13333333333333333, + "literature": 0.3, "politics": 0.6, - "culture": 0.1, - "film": 0.2, - "law": 0.3, - "geography": 0.2 + "culture": 0.3, + "film": 0.3, + "law": 0.5, + "geography": 0.7 } }, "prompt_5": { - "accuracy": 0.27, + "accuracy": 0.38, "category_acc": { - "brand": 0.5, + "brand": 0.4, "demographics": 0.0, - "biology": 0.1, + "biology": 0.3, "history": 0.13333333333333333, - "literature": 0.3, - "politics": 0.4, - "culture": 0.4, - "film": 0.4, - "law": 0.2, - "geography": 0.2 + "literature": 0.5, + "politics": 0.7, + "culture": 0.2, + "film": 0.3, + "law": 0.5, + "geography": 0.7 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.19744028073239198 + "bleu_score": 0.19216763243806428 }, "prompt_2": { - "bleu_score": 0.12117531019246643 + "bleu_score": 0.17617876480146175 }, "prompt_3": { - "bleu_score": 0.08982750108268342 + "bleu_score": 0.1844853053383724 }, "prompt_4": { - "bleu_score": 0.15282472895327717 + "bleu_score": 0.19387297841484136 }, "prompt_5": { - "bleu_score": 0.15072530075901933 + "bleu_score": 0.14092590287296944 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "prompt_1": { + "accuracy": 0.49068696174644505, + "category_acc": { + "History": 0.4879518072289157, + "Geography": 0.46122448979591835, + "Lampungic": 0.3197278911564626, + "Social science": 0.7646076794657763, + "Balinese": 0.29723991507430997, + "Makassarese": 0.3172043010752688, + "Banjarese": 0.4444444444444444, + "Chemistry": 0.29635036496350364, + "Biology": 0.44733727810650886, + "Science": 0.6212590299277606, + "Christian religion": 0.6616915422885572, + "Art": 0.6039933444259568, + "Islam religion": 0.6002844950213371, + "Hindu religion": 0.4666666666666667, + "Madurese": 0.31186440677966104, + "Sport": 0.5337837837837838, + "Indonesian language": 0.5435865504358655, + "Physics": 0.4121212121212121, + "Minangkabau culture": 0.37185929648241206, + "Dayak language": 0.28440366972477066, + "Sociology": 0.5020161290322581, + "Economy": 0.4672131147540984, + "Sundanese": 0.38720829732065687, + "Javanese": 0.36794354838709675, + "Civic education": 0.6094420600858369 + } + }, + "prompt_2": { + "accuracy": 0.4917551238400427, + "category_acc": { + "History": 0.4959839357429719, + "Geography": 0.46938775510204084, + "Lampungic": 0.36054421768707484, + "Social science": 0.7746243739565943, + "Balinese": 0.2951167728237792, + "Makassarese": 0.27956989247311825, + "Banjarese": 0.4166666666666667, + "Chemistry": 0.3021897810218978, + "Biology": 0.4544378698224852, + "Science": 0.6057791537667698, + "Christian religion": 0.6119402985074627, + "Art": 0.6372712146422629, + "Islam religion": 0.604551920341394, + "Hindu religion": 0.49333333333333335, + "Madurese": 0.2677966101694915, + "Sport": 0.5405405405405406, + "Indonesian language": 0.5482565379825654, + "Physics": 0.402020202020202, + "Minangkabau culture": 0.41708542713567837, + "Dayak language": 0.28440366972477066, + "Sociology": 0.4717741935483871, + "Economy": 0.45286885245901637, + "Sundanese": 0.39844425237683667, + "Javanese": 0.36794354838709675, + "Civic education": 0.6065808297567954 + } + }, + "prompt_3": { + "accuracy": 0.5047065892249148, + "category_acc": { + "History": 0.5261044176706827, + "Geography": 0.4816326530612245, + "Lampungic": 0.3741496598639456, + "Social science": 0.7679465776293823, + "Balinese": 0.3205944798301486, + "Makassarese": 0.34408602150537637, + "Banjarese": 0.4444444444444444, + "Chemistry": 0.29635036496350364, + "Biology": 0.4544378698224852, + "Science": 0.6326109391124871, + "Christian religion": 0.6666666666666666, + "Art": 0.6156405990016639, + "Islam religion": 0.6344238975817923, + "Hindu religion": 0.48, + "Madurese": 0.2983050847457627, + "Sport": 0.5540540540540541, + "Indonesian language": 0.5504358655043586, + "Physics": 0.4222222222222222, + "Minangkabau culture": 0.41708542713567837, + "Dayak language": 0.3394495412844037, + "Sociology": 0.5080645161290323, + "Economy": 0.4733606557377049, + "Sundanese": 0.3923941227312014, + "Javanese": 0.40524193548387094, + "Civic education": 0.6294706723891274 + } + }, + "prompt_4": { + "accuracy": 0.5030375859536684, + "category_acc": { + "History": 0.5220883534136547, + "Geography": 0.45714285714285713, + "Lampungic": 0.43537414965986393, + "Social science": 0.7679465776293823, + "Balinese": 0.32908704883227174, + "Makassarese": 0.3870967741935484, + "Banjarese": 0.4375, + "Chemistry": 0.30364963503649633, + "Biology": 0.4662721893491124, + "Science": 0.6357069143446853, + "Christian religion": 0.6567164179104478, + "Art": 0.6289517470881864, + "Islam religion": 0.6301564722617354, + "Hindu religion": 0.46, + "Madurese": 0.288135593220339, + "Sport": 0.5472972972972973, + "Indonesian language": 0.5457658779576587, + "Physics": 0.3939393939393939, + "Minangkabau culture": 0.41708542713567837, + "Dayak language": 0.3486238532110092, + "Sociology": 0.49798387096774194, + "Economy": 0.48155737704918034, + "Sundanese": 0.38202247191011235, + "Javanese": 0.39314516129032256, + "Civic education": 0.6409155937052933 + } + }, + "prompt_5": { + "accuracy": 0.47927097937111957, + "category_acc": { + "History": 0.4879518072289157, + "Geography": 0.42244897959183675, + "Lampungic": 0.3401360544217687, + "Social science": 0.7395659432387313, + "Balinese": 0.28450106157112526, + "Makassarese": 0.3763440860215054, + "Banjarese": 0.3541666666666667, + "Chemistry": 0.2905109489051095, + "Biology": 0.4307692307692308, + "Science": 0.6130030959752322, + "Christian religion": 0.6318407960199005, + "Art": 0.5740432612312812, + "Islam religion": 0.5889046941678521, + "Hindu religion": 0.4266666666666667, + "Madurese": 0.3050847457627119, + "Sport": 0.5540540540540541, + "Indonesian language": 0.5298879202988792, + "Physics": 0.3878787878787879, + "Minangkabau culture": 0.39195979899497485, + "Dayak language": 0.29357798165137616, + "Sociology": 0.4415322580645161, + "Economy": 0.45081967213114754, + "Sundanese": 0.3802938634399309, + "Javanese": 0.38306451612903225, + "Civic education": 0.6280400572246065 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.14945774683204474 + "bleu_score": 0.33974787333771794 }, "prompt_2": { - "bleu_score": 0.10685375440449423 + "bleu_score": 0.3442551398385029 }, "prompt_3": { - "bleu_score": 0.08225686362814823 + "bleu_score": 0.33743740022795504 }, "prompt_4": { - "bleu_score": 0.23735245651287692 + "bleu_score": 0.33896465094708667 }, "prompt_5": { - "bleu_score": 0.14216035520735448 + "bleu_score": 0.3173247794383544 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.1967042668619164 + "bleu_score": 0.2863714706572181 }, "prompt_2": { - "bleu_score": 0.0908269618670985 + "bleu_score": 0.2936750253543749 }, "prompt_3": { - "bleu_score": 0.058990452108142404 + "bleu_score": 0.2885398011099866 }, "prompt_4": { - "bleu_score": 0.23293797472337713 + "bleu_score": 0.29035383506316476 }, "prompt_5": { - "bleu_score": 0.1856398430710031 + "bleu_score": 0.28144461734788845 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.1613802673155159 + "bleu_score": 0.20900964720414072 }, "prompt_2": { - "bleu_score": 0.10470411665984837 + "bleu_score": 0.21451840379245748 }, "prompt_3": { - "bleu_score": 0.05605987655219017 + "bleu_score": 0.20720639696518997 }, "prompt_4": { - "bleu_score": 0.16500401402936918 + "bleu_score": 0.20592897490558038 }, "prompt_5": { - "bleu_score": 0.13832068386608407 + "bleu_score": 0.20436679638061198 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.16397807913043072 + "bleu_score": 0.3528671298807442 }, "prompt_2": { - "bleu_score": 0.09032564560760585 + "bleu_score": 0.3584718766854969 }, "prompt_3": { - "bleu_score": 0.07819718417219963 + "bleu_score": 0.3494575250742662 }, "prompt_4": { - "bleu_score": 0.28903877370609277 + "bleu_score": 0.35256965248509253 }, "prompt_5": { - "bleu_score": 0.16575518435540126 + "bleu_score": 0.3289924681392404 } }, "mmlu": { "prompt_1": { - "accuracy": 0.2602100350058343 + "accuracy": 0.49474912485414235 }, "prompt_2": { - "accuracy": 0.2847141190198366 + "accuracy": 0.5099183197199533 }, "prompt_3": { - "accuracy": 0.2847141190198366 + "accuracy": 0.544924154025671 }, "prompt_4": { - "accuracy": 0.2742123687281214 + "accuracy": 0.5344224037339557 }, "prompt_5": { - "accuracy": 0.26837806301050177 + "accuracy": 0.5344224037339557 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.25927779764032893, + "accuracy": 0.49717554522702895, "category_acc": { - "high_school_european_history": 0.29878048780487804, - "business_ethics": 0.2727272727272727, - "clinical_knowledge": 0.26515151515151514, - "medical_genetics": 0.3434343434343434, - "high_school_us_history": 0.2660098522167488, - "high_school_physics": 0.26, - "high_school_world_history": 0.2627118644067797, - "virology": 0.24242424242424243, - "high_school_microeconomics": 0.22362869198312235, - "econometrics": 0.2743362831858407, - "college_computer_science": 0.26262626262626265, - "high_school_biology": 0.22006472491909385, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.2704626334519573, - "philosophy": 0.22580645161290322, - "professional_medicine": 0.2066420664206642, - "nutrition": 0.26229508196721313, - "global_facts": 0.24242424242424243, - "machine_learning": 0.2882882882882883, - "security_studies": 0.19672131147540983, - "public_relations": 0.3394495412844037, - "professional_psychology": 0.27495908346972175, - "prehistory": 0.23219814241486067, - "anatomy": 0.291044776119403, - "human_sexuality": 0.2230769230769231, - "college_medicine": 0.1686046511627907, - "high_school_government_and_politics": 0.2708333333333333, - "college_chemistry": 0.16161616161616163, - "logical_fallacies": 0.2345679012345679, - "high_school_geography": 0.27411167512690354, - "elementary_mathematics": 0.22546419098143236, - "human_aging": 0.32432432432432434, - "college_mathematics": 0.25252525252525254, - "high_school_psychology": 0.2536764705882353, - "formal_logic": 0.288, - "high_school_statistics": 0.2186046511627907, - "international_law": 0.36666666666666664, - "high_school_mathematics": 0.25650557620817843, - "high_school_computer_science": 0.24242424242424243, - "conceptual_physics": 0.3162393162393162, - "miscellaneous": 0.3145780051150895, - "high_school_chemistry": 0.23267326732673269, - "marketing": 0.3261802575107296, - "professional_law": 0.26418786692759294, - "management": 0.22549019607843138, - "college_physics": 0.3069306930693069, - "jurisprudence": 0.2803738317757009, - "world_religions": 0.27647058823529413, - "sociology": 0.25, - "us_foreign_policy": 0.3434343434343434, - "high_school_macroeconomics": 0.2776349614395887, - "computer_security": 0.29292929292929293, - "moral_scenarios": 0.22818791946308725, - "moral_disputes": 0.23768115942028986, - "electrical_engineering": 0.2222222222222222, - "astronomy": 0.2119205298013245, - "college_biology": 0.2517482517482518 + "high_school_european_history": 0.6402439024390244, + "business_ethics": 0.6767676767676768, + "clinical_knowledge": 0.5075757575757576, + "medical_genetics": 0.5959595959595959, + "high_school_us_history": 0.5369458128078818, + "high_school_physics": 0.34, + "high_school_world_history": 0.5805084745762712, + "virology": 0.4484848484848485, + "high_school_microeconomics": 0.47257383966244726, + "econometrics": 0.3893805309734513, + "college_computer_science": 0.48484848484848486, + "high_school_biology": 0.5922330097087378, + "abstract_algebra": 0.36363636363636365, + "professional_accounting": 0.3594306049822064, + "philosophy": 0.5516129032258065, + "professional_medicine": 0.4833948339483395, + "nutrition": 0.5245901639344263, + "global_facts": 0.36363636363636365, + "machine_learning": 0.45045045045045046, + "security_studies": 0.4139344262295082, + "public_relations": 0.5688073394495413, + "professional_psychology": 0.45990180032733224, + "prehistory": 0.5510835913312694, + "anatomy": 0.48507462686567165, + "human_sexuality": 0.5615384615384615, + "college_medicine": 0.46511627906976744, + "high_school_government_and_politics": 0.6666666666666666, + "college_chemistry": 0.41414141414141414, + "logical_fallacies": 0.5987654320987654, + "high_school_geography": 0.6091370558375635, + "elementary_mathematics": 0.44562334217506633, + "human_aging": 0.5945945945945946, + "college_mathematics": 0.32323232323232326, + "high_school_psychology": 0.7077205882352942, + "formal_logic": 0.352, + "high_school_statistics": 0.386046511627907, + "international_law": 0.6583333333333333, + "high_school_mathematics": 0.2825278810408922, + "high_school_computer_science": 0.5151515151515151, + "conceptual_physics": 0.48717948717948717, + "miscellaneous": 0.7429667519181585, + "high_school_chemistry": 0.3910891089108911, + "marketing": 0.6909871244635193, + "professional_law": 0.3568166992824527, + "management": 0.6274509803921569, + "college_physics": 0.32673267326732675, + "jurisprudence": 0.5046728971962616, + "world_religions": 0.7, + "sociology": 0.725, + "us_foreign_policy": 0.6565656565656566, + "high_school_macroeconomics": 0.4961439588688946, + "computer_security": 0.6464646464646465, + "moral_scenarios": 0.2751677852348993, + "moral_disputes": 0.527536231884058, + "electrical_engineering": 0.4861111111111111, + "astronomy": 0.5099337748344371, + "college_biology": 0.5244755244755245 } }, "prompt_2": { - "accuracy": 0.2557740436181623, + "accuracy": 0.5031104755094744, "category_acc": { - "high_school_european_history": 0.3048780487804878, - "business_ethics": 0.2727272727272727, - "clinical_knowledge": 0.25757575757575757, - "medical_genetics": 0.2727272727272727, - "high_school_us_history": 0.2413793103448276, - "high_school_physics": 0.2, - "high_school_world_history": 0.2330508474576271, - "virology": 0.2545454545454545, - "high_school_microeconomics": 0.25316455696202533, - "econometrics": 0.23008849557522124, - "college_computer_science": 0.24242424242424243, - "high_school_biology": 0.24271844660194175, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.2491103202846975, - "philosophy": 0.23225806451612904, - "professional_medicine": 0.19557195571955718, - "nutrition": 0.28852459016393445, - "global_facts": 0.24242424242424243, - "machine_learning": 0.3063063063063063, - "security_studies": 0.23770491803278687, - "public_relations": 0.23853211009174313, - "professional_psychology": 0.24222585924713586, - "prehistory": 0.25386996904024767, - "anatomy": 0.29850746268656714, - "human_sexuality": 0.23846153846153847, - "college_medicine": 0.19767441860465115, - "high_school_government_and_politics": 0.2708333333333333, - "college_chemistry": 0.21212121212121213, - "logical_fallacies": 0.29012345679012347, - "high_school_geography": 0.20304568527918782, - "elementary_mathematics": 0.23607427055702918, - "human_aging": 0.32432432432432434, - "college_mathematics": 0.18181818181818182, - "high_school_psychology": 0.21875, - "formal_logic": 0.328, - "high_school_statistics": 0.2186046511627907, - "international_law": 0.30833333333333335, + "high_school_european_history": 0.676829268292683, + "business_ethics": 0.6868686868686869, + "clinical_knowledge": 0.553030303030303, + "medical_genetics": 0.6060606060606061, + "high_school_us_history": 0.5566502463054187, + "high_school_physics": 0.36, + "high_school_world_history": 0.614406779661017, + "virology": 0.4121212121212121, + "high_school_microeconomics": 0.48945147679324896, + "econometrics": 0.35398230088495575, + "college_computer_science": 0.47474747474747475, + "high_school_biology": 0.6019417475728155, + "abstract_algebra": 0.31313131313131315, + "professional_accounting": 0.3914590747330961, + "philosophy": 0.5709677419354838, + "professional_medicine": 0.4907749077490775, + "nutrition": 0.5377049180327869, + "global_facts": 0.1919191919191919, + "machine_learning": 0.43243243243243246, + "security_studies": 0.5163934426229508, + "public_relations": 0.5045871559633027, + "professional_psychology": 0.48117839607201307, + "prehistory": 0.4953560371517028, + "anatomy": 0.48507462686567165, + "human_sexuality": 0.5461538461538461, + "college_medicine": 0.47093023255813954, + "high_school_government_and_politics": 0.734375, + "college_chemistry": 0.37373737373737376, + "logical_fallacies": 0.5925925925925926, + "high_school_geography": 0.6751269035532995, + "elementary_mathematics": 0.4403183023872679, + "human_aging": 0.581081081081081, + "college_mathematics": 0.2828282828282828, + "high_school_psychology": 0.7352941176470589, + "formal_logic": 0.384, + "high_school_statistics": 0.4, + "international_law": 0.6416666666666667, "high_school_mathematics": 0.26765799256505574, - "high_school_computer_science": 0.2727272727272727, - "conceptual_physics": 0.27350427350427353, - "miscellaneous": 0.3209718670076726, - "high_school_chemistry": 0.19306930693069307, - "marketing": 0.33476394849785407, - "professional_law": 0.2446183953033268, - "management": 0.3235294117647059, - "college_physics": 0.2079207920792079, - "jurisprudence": 0.2897196261682243, - "world_religions": 0.3411764705882353, - "sociology": 0.22, - "us_foreign_policy": 0.35353535353535354, - "high_school_macroeconomics": 0.2544987146529563, - "computer_security": 0.3434343434343434, - "moral_scenarios": 0.23378076062639822, - "moral_disputes": 0.2579710144927536, - "electrical_engineering": 0.2638888888888889, - "astronomy": 0.24503311258278146, - "college_biology": 0.3146853146853147 + "high_school_computer_science": 0.5353535353535354, + "conceptual_physics": 0.49572649572649574, + "miscellaneous": 0.7468030690537084, + "high_school_chemistry": 0.4603960396039604, + "marketing": 0.6824034334763949, + "professional_law": 0.35094585779517284, + "management": 0.6274509803921569, + "college_physics": 0.33663366336633666, + "jurisprudence": 0.5046728971962616, + "world_religions": 0.7176470588235294, + "sociology": 0.715, + "us_foreign_policy": 0.696969696969697, + "high_school_macroeconomics": 0.5115681233933161, + "computer_security": 0.6464646464646465, + "moral_scenarios": 0.2684563758389262, + "moral_disputes": 0.5391304347826087, + "electrical_engineering": 0.4791666666666667, + "astronomy": 0.5165562913907285, + "college_biology": 0.48951048951048953 } }, "prompt_3": { - "accuracy": 0.25148373257061135, + "accuracy": 0.5221308544869503, "category_acc": { - "high_school_european_history": 0.24390243902439024, - "business_ethics": 0.24242424242424243, - "clinical_knowledge": 0.21212121212121213, - "medical_genetics": 0.30303030303030304, - "high_school_us_history": 0.2660098522167488, - "high_school_physics": 0.20666666666666667, - "high_school_world_history": 0.2245762711864407, - "virology": 0.24848484848484848, - "high_school_microeconomics": 0.31645569620253167, - "econometrics": 0.23008849557522124, - "college_computer_science": 0.25252525252525254, - "high_school_biology": 0.26537216828478966, - "abstract_algebra": 0.21212121212121213, - "professional_accounting": 0.2526690391459075, - "philosophy": 0.22258064516129034, - "professional_medicine": 0.2029520295202952, - "nutrition": 0.28524590163934427, - "global_facts": 0.24242424242424243, - "machine_learning": 0.27927927927927926, - "security_studies": 0.19672131147540983, - "public_relations": 0.23853211009174313, - "professional_psychology": 0.24877250409165302, - "prehistory": 0.21671826625386997, - "anatomy": 0.2537313432835821, - "human_sexuality": 0.2230769230769231, - "college_medicine": 0.20930232558139536, - "high_school_government_and_politics": 0.3020833333333333, - "college_chemistry": 0.21212121212121213, - "logical_fallacies": 0.2839506172839506, - "high_school_geography": 0.25380710659898476, - "elementary_mathematics": 0.21220159151193635, - "human_aging": 0.31981981981981983, - "college_mathematics": 0.21212121212121213, - "high_school_psychology": 0.21507352941176472, - "formal_logic": 0.296, - "high_school_statistics": 0.2372093023255814, - "international_law": 0.3333333333333333, - "high_school_mathematics": 0.275092936802974, - "high_school_computer_science": 0.23232323232323232, - "conceptual_physics": 0.2948717948717949, - "miscellaneous": 0.30434782608695654, - "high_school_chemistry": 0.22277227722772278, - "marketing": 0.30472103004291845, - "professional_law": 0.24331376386170905, - "management": 0.2549019607843137, - "college_physics": 0.16831683168316833, - "jurisprudence": 0.2523364485981308, - "world_religions": 0.32941176470588235, - "sociology": 0.24, - "us_foreign_policy": 0.3939393939393939, - "high_school_macroeconomics": 0.2365038560411311, - "computer_security": 0.3333333333333333, - "moral_scenarios": 0.2371364653243848, - "moral_disputes": 0.25217391304347825, - "electrical_engineering": 0.25, - "astronomy": 0.2185430463576159, - "college_biology": 0.2517482517482518 + "high_school_european_history": 0.6158536585365854, + "business_ethics": 0.6666666666666666, + "clinical_knowledge": 0.5833333333333334, + "medical_genetics": 0.6464646464646465, + "high_school_us_history": 0.6206896551724138, + "high_school_physics": 0.34, + "high_school_world_history": 0.6864406779661016, + "virology": 0.42424242424242425, + "high_school_microeconomics": 0.5147679324894515, + "econometrics": 0.34513274336283184, + "college_computer_science": 0.43434343434343436, + "high_school_biology": 0.6375404530744336, + "abstract_algebra": 0.31313131313131315, + "professional_accounting": 0.39501779359430605, + "philosophy": 0.6064516129032258, + "professional_medicine": 0.4870848708487085, + "nutrition": 0.5475409836065573, + "global_facts": 0.29292929292929293, + "machine_learning": 0.45045045045045046, + "security_studies": 0.5614754098360656, + "public_relations": 0.5779816513761468, + "professional_psychology": 0.486088379705401, + "prehistory": 0.5851393188854489, + "anatomy": 0.44029850746268656, + "human_sexuality": 0.5923076923076923, + "college_medicine": 0.4941860465116279, + "high_school_government_and_politics": 0.75, + "college_chemistry": 0.36363636363636365, + "logical_fallacies": 0.6172839506172839, + "high_school_geography": 0.7055837563451777, + "elementary_mathematics": 0.4509283819628647, + "human_aging": 0.6126126126126126, + "college_mathematics": 0.29292929292929293, + "high_school_psychology": 0.7610294117647058, + "formal_logic": 0.408, + "high_school_statistics": 0.4046511627906977, + "international_law": 0.6416666666666667, + "high_school_mathematics": 0.2825278810408922, + "high_school_computer_science": 0.5252525252525253, + "conceptual_physics": 0.5128205128205128, + "miscellaneous": 0.7583120204603581, + "high_school_chemistry": 0.41089108910891087, + "marketing": 0.7682403433476395, + "professional_law": 0.3724722765818656, + "management": 0.6470588235294118, + "college_physics": 0.27722772277227725, + "jurisprudence": 0.5700934579439252, + "world_religions": 0.7294117647058823, + "sociology": 0.74, + "us_foreign_policy": 0.7070707070707071, + "high_school_macroeconomics": 0.5295629820051414, + "computer_security": 0.696969696969697, + "moral_scenarios": 0.25727069351230425, + "moral_disputes": 0.6202898550724638, + "electrical_engineering": 0.4791666666666667, + "astronomy": 0.5165562913907285, + "college_biology": 0.5034965034965035 } }, "prompt_4": { - "accuracy": 0.2601358598498391, + "accuracy": 0.5174830175187701, "category_acc": { - "high_school_european_history": 0.27439024390243905, - "business_ethics": 0.2828282828282828, - "clinical_knowledge": 0.25757575757575757, - "medical_genetics": 0.35353535353535354, - "high_school_us_history": 0.28078817733990147, - "high_school_physics": 0.25333333333333335, - "high_school_world_history": 0.288135593220339, - "virology": 0.28484848484848485, - "high_school_microeconomics": 0.26582278481012656, - "econometrics": 0.23008849557522124, - "college_computer_science": 0.20202020202020202, - "high_school_biology": 0.255663430420712, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.28825622775800713, - "philosophy": 0.23870967741935484, - "professional_medicine": 0.2066420664206642, - "nutrition": 0.24918032786885247, - "global_facts": 0.29292929292929293, - "machine_learning": 0.24324324324324326, - "security_studies": 0.20081967213114754, - "public_relations": 0.3119266055045872, - "professional_psychology": 0.25859247135842883, - "prehistory": 0.2476780185758514, - "anatomy": 0.2835820895522388, - "human_sexuality": 0.19230769230769232, - "college_medicine": 0.22674418604651161, - "high_school_government_and_politics": 0.2604166666666667, - "college_chemistry": 0.2222222222222222, - "logical_fallacies": 0.24074074074074073, - "high_school_geography": 0.26903553299492383, - "elementary_mathematics": 0.23076923076923078, - "human_aging": 0.2747747747747748, - "college_mathematics": 0.2222222222222222, - "high_school_psychology": 0.2536764705882353, - "formal_logic": 0.288, - "high_school_statistics": 0.19534883720930232, - "international_law": 0.375, - "high_school_mathematics": 0.24907063197026022, - "high_school_computer_science": 0.25252525252525254, - "conceptual_physics": 0.2948717948717949, - "miscellaneous": 0.3209718670076726, - "high_school_chemistry": 0.2623762376237624, - "marketing": 0.2918454935622318, - "professional_law": 0.25440313111545987, - "management": 0.20588235294117646, - "college_physics": 0.1782178217821782, - "jurisprudence": 0.32710280373831774, - "world_religions": 0.35294117647058826, - "sociology": 0.29, - "us_foreign_policy": 0.3333333333333333, - "high_school_macroeconomics": 0.2622107969151671, - "computer_security": 0.25252525252525254, - "moral_scenarios": 0.24496644295302014, - "moral_disputes": 0.21159420289855072, - "electrical_engineering": 0.2361111111111111, - "astronomy": 0.2251655629139073, - "college_biology": 0.2937062937062937 + "high_school_european_history": 0.6585365853658537, + "business_ethics": 0.6565656565656566, + "clinical_knowledge": 0.5416666666666666, + "medical_genetics": 0.6161616161616161, + "high_school_us_history": 0.6009852216748769, + "high_school_physics": 0.32, + "high_school_world_history": 0.6398305084745762, + "virology": 0.44242424242424244, + "high_school_microeconomics": 0.5147679324894515, + "econometrics": 0.37168141592920356, + "college_computer_science": 0.45454545454545453, + "high_school_biology": 0.6148867313915858, + "abstract_algebra": 0.35353535353535354, + "professional_accounting": 0.4128113879003559, + "philosophy": 0.5870967741935483, + "professional_medicine": 0.4833948339483395, + "nutrition": 0.5737704918032787, + "global_facts": 0.32323232323232326, + "machine_learning": 0.45045045045045046, + "security_studies": 0.5573770491803278, + "public_relations": 0.5871559633027523, + "professional_psychology": 0.4877250409165303, + "prehistory": 0.5727554179566563, + "anatomy": 0.4701492537313433, + "human_sexuality": 0.6076923076923076, + "college_medicine": 0.4941860465116279, + "high_school_government_and_politics": 0.7239583333333334, + "college_chemistry": 0.36363636363636365, + "logical_fallacies": 0.6296296296296297, + "high_school_geography": 0.6598984771573604, + "elementary_mathematics": 0.40583554376657827, + "human_aging": 0.6171171171171171, + "college_mathematics": 0.29292929292929293, + "high_school_psychology": 0.7316176470588235, + "formal_logic": 0.384, + "high_school_statistics": 0.42790697674418604, + "international_law": 0.6416666666666667, + "high_school_mathematics": 0.275092936802974, + "high_school_computer_science": 0.5555555555555556, + "conceptual_physics": 0.49145299145299143, + "miscellaneous": 0.7480818414322251, + "high_school_chemistry": 0.45544554455445546, + "marketing": 0.7381974248927039, + "professional_law": 0.3724722765818656, + "management": 0.6470588235294118, + "college_physics": 0.297029702970297, + "jurisprudence": 0.6074766355140186, + "world_religions": 0.7235294117647059, + "sociology": 0.75, + "us_foreign_policy": 0.7070707070707071, + "high_school_macroeconomics": 0.5038560411311054, + "computer_security": 0.6767676767676768, + "moral_scenarios": 0.27628635346756153, + "moral_disputes": 0.5797101449275363, + "electrical_engineering": 0.4791666666666667, + "astronomy": 0.48344370860927155, + "college_biology": 0.5244755244755245 } }, "prompt_5": { - "accuracy": 0.27636753664640684, + "accuracy": 0.5158383982838756, "category_acc": { - "high_school_european_history": 0.2804878048780488, - "business_ethics": 0.2727272727272727, - "clinical_knowledge": 0.2878787878787879, - "medical_genetics": 0.3434343434343434, - "high_school_us_history": 0.29064039408866993, - "high_school_physics": 0.31333333333333335, - "high_school_world_history": 0.25, - "virology": 0.2909090909090909, - "high_school_microeconomics": 0.31645569620253167, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.23232323232323232, - "high_school_biology": 0.2815533980582524, - "abstract_algebra": 0.21212121212121213, - "professional_accounting": 0.2597864768683274, - "philosophy": 0.2161290322580645, - "professional_medicine": 0.23985239852398524, - "nutrition": 0.26229508196721313, - "global_facts": 0.32323232323232326, - "machine_learning": 0.2882882882882883, - "security_studies": 0.1885245901639344, - "public_relations": 0.3577981651376147, - "professional_psychology": 0.265139116202946, - "prehistory": 0.2724458204334365, - "anatomy": 0.2537313432835821, - "human_sexuality": 0.23076923076923078, - "college_medicine": 0.20930232558139536, - "high_school_government_and_politics": 0.34375, - "college_chemistry": 0.2222222222222222, - "logical_fallacies": 0.24074074074074073, - "high_school_geography": 0.3299492385786802, - "elementary_mathematics": 0.2838196286472148, - "human_aging": 0.3153153153153153, - "college_mathematics": 0.2727272727272727, - "high_school_psychology": 0.2647058823529412, - "formal_logic": 0.272, - "high_school_statistics": 0.26976744186046514, - "international_law": 0.3333333333333333, - "high_school_mathematics": 0.26394052044609667, - "high_school_computer_science": 0.21212121212121213, - "conceptual_physics": 0.3418803418803419, - "miscellaneous": 0.31585677749360613, - "high_school_chemistry": 0.27722772277227725, - "marketing": 0.38197424892703863, - "professional_law": 0.25505544683626874, - "management": 0.21568627450980393, - "college_physics": 0.19801980198019803, - "jurisprudence": 0.29906542056074764, - "world_religions": 0.31176470588235294, - "sociology": 0.26, - "us_foreign_policy": 0.3434343434343434, - "high_school_macroeconomics": 0.2647814910025707, - "computer_security": 0.35353535353535354, - "moral_scenarios": 0.2695749440715884, - "moral_disputes": 0.26666666666666666, - "electrical_engineering": 0.2986111111111111, - "astronomy": 0.2980132450331126, - "college_biology": 0.3356643356643357 + "high_school_european_history": 0.6402439024390244, + "business_ethics": 0.6363636363636364, + "clinical_knowledge": 0.5643939393939394, + "medical_genetics": 0.6464646464646465, + "high_school_us_history": 0.625615763546798, + "high_school_physics": 0.32, + "high_school_world_history": 0.635593220338983, + "virology": 0.4121212121212121, + "high_school_microeconomics": 0.5063291139240507, + "econometrics": 0.35398230088495575, + "college_computer_science": 0.41414141414141414, + "high_school_biology": 0.6116504854368932, + "abstract_algebra": 0.29292929292929293, + "professional_accounting": 0.4092526690391459, + "philosophy": 0.5870967741935483, + "professional_medicine": 0.4907749077490775, + "nutrition": 0.5672131147540984, + "global_facts": 0.3333333333333333, + "machine_learning": 0.4864864864864865, + "security_studies": 0.5655737704918032, + "public_relations": 0.5779816513761468, + "professional_psychology": 0.4762684124386252, + "prehistory": 0.56656346749226, + "anatomy": 0.48507462686567165, + "human_sexuality": 0.6230769230769231, + "college_medicine": 0.5348837209302325, + "high_school_government_and_politics": 0.6927083333333334, + "college_chemistry": 0.36363636363636365, + "logical_fallacies": 0.6172839506172839, + "high_school_geography": 0.6903553299492385, + "elementary_mathematics": 0.44297082228116713, + "human_aging": 0.6261261261261262, + "college_mathematics": 0.25252525252525254, + "high_school_psychology": 0.7261029411764706, + "formal_logic": 0.4, + "high_school_statistics": 0.40930232558139534, + "international_law": 0.6583333333333333, + "high_school_mathematics": 0.26022304832713755, + "high_school_computer_science": 0.5252525252525253, + "conceptual_physics": 0.46153846153846156, + "miscellaneous": 0.7442455242966752, + "high_school_chemistry": 0.4405940594059406, + "marketing": 0.7553648068669528, + "professional_law": 0.36268754076973253, + "management": 0.6568627450980392, + "college_physics": 0.25742574257425743, + "jurisprudence": 0.5887850467289719, + "world_religions": 0.7, + "sociology": 0.735, + "us_foreign_policy": 0.7171717171717171, + "high_school_macroeconomics": 0.5115681233933161, + "computer_security": 0.6666666666666666, + "moral_scenarios": 0.28747203579418346, + "moral_disputes": 0.5768115942028985, + "electrical_engineering": 0.4930555555555556, + "astronomy": 0.5033112582781457, + "college_biology": 0.5314685314685315 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.2511144130757801 + "accuracy": 0.5505200594353641 }, "prompt_2": { - "accuracy": 0.2511144130757801 + "accuracy": 0.5586924219910847 }, "prompt_3": { - "accuracy": 0.23699851411589895 + "accuracy": 0.5505200594353641 }, "prompt_4": { - "accuracy": 0.25185735512630014 + "accuracy": 0.5416047548291233 }, "prompt_5": { - "accuracy": 0.24294205052005943 + "accuracy": 0.537890044576523 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.25529265255292655, + "accuracy": 0.5510585305105853, "category_acc": { - "computer_network": 0.25, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.16666666666666666, - "college_physics": 0.25, - "college_chemistry": 0.3448275862068966, + "computer_network": 0.5, + "operating_system": 0.5416666666666666, + "computer_architecture": 0.5384615384615384, + "college_programming": 0.47619047619047616, + "college_physics": 0.4583333333333333, + "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.21739130434782608, + "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.09523809523809523, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.16666666666666666, - "high_school_biology": 0.16666666666666666, + "electrical_engineer": 0.42857142857142855, + "metrology_engineer": 0.5862068965517241, + "high_school_mathematics": 0.2608695652173913, + "high_school_physics": 0.4583333333333333, + "high_school_chemistry": 0.5416666666666666, + "high_school_biology": 0.5, "middle_school_mathematics": 0.4166666666666667, - "middle_school_biology": 0.23076923076923078, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.08, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.26666666666666666, - "business_administration": 0.2894736842105263, - "marxism": 0.375, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.20408163265306123, - "high_school_politics": 0.25, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.34615384615384615, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.07142857142857142, - "ideological_and_moral_cultivation": 0.20833333333333334, - "logic": 0.25925925925925924, - "law": 0.10344827586206896, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.17647058823529413, - "legal_professional": 0.14285714285714285, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.32, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.1346153846153846, - "sports_science": 0.3333333333333333, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.375, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.24074074074074073, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.2037037037037037, - "physician": 0.25925925925925924 + "middle_school_biology": 0.8076923076923077, + "middle_school_physics": 0.7916666666666666, + "middle_school_chemistry": 0.88, + "veterinary_medicine": 0.4642857142857143, + "college_economics": 0.43333333333333335, + "business_administration": 0.47368421052631576, + "marxism": 0.75, + "mao_zedong_thought": 0.7241379310344828, + "education_science": 0.7647058823529411, + "teacher_qualification": 0.7755102040816326, + "high_school_politics": 0.875, + "high_school_geography": 0.7083333333333334, + "middle_school_politics": 0.8076923076923077, + "middle_school_geography": 0.8823529411764706, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.875, + "logic": 0.5185185185185185, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.5357142857142857, + "art_studies": 0.5263157894736842, + "professional_tour_guide": 0.6470588235294118, + "legal_professional": 0.4642857142857143, + "high_school_chinese": 0.5416666666666666, + "high_school_history": 0.8, + "middle_school_history": 0.7777777777777778, + "civil_servant": 0.4230769230769231, + "sports_science": 0.7083333333333334, + "plant_protection": 0.5925925925925926, + "basic_medicine": 0.5, + "clinical_medicine": 0.5555555555555556, + "urban_and_rural_planner": 0.5098039215686274, + "accountant": 0.4444444444444444, + "fire_engineer": 0.4444444444444444, + "environmental_impact_assessment_engineer": 0.5555555555555556, + "tax_accountant": 0.48148148148148145, + "physician": 0.5370370370370371 } }, "prompt_2": { - "accuracy": 0.23785803237858033, + "accuracy": 0.5566625155666252, "category_acc": { - "computer_network": 0.25, - "operating_system": 0.08333333333333333, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.23809523809523808, - "college_physics": 0.25, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.21739130434782608, + "computer_network": 0.4583333333333333, + "operating_system": 0.5416666666666666, + "computer_architecture": 0.46153846153846156, + "college_programming": 0.47619047619047616, + "college_physics": 0.375, + "college_chemistry": 0.3793103448275862, + "advanced_mathematics": 0.4166666666666667, + "probability_and_statistics": 0.43478260869565216, "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.12, - "veterinary_medicine": 0.25, - "college_economics": 0.26666666666666666, - "business_administration": 0.23684210526315788, - "marxism": 0.20833333333333334, - "mao_zedong_thought": 0.3448275862068966, - "education_science": 0.35294117647058826, - "teacher_qualification": 0.20408163265306123, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.25, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.18518518518518517, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.25, - "art_studies": 0.23684210526315788, - "professional_tour_guide": 0.20588235294117646, - "legal_professional": 0.14285714285714285, - "high_school_chinese": 0.25, - "high_school_history": 0.36, - "middle_school_history": 0.25925925925925924, - "civil_servant": 0.21153846153846154, - "sports_science": 0.125, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.18518518518518517, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.2037037037037037, - "physician": 0.25925925925925924 + "electrical_engineer": 0.42857142857142855, + "metrology_engineer": 0.5517241379310345, + "high_school_mathematics": 0.21739130434782608, + "high_school_physics": 0.375, + "high_school_chemistry": 0.5416666666666666, + "high_school_biology": 0.5416666666666666, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.8076923076923077, + "middle_school_physics": 0.8333333333333334, + "middle_school_chemistry": 0.92, + "veterinary_medicine": 0.5, + "college_economics": 0.4666666666666667, + "business_administration": 0.47368421052631576, + "marxism": 0.875, + "mao_zedong_thought": 0.7241379310344828, + "education_science": 0.7058823529411765, + "teacher_qualification": 0.7551020408163265, + "high_school_politics": 0.8333333333333334, + "high_school_geography": 0.75, + "middle_school_politics": 0.9230769230769231, + "middle_school_geography": 0.7647058823529411, + "modern_chinese_history": 0.5, + "ideological_and_moral_cultivation": 0.7916666666666666, + "logic": 0.48148148148148145, + "law": 0.4482758620689655, + "chinese_language_and_literature": 0.6071428571428571, + "art_studies": 0.5263157894736842, + "professional_tour_guide": 0.6470588235294118, + "legal_professional": 0.4642857142857143, + "high_school_chinese": 0.6666666666666666, + "high_school_history": 0.76, + "middle_school_history": 0.7777777777777778, + "civil_servant": 0.4423076923076923, + "sports_science": 0.5833333333333334, + "plant_protection": 0.7037037037037037, + "basic_medicine": 0.5416666666666666, + "clinical_medicine": 0.5185185185185185, + "urban_and_rural_planner": 0.5098039215686274, + "accountant": 0.4444444444444444, + "fire_engineer": 0.4166666666666667, + "environmental_impact_assessment_engineer": 0.5833333333333334, + "tax_accountant": 0.48148148148148145, + "physician": 0.5370370370370371 } }, "prompt_3": { - "accuracy": 0.23785803237858033, + "accuracy": 0.5554171855541719, "category_acc": { - "computer_network": 0.125, - "operating_system": 0.125, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.21428571428571427, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.08695652173913043, - "high_school_physics": 0.25, - "high_school_chemistry": 0.25, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.16666666666666666, - "middle_school_chemistry": 0.08, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.2833333333333333, - "business_administration": 0.21052631578947367, - "marxism": 0.125, - "mao_zedong_thought": 0.3448275862068966, - "education_science": 0.35294117647058826, - "teacher_qualification": 0.24489795918367346, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.2222222222222222, - "law": 0.1724137931034483, - "chinese_language_and_literature": 0.17857142857142858, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.23529411764705882, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.4, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.2692307692307692, - "sports_science": 0.20833333333333334, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.125, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.23529411764705882, - "accountant": 0.2222222222222222, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.16666666666666666, - "tax_accountant": 0.24074074074074073, - "physician": 0.2037037037037037 + "computer_network": 0.5416666666666666, + "operating_system": 0.5416666666666666, + "computer_architecture": 0.46153846153846156, + "college_programming": 0.47619047619047616, + "college_physics": 0.4166666666666667, + "college_chemistry": 0.3448275862068966, + "advanced_mathematics": 0.375, + "probability_and_statistics": 0.43478260869565216, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.4523809523809524, + "metrology_engineer": 0.5517241379310345, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.4166666666666667, + "high_school_chemistry": 0.5, + "high_school_biology": 0.5416666666666666, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.8076923076923077, + "middle_school_physics": 0.8333333333333334, + "middle_school_chemistry": 0.88, + "veterinary_medicine": 0.5, + "college_economics": 0.4166666666666667, + "business_administration": 0.5, + "marxism": 0.875, + "mao_zedong_thought": 0.6896551724137931, + "education_science": 0.7352941176470589, + "teacher_qualification": 0.7346938775510204, + "high_school_politics": 0.8333333333333334, + "high_school_geography": 0.7083333333333334, + "middle_school_politics": 0.8846153846153846, + "middle_school_geography": 0.7647058823529411, + "modern_chinese_history": 0.5357142857142857, + "ideological_and_moral_cultivation": 0.7916666666666666, + "logic": 0.4444444444444444, + "law": 0.4482758620689655, + "chinese_language_and_literature": 0.5714285714285714, + "art_studies": 0.5263157894736842, + "professional_tour_guide": 0.5882352941176471, + "legal_professional": 0.5714285714285714, + "high_school_chinese": 0.625, + "high_school_history": 0.68, + "middle_school_history": 0.7777777777777778, + "civil_servant": 0.46153846153846156, + "sports_science": 0.5833333333333334, + "plant_protection": 0.6666666666666666, + "basic_medicine": 0.5833333333333334, + "clinical_medicine": 0.48148148148148145, + "urban_and_rural_planner": 0.5490196078431373, + "accountant": 0.46296296296296297, + "fire_engineer": 0.4444444444444444, + "environmental_impact_assessment_engineer": 0.5555555555555556, + "tax_accountant": 0.42592592592592593, + "physician": 0.5740740740740741 } }, "prompt_4": { - "accuracy": 0.26089663760896636, + "accuracy": 0.5354919053549191, "category_acc": { - "computer_network": 0.25, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.14285714285714285, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.09523809523809523, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.16666666666666666, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.23076923076923078, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.35, - "business_administration": 0.2894736842105263, - "marxism": 0.375, - "mao_zedong_thought": 0.3448275862068966, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.16326530612244897, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.0, - "modern_chinese_history": 0.14285714285714285, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.25925925925925924, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.20588235294117646, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.32, - "middle_school_history": 0.4074074074074074, - "civil_servant": 0.21153846153846154, - "sports_science": 0.2916666666666667, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.25925925925925924, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.2962962962962963, - "physician": 0.2962962962962963 + "computer_network": 0.4583333333333333, + "operating_system": 0.4583333333333333, + "computer_architecture": 0.5384615384615384, + "college_programming": 0.42857142857142855, + "college_physics": 0.4583333333333333, + "college_chemistry": 0.1724137931034483, + "advanced_mathematics": 0.3333333333333333, + "probability_and_statistics": 0.4782608695652174, + "discrete_mathematics": 0.14285714285714285, + "electrical_engineer": 0.42857142857142855, + "metrology_engineer": 0.5517241379310345, + "high_school_mathematics": 0.2608695652173913, + "high_school_physics": 0.5416666666666666, + "high_school_chemistry": 0.5, + "high_school_biology": 0.4583333333333333, + "middle_school_mathematics": 0.3333333333333333, + "middle_school_biology": 0.8076923076923077, + "middle_school_physics": 0.8333333333333334, + "middle_school_chemistry": 0.76, + "veterinary_medicine": 0.4642857142857143, + "college_economics": 0.5, + "business_administration": 0.5, + "marxism": 0.875, + "mao_zedong_thought": 0.7241379310344828, + "education_science": 0.7058823529411765, + "teacher_qualification": 0.7755102040816326, + "high_school_politics": 0.8333333333333334, + "high_school_geography": 0.7083333333333334, + "middle_school_politics": 0.9230769230769231, + "middle_school_geography": 0.7647058823529411, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.875, + "logic": 0.4444444444444444, + "law": 0.41379310344827586, + "chinese_language_and_literature": 0.5, + "art_studies": 0.5526315789473685, + "professional_tour_guide": 0.6470588235294118, + "legal_professional": 0.42857142857142855, + "high_school_chinese": 0.4583333333333333, + "high_school_history": 0.76, + "middle_school_history": 0.7777777777777778, + "civil_servant": 0.36538461538461536, + "sports_science": 0.5833333333333334, + "plant_protection": 0.5555555555555556, + "basic_medicine": 0.5416666666666666, + "clinical_medicine": 0.5185185185185185, + "urban_and_rural_planner": 0.47058823529411764, + "accountant": 0.42592592592592593, + "fire_engineer": 0.3888888888888889, + "environmental_impact_assessment_engineer": 0.5833333333333334, + "tax_accountant": 0.4074074074074074, + "physician": 0.5185185185185185 } }, "prompt_5": { - "accuracy": 0.24968866749688667, + "accuracy": 0.5473225404732254, "category_acc": { - "computer_network": 0.25, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.14285714285714285, - "college_physics": 0.375, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.13043478260869565, - "discrete_mathematics": 0.047619047619047616, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.1724137931034483, + "computer_network": 0.5, + "operating_system": 0.4583333333333333, + "computer_architecture": 0.5, + "college_programming": 0.5, + "college_physics": 0.3333333333333333, + "college_chemistry": 0.3448275862068966, + "advanced_mathematics": 0.375, + "probability_and_statistics": 0.34782608695652173, + "discrete_mathematics": 0.19047619047619047, + "electrical_engineer": 0.38095238095238093, + "metrology_engineer": 0.5862068965517241, "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.125, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.12, - "veterinary_medicine": 0.42857142857142855, - "college_economics": 0.3, - "business_administration": 0.18421052631578946, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.29411764705882354, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.25, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.11764705882352941, - "modern_chinese_history": 0.25, - "ideological_and_moral_cultivation": 0.20833333333333334, - "logic": 0.25925925925925924, - "law": 0.13793103448275862, - "chinese_language_and_literature": 0.17857142857142858, + "high_school_physics": 0.5, + "high_school_chemistry": 0.5, + "high_school_biology": 0.5416666666666666, + "middle_school_mathematics": 0.3333333333333333, + "middle_school_biology": 0.7307692307692307, + "middle_school_physics": 0.75, + "middle_school_chemistry": 0.8, + "veterinary_medicine": 0.5, + "college_economics": 0.45, + "business_administration": 0.5526315789473685, + "marxism": 0.8333333333333334, + "mao_zedong_thought": 0.7586206896551724, + "education_science": 0.6764705882352942, + "teacher_qualification": 0.7551020408163265, + "high_school_politics": 0.9166666666666666, + "high_school_geography": 0.6666666666666666, + "middle_school_politics": 0.9230769230769231, + "middle_school_geography": 0.7647058823529411, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.8333333333333334, + "logic": 0.4444444444444444, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.5357142857142857, "art_studies": 0.5263157894736842, - "professional_tour_guide": 0.23529411764705882, - "legal_professional": 0.10714285714285714, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.36, - "middle_school_history": 0.25925925925925924, - "civil_servant": 0.19230769230769232, - "sports_science": 0.08333333333333333, - "plant_protection": 0.37037037037037035, - "basic_medicine": 0.25, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.25925925925925924, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.2222222222222222, - "tax_accountant": 0.2222222222222222, - "physician": 0.2777777777777778 + "professional_tour_guide": 0.6470588235294118, + "legal_professional": 0.4642857142857143, + "high_school_chinese": 0.6666666666666666, + "high_school_history": 0.84, + "middle_school_history": 0.8148148148148148, + "civil_servant": 0.46153846153846156, + "sports_science": 0.6666666666666666, + "plant_protection": 0.5555555555555556, + "basic_medicine": 0.5833333333333334, + "clinical_medicine": 0.4074074074074074, + "urban_and_rural_planner": 0.5882352941176471, + "accountant": 0.42592592592592593, + "fire_engineer": 0.3611111111111111, + "environmental_impact_assessment_engineer": 0.5555555555555556, + "tax_accountant": 0.4074074074074074, + "physician": 0.5555555555555556 } } }, "cmmlu": { "prompt_1": { - "accuracy": 0.3225806451612903 + "accuracy": 0.5483870967741935 }, "prompt_2": { - "accuracy": 0.32616487455197135 + "accuracy": 0.5698924731182796 }, "prompt_3": { - "accuracy": 0.2903225806451613 + "accuracy": 0.5412186379928315 }, "prompt_4": { - "accuracy": 0.3225806451612903 + "accuracy": 0.5663082437275986 }, "prompt_5": { - "accuracy": 0.2867383512544803 + "accuracy": 0.5627240143369175 } }, "cmmlu_full": { "prompt_1": { - "accuracy": 0.2712830253842169, + "accuracy": 0.5473147988257642, "category_acc": { - "agronomy": 0.28402366863905326, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.21951219512195122, - "arts": 0.25, - "astronomy": 0.2545454545454545, - "business_ethics": 0.27751196172248804, - "chinese_civil_service_exam": 0.29375, - "chinese_driving_rule": 0.3282442748091603, - "chinese_food_culture": 0.27205882352941174, - "chinese_foreign_policy": 0.32710280373831774, - "chinese_history": 0.24148606811145512, - "chinese_literature": 0.30392156862745096, - "chinese_teacher_qualification": 0.3016759776536313, - "clinical_knowledge": 0.31223628691983124, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.308411214953271, - "college_engineering_hydrology": 0.2830188679245283, - "college_law": 0.2222222222222222, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.32075471698113206, - "college_medicine": 0.2454212454212454, - "computer_science": 0.24509803921568626, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.22448979591836735, - "construction_project_management": 0.23741007194244604, - "economics": 0.2641509433962264, - "education": 0.27607361963190186, - "electrical_engineering": 0.31976744186046513, - "elementary_chinese": 0.25, - "elementary_commonsense": 0.2676767676767677, - "elementary_information_and_technology": 0.23949579831932774, - "elementary_mathematics": 0.2608695652173913, - "ethnology": 0.2740740740740741, - "food_science": 0.40559440559440557, - "genetics": 0.2556818181818182, - "global_facts": 0.2751677852348993, - "high_school_biology": 0.2603550295857988, - "high_school_chemistry": 0.3333333333333333, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.2909090909090909, - "high_school_politics": 0.23076923076923078, - "human_sexuality": 0.30158730158730157, - "international_law": 0.2702702702702703, - "journalism": 0.29069767441860467, - "jurisprudence": 0.2725060827250608, - "legal_and_moral_basis": 0.24766355140186916, - "logical": 0.2926829268292683, - "machine_learning": 0.2459016393442623, - "management": 0.24761904761904763, - "marketing": 0.2611111111111111, - "marxist_theory": 0.30158730158730157, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.2896551724137931, - "philosophy": 0.3238095238095238, - "professional_accounting": 0.24571428571428572, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.2579787234042553, - "professional_psychology": 0.27155172413793105, - "public_relations": 0.27586206896551724, - "security_study": 0.3333333333333333, - "sociology": 0.28761061946902655, - "sports_science": 0.26666666666666666, - "traditional_chinese_medicine": 0.2702702702702703, - "virology": 0.33136094674556216, - "world_history": 0.19875776397515527, - "world_religions": 0.2625 + "agronomy": 0.5266272189349113, + "anatomy": 0.44594594594594594, + "ancient_chinese": 0.35365853658536583, + "arts": 0.7625, + "astronomy": 0.3393939393939394, + "business_ethics": 0.5789473684210527, + "chinese_civil_service_exam": 0.55, + "chinese_driving_rule": 0.7709923664122137, + "chinese_food_culture": 0.5441176470588235, + "chinese_foreign_policy": 0.6074766355140186, + "chinese_history": 0.6749226006191951, + "chinese_literature": 0.43137254901960786, + "chinese_teacher_qualification": 0.7318435754189944, + "clinical_knowledge": 0.4936708860759494, + "college_actuarial_science": 0.27358490566037735, + "college_education": 0.6542056074766355, + "college_engineering_hydrology": 0.5094339622641509, + "college_law": 0.46296296296296297, + "college_mathematics": 0.34285714285714286, + "college_medical_statistics": 0.39622641509433965, + "college_medicine": 0.5018315018315018, + "computer_science": 0.5588235294117647, + "computer_security": 0.6842105263157895, + "conceptual_physics": 0.6462585034013606, + "construction_project_management": 0.4460431654676259, + "economics": 0.5786163522012578, + "education": 0.656441717791411, + "electrical_engineering": 0.4941860465116279, + "elementary_chinese": 0.5317460317460317, + "elementary_commonsense": 0.5656565656565656, + "elementary_information_and_technology": 0.7310924369747899, + "elementary_mathematics": 0.32608695652173914, + "ethnology": 0.5851851851851851, + "food_science": 0.5454545454545454, + "genetics": 0.44886363636363635, + "global_facts": 0.5838926174496645, + "high_school_biology": 0.5088757396449705, + "high_school_chemistry": 0.5075757575757576, + "high_school_geography": 0.5677966101694916, + "high_school_mathematics": 0.2804878048780488, + "high_school_physics": 0.4090909090909091, + "high_school_politics": 0.5524475524475524, + "human_sexuality": 0.5476190476190477, + "international_law": 0.4648648648648649, + "journalism": 0.47093023255813954, + "jurisprudence": 0.537712895377129, + "legal_and_moral_basis": 0.8598130841121495, + "logical": 0.4715447154471545, + "machine_learning": 0.4672131147540984, + "management": 0.6619047619047619, + "marketing": 0.6333333333333333, + "marxist_theory": 0.6613756613756614, + "modern_chinese": 0.39655172413793105, + "nutrition": 0.593103448275862, + "philosophy": 0.638095238095238, + "professional_accounting": 0.6, + "professional_law": 0.5071090047393365, + "professional_medicine": 0.40425531914893614, + "professional_psychology": 0.6293103448275862, + "public_relations": 0.5977011494252874, + "security_study": 0.6666666666666666, + "sociology": 0.5796460176991151, + "sports_science": 0.5151515151515151, + "traditional_chinese_medicine": 0.5243243243243243, + "virology": 0.5325443786982249, + "world_history": 0.6335403726708074, + "world_religions": 0.65625 } }, "prompt_2": { - "accuracy": 0.2640303919875669, + "accuracy": 0.5558625453289587, "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.22972972972972974, - "ancient_chinese": 0.25, - "arts": 0.33125, - "astronomy": 0.23030303030303031, - "business_ethics": 0.28708133971291866, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.2595419847328244, - "chinese_food_culture": 0.25735294117647056, - "chinese_foreign_policy": 0.2523364485981308, - "chinese_history": 0.26006191950464397, - "chinese_literature": 0.27450980392156865, - "chinese_teacher_qualification": 0.2569832402234637, - "clinical_knowledge": 0.23628691983122363, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.2616822429906542, - "college_engineering_hydrology": 0.25471698113207547, - "college_law": 0.28703703703703703, - "college_mathematics": 0.21904761904761905, - "college_medical_statistics": 0.19811320754716982, - "college_medicine": 0.25274725274725274, - "computer_science": 0.2647058823529412, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.2857142857142857, - "construction_project_management": 0.2805755395683453, - "economics": 0.23270440251572327, - "education": 0.3067484662576687, - "electrical_engineering": 0.29651162790697677, - "elementary_chinese": 0.24206349206349206, - "elementary_commonsense": 0.2828282828282828, - "elementary_information_and_technology": 0.29411764705882354, - "elementary_mathematics": 0.28695652173913044, - "ethnology": 0.32592592592592595, - "food_science": 0.25874125874125875, - "genetics": 0.26704545454545453, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.21893491124260356, - "high_school_chemistry": 0.2803030303030303, - "high_school_geography": 0.2288135593220339, - "high_school_mathematics": 0.3048780487804878, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.26573426573426573, - "human_sexuality": 0.24603174603174602, - "international_law": 0.23783783783783785, - "journalism": 0.26744186046511625, - "jurisprudence": 0.24330900243309003, - "legal_and_moral_basis": 0.2850467289719626, - "logical": 0.22764227642276422, - "machine_learning": 0.23770491803278687, - "management": 0.20952380952380953, - "marketing": 0.25, - "marxist_theory": 0.2804232804232804, - "modern_chinese": 0.1810344827586207, - "nutrition": 0.19310344827586207, - "philosophy": 0.2761904761904762, - "professional_accounting": 0.2857142857142857, - "professional_law": 0.26540284360189575, - "professional_medicine": 0.2526595744680851, - "professional_psychology": 0.28448275862068967, - "public_relations": 0.3103448275862069, - "security_study": 0.31851851851851853, - "sociology": 0.28761061946902655, - "sports_science": 0.3333333333333333, - "traditional_chinese_medicine": 0.32432432432432434, - "virology": 0.2781065088757396, - "world_history": 0.2732919254658385, - "world_religions": 0.2625 + "agronomy": 0.5207100591715976, + "anatomy": 0.43243243243243246, + "ancient_chinese": 0.3231707317073171, + "arts": 0.76875, + "astronomy": 0.4, + "business_ethics": 0.5741626794258373, + "chinese_civil_service_exam": 0.55, + "chinese_driving_rule": 0.8015267175572519, + "chinese_food_culture": 0.5735294117647058, + "chinese_foreign_policy": 0.6074766355140186, + "chinese_history": 0.6811145510835913, + "chinese_literature": 0.4215686274509804, + "chinese_teacher_qualification": 0.7374301675977654, + "clinical_knowledge": 0.5147679324894515, + "college_actuarial_science": 0.36792452830188677, + "college_education": 0.6635514018691588, + "college_engineering_hydrology": 0.5188679245283019, + "college_law": 0.49074074074074076, + "college_mathematics": 0.3333333333333333, + "college_medical_statistics": 0.4056603773584906, + "college_medicine": 0.5201465201465202, + "computer_science": 0.5931372549019608, + "computer_security": 0.7017543859649122, + "conceptual_physics": 0.6190476190476191, + "construction_project_management": 0.45323741007194246, + "economics": 0.5911949685534591, + "education": 0.6073619631901841, + "electrical_engineering": 0.48255813953488375, + "elementary_chinese": 0.5515873015873016, + "elementary_commonsense": 0.5707070707070707, + "elementary_information_and_technology": 0.7310924369747899, + "elementary_mathematics": 0.34782608695652173, + "ethnology": 0.5481481481481482, + "food_science": 0.5314685314685315, + "genetics": 0.4318181818181818, + "global_facts": 0.5906040268456376, + "high_school_biology": 0.514792899408284, + "high_school_chemistry": 0.5151515151515151, + "high_school_geography": 0.576271186440678, + "high_school_mathematics": 0.29878048780487804, + "high_school_physics": 0.4636363636363636, + "high_school_politics": 0.5454545454545454, + "human_sexuality": 0.5238095238095238, + "international_law": 0.4810810810810811, + "journalism": 0.5406976744186046, + "jurisprudence": 0.5401459854014599, + "legal_and_moral_basis": 0.8504672897196262, + "logical": 0.4715447154471545, + "machine_learning": 0.47540983606557374, + "management": 0.6761904761904762, + "marketing": 0.65, + "marxist_theory": 0.6666666666666666, + "modern_chinese": 0.4224137931034483, + "nutrition": 0.6344827586206897, + "philosophy": 0.6476190476190476, + "professional_accounting": 0.6057142857142858, + "professional_law": 0.46445497630331756, + "professional_medicine": 0.425531914893617, + "professional_psychology": 0.6508620689655172, + "public_relations": 0.5804597701149425, + "security_study": 0.674074074074074, + "sociology": 0.5752212389380531, + "sports_science": 0.5454545454545454, + "traditional_chinese_medicine": 0.5513513513513514, + "virology": 0.5502958579881657, + "world_history": 0.6770186335403726, + "world_religions": 0.65625 } }, "prompt_3": { - "accuracy": 0.26368502849248837, + "accuracy": 0.5527542738732516, "category_acc": { - "agronomy": 0.28994082840236685, - "anatomy": 0.2905405405405405, - "ancient_chinese": 0.20121951219512196, - "arts": 0.38125, - "astronomy": 0.26666666666666666, - "business_ethics": 0.2583732057416268, - "chinese_civil_service_exam": 0.25, - "chinese_driving_rule": 0.21374045801526717, - "chinese_food_culture": 0.29411764705882354, - "chinese_foreign_policy": 0.205607476635514, - "chinese_history": 0.25386996904024767, - "chinese_literature": 0.25980392156862747, - "chinese_teacher_qualification": 0.2905027932960894, - "clinical_knowledge": 0.2616033755274262, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.2803738317757009, - "college_engineering_hydrology": 0.33962264150943394, - "college_law": 0.2777777777777778, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.19811320754716982, - "college_medicine": 0.23809523809523808, - "computer_science": 0.2696078431372549, - "computer_security": 0.23976608187134502, - "conceptual_physics": 0.29931972789115646, - "construction_project_management": 0.2446043165467626, - "economics": 0.2893081761006289, - "education": 0.3312883435582822, - "electrical_engineering": 0.28488372093023256, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.20707070707070707, - "elementary_information_and_technology": 0.27310924369747897, - "elementary_mathematics": 0.2608695652173913, - "ethnology": 0.31851851851851853, - "food_science": 0.25874125874125875, - "genetics": 0.23863636363636365, - "global_facts": 0.2550335570469799, - "high_school_biology": 0.21893491124260356, - "high_school_chemistry": 0.3106060606060606, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.25609756097560976, - "high_school_physics": 0.3, - "high_school_politics": 0.23776223776223776, - "human_sexuality": 0.23015873015873015, - "international_law": 0.3027027027027027, - "journalism": 0.23837209302325582, - "jurisprudence": 0.23357664233576642, - "legal_and_moral_basis": 0.29439252336448596, - "logical": 0.25203252032520324, - "machine_learning": 0.22131147540983606, - "management": 0.2523809523809524, - "marketing": 0.21666666666666667, - "marxist_theory": 0.2751322751322751, - "modern_chinese": 0.23275862068965517, - "nutrition": 0.21379310344827587, - "philosophy": 0.29523809523809524, - "professional_accounting": 0.2571428571428571, - "professional_law": 0.21800947867298578, - "professional_medicine": 0.2845744680851064, - "professional_psychology": 0.2543103448275862, - "public_relations": 0.29310344827586204, - "security_study": 0.2962962962962963, - "sociology": 0.28761061946902655, - "sports_science": 0.2787878787878788, - "traditional_chinese_medicine": 0.2864864864864865, - "virology": 0.27218934911242604, - "world_history": 0.2670807453416149, - "world_religions": 0.24375 + "agronomy": 0.5443786982248521, + "anatomy": 0.44594594594594594, + "ancient_chinese": 0.31097560975609756, + "arts": 0.75625, + "astronomy": 0.4, + "business_ethics": 0.569377990430622, + "chinese_civil_service_exam": 0.5375, + "chinese_driving_rule": 0.7480916030534351, + "chinese_food_culture": 0.5441176470588235, + "chinese_foreign_policy": 0.6074766355140186, + "chinese_history": 0.6842105263157895, + "chinese_literature": 0.4264705882352941, + "chinese_teacher_qualification": 0.7430167597765364, + "clinical_knowledge": 0.5021097046413502, + "college_actuarial_science": 0.29245283018867924, + "college_education": 0.6355140186915887, + "college_engineering_hydrology": 0.5, + "college_law": 0.4537037037037037, + "college_mathematics": 0.29523809523809524, + "college_medical_statistics": 0.44339622641509435, + "college_medicine": 0.5054945054945055, + "computer_science": 0.5637254901960784, + "computer_security": 0.7017543859649122, + "conceptual_physics": 0.6258503401360545, + "construction_project_management": 0.4748201438848921, + "economics": 0.5911949685534591, + "education": 0.6134969325153374, + "electrical_engineering": 0.5232558139534884, + "elementary_chinese": 0.5476190476190477, + "elementary_commonsense": 0.5555555555555556, + "elementary_information_and_technology": 0.7352941176470589, + "elementary_mathematics": 0.3826086956521739, + "ethnology": 0.5925925925925926, + "food_science": 0.5944055944055944, + "genetics": 0.42613636363636365, + "global_facts": 0.5838926174496645, + "high_school_biology": 0.4970414201183432, + "high_school_chemistry": 0.553030303030303, + "high_school_geography": 0.4915254237288136, + "high_school_mathematics": 0.32926829268292684, + "high_school_physics": 0.41818181818181815, + "high_school_politics": 0.5874125874125874, + "human_sexuality": 0.5555555555555556, + "international_law": 0.4702702702702703, + "journalism": 0.5406976744186046, + "jurisprudence": 0.5547445255474452, + "legal_and_moral_basis": 0.8504672897196262, + "logical": 0.4878048780487805, + "machine_learning": 0.45901639344262296, + "management": 0.6761904761904762, + "marketing": 0.6555555555555556, + "marxist_theory": 0.656084656084656, + "modern_chinese": 0.4396551724137931, + "nutrition": 0.6275862068965518, + "philosophy": 0.5904761904761905, + "professional_accounting": 0.6114285714285714, + "professional_law": 0.44549763033175355, + "professional_medicine": 0.3882978723404255, + "professional_psychology": 0.6551724137931034, + "public_relations": 0.5862068965517241, + "security_study": 0.6814814814814815, + "sociology": 0.5663716814159292, + "sports_science": 0.5212121212121212, + "traditional_chinese_medicine": 0.5459459459459459, + "virology": 0.5443786982248521, + "world_history": 0.6708074534161491, + "world_religions": 0.63125 } }, "prompt_4": { - "accuracy": 0.2694698670350544, + "accuracy": 0.5395441201864963, "category_acc": { - "agronomy": 0.25443786982248523, - "anatomy": 0.28378378378378377, - "ancient_chinese": 0.2621951219512195, - "arts": 0.26875, - "astronomy": 0.2545454545454545, - "business_ethics": 0.2631578947368421, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.3053435114503817, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.29906542056074764, - "chinese_history": 0.25077399380804954, - "chinese_literature": 0.2647058823529412, - "chinese_teacher_qualification": 0.2569832402234637, - "clinical_knowledge": 0.2616033755274262, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.3364485981308411, - "college_engineering_hydrology": 0.2830188679245283, - "college_law": 0.25925925925925924, - "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.32075471698113206, - "college_medicine": 0.27106227106227104, - "computer_science": 0.23039215686274508, - "computer_security": 0.25146198830409355, - "conceptual_physics": 0.2585034013605442, - "construction_project_management": 0.2446043165467626, - "economics": 0.2641509433962264, - "education": 0.27607361963190186, - "electrical_engineering": 0.32558139534883723, - "elementary_chinese": 0.25, - "elementary_commonsense": 0.25757575757575757, - "elementary_information_and_technology": 0.23949579831932774, - "elementary_mathematics": 0.24782608695652175, - "ethnology": 0.25925925925925924, - "food_science": 0.42657342657342656, - "genetics": 0.26704545454545453, - "global_facts": 0.28859060402684567, - "high_school_biology": 0.24260355029585798, - "high_school_chemistry": 0.2803030303030303, - "high_school_geography": 0.288135593220339, - "high_school_mathematics": 0.25, - "high_school_physics": 0.2818181818181818, - "high_school_politics": 0.23776223776223776, - "human_sexuality": 0.2777777777777778, - "international_law": 0.2810810810810811, - "journalism": 0.27325581395348836, - "jurisprudence": 0.26763990267639903, - "legal_and_moral_basis": 0.29439252336448596, - "logical": 0.2845528455284553, - "machine_learning": 0.26229508196721313, - "management": 0.23809523809523808, - "marketing": 0.23333333333333334, - "marxist_theory": 0.30158730158730157, - "modern_chinese": 0.23275862068965517, - "nutrition": 0.27586206896551724, - "philosophy": 0.3142857142857143, - "professional_accounting": 0.2571428571428571, - "professional_law": 0.24644549763033174, - "professional_medicine": 0.26595744680851063, - "professional_psychology": 0.2543103448275862, - "public_relations": 0.27011494252873564, - "security_study": 0.3333333333333333, - "sociology": 0.3008849557522124, - "sports_science": 0.24242424242424243, - "traditional_chinese_medicine": 0.2864864864864865, - "virology": 0.33727810650887574, - "world_history": 0.2422360248447205, - "world_religions": 0.25625 + "agronomy": 0.5088757396449705, + "anatomy": 0.44594594594594594, + "ancient_chinese": 0.32926829268292684, + "arts": 0.75, + "astronomy": 0.3696969696969697, + "business_ethics": 0.569377990430622, + "chinese_civil_service_exam": 0.5375, + "chinese_driving_rule": 0.7480916030534351, + "chinese_food_culture": 0.5514705882352942, + "chinese_foreign_policy": 0.5887850467289719, + "chinese_history": 0.6501547987616099, + "chinese_literature": 0.45098039215686275, + "chinese_teacher_qualification": 0.7430167597765364, + "clinical_knowledge": 0.4978902953586498, + "college_actuarial_science": 0.36792452830188677, + "college_education": 0.6542056074766355, + "college_engineering_hydrology": 0.5283018867924528, + "college_law": 0.46296296296296297, + "college_mathematics": 0.3142857142857143, + "college_medical_statistics": 0.37735849056603776, + "college_medicine": 0.5054945054945055, + "computer_science": 0.5637254901960784, + "computer_security": 0.6666666666666666, + "conceptual_physics": 0.6326530612244898, + "construction_project_management": 0.4460431654676259, + "economics": 0.5660377358490566, + "education": 0.6012269938650306, + "electrical_engineering": 0.5174418604651163, + "elementary_chinese": 0.5317460317460317, + "elementary_commonsense": 0.5606060606060606, + "elementary_information_and_technology": 0.7352941176470589, + "elementary_mathematics": 0.3695652173913043, + "ethnology": 0.5481481481481482, + "food_science": 0.6083916083916084, + "genetics": 0.4431818181818182, + "global_facts": 0.5906040268456376, + "high_school_biology": 0.44970414201183434, + "high_school_chemistry": 0.4696969696969697, + "high_school_geography": 0.5254237288135594, + "high_school_mathematics": 0.2682926829268293, + "high_school_physics": 0.42727272727272725, + "high_school_politics": 0.5244755244755245, + "human_sexuality": 0.5079365079365079, + "international_law": 0.44324324324324327, + "journalism": 0.5116279069767442, + "jurisprudence": 0.5304136253041363, + "legal_and_moral_basis": 0.8411214953271028, + "logical": 0.4715447154471545, + "machine_learning": 0.45901639344262296, + "management": 0.6476190476190476, + "marketing": 0.6277777777777778, + "marxist_theory": 0.6190476190476191, + "modern_chinese": 0.3706896551724138, + "nutrition": 0.5586206896551724, + "philosophy": 0.5904761904761905, + "professional_accounting": 0.5828571428571429, + "professional_law": 0.46919431279620855, + "professional_medicine": 0.39361702127659576, + "professional_psychology": 0.6206896551724138, + "public_relations": 0.5632183908045977, + "security_study": 0.6592592592592592, + "sociology": 0.5530973451327433, + "sports_science": 0.5272727272727272, + "traditional_chinese_medicine": 0.5297297297297298, + "virology": 0.5325443786982249, + "world_history": 0.6211180124223602, + "world_religions": 0.65625 } }, "prompt_5": { - "accuracy": 0.26005871179416334, + "accuracy": 0.5513728198929373, "category_acc": { - "agronomy": 0.2485207100591716, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.24390243902439024, - "arts": 0.25625, - "astronomy": 0.24848484848484848, - "business_ethics": 0.2583732057416268, - "chinese_civil_service_exam": 0.2875, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.29411764705882354, - "chinese_foreign_policy": 0.2616822429906542, - "chinese_history": 0.23839009287925697, - "chinese_literature": 0.27450980392156865, - "chinese_teacher_qualification": 0.24581005586592178, - "clinical_knowledge": 0.27848101265822783, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.2803738317757009, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.28703703703703703, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.2830188679245283, - "college_medicine": 0.2454212454212454, - "computer_science": 0.22549019607843138, - "computer_security": 0.28654970760233917, - "conceptual_physics": 0.2789115646258503, - "construction_project_management": 0.2589928057553957, - "economics": 0.29559748427672955, - "education": 0.26993865030674846, - "electrical_engineering": 0.3023255813953488, - "elementary_chinese": 0.25, - "elementary_commonsense": 0.25757575757575757, - "elementary_information_and_technology": 0.23109243697478993, - "elementary_mathematics": 0.23043478260869565, - "ethnology": 0.2814814814814815, - "food_science": 0.34965034965034963, - "genetics": 0.2159090909090909, - "global_facts": 0.28187919463087246, - "high_school_biology": 0.23076923076923078, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.1864406779661017, - "high_school_mathematics": 0.24390243902439024, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.23776223776223776, - "human_sexuality": 0.2857142857142857, - "international_law": 0.24324324324324326, - "journalism": 0.21511627906976744, - "jurisprudence": 0.26034063260340634, - "legal_and_moral_basis": 0.2616822429906542, - "logical": 0.2682926829268293, - "machine_learning": 0.2459016393442623, - "management": 0.2571428571428571, - "marketing": 0.22777777777777777, - "marxist_theory": 0.25925925925925924, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.20689655172413793, - "philosophy": 0.2761904761904762, - "professional_accounting": 0.2742857142857143, - "professional_law": 0.23696682464454977, - "professional_medicine": 0.2898936170212766, - "professional_psychology": 0.2543103448275862, - "public_relations": 0.25862068965517243, - "security_study": 0.3037037037037037, - "sociology": 0.24336283185840707, - "sports_science": 0.2787878787878788, - "traditional_chinese_medicine": 0.2918918918918919, - "virology": 0.3076923076923077, - "world_history": 0.22981366459627328, - "world_religions": 0.29375 + "agronomy": 0.514792899408284, + "anatomy": 0.5, + "ancient_chinese": 0.3353658536585366, + "arts": 0.7625, + "astronomy": 0.3212121212121212, + "business_ethics": 0.6172248803827751, + "chinese_civil_service_exam": 0.55625, + "chinese_driving_rule": 0.7633587786259542, + "chinese_food_culture": 0.5147058823529411, + "chinese_foreign_policy": 0.6261682242990654, + "chinese_history": 0.6780185758513931, + "chinese_literature": 0.45098039215686275, + "chinese_teacher_qualification": 0.7430167597765364, + "clinical_knowledge": 0.510548523206751, + "college_actuarial_science": 0.3018867924528302, + "college_education": 0.6635514018691588, + "college_engineering_hydrology": 0.49056603773584906, + "college_law": 0.4722222222222222, + "college_mathematics": 0.3238095238095238, + "college_medical_statistics": 0.41509433962264153, + "college_medicine": 0.5054945054945055, + "computer_science": 0.5637254901960784, + "computer_security": 0.7134502923976608, + "conceptual_physics": 0.6258503401360545, + "construction_project_management": 0.4676258992805755, + "economics": 0.5786163522012578, + "education": 0.6073619631901841, + "electrical_engineering": 0.5465116279069767, + "elementary_chinese": 0.5396825396825397, + "elementary_commonsense": 0.5656565656565656, + "elementary_information_and_technology": 0.7563025210084033, + "elementary_mathematics": 0.3695652173913043, + "ethnology": 0.5777777777777777, + "food_science": 0.5454545454545454, + "genetics": 0.4034090909090909, + "global_facts": 0.5436241610738255, + "high_school_biology": 0.4437869822485207, + "high_school_chemistry": 0.5075757575757576, + "high_school_geography": 0.5508474576271186, + "high_school_mathematics": 0.25, + "high_school_physics": 0.4636363636363636, + "high_school_politics": 0.5804195804195804, + "human_sexuality": 0.5476190476190477, + "international_law": 0.43783783783783786, + "journalism": 0.5232558139534884, + "jurisprudence": 0.5669099756690997, + "legal_and_moral_basis": 0.8271028037383178, + "logical": 0.4878048780487805, + "machine_learning": 0.4426229508196721, + "management": 0.6619047619047619, + "marketing": 0.6222222222222222, + "marxist_theory": 0.6455026455026455, + "modern_chinese": 0.4051724137931034, + "nutrition": 0.5310344827586206, + "philosophy": 0.6190476190476191, + "professional_accounting": 0.5771428571428572, + "professional_law": 0.4976303317535545, + "professional_medicine": 0.425531914893617, + "professional_psychology": 0.6379310344827587, + "public_relations": 0.6149425287356322, + "security_study": 0.6814814814814815, + "sociology": 0.584070796460177, + "sports_science": 0.5212121212121212, + "traditional_chinese_medicine": 0.5621621621621622, + "virology": 0.5680473372781065, + "world_history": 0.6521739130434783, + "world_religions": 0.68125 } } }, "zbench": { "prompt_1": { - "accuracy": 0.18181818181818182 + "accuracy": 0.36363636363636365 }, "prompt_2": { - "accuracy": 0.2727272727272727 + "accuracy": 0.30303030303030304 }, "prompt_3": { - "accuracy": 0.21212121212121213 + "accuracy": 0.36363636363636365 }, "prompt_4": { - "accuracy": 0.18181818181818182 + "accuracy": 0.3333333333333333 }, "prompt_5": { - "accuracy": 0.24242424242424243 + "accuracy": 0.36363636363636365 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.21136363636363636 + "accuracy": 0.6 }, "prompt_2": { - "accuracy": 0.18181818181818182 + "accuracy": 0.5590909090909091 }, "prompt_3": { - "accuracy": 0.1590909090909091 + "accuracy": 0.6090909090909091 }, "prompt_4": { - "accuracy": 0.20227272727272727 + "accuracy": 0.6204545454545455 }, "prompt_5": { - "accuracy": 0.17045454545454544 + "accuracy": 0.6 } }, "ocnli": { "prompt_1": { - "accuracy": 0.29389830508474574 + "accuracy": 0.49457627118644065 }, "prompt_2": { - "accuracy": 0.3142372881355932 + "accuracy": 0.4847457627118644 }, "prompt_3": { - "accuracy": 0.2993220338983051 + "accuracy": 0.4942372881355932 }, "prompt_4": { - "accuracy": 0.3206779661016949 + "accuracy": 0.4603389830508475 }, "prompt_5": { - "accuracy": 0.31898305084745765 + "accuracy": 0.4935593220338983 } }, "c3": { "prompt_1": { - "accuracy": 0.27786088257292446 + "accuracy": 0.8534031413612565 }, "prompt_2": { - "accuracy": 0.2801047120418848 + "accuracy": 0.8590127150336574 }, "prompt_3": { - "accuracy": 0.2737471952131638 + "accuracy": 0.8548990276738968 }, "prompt_4": { - "accuracy": 0.2636499626028422 + "accuracy": 0.8515332834704562 }, "prompt_5": { - "accuracy": 0.26215407629020193 + "accuracy": 0.8526551982049364 } }, "dream": { "prompt_1": { - "accuracy": 0.3365997060264576 + "accuracy": 0.7133757961783439 }, "prompt_2": { - "accuracy": 0.3351298383145517 + "accuracy": 0.7506124448799608 }, "prompt_3": { - "accuracy": 0.32533072023517884 + "accuracy": 0.7261146496815286 }, "prompt_4": { - "accuracy": 0.33463988241058307 + "accuracy": 0.7275845173934345 }, "prompt_5": { - "accuracy": 0.3419892209701127 + "accuracy": 0.7207251347378736 } }, "samsum": { "prompt_1": { - "rouge1": 0.15612600162553933, - "rouge2": 0.048596540749105285, - "rougeL": 0.1272555321176518, - "avg_rouge": 0.11065935816409882 + "rouge1": 0.3380038032957861, + "rouge2": 0.11233090757630733, + "rougeL": 0.2525846961222128, + "avg_rouge": 0.23430646899810206 }, "prompt_2": { - "rouge1": 0.16048721015644507, - "rouge2": 0.04804136425724633, - "rougeL": 0.12765300230762966, - "avg_rouge": 0.1120605255737737 + "rouge1": 0.3402688720384836, + "rouge2": 0.11617170717611897, + "rougeL": 0.2556909208109999, + "avg_rouge": 0.23737716667520084 }, "prompt_3": { - "rouge1": 0.13385833863075727, - "rouge2": 0.03682769684726707, - "rougeL": 0.11121519287703845, - "avg_rouge": 0.09396707611835427 + "rouge1": 0.35698629071621746, + "rouge2": 0.11854058162244702, + "rougeL": 0.2686418609896302, + "avg_rouge": 0.2480562444427649 }, "prompt_4": { - "rouge1": 0.16810689032402468, - "rouge2": 0.05148353536120004, - "rougeL": 0.1318350734841346, - "avg_rouge": 0.11714183305645309 + "rouge1": 0.3342994800625261, + "rouge2": 0.11240749315987966, + "rougeL": 0.2490189475006745, + "avg_rouge": 0.23190864024102673 }, "prompt_5": { - "rouge1": 0.1534571410382571, - "rouge2": 0.04578631586822307, - "rougeL": 0.12381631106957348, - "avg_rouge": 0.10768658932535123 + "rouge1": 0.33860545114927265, + "rouge2": 0.11058705018769807, + "rougeL": 0.25443600103582137, + "avg_rouge": 0.23454283412426405 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.2132857887698498, - "rouge2": 0.059607999768275716, - "rougeL": 0.15770212235032663, - "avg_rouge": 0.14353197029615072 + "rouge1": 0.21397562710108725, + "rouge2": 0.057891940912959426, + "rougeL": 0.15740443114264127, + "avg_rouge": 0.14309066638556264 }, "prompt_2": { - "rouge1": 0.19489014825395112, - "rouge2": 0.05316157897090513, - "rougeL": 0.14563384467141377, - "avg_rouge": 0.13122852396542334 + "rouge1": 0.21161044567205908, + "rouge2": 0.054305946671291946, + "rougeL": 0.15502329163680859, + "avg_rouge": 0.14031322799338652 }, "prompt_3": { - "rouge1": 0.1983546222606056, - "rouge2": 0.05400344149002102, - "rougeL": 0.14925484801673988, - "avg_rouge": 0.13387097058912217 + "rouge1": 0.212214415398881, + "rouge2": 0.054087532294718216, + "rougeL": 0.1553242277622777, + "avg_rouge": 0.1405420584852923 }, "prompt_4": { - "rouge1": 0.173951353506643, - "rouge2": 0.04797065637682303, - "rougeL": 0.12738252558875962, - "avg_rouge": 0.11643484515740855 + "rouge1": 0.21398263404053994, + "rouge2": 0.057143774377134346, + "rougeL": 0.15730363685762183, + "avg_rouge": 0.14281001509176536 }, "prompt_5": { - "rouge1": 0.19190121963431314, - "rouge2": 0.05348262869463405, - "rougeL": 0.14374912466578577, - "avg_rouge": 0.12971099099824432 + "rouge1": 0.2109018158847188, + "rouge2": 0.05370712413153861, + "rougeL": 0.15883621981098608, + "avg_rouge": 0.14114838660908116 } }, "sst2": { "prompt_1": { - "accuracy": 0.5045871559633027 + "accuracy": 0.9128440366972477 }, "prompt_2": { - "accuracy": 0.5068807339449541 + "accuracy": 0.911697247706422 }, "prompt_3": { - "accuracy": 0.555045871559633 + "accuracy": 0.8727064220183486 }, "prompt_4": { - "accuracy": 0.5022935779816514 + "accuracy": 0.9139908256880734 }, "prompt_5": { - "accuracy": 0.5252293577981652 + "accuracy": 0.8887614678899083 } }, "cola": { "prompt_1": { - "accuracy": 0.5052732502396932 + "accuracy": 0.7075743048897412 }, "prompt_2": { - "accuracy": 0.663470757430489 + "accuracy": 0.725790987535954 }, "prompt_3": { - "accuracy": 0.5877277085330777 + "accuracy": 0.7181208053691275 }, "prompt_4": { - "accuracy": 0.4956855225311601 + "accuracy": 0.7488015340364333 }, "prompt_5": { - "accuracy": 0.5474592521572388 + "accuracy": 0.7660594439117929 } }, "qqp": { "prompt_1": { - "accuracy": 0.4415 + "accuracy": 0.778 }, "prompt_2": { - "accuracy": 0.4545 + "accuracy": 0.794 }, "prompt_3": { - "accuracy": 0.4695 + "accuracy": 0.779 }, "prompt_4": { - "accuracy": 0.4785 + "accuracy": 0.742 }, "prompt_5": { - "accuracy": 0.4155 + "accuracy": 0.7955 } }, "mnli": { "prompt_1": { - "accuracy": 0.331 + "accuracy": 0.7 }, "prompt_2": { - "accuracy": 0.339 + "accuracy": 0.7 }, "prompt_3": { - "accuracy": 0.3285 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.3195 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.329 + "accuracy": 0.6 } }, "qnli": { "prompt_1": { - "accuracy": 0.4815 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.5095 + "accuracy": 0.8 }, "prompt_3": { - "accuracy": 0.5045 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.4955 + "accuracy": 0.8 }, "prompt_5": { - "accuracy": 0.5205 + "accuracy": 0.9 } }, "wnli": { "prompt_1": { - "accuracy": 0.5211267605633803 + "accuracy": 0.4 }, "prompt_2": { - "accuracy": 0.5352112676056338 + "accuracy": 0.4 }, "prompt_3": { - "accuracy": 0.5352112676056338 + "accuracy": 0.4 }, "prompt_4": { - "accuracy": 0.5211267605633803 + "accuracy": 0.3 }, "prompt_5": { - "accuracy": 0.5211267605633803 + "accuracy": 0.3 } }, "rte": { "prompt_1": { - "accuracy": 0.48375451263537905 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.5270758122743683 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.5270758122743683 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.48375451263537905 + "accuracy": 0.8 }, "prompt_5": { - "accuracy": 0.5126353790613718 + "accuracy": 0.9 } }, "mrpc": { "prompt_1": { - "accuracy": 0.5098039215686274 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.5808823529411765 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.5367647058823529 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.5294117647058824 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.46568627450980393 + "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.2728991596638655, - "language_acc": { - "English": 0.27310924369747897, - "Chinese": 0.26134453781512607, - "Spanish": 0.2865546218487395, - "Vietnamese": 0.27058823529411763 - }, - "consistency_score_2": 0.4081232492997198, - "consistency_score_3": 0.18088235294117647, - "consistency_score_4": 0.07899159663865546, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.40756302521008403, - "English,Spanish": 0.4142857142857143, - "English,Vietnamese": 0.4579831932773109, - "Chinese,Spanish": 0.36134453781512604, - "Chinese,Vietnamese": 0.38823529411764707, - "Spanish,Vietnamese": 0.419327731092437 - }, - "3_combine": { - "English,Chinese,Spanish": 0.16638655462184873, - "English,Chinese,Vietnamese": 0.1915966386554622, - "English,Spanish,Vietnamese": 0.2042016806722689, - "Chinese,Spanish,Vietnamese": 0.16134453781512606 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.07899159663865546 - } - }, - "AC3_2": 0.32708612900513495, - "AC3_3": 0.21756127446185985, - "AC3_4": 0.12251950328890004 - } + "prompt_1": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.2695238095238095, - "language_acc": { - "Spanish": 0.23333333333333334, - "English": 0.29333333333333333, - "Chinese": 0.30666666666666664, - "Vietnamese": 0.22, - "Filipino": 0.2866666666666667, - "Indonesian": 0.26666666666666666, - "Malay": 0.28 - }, - "consistency_score_2": 0.38476190476190486, - "consistency_score_3": 0.1744761904761905, - "consistency_score_4": 0.08419047619047616, - "consistency_score_5": 0.040634920634920635, - "consistency_score_6": 0.018095238095238098, - "consistency_score_7": 0.006666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.49333333333333335, - "Spanish,Chinese": 0.36, - "Spanish,Vietnamese": 0.3333333333333333, - "Spanish,Filipino": 0.34, - "Spanish,Indonesian": 0.34, - "Spanish,Malay": 0.4, - "English,Chinese": 0.37333333333333335, - "English,Vietnamese": 0.38, - "English,Filipino": 0.32666666666666666, - "English,Indonesian": 0.44666666666666666, - "English,Malay": 0.46, - "Chinese,Vietnamese": 0.32, - "Chinese,Filipino": 0.38, - "Chinese,Indonesian": 0.3466666666666667, - "Chinese,Malay": 0.3466666666666667, - "Vietnamese,Filipino": 0.3933333333333333, - "Vietnamese,Indonesian": 0.42, - "Vietnamese,Malay": 0.44, - "Filipino,Indonesian": 0.35333333333333333, - "Filipino,Malay": 0.3933333333333333, - "Indonesian,Malay": 0.43333333333333335 - }, - "3_combine": { - "Spanish,English,Chinese": 0.20666666666666667, - "Spanish,English,Vietnamese": 0.19333333333333333, - "Spanish,English,Filipino": 0.17333333333333334, - "Spanish,English,Indonesian": 0.23333333333333334, - "Spanish,English,Malay": 0.24, - "Spanish,Chinese,Vietnamese": 0.12, - "Spanish,Chinese,Filipino": 0.15333333333333332, - "Spanish,Chinese,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Malay": 0.14, - "Spanish,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,Vietnamese,Malay": 0.17333333333333334, - "Spanish,Filipino,Indonesian": 0.14666666666666667, - "Spanish,Filipino,Malay": 0.16, - "Spanish,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese": 0.14, - "English,Chinese,Filipino": 0.13333333333333333, - "English,Chinese,Indonesian": 0.20666666666666667, - "English,Chinese,Malay": 0.16, - "English,Vietnamese,Filipino": 0.14666666666666667, - "English,Vietnamese,Indonesian": 0.21333333333333335, - "English,Vietnamese,Malay": 0.24666666666666667, - "English,Filipino,Indonesian": 0.18666666666666668, - "English,Filipino,Malay": 0.17333333333333334, - "English,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "Chinese,Vietnamese,Malay": 0.15333333333333332, - "Chinese,Filipino,Indonesian": 0.16666666666666666, - "Chinese,Filipino,Malay": 0.14666666666666667, - "Chinese,Indonesian,Malay": 0.16, - "Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Vietnamese,Filipino,Malay": 0.2, - "Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Filipino,Indonesian,Malay": 0.18666666666666668 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.08666666666666667, - "Spanish,English,Chinese,Filipino": 0.08, - "Spanish,English,Chinese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Malay": 0.08, - "Spanish,English,Vietnamese,Filipino": 0.08666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.10666666666666667, - "Spanish,English,Vietnamese,Malay": 0.12, - "Spanish,English,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Filipino,Malay": 0.08, - "Spanish,English,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.06, - "Spanish,Chinese,Vietnamese,Indonesian": 0.05333333333333334, - "Spanish,Chinese,Vietnamese,Malay": 0.04666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.06, - "Spanish,Chinese,Indonesian,Malay": 0.05333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.06666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.05333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.09333333333333334, - "English,Chinese,Vietnamese,Malay": 0.07333333333333333, - "English,Chinese,Filipino,Indonesian": 0.1, - "English,Chinese,Filipino,Malay": 0.06, - "English,Chinese,Indonesian,Malay": 0.1, - "English,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "English,Vietnamese,Filipino,Malay": 0.09333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.14, - "English,Filipino,Indonesian,Malay": 0.10666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.05333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.06666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.12 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.04, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.05333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.04, - "Spanish,English,Chinese,Filipino,Indonesian": 0.06, - "Spanish,English,Chinese,Filipino,Malay": 0.02, - "Spanish,English,Chinese,Indonesian,Malay": 0.04666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.06, - "Spanish,English,Vietnamese,Filipino,Malay": 0.05333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.06, - "Spanish,English,Filipino,Indonesian,Malay": 0.03333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.02, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.02, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.013333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.04666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.02, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.04, - "English,Chinese,Filipino,Indonesian,Malay": 0.04666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.02666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.013333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.02, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.013333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.02666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667 - } - }, - "AC3_2": 0.3169945241076473, - "AC3_3": 0.21182652035024033, - "AC3_4": 0.12830320277419946, - "AC3_5": 0.0706224106610534, - "AC3_6": 0.03391359191504581, - "AC3_7": 0.013011494248162505 - } + "prompt_1": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.2694805194805195, - "language_acc": { - "English": 0.2897727272727273, - "Filipino": 0.23863636363636365, - "Spanish": 0.2897727272727273, - "Malay": 0.26136363636363635, - "Chinese": 0.23295454545454544, - "Indonesian": 0.2897727272727273, - "Vietnamese": 0.2840909090909091 - }, - "consistency_score_2": 0.48890692640692646, - "consistency_score_3": 0.28084415584415584, - "consistency_score_4": 0.1701298701298701, - "consistency_score_5": 0.10687229437229438, - "consistency_score_6": 0.06899350649350648, - "consistency_score_7": 0.045454545454545456, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3806818181818182, - "English,Spanish": 0.42613636363636365, - "English,Malay": 0.5397727272727273, - "English,Chinese": 0.5113636363636364, - "English,Indonesian": 0.5681818181818182, - "English,Vietnamese": 0.5965909090909091, - "Filipino,Spanish": 0.39204545454545453, - "Filipino,Malay": 0.39204545454545453, - "Filipino,Chinese": 0.3806818181818182, - "Filipino,Indonesian": 0.36363636363636365, - "Filipino,Vietnamese": 0.4147727272727273, - "Spanish,Malay": 0.42045454545454547, - "Spanish,Chinese": 0.4431818181818182, - "Spanish,Indonesian": 0.4715909090909091, - "Spanish,Vietnamese": 0.5113636363636364, - "Malay,Chinese": 0.5511363636363636, - "Malay,Indonesian": 0.6420454545454546, - "Malay,Vietnamese": 0.5795454545454546, - "Chinese,Indonesian": 0.5227272727272727, - "Chinese,Vietnamese": 0.5454545454545454, - "Indonesian,Vietnamese": 0.6136363636363636 - }, - "3_combine": { - "English,Filipino,Spanish": 0.1875, - "English,Filipino,Malay": 0.23863636363636365, - "English,Filipino,Chinese": 0.19318181818181818, - "English,Filipino,Indonesian": 0.23863636363636365, - "English,Filipino,Vietnamese": 0.2556818181818182, - "English,Spanish,Malay": 0.2215909090909091, - "English,Spanish,Chinese": 0.23295454545454544, - "English,Spanish,Indonesian": 0.26704545454545453, - "English,Spanish,Vietnamese": 0.30113636363636365, - "English,Malay,Chinese": 0.32954545454545453, - "English,Malay,Indonesian": 0.39204545454545453, - "English,Malay,Vietnamese": 0.375, - "English,Chinese,Indonesian": 0.3352272727272727, - "English,Chinese,Vietnamese": 0.3522727272727273, - "English,Indonesian,Vietnamese": 0.4034090909090909, - "Filipino,Spanish,Malay": 0.19318181818181818, - "Filipino,Spanish,Chinese": 0.20454545454545456, - "Filipino,Spanish,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese": 0.2159090909090909, - "Filipino,Malay,Indonesian": 0.26136363636363635, - "Filipino,Malay,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian": 0.21022727272727273, - "Filipino,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Indonesian,Vietnamese": 0.24431818181818182, - "Spanish,Malay,Chinese": 0.25, - "Spanish,Malay,Indonesian": 0.29545454545454547, - "Spanish,Malay,Vietnamese": 0.2897727272727273, - "Spanish,Chinese,Indonesian": 0.2727272727272727, - "Spanish,Chinese,Vietnamese": 0.29545454545454547, - "Spanish,Indonesian,Vietnamese": 0.3181818181818182, - "Malay,Chinese,Indonesian": 0.38636363636363635, - "Malay,Chinese,Vietnamese": 0.3522727272727273, - "Malay,Indonesian,Vietnamese": 0.4375, - "Chinese,Indonesian,Vietnamese": 0.3806818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.10795454545454546, - "English,Filipino,Spanish,Chinese": 0.09090909090909091, - "English,Filipino,Spanish,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Malay,Vietnamese": 0.19318181818181818, - "English,Filipino,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Chinese": 0.14204545454545456, - "English,Spanish,Malay,Indonesian": 0.1534090909090909, - "English,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Spanish,Chinese,Indonesian": 0.16477272727272727, - "English,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Spanish,Indonesian,Vietnamese": 0.2159090909090909, - "English,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Malay,Chinese,Vietnamese": 0.22727272727272727, - "English,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Malay,Chinese": 0.11931818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.1875, - "Spanish,Malay,Chinese,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Malay,Chinese,Indonesian,Vietnamese": 0.2840909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.0625, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - } - }, - "AC3_2": 0.34745008821768814, - "AC3_3": 0.2750450139330884, - "AC3_4": 0.2085787181324536, - "AC3_5": 0.15304788665208044, - "AC3_6": 0.10986016378454083, - "AC3_7": 0.07778819116555329 - } + "prompt_1": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.1941747572815534 - } + "prompt_1": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.23809523809523808 - } + "prompt_1": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.2803738317757009 - } + "prompt_1": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.34, - "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.4, - "politics": 0.5, - "culture": 0.1, - "film": 0.4, - "law": 0.2, - "geography": 0.5 - } - } + "prompt_1": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.06335396898813274 - } + "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.3265865379663848 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.2616202581249587 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.1705127081929969 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.32151597352464384 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.25787631271878647 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.2572041473006793, - "category_acc": { - "high_school_european_history": 0.23780487804878048, - "business_ethics": 0.3434343434343434, - "clinical_knowledge": 0.24242424242424243, - "medical_genetics": 0.3434343434343434, - "high_school_us_history": 0.20689655172413793, - "high_school_physics": 0.31333333333333335, - "high_school_world_history": 0.22033898305084745, - "virology": 0.2787878787878788, - "high_school_microeconomics": 0.27848101265822783, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.2222222222222222, - "high_school_biology": 0.26537216828478966, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.25622775800711745, - "philosophy": 0.16774193548387098, - "professional_medicine": 0.34317343173431736, - "nutrition": 0.23934426229508196, - "global_facts": 0.16161616161616163, - "machine_learning": 0.24324324324324326, - "security_studies": 0.26639344262295084, - "public_relations": 0.24770642201834864, - "professional_psychology": 0.24386252045826515, - "prehistory": 0.2724458204334365, - "anatomy": 0.19402985074626866, - "human_sexuality": 0.27692307692307694, - "college_medicine": 0.2558139534883721, - "high_school_government_and_politics": 0.328125, - "college_chemistry": 0.32323232323232326, - "logical_fallacies": 0.2037037037037037, - "high_school_geography": 0.26903553299492383, - "elementary_mathematics": 0.23607427055702918, - "human_aging": 0.22972972972972974, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.25919117647058826, - "formal_logic": 0.28, - "high_school_statistics": 0.3209302325581395, - "international_law": 0.23333333333333334, - "high_school_mathematics": 0.241635687732342, - "high_school_computer_science": 0.24242424242424243, - "conceptual_physics": 0.2692307692307692, - "miscellaneous": 0.23017902813299232, - "high_school_chemistry": 0.21782178217821782, - "marketing": 0.26180257510729615, - "professional_law": 0.25962165688193084, - "management": 0.24509803921568626, - "college_physics": 0.3564356435643564, - "jurisprudence": 0.22429906542056074, - "world_religions": 0.3, - "sociology": 0.26, - "us_foreign_policy": 0.26262626262626265, - "high_school_macroeconomics": 0.3033419023136247, - "computer_security": 0.21212121212121213, - "moral_scenarios": 0.26733780760626397, - "moral_disputes": 0.24057971014492754, - "electrical_engineering": 0.2638888888888889, - "astronomy": 0.2185430463576159, - "college_biology": 0.2517482517482518 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.2511144130757801 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.26650062266500624, - "category_acc": { - "computer_network": 0.08333333333333333, - "operating_system": 0.375, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.2619047619047619, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.13043478260869565, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.375, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.3333333333333333, - "business_administration": 0.23684210526315788, - "marxism": 0.25, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.14705882352941177, - "teacher_qualification": 0.1836734693877551, - "high_school_politics": 0.16666666666666666, - "high_school_geography": 0.25, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.25, - "ideological_and_moral_cultivation": 0.125, - "logic": 0.2222222222222222, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.21052631578947367, - "professional_tour_guide": 0.23529411764705882, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.125, - "high_school_history": 0.24, - "middle_school_history": 0.14814814814814814, - "civil_servant": 0.2692307692307692, - "sports_science": 0.20833333333333334, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.18518518518518517, - "urban_and_rural_planner": 0.43137254901960786, - "accountant": 0.3148148148148148, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.2777777777777778, - "physician": 0.35185185185185186 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.3010752688172043 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.2581592125712312, - "category_acc": { - "agronomy": 0.3076923076923077, - "anatomy": 0.22297297297297297, - "ancient_chinese": 0.29878048780487804, - "arts": 0.25, - "astronomy": 0.296969696969697, - "business_ethics": 0.2631578947368421, - "chinese_civil_service_exam": 0.25625, - "chinese_driving_rule": 0.2748091603053435, - "chinese_food_culture": 0.25735294117647056, - "chinese_foreign_policy": 0.205607476635514, - "chinese_history": 0.23219814241486067, - "chinese_literature": 0.2696078431372549, - "chinese_teacher_qualification": 0.29608938547486036, - "clinical_knowledge": 0.25316455696202533, - "college_actuarial_science": 0.1792452830188679, - "college_education": 0.2616822429906542, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.25925925925925924, - "college_mathematics": 0.20952380952380953, - "college_medical_statistics": 0.20754716981132076, - "college_medicine": 0.2673992673992674, - "computer_science": 0.2549019607843137, - "computer_security": 0.2573099415204678, - "conceptual_physics": 0.2653061224489796, - "construction_project_management": 0.26618705035971224, - "economics": 0.2578616352201258, - "education": 0.26380368098159507, - "electrical_engineering": 0.23837209302325582, - "elementary_chinese": 0.2222222222222222, - "elementary_commonsense": 0.2777777777777778, - "elementary_information_and_technology": 0.2184873949579832, - "elementary_mathematics": 0.2826086956521739, - "ethnology": 0.28888888888888886, - "food_science": 0.26573426573426573, - "genetics": 0.29545454545454547, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.31952662721893493, - "high_school_chemistry": 0.2196969696969697, - "high_school_geography": 0.22033898305084745, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.26573426573426573, - "human_sexuality": 0.23015873015873015, - "international_law": 0.22162162162162163, - "journalism": 0.23255813953488372, - "jurisprudence": 0.22871046228710462, - "legal_and_moral_basis": 0.3130841121495327, - "logical": 0.1951219512195122, - "machine_learning": 0.22131147540983606, - "management": 0.2571428571428571, - "marketing": 0.2722222222222222, - "marxist_theory": 0.25925925925925924, - "modern_chinese": 0.23275862068965517, - "nutrition": 0.2689655172413793, - "philosophy": 0.23809523809523808, - "professional_accounting": 0.21142857142857144, - "professional_law": 0.24170616113744076, - "professional_medicine": 0.2632978723404255, - "professional_psychology": 0.25, - "public_relations": 0.3103448275862069, - "security_study": 0.362962962962963, - "sociology": 0.25663716814159293, - "sports_science": 0.2727272727272727, - "traditional_chinese_medicine": 0.2756756756756757, - "virology": 0.3431952662721893, - "world_history": 0.2422360248447205, - "world_religions": 0.25 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.36363636363636365 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.19090909090909092 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.3206779661016949 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.29693343305908754 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.34835864772170505 - } + "prompt_1": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.161988902166818, - "rouge2": 0.05491271354141179, - "rougeL": 0.12698172316994347, - "avg_rouge": 0.11462777962605775 - } + "prompt_1": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.15843097165373676, - "rouge2": 0.04510014069431016, - "rougeL": 0.12134065642539484, - "avg_rouge": 0.10829058959114725 - } + "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.5080275229357798 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.3825503355704698 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.5255 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.3195 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.509 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.43661971830985913 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.5090252707581228 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.4362745098039216 - } + "prompt_1": -1 } } }, - "sailor_4b": { - "model_size": "4B", - "model_link": "https://huggingface.co/sail/Sailor-4B", + "sea_mistral_highest_acc_inst_7b": { + "model_size": "7B", + "model_link": "https://seaeval.github.io/", "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.4640756302521008, + "overall_acc": 0.8836134453781513, "language_acc": { - "English": 0.4823529411764706, - "Chinese": 0.4672268907563025, - "Spanish": 0.446218487394958, - "Vietnamese": 0.46050420168067224 - }, - "consistency_score_2": 0.6760504201680672, - "consistency_score_3": 0.5367647058823529, - "consistency_score_4": 0.45630252100840335, + "Spanish": 0.9, + "English": 0.9235294117647059, + "Chinese": 0.8705882352941177, + "Vietnamese": 0.8403361344537815 + }, + "consistency_score_2": 0.8586834733893557, + "consistency_score_3": 0.7983193277310924, + "consistency_score_4": 0.7605042016806722, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.6512605042016807, - "English,Spanish": 0.7168067226890756, - "English,Vietnamese": 0.7243697478991596, - "Chinese,Spanish": 0.5957983193277311, - "Chinese,Vietnamese": 0.6504201680672269, - "Spanish,Vietnamese": 0.7176470588235294 + "Spanish,English": 0.9201680672268907, + "Spanish,Chinese": 0.8596638655462185, + "Spanish,Vietnamese": 0.8436974789915966, + "English,Chinese": 0.8680672268907563, + "English,Vietnamese": 0.8411764705882353, + "Chinese,Vietnamese": 0.819327731092437 }, "3_combine": { - "English,Chinese,Spanish": 0.5067226890756302, - "English,Chinese,Vietnamese": 0.5310924369747899, - "English,Spanish,Vietnamese": 0.6016806722689075, - "Chinese,Spanish,Vietnamese": 0.507563025210084 + "Spanish,English,Chinese": 0.8327731092436975, + "Spanish,English,Vietnamese": 0.8109243697478992, + "Spanish,Chinese,Vietnamese": 0.7739495798319328, + "English,Chinese,Vietnamese": 0.7756302521008404 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.45630252100840335 + "Spanish,English,Chinese,Vietnamese": 0.7605042016806722 } }, - "AC3_2": 0.550357611210715, - "AC3_3": 0.4977805353382224, - "AC3_4": 0.4601562514485486 + "AC3_2": 0.8709701018171236, + "AC3_3": 0.838803670306605, + "AC3_4": 0.8174496989712126 }, "prompt_2": { - "overall_acc": 0.44873949579831934, + "overall_acc": 0.8821428571428572, "language_acc": { - "English": 0.4773109243697479, - "Chinese": 0.43025210084033616, - "Spanish": 0.4327731092436975, - "Vietnamese": 0.4546218487394958 - }, - "consistency_score_2": 0.469047619047619, - "consistency_score_3": 0.2796218487394958, - "consistency_score_4": 0.18487394957983194, + "Spanish": 0.9042016806722689, + "English": 0.9184873949579831, + "Chinese": 0.8663865546218488, + "Vietnamese": 0.8394957983193277 + }, + "consistency_score_2": 0.8579831932773109, + "consistency_score_3": 0.7970588235294117, + "consistency_score_4": 0.7571428571428571, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.4050420168067227, - "English,Spanish": 0.5621848739495798, - "English,Vietnamese": 0.5411764705882353, - "Chinese,Spanish": 0.3630252100840336, - "Chinese,Vietnamese": 0.36722689075630255, - "Spanish,Vietnamese": 0.5756302521008403 + "Spanish,English": 0.9159663865546218, + "Spanish,Chinese": 0.8638655462184874, + "Spanish,Vietnamese": 0.8394957983193277, + "English,Chinese": 0.8672268907563025, + "English,Vietnamese": 0.8378151260504202, + "Chinese,Vietnamese": 0.8235294117647058 }, "3_combine": { - "English,Chinese,Spanish": 0.2495798319327731, - "English,Chinese,Vietnamese": 0.246218487394958, - "English,Spanish,Vietnamese": 0.38571428571428573, - "Chinese,Spanish,Vietnamese": 0.23697478991596638 + "Spanish,English,Chinese": 0.8327731092436975, + "Spanish,English,Vietnamese": 0.8058823529411765, + "Spanish,Chinese,Vietnamese": 0.7747899159663866, + "English,Chinese,Vietnamese": 0.7747899159663866 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.18487394957983194 + "Spanish,English,Chinese,Vietnamese": 0.7571428571428571 } }, - "AC3_2": 0.45866887571030873, - "AC3_3": 0.3445470255699803, - "AC3_4": 0.2618638967110063 + "AC3_2": 0.8698953104827705, + "AC3_3": 0.837445264556659, + "AC3_4": 0.8148770618861761 }, "prompt_3": { - "overall_acc": 0.46197478991596636, + "overall_acc": 0.8815126050420169, "language_acc": { - "English": 0.5058823529411764, - "Chinese": 0.4336134453781513, - "Spanish": 0.44789915966386556, - "Vietnamese": 0.46050420168067224 - }, - "consistency_score_2": 0.4878151260504202, - "consistency_score_3": 0.30210084033613444, - "consistency_score_4": 0.2008403361344538, + "Spanish": 0.9, + "English": 0.9159663865546218, + "Chinese": 0.8697478991596639, + "Vietnamese": 0.8403361344537815 + }, + "consistency_score_2": 0.8579831932773109, + "consistency_score_3": 0.7964285714285714, + "consistency_score_4": 0.7571428571428571, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.39243697478991596, - "English,Spanish": 0.5949579831932773, - "English,Vietnamese": 0.5554621848739496, - "Chinese,Spanish": 0.36134453781512604, - "Chinese,Vietnamese": 0.39915966386554624, - "Spanish,Vietnamese": 0.6235294117647059 + "Spanish,English": 0.907563025210084, + "Spanish,Chinese": 0.8638655462184874, + "Spanish,Vietnamese": 0.8403361344537815, + "English,Chinese": 0.8689075630252101, + "English,Vietnamese": 0.8369747899159664, + "Chinese,Vietnamese": 0.8302521008403362 }, "3_combine": { - "English,Chinese,Spanish": 0.2546218487394958, - "English,Chinese,Vietnamese": 0.25546218487394956, - "English,Spanish,Vietnamese": 0.42857142857142855, - "Chinese,Spanish,Vietnamese": 0.26974789915966385 + "Spanish,English,Chinese": 0.8285714285714286, + "Spanish,English,Vietnamese": 0.8025210084033614, + "Spanish,Chinese,Vietnamese": 0.7773109243697479, + "English,Chinese,Vietnamese": 0.7773109243697479 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.2008403361344538 + "Spanish,English,Chinese,Vietnamese": 0.7571428571428571 } }, - "AC3_2": 0.474543447056816, - "AC3_3": 0.36531193170866944, - "AC3_4": 0.27996697248369684 + "AC3_2": 0.8695887629914576, + "AC3_3": 0.8368133931426378, + "AC3_4": 0.8146080585583467 }, "prompt_4": { - "overall_acc": 0.41323529411764703, + "overall_acc": 0.88109243697479, "language_acc": { - "English": 0.45294117647058824, - "Chinese": 0.39327731092436974, - "Spanish": 0.4050420168067227, - "Vietnamese": 0.4016806722689076 - }, - "consistency_score_2": 0.7567226890756302, - "consistency_score_3": 0.6451680672268908, - "consistency_score_4": 0.5714285714285714, + "Spanish": 0.8991596638655462, + "English": 0.9126050420168067, + "Chinese": 0.8714285714285714, + "Vietnamese": 0.8411764705882353 + }, + "consistency_score_2": 0.8586834733893557, + "consistency_score_3": 0.7972689075630252, + "consistency_score_4": 0.7571428571428571, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.7168067226890756, - "English,Spanish": 0.7689075630252101, - "English,Vietnamese": 0.7714285714285715, - "Chinese,Spanish": 0.7218487394957983, - "Chinese,Vietnamese": 0.7596638655462185, - "Spanish,Vietnamese": 0.8016806722689076 + "Spanish,English": 0.9109243697478991, + "Spanish,Chinese": 0.873109243697479, + "Spanish,Vietnamese": 0.8445378151260504, + "English,Chinese": 0.865546218487395, + "English,Vietnamese": 0.8336134453781513, + "Chinese,Vietnamese": 0.8243697478991596 }, "3_combine": { - "English,Chinese,Spanish": 0.6151260504201681, - "English,Chinese,Vietnamese": 0.6352941176470588, - "English,Spanish,Vietnamese": 0.6781512605042017, - "Chinese,Spanish,Vietnamese": 0.6521008403361345 + "Spanish,English,Chinese": 0.8327731092436975, + "Spanish,English,Vietnamese": 0.8042016806722689, + "Spanish,Chinese,Vietnamese": 0.780672268907563, + "English,Chinese,Vietnamese": 0.7714285714285715 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5714285714285714 + "Spanish,English,Chinese,Vietnamese": 0.7571428571428571 } }, - "AC3_2": 0.534556842982417, - "AC3_3": 0.5037894355360826, - "AC3_4": 0.4796244932305736 + "AC3_2": 0.8697436372213716, + "AC3_3": 0.8370874447665793, + "AC3_4": 0.8144286080182891 }, "prompt_5": { - "overall_acc": 0.4569327731092437, + "overall_acc": 0.8741596638655463, "language_acc": { - "English": 0.4907563025210084, - "Chinese": 0.4361344537815126, - "Spanish": 0.45210084033613446, - "Vietnamese": 0.44873949579831934 - }, - "consistency_score_2": 0.7168067226890756, - "consistency_score_3": 0.5943277310924369, - "consistency_score_4": 0.5210084033613446, + "Spanish": 0.8957983193277311, + "English": 0.9025210084033614, + "Chinese": 0.8689075630252101, + "Vietnamese": 0.8294117647058824 + }, + "consistency_score_2": 0.85, + "consistency_score_3": 0.7852941176470588, + "consistency_score_4": 0.7411764705882353, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.6857142857142857, - "English,Spanish": 0.7680672268907563, - "English,Vietnamese": 0.7453781512605042, - "Chinese,Spanish": 0.6478991596638656, - "Chinese,Vietnamese": 0.6848739495798319, - "Spanish,Vietnamese": 0.7689075630252101 + "Spanish,English": 0.9008403361344538, + "Spanish,Chinese": 0.8739495798319328, + "Spanish,Vietnamese": 0.8302521008403362, + "English,Chinese": 0.8630252100840337, + "English,Vietnamese": 0.8168067226890756, + "Chinese,Vietnamese": 0.8151260504201681 }, "3_combine": { - "English,Chinese,Spanish": 0.573109243697479, - "English,Chinese,Vietnamese": 0.5789915966386555, - "English,Spanish,Vietnamese": 0.6563025210084034, - "Chinese,Spanish,Vietnamese": 0.5689075630252101 + "Spanish,English,Chinese": 0.8260504201680672, + "Spanish,English,Vietnamese": 0.7848739495798319, + "Spanish,Chinese,Vietnamese": 0.7714285714285715, + "English,Chinese,Vietnamese": 0.7588235294117647 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5210084033613446 + "Spanish,English,Chinese,Vietnamese": 0.7411764705882353 } }, - "AC3_2": 0.5581008132149317, - "AC3_3": 0.5166518045575889, - "AC3_4": 0.48687144025566015 + "AC3_2": 0.8619105641025625, + "AC3_3": 0.8273474676117238, + "AC3_4": 0.8021941198925023 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.2914285714285714, + "overall_acc": 0.5038095238095238, "language_acc": { - "Spanish": 0.32666666666666666, - "English": 0.32666666666666666, - "Chinese": 0.2866666666666667, - "Vietnamese": 0.23333333333333334, - "Filipino": 0.23333333333333334, - "Indonesian": 0.32666666666666666, - "Malay": 0.30666666666666664 + "Filipino": 0.42, + "Vietnamese": 0.38666666666666666, + "Chinese": 0.5133333333333333, + "Spanish": 0.54, + "Malay": 0.46, + "Indonesian": 0.54, + "English": 0.6666666666666666 }, - "consistency_score_2": 0.5158730158730159, - "consistency_score_3": 0.3339047619047619, - "consistency_score_4": 0.23904761904761906, - "consistency_score_5": 0.18031746031746027, - "consistency_score_6": 0.14095238095238094, - "consistency_score_7": 0.11333333333333333, + "consistency_score_2": 0.5968253968253967, + "consistency_score_3": 0.4401904761904763, + "consistency_score_4": 0.3518095238095238, + "consistency_score_5": 0.29555555555555557, + "consistency_score_6": 0.2580952380952381, + "consistency_score_7": 0.23333333333333334, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5533333333333333, - "Spanish,Chinese": 0.4533333333333333, - "Spanish,Vietnamese": 0.5066666666666667, - "Spanish,Filipino": 0.5, - "Spanish,Indonesian": 0.5666666666666667, - "Spanish,Malay": 0.49333333333333335, - "English,Chinese": 0.48, - "English,Vietnamese": 0.5133333333333333, - "English,Filipino": 0.54, - "English,Indonesian": 0.6133333333333333, - "English,Malay": 0.56, - "Chinese,Vietnamese": 0.4533333333333333, - "Chinese,Filipino": 0.49333333333333335, - "Chinese,Indonesian": 0.42, - "Chinese,Malay": 0.44, - "Vietnamese,Filipino": 0.5333333333333333, - "Vietnamese,Indonesian": 0.5133333333333333, - "Vietnamese,Malay": 0.5266666666666666, - "Filipino,Indonesian": 0.5266666666666666, - "Filipino,Malay": 0.5133333333333333, - "Indonesian,Malay": 0.6333333333333333 + "Filipino,Vietnamese": 0.5, + "Filipino,Chinese": 0.5333333333333333, + "Filipino,Spanish": 0.5266666666666666, + "Filipino,Malay": 0.4866666666666667, + "Filipino,Indonesian": 0.54, + "Filipino,English": 0.49333333333333335, + "Vietnamese,Chinese": 0.5533333333333333, + "Vietnamese,Spanish": 0.5066666666666667, + "Vietnamese,Malay": 0.58, + "Vietnamese,Indonesian": 0.5466666666666666, + "Vietnamese,English": 0.46, + "Chinese,Spanish": 0.6133333333333333, + "Chinese,Malay": 0.6333333333333333, + "Chinese,Indonesian": 0.7066666666666667, + "Chinese,English": 0.6333333333333333, + "Spanish,Malay": 0.68, + "Spanish,Indonesian": 0.7466666666666667, + "Spanish,English": 0.74, + "Malay,Indonesian": 0.7333333333333333, + "Malay,English": 0.62, + "Indonesian,English": 0.7 }, "3_combine": { - "Spanish,English,Chinese": 0.29333333333333333, - "Spanish,English,Vietnamese": 0.3333333333333333, - "Spanish,English,Filipino": 0.34, - "Spanish,English,Indonesian": 0.42, - "Spanish,English,Malay": 0.34, - "Spanish,Chinese,Vietnamese": 0.2733333333333333, - "Spanish,Chinese,Filipino": 0.2866666666666667, - "Spanish,Chinese,Indonesian": 0.28, - "Spanish,Chinese,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino": 0.34, - "Spanish,Vietnamese,Indonesian": 0.3466666666666667, - "Spanish,Vietnamese,Malay": 0.32666666666666666, - "Spanish,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Filipino,Malay": 0.3333333333333333, - "Spanish,Indonesian,Malay": 0.3933333333333333, - "English,Chinese,Vietnamese": 0.30666666666666664, - "English,Chinese,Filipino": 0.32, - "English,Chinese,Indonesian": 0.3, - "English,Chinese,Malay": 0.29333333333333333, - "English,Vietnamese,Filipino": 0.36, - "English,Vietnamese,Indonesian": 0.38666666666666666, - "English,Vietnamese,Malay": 0.3466666666666667, - "English,Filipino,Indonesian": 0.3933333333333333, - "English,Filipino,Malay": 0.36, - "English,Indonesian,Malay": 0.44666666666666666, - "Chinese,Vietnamese,Filipino": 0.30666666666666664, - "Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Malay": 0.2866666666666667, - "Chinese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Filipino,Malay": 0.31333333333333335, - "Chinese,Indonesian,Malay": 0.31333333333333335, - "Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Vietnamese,Filipino,Malay": 0.34, - "Vietnamese,Indonesian,Malay": 0.38, - "Filipino,Indonesian,Malay": 0.3933333333333333 + "Filipino,Vietnamese,Chinese": 0.36666666666666664, + "Filipino,Vietnamese,Spanish": 0.3333333333333333, + "Filipino,Vietnamese,Malay": 0.3466666666666667, + "Filipino,Vietnamese,Indonesian": 0.34, + "Filipino,Vietnamese,English": 0.31333333333333335, + "Filipino,Chinese,Spanish": 0.38666666666666666, + "Filipino,Chinese,Malay": 0.38, + "Filipino,Chinese,Indonesian": 0.4266666666666667, + "Filipino,Chinese,English": 0.38666666666666666, + "Filipino,Spanish,Malay": 0.3933333333333333, + "Filipino,Spanish,Indonesian": 0.43333333333333335, + "Filipino,Spanish,English": 0.4266666666666667, + "Filipino,Malay,Indonesian": 0.41333333333333333, + "Filipino,Malay,English": 0.37333333333333335, + "Filipino,Indonesian,English": 0.3933333333333333, + "Vietnamese,Chinese,Spanish": 0.4, + "Vietnamese,Chinese,Malay": 0.43333333333333335, + "Vietnamese,Chinese,Indonesian": 0.44666666666666666, + "Vietnamese,Chinese,English": 0.38666666666666666, + "Vietnamese,Spanish,Malay": 0.4266666666666667, + "Vietnamese,Spanish,Indonesian": 0.44666666666666666, + "Vietnamese,Spanish,English": 0.4, + "Vietnamese,Malay,Indonesian": 0.47333333333333333, + "Vietnamese,Malay,English": 0.4, + "Vietnamese,Indonesian,English": 0.3933333333333333, + "Chinese,Spanish,Malay": 0.49333333333333335, + "Chinese,Spanish,Indonesian": 0.56, + "Chinese,Spanish,English": 0.52, + "Chinese,Malay,Indonesian": 0.5533333333333333, + "Chinese,Malay,English": 0.48, + "Chinese,Indonesian,English": 0.5466666666666666, + "Spanish,Malay,Indonesian": 0.6066666666666667, + "Spanish,Malay,English": 0.54, + "Spanish,Indonesian,English": 0.6266666666666667, + "Malay,Indonesian,English": 0.56 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.19333333333333333, - "Spanish,English,Chinese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Indonesian": 0.22, - "Spanish,English,Chinese,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,English,Vietnamese,Malay": 0.25333333333333335, - "Spanish,English,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Filipino,Malay": 0.24666666666666667, - "Spanish,English,Indonesian,Malay": 0.3, - "Spanish,Chinese,Vietnamese,Filipino": 0.2, - "Spanish,Chinese,Vietnamese,Indonesian": 0.2, - "Spanish,Chinese,Vietnamese,Malay": 0.19333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.22, - "Spanish,Chinese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.28, - "Spanish,Filipino,Indonesian,Malay": 0.28, - "English,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.22, - "English,Chinese,Vietnamese,Malay": 0.20666666666666667, - "English,Chinese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Filipino,Malay": 0.22, - "English,Chinese,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "English,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "English,Filipino,Indonesian,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.22, - "Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Chinese,Filipino,Indonesian,Malay": 0.22, - "Vietnamese,Filipino,Indonesian,Malay": 0.28 + "Filipino,Vietnamese,Chinese,Spanish": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Malay": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,English": 0.2866666666666667, + "Filipino,Vietnamese,Spanish,Malay": 0.2866666666666667, + "Filipino,Vietnamese,Spanish,Indonesian": 0.3, + "Filipino,Vietnamese,Spanish,English": 0.2733333333333333, + "Filipino,Vietnamese,Malay,Indonesian": 0.3, + "Filipino,Vietnamese,Malay,English": 0.2733333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.26666666666666666, + "Filipino,Chinese,Spanish,Malay": 0.32, + "Filipino,Chinese,Spanish,Indonesian": 0.35333333333333333, + "Filipino,Chinese,Spanish,English": 0.3333333333333333, + "Filipino,Chinese,Malay,Indonesian": 0.3466666666666667, + "Filipino,Chinese,Malay,English": 0.31333333333333335, + "Filipino,Chinese,Indonesian,English": 0.3333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.37333333333333335, + "Filipino,Spanish,Malay,English": 0.3333333333333333, + "Filipino,Spanish,Indonesian,English": 0.36666666666666664, + "Filipino,Malay,Indonesian,English": 0.34, + "Vietnamese,Chinese,Spanish,Malay": 0.36, + "Vietnamese,Chinese,Spanish,Indonesian": 0.38666666666666666, + "Vietnamese,Chinese,Spanish,English": 0.34, + "Vietnamese,Chinese,Malay,Indonesian": 0.4, + "Vietnamese,Chinese,Malay,English": 0.3466666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.3466666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.4066666666666667, + "Vietnamese,Spanish,Malay,English": 0.35333333333333333, + "Vietnamese,Spanish,Indonesian,English": 0.38, + "Vietnamese,Malay,Indonesian,English": 0.35333333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.47333333333333333, + "Chinese,Spanish,Malay,English": 0.42, + "Chinese,Spanish,Indonesian,English": 0.4866666666666667, + "Chinese,Malay,Indonesian,English": 0.44, + "Spanish,Malay,Indonesian,English": 0.5266666666666666 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16, - "Spanish,English,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.2, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.26, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.28, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.28, + "Filipino,Vietnamese,Spanish,Malay,English": 0.24666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.26, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.24666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.31333333333333335, + "Filipino,Chinese,Spanish,Malay,English": 0.28, + "Filipino,Chinese,Spanish,Indonesian,English": 0.30666666666666664, + "Filipino,Chinese,Malay,Indonesian,English": 0.2866666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.36, + "Vietnamese,Chinese,Spanish,Malay,English": 0.31333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.3333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.32, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.3466666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.41333333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.24666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.2733333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.31333333333333335 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334 } }, - "AC3_2": 0.37245098585353326, - "AC3_3": 0.3112240546040752, - "AC3_4": 0.2626519619920366, - "AC3_5": 0.22278792535173544, - "AC3_6": 0.19000629322226095, - "AC3_7": 0.16319999995968 + "AC3_2": 0.5463870231761457, + "AC3_3": 0.469856258837963, + "AC3_4": 0.41430818805317654, + "AC3_5": 0.37255493773470155, + "AC3_6": 0.34133095233615274, + "AC3_7": 0.3189491816966112 }, "prompt_2": { - "overall_acc": 0.3257142857142857, + "overall_acc": 0.5038095238095238, "language_acc": { - "Spanish": 0.31333333333333335, - "English": 0.36666666666666664, - "Chinese": 0.38, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.24666666666666667, - "Indonesian": 0.36, - "Malay": 0.32666666666666666 + "Filipino": 0.4666666666666667, + "Vietnamese": 0.3933333333333333, + "Chinese": 0.52, + "Spanish": 0.52, + "Malay": 0.46, + "Indonesian": 0.5333333333333333, + "English": 0.6333333333333333 }, - "consistency_score_2": 0.45968253968253964, - "consistency_score_3": 0.26780952380952383, - "consistency_score_4": 0.17809523809523814, - "consistency_score_5": 0.12761904761904763, - "consistency_score_6": 0.09523809523809523, - "consistency_score_7": 0.07333333333333333, + "consistency_score_2": 0.5796825396825397, + "consistency_score_3": 0.4180952380952379, + "consistency_score_4": 0.32800000000000007, + "consistency_score_5": 0.2714285714285714, + "consistency_score_6": 0.23333333333333334, + "consistency_score_7": 0.20666666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.4866666666666667, - "Spanish,Chinese": 0.43333333333333335, - "Spanish,Vietnamese": 0.47333333333333333, - "Spanish,Filipino": 0.54, - "Spanish,Indonesian": 0.4533333333333333, - "Spanish,Malay": 0.44666666666666666, - "English,Chinese": 0.4266666666666667, - "English,Vietnamese": 0.4533333333333333, - "English,Filipino": 0.46, - "English,Indonesian": 0.52, - "English,Malay": 0.49333333333333335, - "Chinese,Vietnamese": 0.37333333333333335, - "Chinese,Filipino": 0.3333333333333333, - "Chinese,Indonesian": 0.41333333333333333, - "Chinese,Malay": 0.4, - "Vietnamese,Filipino": 0.4533333333333333, - "Vietnamese,Indonesian": 0.4533333333333333, - "Vietnamese,Malay": 0.4533333333333333, - "Filipino,Indonesian": 0.47333333333333333, - "Filipino,Malay": 0.54, - "Indonesian,Malay": 0.5733333333333334 + "Filipino,Vietnamese": 0.47333333333333333, + "Filipino,Chinese": 0.5333333333333333, + "Filipino,Spanish": 0.5466666666666666, + "Filipino,Malay": 0.5, + "Filipino,Indonesian": 0.56, + "Filipino,English": 0.54, + "Vietnamese,Chinese": 0.52, + "Vietnamese,Spanish": 0.49333333333333335, + "Vietnamese,Malay": 0.5733333333333334, + "Vietnamese,Indonesian": 0.5533333333333333, + "Vietnamese,English": 0.46, + "Chinese,Spanish": 0.6066666666666667, + "Chinese,Malay": 0.6, + "Chinese,Indonesian": 0.64, + "Chinese,English": 0.64, + "Spanish,Malay": 0.6333333333333333, + "Spanish,Indonesian": 0.7, + "Spanish,English": 0.6533333333333333, + "Malay,Indonesian": 0.72, + "Malay,English": 0.56, + "Indonesian,English": 0.6666666666666666 }, "3_combine": { - "Spanish,English,Chinese": 0.25333333333333335, - "Spanish,English,Vietnamese": 0.28, - "Spanish,English,Filipino": 0.29333333333333333, - "Spanish,English,Indonesian": 0.29333333333333333, - "Spanish,English,Malay": 0.3, - "Spanish,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,Chinese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Indonesian": 0.24, - "Spanish,Chinese,Malay": 0.22, - "Spanish,Vietnamese,Filipino": 0.2733333333333333, - "Spanish,Vietnamese,Indonesian": 0.28, - "Spanish,Vietnamese,Malay": 0.2733333333333333, - "Spanish,Filipino,Indonesian": 0.28, - "Spanish,Filipino,Malay": 0.31333333333333335, - "Spanish,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Vietnamese": 0.22, - "English,Chinese,Filipino": 0.19333333333333333, - "English,Chinese,Indonesian": 0.28, - "English,Chinese,Malay": 0.24, - "English,Vietnamese,Filipino": 0.26, - "English,Vietnamese,Indonesian": 0.30666666666666664, - "English,Vietnamese,Malay": 0.29333333333333333, - "English,Filipino,Indonesian": 0.29333333333333333, - "English,Filipino,Malay": 0.32, - "English,Indonesian,Malay": 0.35333333333333333, - "Chinese,Vietnamese,Filipino": 0.16, - "Chinese,Vietnamese,Indonesian": 0.24, - "Chinese,Vietnamese,Malay": 0.22666666666666666, - "Chinese,Filipino,Indonesian": 0.22, - "Chinese,Filipino,Malay": 0.21333333333333335, - "Chinese,Indonesian,Malay": 0.29333333333333333, - "Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Vietnamese,Filipino,Malay": 0.31333333333333335, - "Vietnamese,Indonesian,Malay": 0.32, - "Filipino,Indonesian,Malay": 0.3466666666666667 + "Filipino,Vietnamese,Chinese": 0.34, + "Filipino,Vietnamese,Spanish": 0.32, + "Filipino,Vietnamese,Malay": 0.3466666666666667, + "Filipino,Vietnamese,Indonesian": 0.36, + "Filipino,Vietnamese,English": 0.32, + "Filipino,Chinese,Spanish": 0.38, + "Filipino,Chinese,Malay": 0.36, + "Filipino,Chinese,Indonesian": 0.38666666666666666, + "Filipino,Chinese,English": 0.4, + "Filipino,Spanish,Malay": 0.4, + "Filipino,Spanish,Indonesian": 0.44666666666666666, + "Filipino,Spanish,English": 0.43333333333333335, + "Filipino,Malay,Indonesian": 0.44, + "Filipino,Malay,English": 0.36666666666666664, + "Filipino,Indonesian,English": 0.42, + "Vietnamese,Chinese,Spanish": 0.38, + "Vietnamese,Chinese,Malay": 0.4, + "Vietnamese,Chinese,Indonesian": 0.41333333333333333, + "Vietnamese,Chinese,English": 0.38, + "Vietnamese,Spanish,Malay": 0.4066666666666667, + "Vietnamese,Spanish,Indonesian": 0.4266666666666667, + "Vietnamese,Spanish,English": 0.36, + "Vietnamese,Malay,Indonesian": 0.46, + "Vietnamese,Malay,English": 0.36, + "Vietnamese,Indonesian,English": 0.3933333333333333, + "Chinese,Spanish,Malay": 0.4533333333333333, + "Chinese,Spanish,Indonesian": 0.5066666666666667, + "Chinese,Spanish,English": 0.47333333333333333, + "Chinese,Malay,Indonesian": 0.5066666666666667, + "Chinese,Malay,English": 0.44, + "Chinese,Indonesian,English": 0.5133333333333333, + "Spanish,Malay,Indonesian": 0.5466666666666666, + "Spanish,Malay,English": 0.4533333333333333, + "Spanish,Indonesian,English": 0.5333333333333333, + "Malay,Indonesian,English": 0.5066666666666667 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.14, - "Spanish,English,Chinese,Filipino": 0.14666666666666667, - "Spanish,English,Chinese,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Malay": 0.16, - "Spanish,English,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,English,Vietnamese,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Filipino,Indonesian": 0.19333333333333333, - "Spanish,English,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.14, - "Spanish,Chinese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.2, - "Spanish,Vietnamese,Indonesian,Malay": 0.2, - "Spanish,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.12, - "English,Chinese,Vietnamese,Indonesian": 0.18, - "English,Chinese,Vietnamese,Malay": 0.16, - "English,Chinese,Filipino,Indonesian": 0.16666666666666666, - "English,Chinese,Filipino,Malay": 0.14666666666666667, - "English,Chinese,Indonesian,Malay": 0.21333333333333335, - "English,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "English,Vietnamese,Filipino,Malay": 0.22, - "English,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "English,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 + "Filipino,Vietnamese,Chinese,Spanish": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Malay": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.28, + "Filipino,Vietnamese,Chinese,English": 0.2733333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.2866666666666667, + "Filipino,Vietnamese,Spanish,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Spanish,English": 0.26, + "Filipino,Vietnamese,Malay,Indonesian": 0.32, + "Filipino,Vietnamese,Malay,English": 0.26, + "Filipino,Vietnamese,Indonesian,English": 0.2733333333333333, + "Filipino,Chinese,Spanish,Malay": 0.31333333333333335, + "Filipino,Chinese,Spanish,Indonesian": 0.32666666666666666, + "Filipino,Chinese,Spanish,English": 0.32, + "Filipino,Chinese,Malay,Indonesian": 0.32, + "Filipino,Chinese,Malay,English": 0.29333333333333333, + "Filipino,Chinese,Indonesian,English": 0.32666666666666666, + "Filipino,Spanish,Malay,Indonesian": 0.37333333333333335, + "Filipino,Spanish,Malay,English": 0.32, + "Filipino,Spanish,Indonesian,English": 0.36, + "Filipino,Malay,Indonesian,English": 0.34, + "Vietnamese,Chinese,Spanish,Malay": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian": 0.36, + "Vietnamese,Chinese,Spanish,English": 0.30666666666666664, + "Vietnamese,Chinese,Malay,Indonesian": 0.36666666666666664, + "Vietnamese,Chinese,Malay,English": 0.30666666666666664, + "Vietnamese,Chinese,Indonesian,English": 0.34, + "Vietnamese,Spanish,Malay,Indonesian": 0.37333333333333335, + "Vietnamese,Spanish,Malay,English": 0.30666666666666664, + "Vietnamese,Spanish,Indonesian,English": 0.3333333333333333, + "Vietnamese,Malay,Indonesian,English": 0.3466666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, + "Chinese,Spanish,Malay,English": 0.36666666666666664, + "Chinese,Spanish,Indonesian,English": 0.42, + "Chinese,Malay,Indonesian,English": 0.4, + "Spanish,Malay,Indonesian,English": 0.42 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.12, - "Spanish,English,Chinese,Vietnamese,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.12, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian,Malay": 0.14, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.26, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.26, + "Filipino,Vietnamese,Chinese,Malay,English": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.24, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.25333333333333335, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2866666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.26, + "Filipino,Chinese,Spanish,Indonesian,English": 0.28, + "Filipino,Chinese,Malay,Indonesian,English": 0.26666666666666666, + "Filipino,Spanish,Malay,Indonesian,English": 0.3, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.32, + "Vietnamese,Chinese,Spanish,Malay,English": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.3, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.3, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.3, + "Chinese,Spanish,Malay,Indonesian,English": 0.34 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.22, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.22, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.23333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.24, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667 } }, - "AC3_2": 0.38127266423139466, - "AC3_3": 0.293937282180441, - "AC3_4": 0.23027815280333397, - "AC3_5": 0.183385354101204, - "AC3_6": 0.14738202969995934, - "AC3_7": 0.11971360378861594 + "AC3_2": 0.5390894757640105, + "AC3_3": 0.4569677291908886, + "AC3_4": 0.3973253949602483, + "AC3_5": 0.35279045274494536, + "AC3_6": 0.3189491816966112, + "AC3_7": 0.2931009829792999 }, "prompt_3": { - "overall_acc": 0.31333333333333335, + "overall_acc": 0.4942857142857143, "language_acc": { - "Spanish": 0.36, - "English": 0.36, - "Chinese": 0.2733333333333333, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.22666666666666666, - "Indonesian": 0.37333333333333335, - "Malay": 0.31333333333333335 - }, - "consistency_score_2": 0.45079365079365086, - "consistency_score_3": 0.2655238095238095, - "consistency_score_4": 0.17733333333333332, - "consistency_score_5": 0.1266666666666667, - "consistency_score_6": 0.0942857142857143, - "consistency_score_7": 0.07333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5666666666666667, - "Spanish,Chinese": 0.29333333333333333, - "Spanish,Vietnamese": 0.5133333333333333, - "Spanish,Filipino": 0.5333333333333333, - "Spanish,Indonesian": 0.5, - "Spanish,Malay": 0.48, - "English,Chinese": 0.38, - "English,Vietnamese": 0.5, - "English,Filipino": 0.5, - "English,Indonesian": 0.5266666666666666, - "English,Malay": 0.5733333333333334, - "Chinese,Vietnamese": 0.31333333333333335, - "Chinese,Filipino": 0.3, - "Chinese,Indonesian": 0.36, - "Chinese,Malay": 0.3, - "Vietnamese,Filipino": 0.47333333333333333, - "Vietnamese,Indonesian": 0.4533333333333333, - "Vietnamese,Malay": 0.49333333333333335, - "Filipino,Indonesian": 0.41333333333333333, - "Filipino,Malay": 0.4533333333333333, - "Indonesian,Malay": 0.54 + "Filipino": 0.4, + "Vietnamese": 0.4266666666666667, + "Chinese": 0.47333333333333333, + "Spanish": 0.56, + "Malay": 0.47333333333333333, + "Indonesian": 0.54, + "English": 0.5866666666666667 + }, + "consistency_score_2": 0.5203174603174604, + "consistency_score_3": 0.3432380952380952, + "consistency_score_4": 0.2521904761904762, + "consistency_score_5": 0.19777777777777777, + "consistency_score_6": 0.16285714285714284, + "consistency_score_7": 0.14, + "detailed_consistency_score": { + "2_combine": { + "Filipino,Vietnamese": 0.46, + "Filipino,Chinese": 0.4, + "Filipino,Spanish": 0.5266666666666666, + "Filipino,Malay": 0.42, + "Filipino,Indonesian": 0.43333333333333335, + "Filipino,English": 0.41333333333333333, + "Vietnamese,Chinese": 0.5133333333333333, + "Vietnamese,Spanish": 0.43333333333333335, + "Vietnamese,Malay": 0.5533333333333333, + "Vietnamese,Indonesian": 0.5133333333333333, + "Vietnamese,English": 0.43333333333333335, + "Chinese,Spanish": 0.5133333333333333, + "Chinese,Malay": 0.5533333333333333, + "Chinese,Indonesian": 0.6, + "Chinese,English": 0.5466666666666666, + "Spanish,Malay": 0.6333333333333333, + "Spanish,Indonesian": 0.6266666666666667, + "Spanish,English": 0.6066666666666667, + "Malay,Indonesian": 0.6466666666666666, + "Malay,English": 0.48, + "Indonesian,English": 0.62 }, "3_combine": { - "Spanish,English,Chinese": 0.20666666666666667, - "Spanish,English,Vietnamese": 0.37333333333333335, - "Spanish,English,Filipino": 0.37333333333333335, - "Spanish,English,Indonesian": 0.36666666666666664, - "Spanish,English,Malay": 0.38, - "Spanish,Chinese,Vietnamese": 0.16666666666666666, - "Spanish,Chinese,Filipino": 0.16, - "Spanish,Chinese,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino": 0.31333333333333335, - "Spanish,Vietnamese,Indonesian": 0.30666666666666664, - "Spanish,Vietnamese,Malay": 0.32666666666666666, - "Spanish,Filipino,Indonesian": 0.3, - "Spanish,Filipino,Malay": 0.3, - "Spanish,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese": 0.19333333333333333, - "English,Chinese,Filipino": 0.18666666666666668, - "English,Chinese,Indonesian": 0.24, - "English,Chinese,Malay": 0.21333333333333335, - "English,Vietnamese,Filipino": 0.31333333333333335, - "English,Vietnamese,Indonesian": 0.32666666666666666, - "English,Vietnamese,Malay": 0.35333333333333333, - "English,Filipino,Indonesian": 0.31333333333333335, - "English,Filipino,Malay": 0.3333333333333333, - "English,Indonesian,Malay": 0.3933333333333333, - "Chinese,Vietnamese,Filipino": 0.16666666666666666, - "Chinese,Vietnamese,Indonesian": 0.18, - "Chinese,Vietnamese,Malay": 0.16, - "Chinese,Filipino,Indonesian": 0.15333333333333332, - "Chinese,Filipino,Malay": 0.14, - "Chinese,Indonesian,Malay": 0.18666666666666668, - "Vietnamese,Filipino,Indonesian": 0.28, - "Vietnamese,Filipino,Malay": 0.29333333333333333, - "Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Filipino,Indonesian,Malay": 0.28 + "Filipino,Vietnamese,Chinese": 0.2866666666666667, + "Filipino,Vietnamese,Spanish": 0.29333333333333333, + "Filipino,Vietnamese,Malay": 0.29333333333333333, + "Filipino,Vietnamese,Indonesian": 0.28, + "Filipino,Vietnamese,English": 0.24, + "Filipino,Chinese,Spanish": 0.26666666666666666, + "Filipino,Chinese,Malay": 0.2733333333333333, + "Filipino,Chinese,Indonesian": 0.2733333333333333, + "Filipino,Chinese,English": 0.25333333333333335, + "Filipino,Spanish,Malay": 0.3466666666666667, + "Filipino,Spanish,Indonesian": 0.35333333333333333, + "Filipino,Spanish,English": 0.3333333333333333, + "Filipino,Malay,Indonesian": 0.32666666666666666, + "Filipino,Malay,English": 0.25333333333333335, + "Filipino,Indonesian,English": 0.28, + "Vietnamese,Chinese,Spanish": 0.29333333333333333, + "Vietnamese,Chinese,Malay": 0.36, + "Vietnamese,Chinese,Indonesian": 0.37333333333333335, + "Vietnamese,Chinese,English": 0.32, + "Vietnamese,Spanish,Malay": 0.36, + "Vietnamese,Spanish,Indonesian": 0.36, + "Vietnamese,Spanish,English": 0.30666666666666664, + "Vietnamese,Malay,Indonesian": 0.42, + "Vietnamese,Malay,English": 0.31333333333333335, + "Vietnamese,Indonesian,English": 0.34, + "Chinese,Spanish,Malay": 0.3933333333333333, + "Chinese,Spanish,Indonesian": 0.42, + "Chinese,Spanish,English": 0.38, + "Chinese,Malay,Indonesian": 0.44, + "Chinese,Malay,English": 0.34, + "Chinese,Indonesian,English": 0.44, + "Spanish,Malay,Indonesian": 0.49333333333333335, + "Spanish,Malay,English": 0.4066666666666667, + "Spanish,Indonesian,English": 0.47333333333333333, + "Malay,Indonesian,English": 0.4266666666666667 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.14, - "Spanish,English,Chinese,Filipino": 0.13333333333333333, - "Spanish,English,Chinese,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.26, - "Spanish,English,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,English,Vietnamese,Malay": 0.28, - "Spanish,English,Filipino,Indonesian": 0.26, - "Spanish,English,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.12, - "Spanish,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.09333333333333334, - "Spanish,Chinese,Indonesian,Malay": 0.10666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.26, - "Spanish,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.14, - "English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian": 0.14, - "English,Chinese,Filipino,Malay": 0.12, - "English,Chinese,Indonesian,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "English,Vietnamese,Filipino,Malay": 0.24, - "English,Vietnamese,Indonesian,Malay": 0.26, - "English,Filipino,Indonesian,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.1, - "Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 + "Filipino,Vietnamese,Chinese,Spanish": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Malay": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,Indonesian": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,English": 0.19333333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.24, + "Filipino,Vietnamese,Spanish,Indonesian": 0.24, + "Filipino,Vietnamese,Spanish,English": 0.18666666666666668, + "Filipino,Vietnamese,Malay,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Malay,English": 0.18666666666666668, + "Filipino,Vietnamese,Indonesian,English": 0.19333333333333333, + "Filipino,Chinese,Spanish,Malay": 0.22666666666666666, + "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, + "Filipino,Chinese,Spanish,English": 0.2, + "Filipino,Chinese,Malay,Indonesian": 0.23333333333333334, + "Filipino,Chinese,Malay,English": 0.19333333333333333, + "Filipino,Chinese,Indonesian,English": 0.20666666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.29333333333333333, + "Filipino,Spanish,Malay,English": 0.22, + "Filipino,Spanish,Indonesian,English": 0.24666666666666667, + "Filipino,Malay,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Malay": 0.26, + "Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,English": 0.24, + "Vietnamese,Chinese,Malay,Indonesian": 0.32, + "Vietnamese,Chinese,Malay,English": 0.25333333333333335, + "Vietnamese,Chinese,Indonesian,English": 0.2866666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.32666666666666666, + "Vietnamese,Spanish,Malay,English": 0.26, + "Vietnamese,Spanish,Indonesian,English": 0.2733333333333333, + "Vietnamese,Malay,Indonesian,English": 0.28, + "Chinese,Spanish,Malay,Indonesian": 0.3466666666666667, + "Chinese,Spanish,Malay,English": 0.29333333333333333, + "Chinese,Spanish,Indonesian,English": 0.3466666666666667, + "Chinese,Malay,Indonesian,English": 0.32, + "Spanish,Malay,Indonesian,English": 0.37333333333333335 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.08, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "English,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22, + "Filipino,Vietnamese,Spanish,Malay,English": 0.16, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Indonesian,English": 0.18, + "Filipino,Chinese,Malay,Indonesian,English": 0.18, + "Filipino,Spanish,Malay,Indonesian,English": 0.20666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.21333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.24666666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.24666666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.28 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.08, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.16, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14 } }, - "AC3_2": 0.36969948755721294, - "AC3_3": 0.287454206377894, - "AC3_4": 0.22648550720021807, - "AC3_5": 0.18040404036303953, - "AC3_6": 0.14495327099247646, - "AC3_7": 0.11885057468190646 + "AC3_2": 0.5069676380691103, + "AC3_3": 0.4051411676308881, + "AC3_4": 0.3339802427285613, + "AC3_5": 0.28251376142706797, + "AC3_6": 0.2449937887825941, + "AC3_7": 0.2181981981637976 }, "prompt_4": { - "overall_acc": 0.3257142857142857, + "overall_acc": 0.499047619047619, "language_acc": { - "Spanish": 0.34, - "English": 0.38, - "Chinese": 0.31333333333333335, - "Vietnamese": 0.3, - "Filipino": 0.2733333333333333, - "Indonesian": 0.32666666666666666, - "Malay": 0.3466666666666667 + "Filipino": 0.43333333333333335, + "Vietnamese": 0.38666666666666666, + "Chinese": 0.5133333333333333, + "Spanish": 0.5266666666666666, + "Malay": 0.4866666666666667, + "Indonesian": 0.5333333333333333, + "English": 0.6133333333333333 }, - "consistency_score_2": 0.5980952380952379, - "consistency_score_3": 0.4371428571428572, - "consistency_score_4": 0.3493333333333333, - "consistency_score_5": 0.29428571428571426, - "consistency_score_6": 0.2561904761904762, - "consistency_score_7": 0.22666666666666666, + "consistency_score_2": 0.5647619047619048, + "consistency_score_3": 0.3992380952380954, + "consistency_score_4": 0.30876190476190474, + "consistency_score_5": 0.25238095238095243, + "consistency_score_6": 0.21428571428571427, + "consistency_score_7": 0.18666666666666668, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6133333333333333, - "Spanish,Chinese": 0.5533333333333333, - "Spanish,Vietnamese": 0.5666666666666667, - "Spanish,Filipino": 0.6333333333333333, - "Spanish,Indonesian": 0.58, - "Spanish,Malay": 0.6, - "English,Chinese": 0.5533333333333333, - "English,Vietnamese": 0.58, - "English,Filipino": 0.5933333333333334, - "English,Indonesian": 0.6266666666666667, - "English,Malay": 0.6, - "Chinese,Vietnamese": 0.5933333333333334, - "Chinese,Filipino": 0.5266666666666666, - "Chinese,Indonesian": 0.58, - "Chinese,Malay": 0.6, - "Vietnamese,Filipino": 0.5733333333333334, - "Vietnamese,Indonesian": 0.5866666666666667, - "Vietnamese,Malay": 0.6, - "Filipino,Indonesian": 0.6333333333333333, - "Filipino,Malay": 0.6333333333333333, - "Indonesian,Malay": 0.7333333333333333 + "Filipino,Vietnamese": 0.4533333333333333, + "Filipino,Chinese": 0.4866666666666667, + "Filipino,Spanish": 0.4866666666666667, + "Filipino,Malay": 0.4666666666666667, + "Filipino,Indonesian": 0.5066666666666667, + "Filipino,English": 0.46, + "Vietnamese,Chinese": 0.5333333333333333, + "Vietnamese,Spanish": 0.4866666666666667, + "Vietnamese,Malay": 0.5733333333333334, + "Vietnamese,Indonesian": 0.5466666666666666, + "Vietnamese,English": 0.46, + "Chinese,Spanish": 0.5733333333333334, + "Chinese,Malay": 0.58, + "Chinese,Indonesian": 0.6466666666666666, + "Chinese,English": 0.58, + "Spanish,Malay": 0.6466666666666666, + "Spanish,Indonesian": 0.74, + "Spanish,English": 0.6666666666666666, + "Malay,Indonesian": 0.7333333333333333, + "Malay,English": 0.56, + "Indonesian,English": 0.6733333333333333 }, "3_combine": { - "Spanish,English,Chinese": 0.4066666666666667, - "Spanish,English,Vietnamese": 0.42, - "Spanish,English,Filipino": 0.46, - "Spanish,English,Indonesian": 0.4666666666666667, - "Spanish,English,Malay": 0.4533333333333333, - "Spanish,Chinese,Vietnamese": 0.3933333333333333, - "Spanish,Chinese,Filipino": 0.3933333333333333, - "Spanish,Chinese,Indonesian": 0.4, - "Spanish,Chinese,Malay": 0.41333333333333333, - "Spanish,Vietnamese,Filipino": 0.43333333333333335, - "Spanish,Vietnamese,Indonesian": 0.43333333333333335, - "Spanish,Vietnamese,Malay": 0.44666666666666666, - "Spanish,Filipino,Indonesian": 0.47333333333333333, - "Spanish,Filipino,Malay": 0.4666666666666667, - "Spanish,Indonesian,Malay": 0.49333333333333335, - "English,Chinese,Vietnamese": 0.4, - "English,Chinese,Filipino": 0.38, - "English,Chinese,Indonesian": 0.4266666666666667, - "English,Chinese,Malay": 0.4066666666666667, - "English,Vietnamese,Filipino": 0.42, - "English,Vietnamese,Indonesian": 0.44, - "English,Vietnamese,Malay": 0.42, - "English,Filipino,Indonesian": 0.4666666666666667, - "English,Filipino,Malay": 0.44666666666666666, - "English,Indonesian,Malay": 0.5066666666666667, - "Chinese,Vietnamese,Filipino": 0.38666666666666666, - "Chinese,Vietnamese,Indonesian": 0.4266666666666667, - "Chinese,Vietnamese,Malay": 0.43333333333333335, - "Chinese,Filipino,Indonesian": 0.41333333333333333, - "Chinese,Filipino,Malay": 0.41333333333333333, - "Chinese,Indonesian,Malay": 0.48, - "Vietnamese,Filipino,Indonesian": 0.44666666666666666, - "Vietnamese,Filipino,Malay": 0.44, - "Vietnamese,Indonesian,Malay": 0.48, - "Filipino,Indonesian,Malay": 0.5133333333333333 + "Filipino,Vietnamese,Chinese": 0.32666666666666666, + "Filipino,Vietnamese,Spanish": 0.2866666666666667, + "Filipino,Vietnamese,Malay": 0.30666666666666664, + "Filipino,Vietnamese,Indonesian": 0.31333333333333335, + "Filipino,Vietnamese,English": 0.28, + "Filipino,Chinese,Spanish": 0.31333333333333335, + "Filipino,Chinese,Malay": 0.34, + "Filipino,Chinese,Indonesian": 0.36, + "Filipino,Chinese,English": 0.32, + "Filipino,Spanish,Malay": 0.36, + "Filipino,Spanish,Indonesian": 0.4066666666666667, + "Filipino,Spanish,English": 0.36, + "Filipino,Malay,Indonesian": 0.4, + "Filipino,Malay,English": 0.32, + "Filipino,Indonesian,English": 0.36, + "Vietnamese,Chinese,Spanish": 0.36, + "Vietnamese,Chinese,Malay": 0.4, + "Vietnamese,Chinese,Indonesian": 0.4066666666666667, + "Vietnamese,Chinese,English": 0.35333333333333333, + "Vietnamese,Spanish,Malay": 0.3933333333333333, + "Vietnamese,Spanish,Indonesian": 0.43333333333333335, + "Vietnamese,Spanish,English": 0.36666666666666664, + "Vietnamese,Malay,Indonesian": 0.4666666666666667, + "Vietnamese,Malay,English": 0.38, + "Vietnamese,Indonesian,English": 0.4, + "Chinese,Spanish,Malay": 0.43333333333333335, + "Chinese,Spanish,Indonesian": 0.5133333333333333, + "Chinese,Spanish,English": 0.46, + "Chinese,Malay,Indonesian": 0.5, + "Chinese,Malay,English": 0.4066666666666667, + "Chinese,Indonesian,English": 0.49333333333333335, + "Spanish,Malay,Indonesian": 0.58, + "Spanish,Malay,English": 0.48, + "Spanish,Indonesian,English": 0.5666666666666667, + "Malay,Indonesian,English": 0.5266666666666666 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.30666666666666664, - "Spanish,English,Chinese,Filipino": 0.32, - "Spanish,English,Chinese,Indonesian": 0.3333333333333333, - "Spanish,English,Chinese,Malay": 0.3333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.3466666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.36, - "Spanish,English,Vietnamese,Malay": 0.3466666666666667, - "Spanish,English,Filipino,Indonesian": 0.38666666666666666, - "Spanish,English,Filipino,Malay": 0.36666666666666664, - "Spanish,English,Indonesian,Malay": 0.4, - "Spanish,Chinese,Vietnamese,Filipino": 0.3, - "Spanish,Chinese,Vietnamese,Indonesian": 0.32, - "Spanish,Chinese,Vietnamese,Malay": 0.3333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.3466666666666667, - "Spanish,Chinese,Filipino,Malay": 0.32, - "Spanish,Chinese,Indonesian,Malay": 0.36, - "Spanish,Vietnamese,Filipino,Indonesian": 0.38, - "Spanish,Vietnamese,Filipino,Malay": 0.36666666666666664, - "Spanish,Vietnamese,Indonesian,Malay": 0.38, - "Spanish,Filipino,Indonesian,Malay": 0.41333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.30666666666666664, - "English,Chinese,Vietnamese,Indonesian": 0.34, - "English,Chinese,Vietnamese,Malay": 0.32666666666666666, - "English,Chinese,Filipino,Indonesian": 0.3333333333333333, - "English,Chinese,Filipino,Malay": 0.31333333333333335, - "English,Chinese,Indonesian,Malay": 0.36, - "English,Vietnamese,Filipino,Indonesian": 0.36, - "English,Vietnamese,Filipino,Malay": 0.34, - "English,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "English,Filipino,Indonesian,Malay": 0.4066666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.32, - "Chinese,Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian,Malay": 0.38 + "Filipino,Vietnamese,Chinese,Spanish": 0.24, + "Filipino,Vietnamese,Chinese,Malay": 0.26, + "Filipino,Vietnamese,Chinese,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,English": 0.24, + "Filipino,Vietnamese,Spanish,Malay": 0.24, + "Filipino,Vietnamese,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Spanish,English": 0.22666666666666666, + "Filipino,Vietnamese,Malay,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Malay,English": 0.23333333333333334, + "Filipino,Vietnamese,Indonesian,English": 0.24666666666666667, + "Filipino,Chinese,Spanish,Malay": 0.26666666666666666, + "Filipino,Chinese,Spanish,Indonesian": 0.29333333333333333, + "Filipino,Chinese,Spanish,English": 0.26, + "Filipino,Chinese,Malay,Indonesian": 0.3, + "Filipino,Chinese,Malay,English": 0.25333333333333335, + "Filipino,Chinese,Indonesian,English": 0.2866666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.34, + "Filipino,Spanish,Malay,English": 0.28, + "Filipino,Spanish,Indonesian,English": 0.31333333333333335, + "Filipino,Malay,Indonesian,English": 0.31333333333333335, + "Vietnamese,Chinese,Spanish,Malay": 0.30666666666666664, + "Vietnamese,Chinese,Spanish,Indonesian": 0.34, + "Vietnamese,Chinese,Spanish,English": 0.29333333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.35333333333333333, + "Vietnamese,Chinese,Malay,English": 0.3, + "Vietnamese,Chinese,Indonesian,English": 0.32666666666666666, + "Vietnamese,Spanish,Malay,Indonesian": 0.37333333333333335, + "Vietnamese,Spanish,Malay,English": 0.32666666666666666, + "Vietnamese,Spanish,Indonesian,English": 0.35333333333333333, + "Vietnamese,Malay,Indonesian,English": 0.36, + "Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, + "Chinese,Spanish,Malay,English": 0.36666666666666664, + "Chinese,Spanish,Indonesian,English": 0.4266666666666667, + "Chinese,Malay,Indonesian,English": 0.4, + "Spanish,Malay,Indonesian,English": 0.4666666666666667 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.3, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.32, - "Spanish,English,Vietnamese,Filipino,Malay": 0.3, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.34, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.34, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.28, - "English,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.3, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.32, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.2, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Malay,English": 0.2, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,Malay,English": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.22, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.22666666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26, + "Filipino,Chinese,Spanish,Malay,English": 0.22666666666666666, + "Filipino,Chinese,Spanish,Indonesian,English": 0.24666666666666667, + "Filipino,Chinese,Malay,Indonesian,English": 0.25333333333333335, + "Filipino,Spanish,Malay,Indonesian,English": 0.28, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.3, + "Vietnamese,Chinese,Spanish,Malay,English": 0.2733333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.29333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.3, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.32, + "Chinese,Spanish,Malay,Indonesian,English": 0.36 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.2, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.2, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668 } }, - "AC3_2": 0.4217496317658341, - "AC3_3": 0.3732905296460909, - "AC3_4": 0.33711060943087384, - "AC3_5": 0.3092034232549343, - "AC3_6": 0.2867991582392337, - "AC3_7": 0.2673103447791938 + "AC3_2": 0.5298750905414855, + "AC3_3": 0.443597883548501, + "AC3_4": 0.3814931338722228, + "AC3_5": 0.3352284386058798, + "AC3_6": 0.29982834250994206, + "AC3_7": 0.27170370366408025 }, "prompt_5": { - "overall_acc": 0.3257142857142857, + "overall_acc": 0.49714285714285716, "language_acc": { - "Spanish": 0.32666666666666666, - "English": 0.38, - "Chinese": 0.31333333333333335, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.26666666666666666, - "Indonesian": 0.34, - "Malay": 0.36666666666666664 - }, - "consistency_score_2": 0.6374603174603175, - "consistency_score_3": 0.48704761904761906, - "consistency_score_4": 0.4053333333333334, - "consistency_score_5": 0.3536507936507937, - "consistency_score_6": 0.3180952380952381, - "consistency_score_7": 0.29333333333333333, + "Filipino": 0.41333333333333333, + "Vietnamese": 0.4, + "Chinese": 0.49333333333333335, + "Spanish": 0.5133333333333333, + "Malay": 0.5, + "Indonesian": 0.54, + "English": 0.62 + }, + "consistency_score_2": 0.5552380952380952, + "consistency_score_3": 0.3872380952380953, + "consistency_score_4": 0.29999999999999993, + "consistency_score_5": 0.2485714285714286, + "consistency_score_6": 0.2161904761904762, + "consistency_score_7": 0.19333333333333333, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.64, - "Spanish,Chinese": 0.6866666666666666, - "Spanish,Vietnamese": 0.7066666666666667, - "Spanish,Filipino": 0.68, - "Spanish,Indonesian": 0.6666666666666666, - "Spanish,Malay": 0.6066666666666667, - "English,Chinese": 0.5866666666666667, - "English,Vietnamese": 0.6133333333333333, - "English,Filipino": 0.6533333333333333, - "English,Indonesian": 0.68, - "English,Malay": 0.6866666666666666, - "Chinese,Vietnamese": 0.62, - "Chinese,Filipino": 0.5866666666666667, - "Chinese,Indonesian": 0.5466666666666666, - "Chinese,Malay": 0.5733333333333334, - "Vietnamese,Filipino": 0.66, - "Vietnamese,Indonesian": 0.5733333333333334, - "Vietnamese,Malay": 0.6133333333333333, - "Filipino,Indonesian": 0.6066666666666667, - "Filipino,Malay": 0.6, - "Indonesian,Malay": 0.8 + "Filipino,Vietnamese": 0.4866666666666667, + "Filipino,Chinese": 0.49333333333333335, + "Filipino,Spanish": 0.48, + "Filipino,Malay": 0.5133333333333333, + "Filipino,Indonesian": 0.4533333333333333, + "Filipino,English": 0.47333333333333333, + "Vietnamese,Chinese": 0.5, + "Vietnamese,Spanish": 0.49333333333333335, + "Vietnamese,Malay": 0.6, + "Vietnamese,Indonesian": 0.58, + "Vietnamese,English": 0.44, + "Chinese,Spanish": 0.5333333333333333, + "Chinese,Malay": 0.58, + "Chinese,Indonesian": 0.6466666666666666, + "Chinese,English": 0.5466666666666666, + "Spanish,Malay": 0.66, + "Spanish,Indonesian": 0.6866666666666666, + "Spanish,English": 0.62, + "Malay,Indonesian": 0.7333333333333333, + "Malay,English": 0.5333333333333333, + "Indonesian,English": 0.6066666666666667 }, "3_combine": { - "Spanish,English,Chinese": 0.4866666666666667, - "Spanish,English,Vietnamese": 0.5066666666666667, - "Spanish,English,Filipino": 0.5133333333333333, - "Spanish,English,Indonesian": 0.5266666666666666, - "Spanish,English,Malay": 0.5066666666666667, - "Spanish,Chinese,Vietnamese": 0.52, - "Spanish,Chinese,Filipino": 0.4866666666666667, - "Spanish,Chinese,Indonesian": 0.48, - "Spanish,Chinese,Malay": 0.46, - "Spanish,Vietnamese,Filipino": 0.54, - "Spanish,Vietnamese,Indonesian": 0.5133333333333333, - "Spanish,Vietnamese,Malay": 0.5, - "Spanish,Filipino,Indonesian": 0.5133333333333333, - "Spanish,Filipino,Malay": 0.48, - "Spanish,Indonesian,Malay": 0.56, - "English,Chinese,Vietnamese": 0.44, - "English,Chinese,Filipino": 0.44, - "English,Chinese,Indonesian": 0.44, - "English,Chinese,Malay": 0.4533333333333333, - "English,Vietnamese,Filipino": 0.49333333333333335, - "English,Vietnamese,Indonesian": 0.47333333333333333, - "English,Vietnamese,Malay": 0.49333333333333335, - "English,Filipino,Indonesian": 0.5066666666666667, - "English,Filipino,Malay": 0.49333333333333335, - "English,Indonesian,Malay": 0.6, - "Chinese,Vietnamese,Filipino": 0.46, - "Chinese,Vietnamese,Indonesian": 0.41333333333333333, - "Chinese,Vietnamese,Malay": 0.44, - "Chinese,Filipino,Indonesian": 0.4066666666666667, - "Chinese,Filipino,Malay": 0.41333333333333333, - "Chinese,Indonesian,Malay": 0.5, - "Vietnamese,Filipino,Indonesian": 0.47333333333333333, - "Vietnamese,Filipino,Malay": 0.48, - "Vietnamese,Indonesian,Malay": 0.5133333333333333, - "Filipino,Indonesian,Malay": 0.52 + "Filipino,Vietnamese,Chinese": 0.32, + "Filipino,Vietnamese,Spanish": 0.3, + "Filipino,Vietnamese,Malay": 0.35333333333333333, + "Filipino,Vietnamese,Indonesian": 0.32666666666666666, + "Filipino,Vietnamese,English": 0.28, + "Filipino,Chinese,Spanish": 0.32, + "Filipino,Chinese,Malay": 0.36, + "Filipino,Chinese,Indonesian": 0.3466666666666667, + "Filipino,Chinese,English": 0.32666666666666666, + "Filipino,Spanish,Malay": 0.38, + "Filipino,Spanish,Indonesian": 0.37333333333333335, + "Filipino,Spanish,English": 0.32, + "Filipino,Malay,Indonesian": 0.4, + "Filipino,Malay,English": 0.32666666666666666, + "Filipino,Indonesian,English": 0.31333333333333335, + "Vietnamese,Chinese,Spanish": 0.3466666666666667, + "Vietnamese,Chinese,Malay": 0.41333333333333333, + "Vietnamese,Chinese,Indonesian": 0.42, + "Vietnamese,Chinese,English": 0.34, + "Vietnamese,Spanish,Malay": 0.42, + "Vietnamese,Spanish,Indonesian": 0.4266666666666667, + "Vietnamese,Spanish,English": 0.3333333333333333, + "Vietnamese,Malay,Indonesian": 0.4866666666666667, + "Vietnamese,Malay,English": 0.3466666666666667, + "Vietnamese,Indonesian,English": 0.36666666666666664, + "Chinese,Spanish,Malay": 0.43333333333333335, + "Chinese,Spanish,Indonesian": 0.47333333333333333, + "Chinese,Spanish,English": 0.4066666666666667, + "Chinese,Malay,Indonesian": 0.5133333333333333, + "Chinese,Malay,English": 0.38, + "Chinese,Indonesian,English": 0.44, + "Spanish,Malay,Indonesian": 0.5666666666666667, + "Spanish,Malay,English": 0.43333333333333335, + "Spanish,Indonesian,English": 0.49333333333333335, + "Malay,Indonesian,English": 0.4666666666666667 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4066666666666667, - "Spanish,English,Chinese,Filipino": 0.3933333333333333, - "Spanish,English,Chinese,Indonesian": 0.4066666666666667, - "Spanish,English,Chinese,Malay": 0.4066666666666667, - "Spanish,English,Vietnamese,Filipino": 0.43333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.4266666666666667, - "Spanish,English,Vietnamese,Malay": 0.4266666666666667, - "Spanish,English,Filipino,Indonesian": 0.44, - "Spanish,English,Filipino,Malay": 0.4266666666666667, - "Spanish,English,Indonesian,Malay": 0.48, - "Spanish,Chinese,Vietnamese,Filipino": 0.41333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.38666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.38666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.36666666666666664, - "Spanish,Chinese,Filipino,Malay": 0.35333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.43333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.4266666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.41333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.4666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.44, - "English,Chinese,Vietnamese,Filipino": 0.37333333333333335, - "English,Chinese,Vietnamese,Indonesian": 0.35333333333333333, - "English,Chinese,Vietnamese,Malay": 0.38, - "English,Chinese,Filipino,Indonesian": 0.35333333333333333, - "English,Chinese,Filipino,Malay": 0.36, - "English,Chinese,Indonesian,Malay": 0.42, - "English,Vietnamese,Filipino,Indonesian": 0.4066666666666667, - "English,Vietnamese,Filipino,Malay": 0.41333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.44, - "English,Filipino,Indonesian,Malay": 0.46, - "Chinese,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.35333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.37333333333333335, - "Vietnamese,Filipino,Indonesian,Malay": 0.4266666666666667 + "Filipino,Vietnamese,Chinese,Spanish": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.28, + "Filipino,Vietnamese,Chinese,Indonesian": 0.28, + "Filipino,Vietnamese,Chinese,English": 0.24, + "Filipino,Vietnamese,Spanish,Malay": 0.28, + "Filipino,Vietnamese,Spanish,Indonesian": 0.28, + "Filipino,Vietnamese,Spanish,English": 0.22, + "Filipino,Vietnamese,Malay,Indonesian": 0.3, + "Filipino,Vietnamese,Malay,English": 0.24666666666666667, + "Filipino,Vietnamese,Indonesian,English": 0.23333333333333334, + "Filipino,Chinese,Spanish,Malay": 0.2866666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.29333333333333333, + "Filipino,Chinese,Spanish,English": 0.25333333333333335, + "Filipino,Chinese,Malay,Indonesian": 0.31333333333333335, + "Filipino,Chinese,Malay,English": 0.25333333333333335, + "Filipino,Chinese,Indonesian,English": 0.25333333333333335, + "Filipino,Spanish,Malay,Indonesian": 0.3466666666666667, + "Filipino,Spanish,Malay,English": 0.26, + "Filipino,Spanish,Indonesian,English": 0.2733333333333333, + "Filipino,Malay,Indonesian,English": 0.28, + "Vietnamese,Chinese,Spanish,Malay": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,English": 0.2733333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.38666666666666666, + "Vietnamese,Chinese,Malay,English": 0.3, + "Vietnamese,Chinese,Indonesian,English": 0.31333333333333335, + "Vietnamese,Spanish,Malay,Indonesian": 0.38, + "Vietnamese,Spanish,Malay,English": 0.2866666666666667, + "Vietnamese,Spanish,Indonesian,English": 0.31333333333333335, + "Vietnamese,Malay,Indonesian,English": 0.31333333333333335, + "Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, + "Chinese,Spanish,Malay,English": 0.32, + "Chinese,Spanish,Indonesian,English": 0.36666666666666664, + "Chinese,Malay,Indonesian,English": 0.35333333333333333, + "Spanish,Malay,Indonesian,English": 0.4066666666666667 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.35333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.34, - "Spanish,English,Chinese,Vietnamese,Malay": 0.35333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.3333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.3333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.38666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.37333333333333335, - "Spanish,English,Vietnamese,Filipino,Malay": 0.37333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.4066666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.32, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.36, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.38666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.34, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.38666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.32666666666666666 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.24, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.2, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Malay,English": 0.22, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Spanish,Malay,English": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.21333333333333335, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.22, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.28, + "Filipino,Chinese,Spanish,Malay,English": 0.22, + "Filipino,Chinese,Spanish,Indonesian,English": 0.22666666666666666, + "Filipino,Chinese,Malay,Indonesian,English": 0.22666666666666666, + "Filipino,Spanish,Malay,Indonesian,English": 0.25333333333333335, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.31333333333333335, + "Vietnamese,Chinese,Spanish,Malay,English": 0.25333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.26666666666666666, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.2866666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.28, + "Chinese,Spanish,Malay,Indonesian,English": 0.31333333333333335 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.30666666666666664, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.32, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.36, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.2, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.25333333333333335 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 } }, - "AC3_2": 0.4311366418235872, - "AC3_3": 0.39036860954720254, - "AC3_4": 0.36118811876247425, - "AC3_5": 0.3391081441423409, - "AC3_6": 0.32185967873276267, - "AC3_7": 0.30867692302705985 + "AC3_2": 0.5245869424194477, + "AC3_3": 0.43536137344697234, + "AC3_4": 0.37419354834015484, + "AC3_5": 0.331428571384127, + "AC3_6": 0.30133892805235857, + "AC3_7": 0.27839999995968 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.2792207792207792, + "overall_acc": 0.452922077922078, "language_acc": { - "English": 0.29545454545454547, - "Filipino": 0.25, - "Spanish": 0.2556818181818182, - "Malay": 0.3068181818181818, - "Chinese": 0.30113636363636365, - "Indonesian": 0.2784090909090909, - "Vietnamese": 0.26704545454545453 + "Indonesian": 0.5, + "English": 0.5397727272727273, + "Filipino": 0.3522727272727273, + "Spanish": 0.4943181818181818, + "Chinese": 0.48863636363636365, + "Malay": 0.4147727272727273, + "Vietnamese": 0.3806818181818182 }, - "consistency_score_2": 0.6439393939393939, - "consistency_score_3": 0.4931818181818183, - "consistency_score_4": 0.39788961038961046, - "consistency_score_5": 0.3292748917748919, - "consistency_score_6": 0.2775974025974026, - "consistency_score_7": 0.23863636363636365, + "consistency_score_2": 0.5662878787878788, + "consistency_score_3": 0.39577922077922073, + "consistency_score_4": 0.3006493506493506, + "consistency_score_5": 0.23917748917748916, + "consistency_score_6": 0.19642857142857142, + "consistency_score_7": 0.16477272727272727, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.5795454545454546, - "English,Spanish": 0.6477272727272727, - "English,Malay": 0.5625, - "English,Chinese": 0.5113636363636364, - "English,Indonesian": 0.5909090909090909, - "English,Vietnamese": 0.5738636363636364, - "Filipino,Spanish": 0.8693181818181818, - "Filipino,Malay": 0.6988636363636364, + "Indonesian,English": 0.7215909090909091, + "Indonesian,Filipino": 0.4943181818181818, + "Indonesian,Spanish": 0.7443181818181818, + "Indonesian,Chinese": 0.6420454545454546, + "Indonesian,Malay": 0.7045454545454546, + "Indonesian,Vietnamese": 0.5056818181818182, + "English,Filipino": 0.45454545454545453, + "English,Spanish": 0.6875, + "English,Chinese": 0.6306818181818182, + "English,Malay": 0.6079545454545454, + "English,Vietnamese": 0.48295454545454547, + "Filipino,Spanish": 0.45454545454545453, "Filipino,Chinese": 0.4318181818181818, - "Filipino,Indonesian": 0.875, - "Filipino,Vietnamese": 0.8465909090909091, - "Spanish,Malay": 0.6875, - "Spanish,Chinese": 0.5, - "Spanish,Indonesian": 0.8181818181818182, - "Spanish,Vietnamese": 0.7840909090909091, - "Malay,Chinese": 0.4431818181818182, - "Malay,Indonesian": 0.7159090909090909, - "Malay,Vietnamese": 0.6534090909090909, - "Chinese,Indonesian": 0.4715909090909091, - "Chinese,Vietnamese": 0.4715909090909091, - "Indonesian,Vietnamese": 0.7897727272727273 + "Filipino,Malay": 0.5, + "Filipino,Vietnamese": 0.5170454545454546, + "Spanish,Chinese": 0.6136363636363636, + "Spanish,Malay": 0.6079545454545454, + "Spanish,Vietnamese": 0.5113636363636364, + "Chinese,Malay": 0.5852272727272727, + "Chinese,Vietnamese": 0.48295454545454547, + "Malay,Vietnamese": 0.5113636363636364 }, "3_combine": { - "English,Filipino,Spanish": 0.5568181818181818, - "English,Filipino,Malay": 0.4602272727272727, - "English,Filipino,Chinese": 0.30113636363636365, - "English,Filipino,Indonesian": 0.5397727272727273, - "English,Filipino,Vietnamese": 0.5113636363636364, - "English,Spanish,Malay": 0.4772727272727273, - "English,Spanish,Chinese": 0.3693181818181818, - "English,Spanish,Indonesian": 0.5454545454545454, - "English,Spanish,Vietnamese": 0.5170454545454546, - "English,Malay,Chinese": 0.32386363636363635, - "English,Malay,Indonesian": 0.4659090909090909, - "English,Malay,Vietnamese": 0.4431818181818182, - "English,Chinese,Indonesian": 0.3352272727272727, - "English,Chinese,Vietnamese": 0.32386363636363635, - "English,Indonesian,Vietnamese": 0.5, - "Filipino,Spanish,Malay": 0.6477272727272727, - "Filipino,Spanish,Chinese": 0.4034090909090909, - "Filipino,Spanish,Indonesian": 0.7840909090909091, - "Filipino,Spanish,Vietnamese": 0.7556818181818182, - "Filipino,Malay,Chinese": 0.3522727272727273, - "Filipino,Malay,Indonesian": 0.6534090909090909, - "Filipino,Malay,Vietnamese": 0.6079545454545454, - "Filipino,Chinese,Indonesian": 0.4090909090909091, - "Filipino,Chinese,Vietnamese": 0.3977272727272727, - "Filipino,Indonesian,Vietnamese": 0.7613636363636364, - "Spanish,Malay,Chinese": 0.36363636363636365, - "Spanish,Malay,Indonesian": 0.6363636363636364, - "Spanish,Malay,Vietnamese": 0.5852272727272727, - "Spanish,Chinese,Indonesian": 0.4090909090909091, - "Spanish,Chinese,Vietnamese": 0.3977272727272727, - "Spanish,Indonesian,Vietnamese": 0.7045454545454546, - "Malay,Chinese,Indonesian": 0.3693181818181818, - "Malay,Chinese,Vietnamese": 0.35795454545454547, - "Malay,Indonesian,Vietnamese": 0.5965909090909091, - "Chinese,Indonesian,Vietnamese": 0.3977272727272727 + "Indonesian,English,Filipino": 0.38636363636363635, + "Indonesian,English,Spanish": 0.5909090909090909, + "Indonesian,English,Chinese": 0.5340909090909091, + "Indonesian,English,Malay": 0.5454545454545454, + "Indonesian,English,Vietnamese": 0.38636363636363635, + "Indonesian,Filipino,Spanish": 0.38636363636363635, + "Indonesian,Filipino,Chinese": 0.3522727272727273, + "Indonesian,Filipino,Malay": 0.4034090909090909, + "Indonesian,Filipino,Vietnamese": 0.3181818181818182, + "Indonesian,Spanish,Chinese": 0.5227272727272727, + "Indonesian,Spanish,Malay": 0.5454545454545454, + "Indonesian,Spanish,Vietnamese": 0.4090909090909091, + "Indonesian,Chinese,Malay": 0.48863636363636365, + "Indonesian,Chinese,Vietnamese": 0.3522727272727273, + "Indonesian,Malay,Vietnamese": 0.3977272727272727, + "English,Filipino,Spanish": 0.3465909090909091, + "English,Filipino,Chinese": 0.3352272727272727, + "English,Filipino,Malay": 0.3465909090909091, + "English,Filipino,Vietnamese": 0.2897727272727273, + "English,Spanish,Chinese": 0.5113636363636364, + "English,Spanish,Malay": 0.48295454545454547, + "English,Spanish,Vietnamese": 0.3806818181818182, + "English,Chinese,Malay": 0.45454545454545453, + "English,Chinese,Vietnamese": 0.35795454545454547, + "English,Malay,Vietnamese": 0.3522727272727273, + "Filipino,Spanish,Chinese": 0.3181818181818182, + "Filipino,Spanish,Malay": 0.3409090909090909, + "Filipino,Spanish,Vietnamese": 0.3068181818181818, + "Filipino,Chinese,Malay": 0.3181818181818182, + "Filipino,Chinese,Vietnamese": 0.2840909090909091, + "Filipino,Malay,Vietnamese": 0.3125, + "Spanish,Chinese,Malay": 0.4375, + "Spanish,Chinese,Vietnamese": 0.3522727272727273, + "Spanish,Malay,Vietnamese": 0.36363636363636365, + "Chinese,Malay,Vietnamese": 0.3409090909090909 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.44886363636363635, - "English,Filipino,Spanish,Chinese": 0.2897727272727273, - "English,Filipino,Spanish,Indonesian": 0.5170454545454546, - "English,Filipino,Spanish,Vietnamese": 0.5, - "English,Filipino,Malay,Chinese": 0.26136363636363635, - "English,Filipino,Malay,Indonesian": 0.4318181818181818, - "English,Filipino,Malay,Vietnamese": 0.4147727272727273, - "English,Filipino,Chinese,Indonesian": 0.2897727272727273, - "English,Filipino,Chinese,Vietnamese": 0.2784090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.4772727272727273, - "English,Spanish,Malay,Chinese": 0.2840909090909091, - "English,Spanish,Malay,Indonesian": 0.4431818181818182, - "English,Spanish,Malay,Vietnamese": 0.42613636363636365, - "English,Spanish,Chinese,Indonesian": 0.3068181818181818, - "English,Spanish,Chinese,Vietnamese": 0.2897727272727273, - "English,Spanish,Indonesian,Vietnamese": 0.4772727272727273, - "English,Malay,Chinese,Indonesian": 0.2784090909090909, - "English,Malay,Chinese,Vietnamese": 0.2727272727272727, - "English,Malay,Indonesian,Vietnamese": 0.4034090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Malay,Chinese": 0.3352272727272727, - "Filipino,Spanish,Malay,Indonesian": 0.6079545454545454, - "Filipino,Spanish,Malay,Vietnamese": 0.5738636363636364, - "Filipino,Spanish,Chinese,Indonesian": 0.3806818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.3806818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.6931818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.3409090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.3352272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.5738636363636364, - "Filipino,Chinese,Indonesian,Vietnamese": 0.375, - "Spanish,Malay,Chinese,Indonesian": 0.3465909090909091, - "Spanish,Malay,Chinese,Vietnamese": 0.3352272727272727, - "Spanish,Malay,Indonesian,Vietnamese": 0.5568181818181818, - "Spanish,Chinese,Indonesian,Vietnamese": 0.3693181818181818, - "Malay,Chinese,Indonesian,Vietnamese": 0.3409090909090909 + "Indonesian,English,Filipino,Spanish": 0.3181818181818182, + "Indonesian,English,Filipino,Chinese": 0.3125, + "Indonesian,English,Filipino,Malay": 0.32386363636363635, + "Indonesian,English,Filipino,Vietnamese": 0.25, + "Indonesian,English,Spanish,Chinese": 0.4659090909090909, + "Indonesian,English,Spanish,Malay": 0.45454545454545453, + "Indonesian,English,Spanish,Vietnamese": 0.3352272727272727, + "Indonesian,English,Chinese,Malay": 0.42045454545454547, + "Indonesian,English,Chinese,Vietnamese": 0.30113636363636365, + "Indonesian,English,Malay,Vietnamese": 0.32386363636363635, + "Indonesian,Filipino,Spanish,Chinese": 0.29545454545454547, + "Indonesian,Filipino,Spanish,Malay": 0.3181818181818182, + "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.30113636363636365, + "Indonesian,Filipino,Chinese,Vietnamese": 0.24431818181818182, + "Indonesian,Filipino,Malay,Vietnamese": 0.26136363636363635, + "Indonesian,Spanish,Chinese,Malay": 0.3977272727272727, + "Indonesian,Spanish,Chinese,Vietnamese": 0.30113636363636365, + "Indonesian,Spanish,Malay,Vietnamese": 0.32386363636363635, + "Indonesian,Chinese,Malay,Vietnamese": 0.29545454545454547, + "English,Filipino,Spanish,Chinese": 0.29545454545454547, + "English,Filipino,Spanish,Malay": 0.2727272727272727, + "English,Filipino,Spanish,Vietnamese": 0.23295454545454544, + "English,Filipino,Chinese,Malay": 0.2784090909090909, + "English,Filipino,Chinese,Vietnamese": 0.23863636363636365, + "English,Filipino,Malay,Vietnamese": 0.22727272727272727, + "English,Spanish,Chinese,Malay": 0.38636363636363635, + "English,Spanish,Chinese,Vietnamese": 0.30113636363636365, + "English,Spanish,Malay,Vietnamese": 0.29545454545454547, + "English,Chinese,Malay,Vietnamese": 0.2727272727272727, + "Filipino,Spanish,Chinese,Malay": 0.2556818181818182, + "Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, + "Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, + "Filipino,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.2556818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.42045454545454547, - "English,Filipino,Spanish,Malay,Vietnamese": 0.4147727272727273, - "English,Filipino,Spanish,Chinese,Indonesian": 0.2784090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.2727272727272727, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.4659090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.25, - "English,Filipino,Malay,Chinese,Vietnamese": 0.25, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.38636363636363635, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "English,Spanish,Malay,Chinese,Indonesian": 0.26704545454545453, - "English,Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.3977272727272727, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.32386363636363635, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.32386363636363635, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.5454545454545454, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.35795454545454547, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.32386363636363635, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.32386363636363635 + "Indonesian,English,Filipino,Spanish,Chinese": 0.2784090909090909, + "Indonesian,English,Filipino,Spanish,Malay": 0.26704545454545453, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.21022727272727273, + "Indonesian,English,Filipino,Chinese,Malay": 0.26704545454545453, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2159090909090909, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.2159090909090909, + "Indonesian,English,Spanish,Chinese,Malay": 0.36363636363636365, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.25, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2159090909090909, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.25, + "English,Filipino,Spanish,Chinese,Malay": 0.23863636363636365, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, + "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, + "English,Filipino,Chinese,Malay,Vietnamese": 0.19886363636363635, + "English,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1875 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.25, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.38636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.25, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.3125 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.23295454545454544, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1875, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727 } }, - "AC3_2": 0.3895342640524435, - "AC3_3": 0.3565669302912588, - "AC3_4": 0.3281563796543073, - "AC3_5": 0.30218914028976773, - "AC3_6": 0.27840672442086484, - "AC3_7": 0.25733827296256134 + "AC3_2": 0.5033001906014862, + "AC3_3": 0.42242694179185863, + "AC3_4": 0.36140098586832264, + "AC3_5": 0.3130438755475109, + "AC3_6": 0.2740178571006584, + "AC3_7": 0.24163779711773178 }, "prompt_2": { - "overall_acc": 0.26866883116883117, + "overall_acc": 0.43262987012987014, "language_acc": { - "English": 0.2897727272727273, - "Filipino": 0.23295454545454544, - "Spanish": 0.3068181818181818, - "Malay": 0.24431818181818182, - "Chinese": 0.22727272727272727, - "Indonesian": 0.2784090909090909, - "Vietnamese": 0.30113636363636365 + "Indonesian": 0.48863636363636365, + "English": 0.4772727272727273, + "Filipino": 0.3465909090909091, + "Spanish": 0.4943181818181818, + "Chinese": 0.48863636363636365, + "Malay": 0.4090909090909091, + "Vietnamese": 0.32386363636363635 }, - "consistency_score_2": 0.4829545454545454, - "consistency_score_3": 0.3034090909090909, - "consistency_score_4": 0.2105519480519481, - "consistency_score_5": 0.1512445887445887, - "consistency_score_6": 0.10957792207792208, - "consistency_score_7": 0.07954545454545454, + "consistency_score_2": 0.5546536796536797, + "consistency_score_3": 0.37727272727272726, + "consistency_score_4": 0.274025974025974, + "consistency_score_5": 0.20427489177489178, + "consistency_score_6": 0.15503246753246755, + "consistency_score_7": 0.11931818181818182, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.5625, - "English,Spanish": 0.5625, - "English,Malay": 0.5681818181818182, - "English,Chinese": 0.3181818181818182, - "English,Indonesian": 0.48295454545454547, - "English,Vietnamese": 0.42045454545454547, - "Filipino,Spanish": 0.6875, - "Filipino,Malay": 0.6590909090909091, - "Filipino,Chinese": 0.39204545454545453, - "Filipino,Indonesian": 0.5227272727272727, - "Filipino,Vietnamese": 0.4431818181818182, - "Spanish,Malay": 0.5795454545454546, - "Spanish,Chinese": 0.3409090909090909, - "Spanish,Indonesian": 0.5284090909090909, - "Spanish,Vietnamese": 0.5113636363636364, - "Malay,Chinese": 0.3693181818181818, - "Malay,Indonesian": 0.5568181818181818, - "Malay,Vietnamese": 0.4375, - "Chinese,Indonesian": 0.36363636363636365, - "Chinese,Vietnamese": 0.36363636363636365, - "Indonesian,Vietnamese": 0.4715909090909091 + "Indonesian,English": 0.6931818181818182, + "Indonesian,Filipino": 0.4715909090909091, + "Indonesian,Spanish": 0.7159090909090909, + "Indonesian,Chinese": 0.6363636363636364, + "Indonesian,Malay": 0.6988636363636364, + "Indonesian,Vietnamese": 0.5056818181818182, + "English,Filipino": 0.4602272727272727, + "English,Spanish": 0.6420454545454546, + "English,Chinese": 0.5965909090909091, + "English,Malay": 0.5965909090909091, + "English,Vietnamese": 0.44886363636363635, + "Filipino,Spanish": 0.4772727272727273, + "Filipino,Chinese": 0.45454545454545453, + "Filipino,Malay": 0.5056818181818182, + "Filipino,Vietnamese": 0.5284090909090909, + "Spanish,Chinese": 0.5681818181818182, + "Spanish,Malay": 0.6477272727272727, + "Spanish,Vietnamese": 0.48295454545454547, + "Chinese,Malay": 0.5795454545454546, + "Chinese,Vietnamese": 0.4318181818181818, + "Malay,Vietnamese": 0.5056818181818182 }, "3_combine": { - "English,Filipino,Spanish": 0.4602272727272727, - "English,Filipino,Malay": 0.4375, - "English,Filipino,Chinese": 0.2159090909090909, - "English,Filipino,Indonesian": 0.3352272727272727, - "English,Filipino,Vietnamese": 0.2840909090909091, - "English,Spanish,Malay": 0.42045454545454547, - "English,Spanish,Chinese": 0.2159090909090909, - "English,Spanish,Indonesian": 0.35795454545454547, - "English,Spanish,Vietnamese": 0.32954545454545453, - "English,Malay,Chinese": 0.23295454545454544, - "English,Malay,Indonesian": 0.36363636363636365, - "English,Malay,Vietnamese": 0.29545454545454547, - "English,Chinese,Indonesian": 0.19318181818181818, - "English,Chinese,Vietnamese": 0.17045454545454544, - "English,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay": 0.5170454545454546, - "Filipino,Spanish,Chinese": 0.2784090909090909, - "Filipino,Spanish,Indonesian": 0.42613636363636365, - "Filipino,Spanish,Vietnamese": 0.3693181818181818, - "Filipino,Malay,Chinese": 0.2897727272727273, - "Filipino,Malay,Indonesian": 0.4147727272727273, - "Filipino,Malay,Vietnamese": 0.3352272727272727, - "Filipino,Chinese,Indonesian": 0.23295454545454544, - "Filipino,Chinese,Vietnamese": 0.1875, - "Filipino,Indonesian,Vietnamese": 0.2897727272727273, - "Spanish,Malay,Chinese": 0.2556818181818182, - "Spanish,Malay,Indonesian": 0.4147727272727273, - "Spanish,Malay,Vietnamese": 0.3352272727272727, - "Spanish,Chinese,Indonesian": 0.22727272727272727, - "Spanish,Chinese,Vietnamese": 0.20454545454545456, - "Spanish,Indonesian,Vietnamese": 0.3352272727272727, - "Malay,Chinese,Indonesian": 0.23863636363636365, - "Malay,Chinese,Vietnamese": 0.19318181818181818, - "Malay,Indonesian,Vietnamese": 0.30113636363636365, - "Chinese,Indonesian,Vietnamese": 0.19318181818181818 + "Indonesian,English,Filipino": 0.36363636363636365, + "Indonesian,English,Spanish": 0.5568181818181818, + "Indonesian,English,Chinese": 0.5, + "Indonesian,English,Malay": 0.5170454545454546, + "Indonesian,English,Vietnamese": 0.3522727272727273, + "Indonesian,Filipino,Spanish": 0.375, + "Indonesian,Filipino,Chinese": 0.32386363636363635, + "Indonesian,Filipino,Malay": 0.39204545454545453, + "Indonesian,Filipino,Vietnamese": 0.3125, + "Indonesian,Spanish,Chinese": 0.48863636363636365, + "Indonesian,Spanish,Malay": 0.5454545454545454, + "Indonesian,Spanish,Vietnamese": 0.39204545454545453, + "Indonesian,Chinese,Malay": 0.48295454545454547, + "Indonesian,Chinese,Vietnamese": 0.32386363636363635, + "Indonesian,Malay,Vietnamese": 0.38636363636363635, + "English,Filipino,Spanish": 0.3409090909090909, + "English,Filipino,Chinese": 0.32386363636363635, + "English,Filipino,Malay": 0.3465909090909091, + "English,Filipino,Vietnamese": 0.2784090909090909, + "English,Spanish,Chinese": 0.45454545454545453, + "English,Spanish,Malay": 0.4659090909090909, + "English,Spanish,Vietnamese": 0.3352272727272727, + "English,Chinese,Malay": 0.42045454545454547, + "English,Chinese,Vietnamese": 0.3068181818181818, + "English,Malay,Vietnamese": 0.32954545454545453, + "Filipino,Spanish,Chinese": 0.30113636363636365, + "Filipino,Spanish,Malay": 0.3693181818181818, + "Filipino,Spanish,Vietnamese": 0.3181818181818182, + "Filipino,Chinese,Malay": 0.3125, + "Filipino,Chinese,Vietnamese": 0.2840909090909091, + "Filipino,Malay,Vietnamese": 0.3181818181818182, + "Spanish,Chinese,Malay": 0.42045454545454547, + "Spanish,Chinese,Vietnamese": 0.30113636363636365, + "Spanish,Malay,Vietnamese": 0.36363636363636365, + "Chinese,Malay,Vietnamese": 0.30113636363636365 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.3693181818181818, - "English,Filipino,Spanish,Chinese": 0.18181818181818182, - "English,Filipino,Spanish,Indonesian": 0.30113636363636365, - "English,Filipino,Spanish,Vietnamese": 0.26704545454545453, - "English,Filipino,Malay,Chinese": 0.19318181818181818, - "English,Filipino,Malay,Indonesian": 0.29545454545454547, - "English,Filipino,Malay,Vietnamese": 0.24431818181818182, - "English,Filipino,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Indonesian,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay,Chinese": 0.1875, - "English,Spanish,Malay,Indonesian": 0.3068181818181818, - "English,Spanish,Malay,Vietnamese": 0.26136363636363635, - "English,Spanish,Chinese,Indonesian": 0.1534090909090909, - "English,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "English,Spanish,Indonesian,Vietnamese": 0.2215909090909091, - "English,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "English,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Chinese": 0.23295454545454544, - "Filipino,Spanish,Malay,Indonesian": 0.375, - "Filipino,Spanish,Malay,Vietnamese": 0.29545454545454547, - "Filipino,Spanish,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Malay,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Indonesian,Vietnamese": 0.25, - "Spanish,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 + "Indonesian,English,Filipino,Spanish": 0.29545454545454547, + "Indonesian,English,Filipino,Chinese": 0.2840909090909091, + "Indonesian,English,Filipino,Malay": 0.30113636363636365, + "Indonesian,English,Filipino,Vietnamese": 0.22727272727272727, + "Indonesian,English,Spanish,Chinese": 0.42045454545454547, + "Indonesian,English,Spanish,Malay": 0.4318181818181818, + "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, + "Indonesian,English,Chinese,Malay": 0.38636363636363635, + "Indonesian,English,Chinese,Vietnamese": 0.26704545454545453, + "Indonesian,English,Malay,Vietnamese": 0.2840909090909091, + "Indonesian,Filipino,Spanish,Chinese": 0.2556818181818182, + "Indonesian,Filipino,Spanish,Malay": 0.3125, + "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.2727272727272727, + "Indonesian,Filipino,Chinese,Vietnamese": 0.2215909090909091, + "Indonesian,Filipino,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,Spanish,Chinese,Malay": 0.38636363636363635, + "Indonesian,Spanish,Chinese,Vietnamese": 0.2556818181818182, + "Indonesian,Spanish,Malay,Vietnamese": 0.3181818181818182, + "Indonesian,Chinese,Malay,Vietnamese": 0.2556818181818182, + "English,Filipino,Spanish,Chinese": 0.25, + "English,Filipino,Spanish,Malay": 0.25, + "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, + "English,Filipino,Chinese,Malay": 0.26136363636363635, + "English,Filipino,Chinese,Vietnamese": 0.2159090909090909, + "English,Filipino,Malay,Vietnamese": 0.21022727272727273, + "English,Spanish,Chinese,Malay": 0.3522727272727273, + "English,Spanish,Chinese,Vietnamese": 0.25, + "English,Spanish,Malay,Vietnamese": 0.25, + "English,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Filipino,Spanish,Chinese,Malay": 0.23863636363636365, + "Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, + "Filipino,Spanish,Malay,Vietnamese": 0.24431818181818182, + "Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, + "Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.16477272727272727, - "English,Filipino,Spanish,Malay,Indonesian": 0.2727272727272727, - "English,Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, - "English,Filipino,Spanish,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.1875, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125 + "Indonesian,English,Filipino,Spanish,Chinese": 0.23295454545454544, + "Indonesian,English,Filipino,Spanish,Malay": 0.23863636363636365, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.18181818181818182, + "Indonesian,English,Filipino,Chinese,Malay": 0.23863636363636365, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1875, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.1875, + "Indonesian,English,Spanish,Chinese,Malay": 0.3352272727272727, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2215909090909091, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.2159090909090909, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1875, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, + "English,Filipino,Spanish,Chinese,Malay": 0.19886363636363635, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, + "English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, + "English,Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, + "English,Spanish,Chinese,Malay,Vietnamese": 0.1875, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 } }, - "AC3_2": 0.34526556052739993, - "AC3_3": 0.2849841347143986, - "AC3_4": 0.23608636446697795, - "AC3_5": 0.19353850075723025, - "AC3_6": 0.15566649012104355, - "AC3_7": 0.12274846362230372 + "AC3_2": 0.48610097762477517, + "AC3_3": 0.40305945803073795, + "AC3_4": 0.3355291618083518, + "AC3_5": 0.2775153372105073, + "AC3_6": 0.22826603641089505, + "AC3_7": 0.187048796757555 }, "prompt_3": { - "overall_acc": 0.2824675324675324, + "overall_acc": 0.43749999999999994, "language_acc": { - "English": 0.3181818181818182, - "Filipino": 0.26704545454545453, - "Spanish": 0.2897727272727273, - "Malay": 0.2727272727272727, - "Chinese": 0.23295454545454544, - "Indonesian": 0.30113636363636365, - "Vietnamese": 0.29545454545454547 + "Indonesian": 0.4659090909090909, + "English": 0.4943181818181818, + "Filipino": 0.36363636363636365, + "Spanish": 0.4715909090909091, + "Chinese": 0.4943181818181818, + "Malay": 0.4318181818181818, + "Vietnamese": 0.3409090909090909 }, - "consistency_score_2": 0.48106060606060613, - "consistency_score_3": 0.2970779220779221, - "consistency_score_4": 0.2037337662337662, - "consistency_score_5": 0.14664502164502163, - "consistency_score_6": 0.10795454545454544, - "consistency_score_7": 0.07954545454545454, + "consistency_score_2": 0.5451839826839826, + "consistency_score_3": 0.3657467532467532, + "consistency_score_4": 0.26233766233766226, + "consistency_score_5": 0.19291125541125545, + "consistency_score_6": 0.14204545454545453, + "consistency_score_7": 0.10227272727272728, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.5625, - "English,Spanish": 0.5227272727272727, - "English,Malay": 0.5, - "English,Chinese": 0.4147727272727273, - "English,Indonesian": 0.5056818181818182, - "English,Vietnamese": 0.4943181818181818, - "Filipino,Spanish": 0.6931818181818182, - "Filipino,Malay": 0.6534090909090909, - "Filipino,Chinese": 0.35795454545454547, - "Filipino,Indonesian": 0.48295454545454547, - "Filipino,Vietnamese": 0.5738636363636364, - "Spanish,Malay": 0.5681818181818182, - "Spanish,Chinese": 0.30113636363636365, - "Spanish,Indonesian": 0.44886363636363635, - "Spanish,Vietnamese": 0.5, - "Malay,Chinese": 0.30113636363636365, - "Malay,Indonesian": 0.4943181818181818, - "Malay,Vietnamese": 0.5, - "Chinese,Indonesian": 0.35795454545454547, - "Chinese,Vietnamese": 0.3465909090909091, - "Indonesian,Vietnamese": 0.5227272727272727 - }, - "3_combine": { - "English,Filipino,Spanish": 0.4431818181818182, - "English,Filipino,Malay": 0.3977272727272727, - "English,Filipino,Chinese": 0.25, - "English,Filipino,Indonesian": 0.3352272727272727, - "English,Filipino,Vietnamese": 0.3693181818181818, - "English,Spanish,Malay": 0.3693181818181818, - "English,Spanish,Chinese": 0.21022727272727273, - "English,Spanish,Indonesian": 0.3125, - "English,Spanish,Vietnamese": 0.32954545454545453, - "English,Malay,Chinese": 0.2159090909090909, - "English,Malay,Indonesian": 0.3125, - "English,Malay,Vietnamese": 0.3181818181818182, - "English,Chinese,Indonesian": 0.23295454545454544, - "English,Chinese,Vietnamese": 0.2215909090909091, - "English,Indonesian,Vietnamese": 0.3181818181818182, - "Filipino,Spanish,Malay": 0.4943181818181818, - "Filipino,Spanish,Chinese": 0.23863636363636365, - "Filipino,Spanish,Indonesian": 0.3806818181818182, - "Filipino,Spanish,Vietnamese": 0.4090909090909091, - "Filipino,Malay,Chinese": 0.23863636363636365, - "Filipino,Malay,Indonesian": 0.35795454545454547, - "Filipino,Malay,Vietnamese": 0.42045454545454547, - "Filipino,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Chinese,Vietnamese": 0.21022727272727273, - "Filipino,Indonesian,Vietnamese": 0.3465909090909091, - "Spanish,Malay,Chinese": 0.1875, - "Spanish,Malay,Indonesian": 0.32954545454545453, - "Spanish,Malay,Vietnamese": 0.36363636363636365, - "Spanish,Chinese,Indonesian": 0.1875, - "Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Spanish,Indonesian,Vietnamese": 0.3125, - "Malay,Chinese,Indonesian": 0.19318181818181818, - "Malay,Chinese,Vietnamese": 0.17613636363636365, - "Malay,Indonesian,Vietnamese": 0.3125, - "Chinese,Indonesian,Vietnamese": 0.22727272727272727 + "Indonesian,English": 0.6761363636363636, + "Indonesian,Filipino": 0.4772727272727273, + "Indonesian,Spanish": 0.7045454545454546, + "Indonesian,Chinese": 0.6079545454545454, + "Indonesian,Malay": 0.6875, + "Indonesian,Vietnamese": 0.5113636363636364, + "English,Filipino": 0.4318181818181818, + "English,Spanish": 0.6306818181818182, + "English,Chinese": 0.5965909090909091, + "English,Malay": 0.5511363636363636, + "English,Vietnamese": 0.4147727272727273, + "Filipino,Spanish": 0.45454545454545453, + "Filipino,Chinese": 0.42613636363636365, + "Filipino,Malay": 0.5284090909090909, + "Filipino,Vietnamese": 0.5227272727272727, + "Spanish,Chinese": 0.5852272727272727, + "Spanish,Malay": 0.6193181818181818, + "Spanish,Vietnamese": 0.5, + "Chinese,Malay": 0.5625, + "Chinese,Vietnamese": 0.4147727272727273, + "Malay,Vietnamese": 0.5454545454545454 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.3465909090909091, + "Indonesian,English,Spanish": 0.5227272727272727, + "Indonesian,English,Chinese": 0.4715909090909091, + "Indonesian,English,Malay": 0.48863636363636365, + "Indonesian,English,Vietnamese": 0.3465909090909091, + "Indonesian,Filipino,Spanish": 0.3693181818181818, + "Indonesian,Filipino,Chinese": 0.32954545454545453, + "Indonesian,Filipino,Malay": 0.4034090909090909, + "Indonesian,Filipino,Vietnamese": 0.3125, + "Indonesian,Spanish,Chinese": 0.4659090909090909, + "Indonesian,Spanish,Malay": 0.5227272727272727, + "Indonesian,Spanish,Vietnamese": 0.4034090909090909, + "Indonesian,Chinese,Malay": 0.4602272727272727, + "Indonesian,Chinese,Vietnamese": 0.3068181818181818, + "Indonesian,Malay,Vietnamese": 0.4034090909090909, + "English,Filipino,Spanish": 0.3125, + "English,Filipino,Chinese": 0.29545454545454547, + "English,Filipino,Malay": 0.3352272727272727, + "English,Filipino,Vietnamese": 0.24431818181818182, + "English,Spanish,Chinese": 0.4375, + "English,Spanish,Malay": 0.42045454545454547, + "English,Spanish,Vietnamese": 0.32386363636363635, + "English,Chinese,Malay": 0.39204545454545453, + "English,Chinese,Vietnamese": 0.2727272727272727, + "English,Malay,Vietnamese": 0.3125, + "Filipino,Spanish,Chinese": 0.30113636363636365, + "Filipino,Spanish,Malay": 0.36363636363636365, + "Filipino,Spanish,Vietnamese": 0.29545454545454547, + "Filipino,Chinese,Malay": 0.3181818181818182, + "Filipino,Chinese,Vietnamese": 0.2784090909090909, + "Filipino,Malay,Vietnamese": 0.3465909090909091, + "Spanish,Chinese,Malay": 0.4090909090909091, + "Spanish,Chinese,Vietnamese": 0.29545454545454547, + "Spanish,Malay,Vietnamese": 0.39204545454545453, + "Chinese,Malay,Vietnamese": 0.30113636363636365 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.3352272727272727, - "English,Filipino,Spanish,Chinese": 0.1875, - "English,Filipino,Spanish,Indonesian": 0.2784090909090909, - "English,Filipino,Spanish,Vietnamese": 0.2840909090909091, - "English,Filipino,Malay,Chinese": 0.18181818181818182, - "English,Filipino,Malay,Indonesian": 0.2556818181818182, - "English,Filipino,Malay,Vietnamese": 0.2840909090909091, - "English,Filipino,Chinese,Indonesian": 0.1875, - "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.2556818181818182, - "English,Spanish,Malay,Chinese": 0.1590909090909091, - "English,Spanish,Malay,Indonesian": 0.24431818181818182, - "English,Spanish,Malay,Vietnamese": 0.26136363636363635, - "English,Spanish,Chinese,Indonesian": 0.16477272727272727, - "English,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.23863636363636365, - "English,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese": 0.17613636363636365, - "Filipino,Spanish,Malay,Indonesian": 0.29545454545454547, - "Filipino,Spanish,Malay,Vietnamese": 0.3125, - "Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Malay,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Spanish,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635 + "Indonesian,English,Filipino,Spanish": 0.26704545454545453, + "Indonesian,English,Filipino,Chinese": 0.26136363636363635, + "Indonesian,English,Filipino,Malay": 0.29545454545454547, + "Indonesian,English,Filipino,Vietnamese": 0.19886363636363635, + "Indonesian,English,Spanish,Chinese": 0.38636363636363635, + "Indonesian,English,Spanish,Malay": 0.38636363636363635, + "Indonesian,English,Spanish,Vietnamese": 0.2840909090909091, + "Indonesian,English,Chinese,Malay": 0.36363636363636365, + "Indonesian,English,Chinese,Vietnamese": 0.23863636363636365, + "Indonesian,English,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,Filipino,Spanish,Chinese": 0.26704545454545453, + "Indonesian,Filipino,Spanish,Malay": 0.3125, + "Indonesian,Filipino,Spanish,Vietnamese": 0.24431818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.2840909090909091, + "Indonesian,Filipino,Chinese,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Malay,Vietnamese": 0.2727272727272727, + "Indonesian,Spanish,Chinese,Malay": 0.36363636363636365, + "Indonesian,Spanish,Chinese,Vietnamese": 0.24431818181818182, + "Indonesian,Spanish,Malay,Vietnamese": 0.3352272727272727, + "Indonesian,Chinese,Malay,Vietnamese": 0.25, + "English,Filipino,Spanish,Chinese": 0.22727272727272727, + "English,Filipino,Spanish,Malay": 0.23863636363636365, + "English,Filipino,Spanish,Vietnamese": 0.18181818181818182, + "English,Filipino,Chinese,Malay": 0.25, + "English,Filipino,Chinese,Vietnamese": 0.18181818181818182, + "English,Filipino,Malay,Vietnamese": 0.19886363636363635, + "English,Spanish,Chinese,Malay": 0.3125, + "English,Spanish,Chinese,Vietnamese": 0.2215909090909091, + "English,Spanish,Malay,Vietnamese": 0.24431818181818182, + "English,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Filipino,Spanish,Chinese,Malay": 0.25, + "Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, + "Filipino,Spanish,Malay,Vietnamese": 0.25, + "Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Indonesian": 0.2215909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, - "English,Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.21022727272727273, - "English,Filipino,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 + "Indonesian,English,Filipino,Spanish,Chinese": 0.21022727272727273, + "Indonesian,English,Filipino,Spanish,Malay": 0.2215909090909091, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1590909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.22727272727272727, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1534090909090909, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,English,Spanish,Chinese,Malay": 0.29545454545454547, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.19886363636363635, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.22727272727272727, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.22727272727272727, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1875, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, + "English,Filipino,Spanish,Chinese,Malay": 0.1875, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, + "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, + "English,Filipino,Chinese,Malay,Vietnamese": 0.1590909090909091, + "English,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.17613636363636365, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 } }, - "AC3_2": 0.35593711740715495, - "AC3_3": 0.2895885626478381, - "AC3_4": 0.2367257116352866, - "AC3_5": 0.1930610372783416, - "AC3_6": 0.156208656168646, - "AC3_7": 0.12413371378552261 + "AC3_2": 0.4854419052369439, + "AC3_3": 0.39841855290108613, + "AC3_4": 0.3279981442352013, + "AC3_5": 0.2677575106871403, + "AC3_6": 0.21446078427672047, + "AC3_7": 0.16578947365349586 }, "prompt_4": { - "overall_acc": 0.2857142857142857, + "overall_acc": 0.439935064935065, "language_acc": { - "English": 0.2897727272727273, - "Filipino": 0.25, - "Spanish": 0.2897727272727273, - "Malay": 0.3409090909090909, - "Chinese": 0.29545454545454547, - "Indonesian": 0.30113636363636365, - "Vietnamese": 0.23295454545454544 + "Indonesian": 0.4715909090909091, + "English": 0.5, + "Filipino": 0.3522727272727273, + "Spanish": 0.48295454545454547, + "Chinese": 0.4943181818181818, + "Malay": 0.3977272727272727, + "Vietnamese": 0.3806818181818182 }, - "consistency_score_2": 0.49810606060606066, - "consistency_score_3": 0.28603896103896104, - "consistency_score_4": 0.17548701298701297, - "consistency_score_5": 0.11011904761904762, - "consistency_score_6": 0.06818181818181819, - "consistency_score_7": 0.03977272727272727, + "consistency_score_2": 0.5497835497835497, + "consistency_score_3": 0.3689935064935065, + "consistency_score_4": 0.2631493506493507, + "consistency_score_5": 0.19291125541125545, + "consistency_score_6": 0.14366883116883117, + "consistency_score_7": 0.10795454545454546, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.32954545454545453, - "English,Spanish": 0.5681818181818182, - "English,Malay": 0.4772727272727273, - "English,Chinese": 0.5681818181818182, - "English,Indonesian": 0.44886363636363635, - "English,Vietnamese": 0.39204545454545453, - "Filipino,Spanish": 0.6363636363636364, - "Filipino,Malay": 0.4602272727272727, - "Filipino,Chinese": 0.13068181818181818, - "Filipino,Indonesian": 0.5965909090909091, - "Filipino,Vietnamese": 0.7556818181818182, - "Spanish,Malay": 0.5625, - "Spanish,Chinese": 0.4318181818181818, - "Spanish,Indonesian": 0.625, - "Spanish,Vietnamese": 0.6590909090909091, - "Malay,Chinese": 0.4034090909090909, - "Malay,Indonesian": 0.6420454545454546, - "Malay,Vietnamese": 0.5, - "Chinese,Indonesian": 0.3693181818181818, - "Chinese,Vietnamese": 0.30113636363636365, - "Indonesian,Vietnamese": 0.6022727272727273 + "Indonesian,English": 0.6534090909090909, + "Indonesian,Filipino": 0.5056818181818182, + "Indonesian,Spanish": 0.7045454545454546, + "Indonesian,Chinese": 0.6420454545454546, + "Indonesian,Malay": 0.7102272727272727, + "Indonesian,Vietnamese": 0.5568181818181818, + "English,Filipino": 0.38636363636363635, + "English,Spanish": 0.6079545454545454, + "English,Chinese": 0.5738636363636364, + "English,Malay": 0.5568181818181818, + "English,Vietnamese": 0.4147727272727273, + "Filipino,Spanish": 0.4659090909090909, + "Filipino,Chinese": 0.4431818181818182, + "Filipino,Malay": 0.5, + "Filipino,Vietnamese": 0.5, + "Spanish,Chinese": 0.6022727272727273, + "Spanish,Malay": 0.6079545454545454, + "Spanish,Vietnamese": 0.5170454545454546, + "Chinese,Malay": 0.5625, + "Chinese,Vietnamese": 0.4943181818181818, + "Malay,Vietnamese": 0.5397727272727273 }, "3_combine": { - "English,Filipino,Spanish": 0.2727272727272727, - "English,Filipino,Malay": 0.21022727272727273, - "English,Filipino,Chinese": 0.05113636363636364, - "English,Filipino,Indonesian": 0.23295454545454544, - "English,Filipino,Vietnamese": 0.2556818181818182, - "English,Spanish,Malay": 0.3465909090909091, - "English,Spanish,Chinese": 0.3125, - "English,Spanish,Indonesian": 0.3522727272727273, - "English,Spanish,Vietnamese": 0.32386363636363635, - "English,Malay,Chinese": 0.25, - "English,Malay,Indonesian": 0.32954545454545453, - "English,Malay,Vietnamese": 0.25, - "English,Chinese,Indonesian": 0.22727272727272727, - "English,Chinese,Vietnamese": 0.16477272727272727, - "English,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay": 0.3693181818181818, - "Filipino,Spanish,Chinese": 0.10227272727272728, - "Filipino,Spanish,Indonesian": 0.44886363636363635, - "Filipino,Spanish,Vietnamese": 0.5340909090909091, - "Filipino,Malay,Chinese": 0.10795454545454546, - "Filipino,Malay,Indonesian": 0.39204545454545453, - "Filipino,Malay,Vietnamese": 0.3806818181818182, - "Filipino,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Chinese,Vietnamese": 0.125, - "Filipino,Indonesian,Vietnamese": 0.4943181818181818, - "Spanish,Malay,Chinese": 0.26704545454545453, - "Spanish,Malay,Indonesian": 0.45454545454545453, - "Spanish,Malay,Vietnamese": 0.3977272727272727, - "Spanish,Chinese,Indonesian": 0.24431818181818182, - "Spanish,Chinese,Vietnamese": 0.2215909090909091, - "Spanish,Indonesian,Vietnamese": 0.4602272727272727, - "Malay,Chinese,Indonesian": 0.2727272727272727, - "Malay,Chinese,Vietnamese": 0.19886363636363635, - "Malay,Indonesian,Vietnamese": 0.4090909090909091, - "Chinese,Indonesian,Vietnamese": 0.18181818181818182 + "Indonesian,English,Filipino": 0.3352272727272727, + "Indonesian,English,Spanish": 0.5056818181818182, + "Indonesian,English,Chinese": 0.48295454545454547, + "Indonesian,English,Malay": 0.4943181818181818, + "Indonesian,English,Vietnamese": 0.3465909090909091, + "Indonesian,Filipino,Spanish": 0.38636363636363635, + "Indonesian,Filipino,Chinese": 0.3409090909090909, + "Indonesian,Filipino,Malay": 0.39204545454545453, + "Indonesian,Filipino,Vietnamese": 0.32954545454545453, + "Indonesian,Spanish,Chinese": 0.5, + "Indonesian,Spanish,Malay": 0.5227272727272727, + "Indonesian,Spanish,Vietnamese": 0.4090909090909091, + "Indonesian,Chinese,Malay": 0.4772727272727273, + "Indonesian,Chinese,Vietnamese": 0.375, + "Indonesian,Malay,Vietnamese": 0.42613636363636365, + "English,Filipino,Spanish": 0.2840909090909091, + "English,Filipino,Chinese": 0.2897727272727273, + "English,Filipino,Malay": 0.2784090909090909, + "English,Filipino,Vietnamese": 0.22727272727272727, + "English,Spanish,Chinese": 0.4431818181818182, + "English,Spanish,Malay": 0.4147727272727273, + "English,Spanish,Vietnamese": 0.3181818181818182, + "English,Chinese,Malay": 0.4034090909090909, + "English,Chinese,Vietnamese": 0.3068181818181818, + "English,Malay,Vietnamese": 0.30113636363636365, + "Filipino,Spanish,Chinese": 0.3068181818181818, + "Filipino,Spanish,Malay": 0.3352272727272727, + "Filipino,Spanish,Vietnamese": 0.30113636363636365, + "Filipino,Chinese,Malay": 0.2897727272727273, + "Filipino,Chinese,Vietnamese": 0.2840909090909091, + "Filipino,Malay,Vietnamese": 0.3181818181818182, + "Spanish,Chinese,Malay": 0.42613636363636365, + "Spanish,Chinese,Vietnamese": 0.3465909090909091, + "Spanish,Malay,Vietnamese": 0.375, + "Chinese,Malay,Vietnamese": 0.3409090909090909 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.18181818181818182, - "English,Filipino,Spanish,Chinese": 0.05113636363636364, - "English,Filipino,Spanish,Indonesian": 0.20454545454545456, - "English,Filipino,Spanish,Vietnamese": 0.22727272727272727, - "English,Filipino,Malay,Chinese": 0.045454545454545456, - "English,Filipino,Malay,Indonesian": 0.1875, - "English,Filipino,Malay,Vietnamese": 0.17045454545454544, - "English,Filipino,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay,Chinese": 0.1875, - "English,Spanish,Malay,Indonesian": 0.2897727272727273, - "English,Spanish,Malay,Vietnamese": 0.2215909090909091, - "English,Spanish,Chinese,Indonesian": 0.17613636363636365, - "English,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "English,Spanish,Indonesian,Vietnamese": 0.24431818181818182, - "English,Malay,Chinese,Indonesian": 0.16477272727272727, - "English,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "English,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese": 0.09659090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.3125, - "Filipino,Spanish,Malay,Vietnamese": 0.3181818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.39204545454545453, - "Filipino,Malay,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.10227272727272728, - "Filipino,Malay,Indonesian,Vietnamese": 0.32954545454545453, - "Filipino,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.21022727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.17045454545454544, - "Spanish,Malay,Indonesian,Vietnamese": 0.32386363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091 + "Indonesian,English,Filipino,Spanish": 0.2556818181818182, + "Indonesian,English,Filipino,Chinese": 0.26136363636363635, + "Indonesian,English,Filipino,Malay": 0.25, + "Indonesian,English,Filipino,Vietnamese": 0.19886363636363635, + "Indonesian,English,Spanish,Chinese": 0.3977272727272727, + "Indonesian,English,Spanish,Malay": 0.3977272727272727, + "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, + "Indonesian,English,Chinese,Malay": 0.375, + "Indonesian,English,Chinese,Vietnamese": 0.26704545454545453, + "Indonesian,English,Malay,Vietnamese": 0.2727272727272727, + "Indonesian,Filipino,Spanish,Chinese": 0.2727272727272727, + "Indonesian,Filipino,Spanish,Malay": 0.30113636363636365, + "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.26136363636363635, + "Indonesian,Filipino,Chinese,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Malay,Vietnamese": 0.26136363636363635, + "Indonesian,Spanish,Chinese,Malay": 0.38636363636363635, + "Indonesian,Spanish,Chinese,Vietnamese": 0.2897727272727273, + "Indonesian,Spanish,Malay,Vietnamese": 0.32954545454545453, + "Indonesian,Chinese,Malay,Vietnamese": 0.29545454545454547, + "English,Filipino,Spanish,Chinese": 0.2215909090909091, + "English,Filipino,Spanish,Malay": 0.20454545454545456, + "English,Filipino,Spanish,Vietnamese": 0.17045454545454544, + "English,Filipino,Chinese,Malay": 0.2159090909090909, + "English,Filipino,Chinese,Vietnamese": 0.19318181818181818, + "English,Filipino,Malay,Vietnamese": 0.16477272727272727, + "English,Spanish,Chinese,Malay": 0.3352272727272727, + "English,Spanish,Chinese,Vietnamese": 0.25, + "English,Spanish,Malay,Vietnamese": 0.23863636363636365, + "English,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Filipino,Spanish,Chinese,Malay": 0.22727272727272727, + "Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, + "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, + "Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, + "Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 + "Indonesian,English,Filipino,Spanish,Chinese": 0.20454545454545456, + "Indonesian,English,Filipino,Spanish,Malay": 0.19318181818181818, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1590909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.19886363636363635, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.17045454545454544, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,English,Spanish,Chinese,Malay": 0.32386363636363635, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.22727272727272727, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.2159090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.2159090909090909, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1875, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1875, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, + "English,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, + "English,Filipino,Spanish,Malay,Vietnamese": 0.125, + "English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, + "English,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.1875, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 } }, - "AC3_2": 0.3631342767917796, - "AC3_3": 0.28587653114169304, - "AC3_4": 0.21742847081321706, - "AC3_5": 0.15896885065801347, - "AC3_6": 0.11009174308815758, - "AC3_7": 0.06982543638752495 + "AC3_2": 0.488763286964313, + "AC3_3": 0.40135356310406817, + "AC3_4": 0.3293164350765693, + "AC3_5": 0.2682118009825873, + "AC3_6": 0.216602243338425, + "AC3_7": 0.17336700333536068 }, "prompt_5": { - "overall_acc": 0.28652597402597396, + "overall_acc": 0.44237012987012986, "language_acc": { - "English": 0.29545454545454547, - "Filipino": 0.25, - "Spanish": 0.26136363636363635, - "Malay": 0.3125, - "Chinese": 0.3409090909090909, - "Indonesian": 0.29545454545454547, - "Vietnamese": 0.25 + "Indonesian": 0.4772727272727273, + "English": 0.5227272727272727, + "Filipino": 0.32954545454545453, + "Spanish": 0.5056818181818182, + "Chinese": 0.48295454545454547, + "Malay": 0.39204545454545453, + "Vietnamese": 0.38636363636363635 }, - "consistency_score_2": 0.5844155844155844, - "consistency_score_3": 0.400974025974026, - "consistency_score_4": 0.29399350649350653, - "consistency_score_5": 0.22294372294372303, - "consistency_score_6": 0.17288961038961043, - "consistency_score_7": 0.13636363636363635, + "consistency_score_2": 0.5549242424242424, + "consistency_score_3": 0.3759740259740261, + "consistency_score_4": 0.2725649350649351, + "consistency_score_5": 0.2031926406926407, + "consistency_score_6": 0.15259740259740262, + "consistency_score_7": 0.11363636363636363, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4659090909090909, - "English,Spanish": 0.6306818181818182, - "English,Malay": 0.5284090909090909, - "English,Chinese": 0.5738636363636364, - "English,Indonesian": 0.5795454545454546, - "English,Vietnamese": 0.5284090909090909, - "Filipino,Spanish": 0.7045454545454546, - "Filipino,Malay": 0.6193181818181818, - "Filipino,Chinese": 0.3409090909090909, - "Filipino,Indonesian": 0.6761363636363636, - "Filipino,Vietnamese": 0.7386363636363636, - "Spanish,Malay": 0.625, - "Spanish,Chinese": 0.5056818181818182, - "Spanish,Indonesian": 0.6988636363636364, - "Spanish,Vietnamese": 0.7215909090909091, - "Malay,Chinese": 0.4431818181818182, - "Malay,Indonesian": 0.7045454545454546, - "Malay,Vietnamese": 0.6306818181818182, - "Chinese,Indonesian": 0.44886363636363635, - "Chinese,Vietnamese": 0.4375, - "Indonesian,Vietnamese": 0.6704545454545454 + "Indonesian,English": 0.6477272727272727, + "Indonesian,Filipino": 0.4602272727272727, + "Indonesian,Spanish": 0.7045454545454546, + "Indonesian,Chinese": 0.6477272727272727, + "Indonesian,Malay": 0.6988636363636364, + "Indonesian,Vietnamese": 0.5397727272727273, + "English,Filipino": 0.4147727272727273, + "English,Spanish": 0.6818181818181818, + "English,Chinese": 0.5909090909090909, + "English,Malay": 0.5909090909090909, + "English,Vietnamese": 0.4772727272727273, + "Filipino,Spanish": 0.42613636363636365, + "Filipino,Chinese": 0.4318181818181818, + "Filipino,Malay": 0.4715909090909091, + "Filipino,Vietnamese": 0.5170454545454546, + "Spanish,Chinese": 0.5852272727272727, + "Spanish,Malay": 0.6306818181818182, + "Spanish,Vietnamese": 0.5568181818181818, + "Chinese,Malay": 0.5568181818181818, + "Chinese,Vietnamese": 0.4943181818181818, + "Malay,Vietnamese": 0.5284090909090909 }, "3_combine": { - "English,Filipino,Spanish": 0.4034090909090909, - "English,Filipino,Malay": 0.3522727272727273, - "English,Filipino,Chinese": 0.21022727272727273, - "English,Filipino,Indonesian": 0.38636363636363635, - "English,Filipino,Vietnamese": 0.375, - "English,Spanish,Malay": 0.42045454545454547, - "English,Spanish,Chinese": 0.3693181818181818, - "English,Spanish,Indonesian": 0.4772727272727273, - "English,Spanish,Vietnamese": 0.44886363636363635, - "English,Malay,Chinese": 0.29545454545454547, - "English,Malay,Indonesian": 0.4375, - "English,Malay,Vietnamese": 0.3693181818181818, - "English,Chinese,Indonesian": 0.32954545454545453, - "English,Chinese,Vietnamese": 0.30113636363636365, - "English,Indonesian,Vietnamese": 0.42045454545454547, - "Filipino,Spanish,Malay": 0.5, - "Filipino,Spanish,Chinese": 0.2784090909090909, - "Filipino,Spanish,Indonesian": 0.5454545454545454, - "Filipino,Spanish,Vietnamese": 0.5909090909090909, - "Filipino,Malay,Chinese": 0.2556818181818182, - "Filipino,Malay,Indonesian": 0.5227272727272727, - "Filipino,Malay,Vietnamese": 0.5113636363636364, - "Filipino,Chinese,Indonesian": 0.26136363636363635, - "Filipino,Chinese,Vietnamese": 0.2840909090909091, - "Filipino,Indonesian,Vietnamese": 0.5568181818181818, - "Spanish,Malay,Chinese": 0.32386363636363635, - "Spanish,Malay,Indonesian": 0.5284090909090909, - "Spanish,Malay,Vietnamese": 0.5113636363636364, - "Spanish,Chinese,Indonesian": 0.3522727272727273, - "Spanish,Chinese,Vietnamese": 0.3522727272727273, - "Spanish,Indonesian,Vietnamese": 0.5681818181818182, - "Malay,Chinese,Indonesian": 0.3352272727272727, - "Malay,Chinese,Vietnamese": 0.3125, - "Malay,Indonesian,Vietnamese": 0.5284090909090909, - "Chinese,Indonesian,Vietnamese": 0.3181818181818182 + "Indonesian,English,Filipino": 0.32386363636363635, + "Indonesian,English,Spanish": 0.5340909090909091, + "Indonesian,English,Chinese": 0.48863636363636365, + "Indonesian,English,Malay": 0.4943181818181818, + "Indonesian,English,Vietnamese": 0.375, + "Indonesian,Filipino,Spanish": 0.3352272727272727, + "Indonesian,Filipino,Chinese": 0.3125, + "Indonesian,Filipino,Malay": 0.375, + "Indonesian,Filipino,Vietnamese": 0.3181818181818182, + "Indonesian,Spanish,Chinese": 0.5, + "Indonesian,Spanish,Malay": 0.5397727272727273, + "Indonesian,Spanish,Vietnamese": 0.42045454545454547, + "Indonesian,Chinese,Malay": 0.48295454545454547, + "Indonesian,Chinese,Vietnamese": 0.3693181818181818, + "Indonesian,Malay,Vietnamese": 0.4090909090909091, + "English,Filipino,Spanish": 0.3125, + "English,Filipino,Chinese": 0.2840909090909091, + "English,Filipino,Malay": 0.30113636363636365, + "English,Filipino,Vietnamese": 0.2556818181818182, + "English,Spanish,Chinese": 0.4659090909090909, + "English,Spanish,Malay": 0.48863636363636365, + "English,Spanish,Vietnamese": 0.3977272727272727, + "English,Chinese,Malay": 0.4034090909090909, + "English,Chinese,Vietnamese": 0.3409090909090909, + "English,Malay,Vietnamese": 0.3409090909090909, + "Filipino,Spanish,Chinese": 0.2727272727272727, + "Filipino,Spanish,Malay": 0.3181818181818182, + "Filipino,Spanish,Vietnamese": 0.30113636363636365, + "Filipino,Chinese,Malay": 0.2784090909090909, + "Filipino,Chinese,Vietnamese": 0.3068181818181818, + "Filipino,Malay,Vietnamese": 0.3068181818181818, + "Spanish,Chinese,Malay": 0.42613636363636365, + "Spanish,Chinese,Vietnamese": 0.35795454545454547, + "Spanish,Malay,Vietnamese": 0.39204545454545453, + "Chinese,Malay,Vietnamese": 0.32954545454545453 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.32386363636363635, - "English,Filipino,Spanish,Chinese": 0.1875, - "English,Filipino,Spanish,Indonesian": 0.3352272727272727, - "English,Filipino,Spanish,Vietnamese": 0.3352272727272727, - "English,Filipino,Malay,Chinese": 0.17045454545454544, - "English,Filipino,Malay,Indonesian": 0.32386363636363635, - "English,Filipino,Malay,Vietnamese": 0.2897727272727273, - "English,Filipino,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Indonesian,Vietnamese": 0.3181818181818182, - "English,Spanish,Malay,Chinese": 0.2556818181818182, - "English,Spanish,Malay,Indonesian": 0.375, - "English,Spanish,Malay,Vietnamese": 0.3352272727272727, - "English,Spanish,Chinese,Indonesian": 0.2840909090909091, - "English,Spanish,Chinese,Vietnamese": 0.26704545454545453, - "English,Spanish,Indonesian,Vietnamese": 0.3693181818181818, - "English,Malay,Chinese,Indonesian": 0.25, - "English,Malay,Chinese,Vietnamese": 0.2215909090909091, - "English,Malay,Indonesian,Vietnamese": 0.3465909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay,Chinese": 0.2215909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.4375, - "Filipino,Spanish,Malay,Vietnamese": 0.42613636363636365, - "Filipino,Spanish,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.4772727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Malay,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.4431818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.2840909090909091, - "Spanish,Malay,Chinese,Vietnamese": 0.26704545454545453, - "Spanish,Malay,Indonesian,Vietnamese": 0.45454545454545453, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.2727272727272727 + "Indonesian,English,Filipino,Spanish": 0.2556818181818182, + "Indonesian,English,Filipino,Chinese": 0.2556818181818182, + "Indonesian,English,Filipino,Malay": 0.2556818181818182, + "Indonesian,English,Filipino,Vietnamese": 0.2215909090909091, + "Indonesian,English,Spanish,Chinese": 0.4090909090909091, + "Indonesian,English,Spanish,Malay": 0.4375, + "Indonesian,English,Spanish,Vietnamese": 0.3181818181818182, + "Indonesian,English,Chinese,Malay": 0.375, + "Indonesian,English,Chinese,Vietnamese": 0.2840909090909091, + "Indonesian,English,Malay,Vietnamese": 0.30113636363636365, + "Indonesian,Filipino,Spanish,Chinese": 0.24431818181818182, + "Indonesian,Filipino,Spanish,Malay": 0.2840909090909091, + "Indonesian,Filipino,Spanish,Vietnamese": 0.24431818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.25, + "Indonesian,Filipino,Chinese,Vietnamese": 0.23863636363636365, + "Indonesian,Filipino,Malay,Vietnamese": 0.24431818181818182, + "Indonesian,Spanish,Chinese,Malay": 0.4034090909090909, + "Indonesian,Spanish,Chinese,Vietnamese": 0.30113636363636365, + "Indonesian,Spanish,Malay,Vietnamese": 0.3465909090909091, + "Indonesian,Chinese,Malay,Vietnamese": 0.2897727272727273, + "English,Filipino,Spanish,Chinese": 0.23295454545454544, + "English,Filipino,Spanish,Malay": 0.23295454545454544, + "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, + "English,Filipino,Chinese,Malay": 0.20454545454545456, + "English,Filipino,Chinese,Vietnamese": 0.21022727272727273, + "English,Filipino,Malay,Vietnamese": 0.18181818181818182, + "English,Spanish,Chinese,Malay": 0.36363636363636365, + "English,Spanish,Chinese,Vietnamese": 0.2897727272727273, + "English,Spanish,Malay,Vietnamese": 0.30113636363636365, + "English,Chinese,Malay,Vietnamese": 0.23863636363636365, + "Filipino,Spanish,Chinese,Malay": 0.21022727272727273, + "Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, + "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, + "Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, + "Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.16477272727272727, - "English,Filipino,Spanish,Malay,Indonesian": 0.29545454545454547, - "English,Filipino,Spanish,Malay,Vietnamese": 0.26704545454545453, - "English,Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.2840909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.2727272727272727, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese,Indonesian": 0.23295454545454544, - "English,Spanish,Malay,Chinese,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.3125, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.20454545454545456, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.38636363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182 + "Indonesian,English,Filipino,Spanish,Chinese": 0.21022727272727273, + "Indonesian,English,Filipino,Spanish,Malay": 0.21022727272727273, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.18181818181818182, + "Indonesian,English,Filipino,Chinese,Malay": 0.19318181818181818, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1875, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,English,Spanish,Chinese,Malay": 0.3465909090909091, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.24431818181818182, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2727272727272727, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1875, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.25, + "English,Filipino,Spanish,Chinese,Malay": 0.17613636363636365, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, + "English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, + "English,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, + "English,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.25, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.17045454545454544, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 } }, - "AC3_2": 0.3845269362069234, - "AC3_3": 0.33422392242796445, - "AC3_4": 0.29021171077832986, - "AC3_5": 0.25076728895492384, - "AC3_6": 0.21565382492667207, - "AC3_7": 0.18478450527823762 + "AC3_2": 0.4922957874512534, + "AC3_3": 0.40647856403651084, + "AC3_4": 0.33730220161128927, + "AC3_5": 0.27847440694132897, + "AC3_6": 0.22691837490206343, + "AC3_7": 0.18082282677570657 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.3300970873786408 + "accuracy": 0.6116504854368932 }, "prompt_2": { - "accuracy": 0.24271844660194175 + "accuracy": 0.6116504854368932 }, "prompt_3": { - "accuracy": 0.2815533980582524 + "accuracy": 0.6116504854368932 }, "prompt_4": { - "accuracy": 0.22330097087378642 + "accuracy": 0.5728155339805825 }, "prompt_5": { - "accuracy": 0.27184466019417475 + "accuracy": 0.6310679611650486 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.2 + "accuracy": 0.4 }, "prompt_2": { - "accuracy": 0.2 + "accuracy": 0.38095238095238093 }, "prompt_3": { - "accuracy": 0.19047619047619047 + "accuracy": 0.41904761904761906 }, "prompt_4": { - "accuracy": 0.22857142857142856 + "accuracy": 0.3904761904761905 }, "prompt_5": { - "accuracy": 0.1619047619047619 + "accuracy": 0.44761904761904764 } }, "us_eval": { "prompt_1": { - "accuracy": 0.3644859813084112 + "accuracy": 0.6261682242990654 }, "prompt_2": { - "accuracy": 0.308411214953271 + "accuracy": 0.6261682242990654 }, "prompt_3": { - "accuracy": 0.29906542056074764 + "accuracy": 0.6448598130841121 }, "prompt_4": { - "accuracy": 0.2616822429906542 + "accuracy": 0.6542056074766355 }, "prompt_5": { - "accuracy": 0.3644859813084112 + "accuracy": 0.6542056074766355 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.28, + "accuracy": 0.38, "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.4, - "history": 0.3333333333333333, - "literature": 0.0, - "politics": 0.3, - "culture": 0.3, + "brand": 0.4, + "demographics": 0.0, + "biology": 0.2, + "history": 0.2, + "literature": 0.2, + "politics": 0.6, + "culture": 0.8, "film": 0.4, - "law": 0.1, - "geography": 0.3 + "law": 0.2, + "geography": 0.7 } }, "prompt_2": { - "accuracy": 0.32, + "accuracy": 0.4, "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.1, - "politics": 0.3, - "culture": 0.3, - "film": 0.4, - "law": 0.3, - "geography": 0.4 + "brand": 0.4, + "demographics": 0.2, + "biology": 0.4, + "history": 0.2, + "literature": 0.2, + "politics": 0.6, + "culture": 0.6, + "film": 0.5, + "law": 0.2, + "geography": 0.7 } }, "prompt_3": { - "accuracy": 0.31, + "accuracy": 0.37, "category_acc": { - "brand": 0.4, + "brand": 0.3, "demographics": 0.2, "biology": 0.4, - "history": 0.13333333333333333, - "literature": 0.3, - "politics": 0.4, - "culture": 0.3, + "history": 0.26666666666666666, + "literature": 0.2, + "politics": 0.5, + "culture": 0.6, "film": 0.3, - "law": 0.1, - "geography": 0.6 + "law": 0.2, + "geography": 0.7 } }, "prompt_4": { - "accuracy": 0.28, + "accuracy": 0.39, "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.1, - "history": 0.2, - "literature": 0.4, - "politics": 0.5, - "culture": 0.2, - "film": 0.4, + "brand": 0.3, + "demographics": 0.0, + "biology": 0.2, + "history": 0.26666666666666666, + "literature": 0.3, + "politics": 0.6, + "culture": 0.6, + "film": 0.5, "law": 0.3, - "geography": 0.3 + "geography": 0.7 } }, "prompt_5": { - "accuracy": 0.25, + "accuracy": 0.38, "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.3, + "brand": 0.4, + "demographics": 0.0, + "biology": 0.2, "history": 0.26666666666666666, - "literature": 0.1, - "politics": 0.3, - "culture": 0.3, - "film": 0.3, - "law": 0.1, - "geography": 0.2 + "literature": 0.2, + "politics": 0.6, + "culture": 0.6, + "film": 0.4, + "law": 0.3, + "geography": 0.7 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.17993847513879632 + "bleu_score": 0.2574017356444316 }, "prompt_2": { - "bleu_score": 0.176710165815627 + "bleu_score": 0.2697397453044214 }, "prompt_3": { - "bleu_score": 0.12190675760059437 + "bleu_score": 0.2710806888883188 }, "prompt_4": { - "bleu_score": 0.1852490165929848 + "bleu_score": 0.252810003719897 }, "prompt_5": { - "bleu_score": 0.06261140000984902 + "bleu_score": 0.223438870981906 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.33038771287173935 + "accuracy": 0.49495961012083584, + "category_acc": { + "History": 0.4819277108433735, + "Geography": 0.46938775510204084, + "Lampungic": 0.3877551020408163, + "Social science": 0.7328881469115192, + "Balinese": 0.34182590233545646, + "Makassarese": 0.34946236559139787, + "Banjarese": 0.4027777777777778, + "Chemistry": 0.32116788321167883, + "Biology": 0.43431952662721895, + "Science": 0.6171310629514963, + "Christian religion": 0.5920398009950248, + "Art": 0.6039933444259568, + "Islam religion": 0.6017069701280228, + "Hindu religion": 0.5066666666666667, + "Madurese": 0.3152542372881356, + "Sport": 0.5337837837837838, + "Indonesian language": 0.5709838107098381, + "Physics": 0.4202020202020202, + "Minangkabau culture": 0.39195979899497485, + "Dayak language": 0.26605504587155965, + "Sociology": 0.4879032258064516, + "Economy": 0.4733606557377049, + "Sundanese": 0.3777009507346586, + "Javanese": 0.3538306451612903, + "Civic education": 0.5951359084406295 + } }, "prompt_2": { - "bleu_score": 0.14310494948430946 - }, - "prompt_3": { - "bleu_score": 0.0826987517783187 - }, - "prompt_4": { - "bleu_score": 0.3638559322138536 - }, - "prompt_5": { - "bleu_score": 0.32275127771999856 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.2630624168562108 - }, - "prompt_2": { - "bleu_score": 0.22342425885632253 + "accuracy": 0.4968956539154817, + "category_acc": { + "History": 0.4799196787148594, + "Geography": 0.47346938775510206, + "Lampungic": 0.3469387755102041, + "Social science": 0.7445742904841403, + "Balinese": 0.3375796178343949, + "Makassarese": 0.3602150537634409, + "Banjarese": 0.3680555555555556, + "Chemistry": 0.32262773722627736, + "Biology": 0.44260355029585796, + "Science": 0.6243550051599587, + "Christian religion": 0.6119402985074627, + "Art": 0.6173044925124792, + "Islam religion": 0.6273115220483642, + "Hindu religion": 0.52, + "Madurese": 0.3152542372881356, + "Sport": 0.527027027027027, + "Indonesian language": 0.5628891656288917, + "Physics": 0.40404040404040403, + "Minangkabau culture": 0.3768844221105528, + "Dayak language": 0.27522935779816515, + "Sociology": 0.48185483870967744, + "Economy": 0.4651639344262295, + "Sundanese": 0.38547968885047534, + "Javanese": 0.3719758064516129, + "Civic education": 0.597997138769671 + } }, "prompt_3": { - "bleu_score": 0.0741754028007892 + "accuracy": 0.4931570865878897, + "category_acc": { + "History": 0.4718875502008032, + "Geography": 0.463265306122449, + "Lampungic": 0.3401360544217687, + "Social science": 0.7345575959933222, + "Balinese": 0.31210191082802546, + "Makassarese": 0.3548387096774194, + "Banjarese": 0.4166666666666667, + "Chemistry": 0.31532846715328466, + "Biology": 0.44023668639053254, + "Science": 0.6181630546955624, + "Christian religion": 0.6019900497512438, + "Art": 0.610648918469218, + "Islam religion": 0.6216216216216216, + "Hindu religion": 0.5133333333333333, + "Madurese": 0.3050847457627119, + "Sport": 0.527027027027027, + "Indonesian language": 0.562266500622665, + "Physics": 0.4121212121212121, + "Minangkabau culture": 0.40703517587939697, + "Dayak language": 0.3211009174311927, + "Sociology": 0.4798387096774194, + "Economy": 0.45901639344262296, + "Sundanese": 0.3742437337942956, + "Javanese": 0.37701612903225806, + "Civic education": 0.586552217453505 + } }, "prompt_4": { - "bleu_score": 0.29833410329049664 + "accuracy": 0.49355764737298885, + "category_acc": { + "History": 0.4738955823293173, + "Geography": 0.45510204081632655, + "Lampungic": 0.35374149659863946, + "Social science": 0.7245409015025042, + "Balinese": 0.34182590233545646, + "Makassarese": 0.3763440860215054, + "Banjarese": 0.3888888888888889, + "Chemistry": 0.3167883211678832, + "Biology": 0.4461538461538462, + "Science": 0.6160990712074303, + "Christian religion": 0.6069651741293532, + "Art": 0.6056572379367721, + "Islam religion": 0.6116642958748222, + "Hindu religion": 0.5, + "Madurese": 0.2847457627118644, + "Sport": 0.5337837837837838, + "Indonesian language": 0.5675591531755916, + "Physics": 0.402020202020202, + "Minangkabau culture": 0.38190954773869346, + "Dayak language": 0.27522935779816515, + "Sociology": 0.49193548387096775, + "Economy": 0.4672131147540984, + "Sundanese": 0.3777009507346586, + "Javanese": 0.3629032258064516, + "Civic education": 0.5994277539341917 + } }, "prompt_5": { - "bleu_score": 0.2696473010768914 + "accuracy": 0.4947593297282863, + "category_acc": { + "History": 0.4779116465863454, + "Geography": 0.4448979591836735, + "Lampungic": 0.3469387755102041, + "Social science": 0.7128547579298832, + "Balinese": 0.3333333333333333, + "Makassarese": 0.3709677419354839, + "Banjarese": 0.4097222222222222, + "Chemistry": 0.3284671532846715, + "Biology": 0.44260355029585796, + "Science": 0.6202270381836945, + "Christian religion": 0.6268656716417911, + "Art": 0.6023294509151415, + "Islam religion": 0.6031294452347084, + "Hindu religion": 0.5, + "Madurese": 0.3152542372881356, + "Sport": 0.5540540540540541, + "Indonesian language": 0.5691158156911582, + "Physics": 0.4161616161616162, + "Minangkabau culture": 0.3969849246231156, + "Dayak language": 0.28440366972477066, + "Sociology": 0.4717741935483871, + "Economy": 0.4672131147540984, + "Sundanese": 0.3802938634399309, + "Javanese": 0.37399193548387094, + "Civic education": 0.5908440629470673 + } } }, - "flores_zho2eng": { + "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.18241508450761315 + "bleu_score": 0.2904210030825523 }, "prompt_2": { - "bleu_score": 0.18899562698687283 + "bleu_score": 0.36955498730132713 }, "prompt_3": { - "bleu_score": 0.1047252907802861 + "bleu_score": 0.36547048717499736 }, "prompt_4": { - "bleu_score": 0.18903477803021426 + "bleu_score": 0.3656155033164473 }, "prompt_5": { - "bleu_score": 0.17983592290554168 + "bleu_score": 0.3613834087838442 } }, - "flores_zsm2eng": { + "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.327004632559999 + "bleu_score": 0.1361309104530537 }, "prompt_2": { - "bleu_score": 0.2228890470849816 + "bleu_score": 0.22571437892643548 }, "prompt_3": { - "bleu_score": 0.08643629245260406 + "bleu_score": 0.2207638338470559 }, "prompt_4": { - "bleu_score": 0.3589308406386717 + "bleu_score": 0.21907103091395946 }, "prompt_5": { - "bleu_score": 0.2992832990361425 + "bleu_score": 0.18624794146447873 } }, - "mmlu": { + "flores_zho2eng": { "prompt_1": { - "accuracy": 0.3162193698949825 + "bleu_score": 0.2141645930096053 }, "prompt_2": { - "accuracy": 0.32438739789964993 + "bleu_score": 0.22386319490546705 }, "prompt_3": { - "accuracy": 0.3348891481913652 + "bleu_score": 0.22159896503568985 }, "prompt_4": { - "accuracy": 0.3103850641773629 + "bleu_score": 0.22119680625556593 }, "prompt_5": { - "accuracy": 0.3103850641773629 + "bleu_score": 0.20898856132141055 } }, - "mmlu_full": { + "flores_zsm2eng": { "prompt_1": { - "accuracy": 0.290525563103325, - "category_acc": { - "high_school_european_history": 0.3475609756097561, - "business_ethics": 0.23232323232323232, - "clinical_knowledge": 0.2765151515151515, - "medical_genetics": 0.36363636363636365, - "high_school_us_history": 0.35960591133004927, - "high_school_physics": 0.22666666666666666, - "high_school_world_history": 0.3474576271186441, - "virology": 0.2606060606060606, - "high_school_microeconomics": 0.26582278481012656, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.2828282828282828, - "high_school_biology": 0.27184466019417475, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.2775800711743772, - "philosophy": 0.27419354838709675, - "professional_medicine": 0.22509225092250923, - "nutrition": 0.29508196721311475, - "global_facts": 0.23232323232323232, - "machine_learning": 0.2882882882882883, - "security_studies": 0.22950819672131148, - "public_relations": 0.3302752293577982, - "professional_psychology": 0.29296235679214405, - "prehistory": 0.3188854489164087, - "anatomy": 0.31343283582089554, - "human_sexuality": 0.3076923076923077, - "college_medicine": 0.2558139534883721, - "high_school_government_and_politics": 0.296875, - "college_chemistry": 0.20202020202020202, - "logical_fallacies": 0.2654320987654321, - "high_school_geography": 0.38071065989847713, - "elementary_mathematics": 0.23076923076923078, - "human_aging": 0.32882882882882886, - "college_mathematics": 0.35353535353535354, - "high_school_psychology": 0.3161764705882353, - "formal_logic": 0.296, - "high_school_statistics": 0.2, - "international_law": 0.38333333333333336, - "high_school_mathematics": 0.2527881040892193, - "high_school_computer_science": 0.2828282828282828, - "conceptual_physics": 0.2905982905982906, - "miscellaneous": 0.3567774936061381, - "high_school_chemistry": 0.21287128712871287, - "marketing": 0.4206008583690987, - "professional_law": 0.26614481409001955, - "management": 0.21568627450980393, - "college_physics": 0.21782178217821782, - "jurisprudence": 0.35514018691588783, - "world_religions": 0.4235294117647059, - "sociology": 0.315, - "us_foreign_policy": 0.43434343434343436, - "high_school_macroeconomics": 0.2570694087403599, - "computer_security": 0.42424242424242425, - "moral_scenarios": 0.24608501118568232, - "moral_disputes": 0.2927536231884058, - "electrical_engineering": 0.2986111111111111, - "astronomy": 0.2980132450331126, - "college_biology": 0.34965034965034963 - } + "bleu_score": 0.20246195784882284 }, "prompt_2": { - "accuracy": 0.29803360743653917, - "category_acc": { - "high_school_european_history": 0.3475609756097561, - "business_ethics": 0.36363636363636365, - "clinical_knowledge": 0.3068181818181818, - "medical_genetics": 0.3434343434343434, - "high_school_us_history": 0.3694581280788177, - "high_school_physics": 0.20666666666666667, - "high_school_world_history": 0.3686440677966102, - "virology": 0.26666666666666666, - "high_school_microeconomics": 0.26582278481012656, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.3434343434343434, - "high_school_biology": 0.284789644012945, - "abstract_algebra": 0.21212121212121213, - "professional_accounting": 0.27402135231316727, - "philosophy": 0.21935483870967742, - "professional_medicine": 0.28044280442804426, - "nutrition": 0.34098360655737703, - "global_facts": 0.2727272727272727, - "machine_learning": 0.32432432432432434, - "security_studies": 0.21721311475409835, - "public_relations": 0.3761467889908257, - "professional_psychology": 0.28805237315875615, - "prehistory": 0.2848297213622291, - "anatomy": 0.3283582089552239, - "human_sexuality": 0.2692307692307692, - "college_medicine": 0.27906976744186046, - "high_school_government_and_politics": 0.3229166666666667, - "college_chemistry": 0.23232323232323232, - "logical_fallacies": 0.2654320987654321, - "high_school_geography": 0.39086294416243655, - "elementary_mathematics": 0.26790450928381965, - "human_aging": 0.32882882882882886, - "college_mathematics": 0.2727272727272727, - "high_school_psychology": 0.30330882352941174, - "formal_logic": 0.296, - "high_school_statistics": 0.23255813953488372, - "international_law": 0.3333333333333333, - "high_school_mathematics": 0.26022304832713755, - "high_school_computer_science": 0.3333333333333333, - "conceptual_physics": 0.2948717948717949, - "miscellaneous": 0.3874680306905371, - "high_school_chemistry": 0.21782178217821782, - "marketing": 0.39914163090128757, - "professional_law": 0.2654924983692107, - "management": 0.28431372549019607, - "college_physics": 0.27722772277227725, - "jurisprudence": 0.3644859813084112, - "world_religions": 0.48823529411764705, - "sociology": 0.34, - "us_foreign_policy": 0.42424242424242425, - "high_school_macroeconomics": 0.22879177377892032, - "computer_security": 0.35353535353535354, - "moral_scenarios": 0.24384787472035793, - "moral_disputes": 0.336231884057971, - "electrical_engineering": 0.3472222222222222, - "astronomy": 0.3509933774834437, - "college_biology": 0.3076923076923077 - } + "bleu_score": 0.34339277822910913 }, "prompt_3": { - "accuracy": 0.3154093671791205, - "category_acc": { - "high_school_european_history": 0.4024390243902439, - "business_ethics": 0.3939393939393939, - "clinical_knowledge": 0.32954545454545453, - "medical_genetics": 0.3939393939393939, - "high_school_us_history": 0.3497536945812808, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.3771186440677966, - "virology": 0.2606060606060606, - "high_school_microeconomics": 0.24472573839662448, - "econometrics": 0.2743362831858407, - "college_computer_science": 0.2727272727272727, - "high_school_biology": 0.31715210355987056, - "abstract_algebra": 0.21212121212121213, - "professional_accounting": 0.2775800711743772, - "philosophy": 0.32903225806451614, - "professional_medicine": 0.2693726937269373, - "nutrition": 0.3704918032786885, - "global_facts": 0.2828282828282828, - "machine_learning": 0.26126126126126126, - "security_studies": 0.2336065573770492, - "public_relations": 0.3302752293577982, - "professional_psychology": 0.3060556464811784, - "prehistory": 0.33126934984520123, - "anatomy": 0.3582089552238806, - "human_sexuality": 0.27692307692307694, - "college_medicine": 0.29069767441860467, - "high_school_government_and_politics": 0.3802083333333333, - "college_chemistry": 0.25252525252525254, - "logical_fallacies": 0.24074074074074073, - "high_school_geography": 0.4263959390862944, - "elementary_mathematics": 0.2864721485411141, - "human_aging": 0.32432432432432434, - "college_mathematics": 0.29292929292929293, - "high_school_psychology": 0.37683823529411764, - "formal_logic": 0.248, - "high_school_statistics": 0.23255813953488372, - "international_law": 0.39166666666666666, - "high_school_mathematics": 0.21933085501858737, - "high_school_computer_science": 0.3434343434343434, - "conceptual_physics": 0.28205128205128205, - "miscellaneous": 0.3989769820971867, - "high_school_chemistry": 0.28217821782178215, - "marketing": 0.4592274678111588, - "professional_law": 0.27266797129810827, - "management": 0.3235294117647059, - "college_physics": 0.2376237623762376, - "jurisprudence": 0.38317757009345793, - "world_religions": 0.5294117647058824, - "sociology": 0.38, - "us_foreign_policy": 0.43434343434343436, - "high_school_macroeconomics": 0.2570694087403599, - "computer_security": 0.40404040404040403, - "moral_scenarios": 0.24720357941834453, - "moral_disputes": 0.3333333333333333, - "electrical_engineering": 0.3125, - "astronomy": 0.39072847682119205, - "college_biology": 0.35664335664335667 - } + "bleu_score": 0.3414705878211933 }, "prompt_4": { - "accuracy": 0.29209867715409366, - "category_acc": { - "high_school_european_history": 0.3353658536585366, - "business_ethics": 0.31313131313131315, - "clinical_knowledge": 0.3181818181818182, - "medical_genetics": 0.36363636363636365, - "high_school_us_history": 0.33004926108374383, - "high_school_physics": 0.24666666666666667, - "high_school_world_history": 0.3644067796610169, - "virology": 0.23636363636363636, - "high_school_microeconomics": 0.25316455696202533, - "econometrics": 0.22123893805309736, - "college_computer_science": 0.29292929292929293, - "high_school_biology": 0.27184466019417475, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.2846975088967972, - "philosophy": 0.3193548387096774, - "professional_medicine": 0.2177121771217712, - "nutrition": 0.31475409836065577, - "global_facts": 0.30303030303030304, - "machine_learning": 0.23423423423423423, - "security_studies": 0.26639344262295084, - "public_relations": 0.27522935779816515, - "professional_psychology": 0.2978723404255319, - "prehistory": 0.29411764705882354, - "anatomy": 0.3208955223880597, - "human_sexuality": 0.2692307692307692, - "college_medicine": 0.28488372093023256, - "high_school_government_and_politics": 0.3177083333333333, - "college_chemistry": 0.23232323232323232, - "logical_fallacies": 0.30864197530864196, - "high_school_geography": 0.3350253807106599, - "elementary_mathematics": 0.23872679045092837, - "human_aging": 0.34234234234234234, - "college_mathematics": 0.29292929292929293, - "high_school_psychology": 0.27941176470588236, - "formal_logic": 0.216, - "high_school_statistics": 0.19534883720930232, - "international_law": 0.36666666666666664, - "high_school_mathematics": 0.241635687732342, - "high_school_computer_science": 0.29292929292929293, - "conceptual_physics": 0.24358974358974358, - "miscellaneous": 0.3388746803069054, - "high_school_chemistry": 0.2623762376237624, - "marketing": 0.3948497854077253, - "professional_law": 0.2857142857142857, - "management": 0.22549019607843138, - "college_physics": 0.24752475247524752, - "jurisprudence": 0.34579439252336447, - "world_religions": 0.40588235294117647, - "sociology": 0.33, - "us_foreign_policy": 0.3939393939393939, - "high_school_macroeconomics": 0.2930591259640103, - "computer_security": 0.40404040404040403, - "moral_scenarios": 0.2505592841163311, - "moral_disputes": 0.2927536231884058, - "electrical_engineering": 0.3263888888888889, - "astronomy": 0.31125827814569534, - "college_biology": 0.32167832167832167 - } + "bleu_score": 0.3421444160411124 }, "prompt_5": { - "accuracy": 0.2994637111190561, - "category_acc": { - "high_school_european_history": 0.39634146341463417, - "business_ethics": 0.2828282828282828, - "clinical_knowledge": 0.2765151515151515, - "medical_genetics": 0.3434343434343434, - "high_school_us_history": 0.3448275862068966, - "high_school_physics": 0.26, - "high_school_world_history": 0.3728813559322034, - "virology": 0.2545454545454545, - "high_school_microeconomics": 0.26582278481012656, - "econometrics": 0.23008849557522124, - "college_computer_science": 0.32323232323232326, - "high_school_biology": 0.26537216828478966, - "abstract_algebra": 0.2727272727272727, - "professional_accounting": 0.28113879003558717, - "philosophy": 0.3225806451612903, - "professional_medicine": 0.2140221402214022, - "nutrition": 0.30491803278688523, - "global_facts": 0.31313131313131315, - "machine_learning": 0.2702702702702703, - "security_studies": 0.24180327868852458, - "public_relations": 0.3394495412844037, - "professional_psychology": 0.31260229132569556, - "prehistory": 0.32507739938080493, - "anatomy": 0.3283582089552239, - "human_sexuality": 0.2846153846153846, - "college_medicine": 0.25, - "high_school_government_and_politics": 0.3020833333333333, - "college_chemistry": 0.1717171717171717, - "logical_fallacies": 0.24691358024691357, - "high_school_geography": 0.38578680203045684, - "elementary_mathematics": 0.23872679045092837, - "human_aging": 0.35585585585585583, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.31066176470588236, - "formal_logic": 0.272, - "high_school_statistics": 0.2186046511627907, - "international_law": 0.375, - "high_school_mathematics": 0.26394052044609667, - "high_school_computer_science": 0.35353535353535354, - "conceptual_physics": 0.28205128205128205, - "miscellaneous": 0.37595907928388744, - "high_school_chemistry": 0.23267326732673269, - "marketing": 0.4206008583690987, - "professional_law": 0.27723418134377037, - "management": 0.2647058823529412, - "college_physics": 0.2376237623762376, - "jurisprudence": 0.3364485981308411, - "world_religions": 0.4411764705882353, - "sociology": 0.33, - "us_foreign_policy": 0.41414141414141414, - "high_school_macroeconomics": 0.2802056555269923, - "computer_security": 0.4444444444444444, - "moral_scenarios": 0.24608501118568232, - "moral_disputes": 0.32463768115942027, - "electrical_engineering": 0.3333333333333333, - "astronomy": 0.31788079470198677, - "college_biology": 0.32867132867132864 - } + "bleu_score": 0.31648254812909576 } }, - "c_eval": { + "mmlu": { "prompt_1": { - "accuracy": 0.312778603268945 + "accuracy": 0.5472578763127188 }, "prompt_2": { - "accuracy": 0.2890044576523031 + "accuracy": 0.5694282380396732 }, "prompt_3": { - "accuracy": 0.27786032689450224 + "accuracy": 0.5530921820303384 }, "prompt_4": { - "accuracy": 0.2912332838038633 + "accuracy": 0.558926487747958 }, "prompt_5": { - "accuracy": 0.29420505200594355 + "accuracy": 0.5320886814469078 } }, - "c_eval_full": { + "mmlu_full": { "prompt_1": { - "accuracy": 0.29638854296388545, + "accuracy": 0.5574544154451198, "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.125, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.2857142857142857, - "college_physics": 0.375, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.27586206896551724, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.25, - "high_school_chemistry": 0.16666666666666666, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.4642857142857143, - "college_economics": 0.2833333333333333, - "business_administration": 0.23684210526315788, - "marxism": 0.375, - "mao_zedong_thought": 0.3448275862068966, - "education_science": 0.11764705882352941, - "teacher_qualification": 0.32653061224489793, - "high_school_politics": 0.4166666666666667, - "high_school_geography": 0.08333333333333333, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.4583333333333333, - "logic": 0.2962962962962963, - "law": 0.41379310344827586, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.21052631578947367, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.25, - "high_school_chinese": 0.125, - "high_school_history": 0.52, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.3076923076923077, - "sports_science": 0.2916666666666667, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.3333333333333333, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.3148148148148148, - "physician": 0.2777777777777778 + "high_school_european_history": 0.7073170731707317, + "business_ethics": 0.6161616161616161, + "clinical_knowledge": 0.6287878787878788, + "medical_genetics": 0.6565656565656566, + "high_school_us_history": 0.7438423645320197, + "high_school_physics": 0.2866666666666667, + "high_school_world_history": 0.7288135593220338, + "virology": 0.4727272727272727, + "high_school_microeconomics": 0.5780590717299579, + "econometrics": 0.336283185840708, + "college_computer_science": 0.5656565656565656, + "high_school_biology": 0.6699029126213593, + "abstract_algebra": 0.35353535353535354, + "professional_accounting": 0.4128113879003559, + "philosophy": 0.5838709677419355, + "professional_medicine": 0.6568265682656826, + "nutrition": 0.5836065573770491, + "global_facts": 0.35353535353535354, + "machine_learning": 0.3963963963963964, + "security_studies": 0.5819672131147541, + "public_relations": 0.6513761467889908, + "professional_psychology": 0.5319148936170213, + "prehistory": 0.5820433436532507, + "anatomy": 0.5447761194029851, + "human_sexuality": 0.6461538461538462, + "college_medicine": 0.6162790697674418, + "high_school_government_and_politics": 0.8020833333333334, + "college_chemistry": 0.3939393939393939, + "logical_fallacies": 0.7037037037037037, + "high_school_geography": 0.7258883248730964, + "elementary_mathematics": 0.3793103448275862, + "human_aging": 0.6396396396396397, + "college_mathematics": 0.3434343434343434, + "high_school_psychology": 0.78125, + "formal_logic": 0.48, + "high_school_statistics": 0.40930232558139534, + "international_law": 0.7166666666666667, + "high_school_mathematics": 0.31226765799256506, + "high_school_computer_science": 0.5757575757575758, + "conceptual_physics": 0.45726495726495725, + "miscellaneous": 0.7851662404092071, + "high_school_chemistry": 0.43564356435643564, + "marketing": 0.8454935622317596, + "professional_law": 0.42204827136333983, + "management": 0.7647058823529411, + "college_physics": 0.36633663366336633, + "jurisprudence": 0.616822429906542, + "world_religions": 0.8411764705882353, + "sociology": 0.76, + "us_foreign_policy": 0.7474747474747475, + "high_school_macroeconomics": 0.5347043701799485, + "computer_security": 0.7272727272727273, + "moral_scenarios": 0.2550335570469799, + "moral_disputes": 0.6231884057971014, + "electrical_engineering": 0.5138888888888888, + "astronomy": 0.5761589403973509, + "college_biology": 0.6573426573426573 } }, "prompt_2": { - "accuracy": 0.2727272727272727, + "accuracy": 0.560672148730783, "category_acc": { - "computer_network": 0.08333333333333333, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.35714285714285715, - "college_physics": 0.125, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.25, - "high_school_chemistry": 0.25, - "high_school_biology": 0.16666666666666666, - "middle_school_mathematics": 0.08333333333333333, - "middle_school_biology": 0.5, - "middle_school_physics": 0.4166666666666667, - "middle_school_chemistry": 0.12, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.26666666666666666, - "business_administration": 0.23684210526315788, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.29411764705882354, - "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.125, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.4166666666666667, - "logic": 0.25925925925925924, - "law": 0.1724137931034483, - "chinese_language_and_literature": 0.25, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.4117647058823529, - "legal_professional": 0.25, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.36, - "middle_school_history": 0.4444444444444444, - "civil_servant": 0.3076923076923077, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.375, - "clinical_medicine": 0.18518518518518517, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.2037037037037037, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.2037037037037037, - "physician": 0.2962962962962963 + "high_school_european_history": 0.7195121951219512, + "business_ethics": 0.5555555555555556, + "clinical_knowledge": 0.6553030303030303, + "medical_genetics": 0.6262626262626263, + "high_school_us_history": 0.7536945812807881, + "high_school_physics": 0.3333333333333333, + "high_school_world_history": 0.7372881355932204, + "virology": 0.4727272727272727, + "high_school_microeconomics": 0.6075949367088608, + "econometrics": 0.3805309734513274, + "college_computer_science": 0.5454545454545454, + "high_school_biology": 0.6731391585760518, + "abstract_algebra": 0.31313131313131315, + "professional_accounting": 0.4199288256227758, + "philosophy": 0.6096774193548387, + "professional_medicine": 0.6346863468634686, + "nutrition": 0.5737704918032787, + "global_facts": 0.36363636363636365, + "machine_learning": 0.4144144144144144, + "security_studies": 0.6311475409836066, + "public_relations": 0.6697247706422018, + "professional_psychology": 0.5515548281505729, + "prehistory": 0.6099071207430341, + "anatomy": 0.5223880597014925, + "human_sexuality": 0.6384615384615384, + "college_medicine": 0.5872093023255814, + "high_school_government_and_politics": 0.796875, + "college_chemistry": 0.42424242424242425, + "logical_fallacies": 0.6975308641975309, + "high_school_geography": 0.751269035532995, + "elementary_mathematics": 0.3819628647214854, + "human_aging": 0.6351351351351351, + "college_mathematics": 0.30303030303030304, + "high_school_psychology": 0.7830882352941176, + "formal_logic": 0.464, + "high_school_statistics": 0.4418604651162791, + "international_law": 0.7, + "high_school_mathematics": 0.2825278810408922, + "high_school_computer_science": 0.5959595959595959, + "conceptual_physics": 0.48717948717948717, + "miscellaneous": 0.7902813299232737, + "high_school_chemistry": 0.4405940594059406, + "marketing": 0.8669527896995708, + "professional_law": 0.395955642530985, + "management": 0.7647058823529411, + "college_physics": 0.4158415841584158, + "jurisprudence": 0.6448598130841121, + "world_religions": 0.8176470588235294, + "sociology": 0.77, + "us_foreign_policy": 0.7777777777777778, + "high_school_macroeconomics": 0.5449871465295629, + "computer_security": 0.7171717171717171, + "moral_scenarios": 0.25279642058165547, + "moral_disputes": 0.6434782608695652, + "electrical_engineering": 0.5416666666666666, + "astronomy": 0.5629139072847682, + "college_biology": 0.6643356643356644 } }, "prompt_3": { - "accuracy": 0.2901618929016189, + "accuracy": 0.5483732570611369, "category_acc": { - "computer_network": 0.16666666666666666, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.21428571428571427, - "college_physics": 0.041666666666666664, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.4523809523809524, - "metrology_engineer": 0.27586206896551724, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.19230769230769232, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.25, - "business_administration": 0.2631578947368421, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.3235294117647059, - "teacher_qualification": 0.3673469387755102, - "high_school_politics": 0.08333333333333333, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.25, - "ideological_and_moral_cultivation": 0.375, - "logic": 0.2962962962962963, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.4117647058823529, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.25, - "high_school_history": 0.28, - "middle_school_history": 0.2222222222222222, - "civil_servant": 0.3076923076923077, - "sports_science": 0.375, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.4117647058823529, - "accountant": 0.2222222222222222, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.14814814814814814, - "physician": 0.2777777777777778 + "high_school_european_history": 0.6951219512195121, + "business_ethics": 0.6161616161616161, + "clinical_knowledge": 0.625, + "medical_genetics": 0.6464646464646465, + "high_school_us_history": 0.7487684729064039, + "high_school_physics": 0.34, + "high_school_world_history": 0.711864406779661, + "virology": 0.48484848484848486, + "high_school_microeconomics": 0.6118143459915611, + "econometrics": 0.35398230088495575, + "college_computer_science": 0.5050505050505051, + "high_school_biology": 0.656957928802589, + "abstract_algebra": 0.29292929292929293, + "professional_accounting": 0.40569395017793597, + "philosophy": 0.6225806451612903, + "professional_medicine": 0.6088560885608856, + "nutrition": 0.5606557377049181, + "global_facts": 0.35353535353535354, + "machine_learning": 0.38738738738738737, + "security_studies": 0.6311475409836066, + "public_relations": 0.6513761467889908, + "professional_psychology": 0.5368248772504092, + "prehistory": 0.6037151702786377, + "anatomy": 0.5074626865671642, + "human_sexuality": 0.6076923076923076, + "college_medicine": 0.5581395348837209, + "high_school_government_and_politics": 0.7760416666666666, + "college_chemistry": 0.41414141414141414, + "logical_fallacies": 0.654320987654321, + "high_school_geography": 0.7157360406091371, + "elementary_mathematics": 0.3713527851458886, + "human_aging": 0.6396396396396397, + "college_mathematics": 0.2828282828282828, + "high_school_psychology": 0.7573529411764706, + "formal_logic": 0.48, + "high_school_statistics": 0.3953488372093023, + "international_law": 0.7, + "high_school_mathematics": 0.3048327137546468, + "high_school_computer_science": 0.5252525252525253, + "conceptual_physics": 0.44871794871794873, + "miscellaneous": 0.7749360613810742, + "high_school_chemistry": 0.42574257425742573, + "marketing": 0.8154506437768241, + "professional_law": 0.4116112198303979, + "management": 0.7450980392156863, + "college_physics": 0.36633663366336633, + "jurisprudence": 0.6448598130841121, + "world_religions": 0.8058823529411765, + "sociology": 0.73, + "us_foreign_policy": 0.7272727272727273, + "high_school_macroeconomics": 0.4961439588688946, + "computer_security": 0.7070707070707071, + "moral_scenarios": 0.25615212527964204, + "moral_disputes": 0.6202898550724638, + "electrical_engineering": 0.5277777777777778, + "astronomy": 0.5629139072847682, + "college_biology": 0.6363636363636364 } }, "prompt_4": { - "accuracy": 0.28829389788293897, + "accuracy": 0.542652842331069, "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.25, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.14285714285714285, - "college_physics": 0.375, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.4782608695652174, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.21428571428571427, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.125, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.4166666666666667, - "middle_school_chemistry": 0.24, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.25, - "business_administration": 0.2631578947368421, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.5517241379310345, - "education_science": 0.17647058823529413, - "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.08333333333333333, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.25, - "logic": 0.25925925925925924, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.52, - "middle_school_history": 0.25925925925925924, - "civil_servant": 0.2692307692307692, - "sports_science": 0.3333333333333333, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.48148148148148145, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.25925925925925924, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.3148148148148148, - "physician": 0.24074074074074073 + "high_school_european_history": 0.7195121951219512, + "business_ethics": 0.5555555555555556, + "clinical_knowledge": 0.571969696969697, + "medical_genetics": 0.6262626262626263, + "high_school_us_history": 0.7586206896551724, + "high_school_physics": 0.29333333333333333, + "high_school_world_history": 0.7203389830508474, + "virology": 0.48484848484848486, + "high_school_microeconomics": 0.5780590717299579, + "econometrics": 0.3274336283185841, + "college_computer_science": 0.5454545454545454, + "high_school_biology": 0.656957928802589, + "abstract_algebra": 0.29292929292929293, + "professional_accounting": 0.4128113879003559, + "philosophy": 0.567741935483871, + "professional_medicine": 0.5867158671586716, + "nutrition": 0.5967213114754099, + "global_facts": 0.36363636363636365, + "machine_learning": 0.36936936936936937, + "security_studies": 0.5983606557377049, + "public_relations": 0.6513761467889908, + "professional_psychology": 0.5122749590834698, + "prehistory": 0.6006191950464397, + "anatomy": 0.48507462686567165, + "human_sexuality": 0.6538461538461539, + "college_medicine": 0.5581395348837209, + "high_school_government_and_politics": 0.7864583333333334, + "college_chemistry": 0.42424242424242425, + "logical_fallacies": 0.6790123456790124, + "high_school_geography": 0.7360406091370558, + "elementary_mathematics": 0.3713527851458886, + "human_aging": 0.6486486486486487, + "college_mathematics": 0.31313131313131315, + "high_school_psychology": 0.7536764705882353, + "formal_logic": 0.48, + "high_school_statistics": 0.4186046511627907, + "international_law": 0.6916666666666667, + "high_school_mathematics": 0.30111524163568776, + "high_school_computer_science": 0.5555555555555556, + "conceptual_physics": 0.44871794871794873, + "miscellaneous": 0.7595907928388747, + "high_school_chemistry": 0.42574257425742573, + "marketing": 0.8025751072961373, + "professional_law": 0.4044357469015003, + "management": 0.7549019607843137, + "college_physics": 0.37623762376237624, + "jurisprudence": 0.616822429906542, + "world_religions": 0.8235294117647058, + "sociology": 0.735, + "us_foreign_policy": 0.7272727272727273, + "high_school_macroeconomics": 0.4961439588688946, + "computer_security": 0.6767676767676768, + "moral_scenarios": 0.2550335570469799, + "moral_disputes": 0.5884057971014492, + "electrical_engineering": 0.5208333333333334, + "astronomy": 0.5496688741721855, + "college_biology": 0.6433566433566433 } }, "prompt_5": { - "accuracy": 0.28019925280199254, + "accuracy": 0.5278512692170182, "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.15384615384615385, - "college_programming": 0.2619047619047619, - "college_physics": 0.375, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.5416666666666666, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.125, - "high_school_chemistry": 0.16666666666666666, - "high_school_biology": 0.16666666666666666, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.25, - "college_economics": 0.35, - "business_administration": 0.18421052631578946, - "marxism": 0.375, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.14705882352941177, - "teacher_qualification": 0.24489795918367346, - "high_school_politics": 0.375, - "high_school_geography": 0.08333333333333333, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.32142857142857145, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.2962962962962963, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.23684210526315788, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.125, - "high_school_history": 0.56, - "middle_school_history": 0.25925925925925924, - "civil_servant": 0.36538461538461536, - "sports_science": 0.3333333333333333, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.3148148148148148, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.3611111111111111, - "tax_accountant": 0.2962962962962963, - "physician": 0.18518518518518517 + "high_school_european_history": 0.6829268292682927, + "business_ethics": 0.5555555555555556, + "clinical_knowledge": 0.5795454545454546, + "medical_genetics": 0.5252525252525253, + "high_school_us_history": 0.7438423645320197, + "high_school_physics": 0.28, + "high_school_world_history": 0.7033898305084746, + "virology": 0.44242424242424244, + "high_school_microeconomics": 0.5738396624472574, + "econometrics": 0.37168141592920356, + "college_computer_science": 0.5151515151515151, + "high_school_biology": 0.6310679611650486, + "abstract_algebra": 0.2727272727272727, + "professional_accounting": 0.4306049822064057, + "philosophy": 0.5612903225806452, + "professional_medicine": 0.5756457564575646, + "nutrition": 0.5442622950819672, + "global_facts": 0.35353535353535354, + "machine_learning": 0.3963963963963964, + "security_studies": 0.5901639344262295, + "public_relations": 0.6238532110091743, + "professional_psychology": 0.5008183306055647, + "prehistory": 0.5944272445820433, + "anatomy": 0.5, + "human_sexuality": 0.6307692307692307, + "college_medicine": 0.5116279069767442, + "high_school_government_and_politics": 0.7604166666666666, + "college_chemistry": 0.40404040404040403, + "logical_fallacies": 0.6296296296296297, + "high_school_geography": 0.7208121827411168, + "elementary_mathematics": 0.38992042440318303, + "human_aging": 0.6126126126126126, + "college_mathematics": 0.2828282828282828, + "high_school_psychology": 0.7113970588235294, + "formal_logic": 0.464, + "high_school_statistics": 0.4232558139534884, + "international_law": 0.6583333333333333, + "high_school_mathematics": 0.30855018587360594, + "high_school_computer_science": 0.5454545454545454, + "conceptual_physics": 0.46153846153846156, + "miscellaneous": 0.7544757033248082, + "high_school_chemistry": 0.42574257425742573, + "marketing": 0.7725321888412017, + "professional_law": 0.4050880626223092, + "management": 0.7058823529411765, + "college_physics": 0.3564356435643564, + "jurisprudence": 0.6261682242990654, + "world_religions": 0.7823529411764706, + "sociology": 0.73, + "us_foreign_policy": 0.6565656565656566, + "high_school_macroeconomics": 0.4987146529562982, + "computer_security": 0.6767676767676768, + "moral_scenarios": 0.22930648769574943, + "moral_disputes": 0.5652173913043478, + "electrical_engineering": 0.5069444444444444, + "astronomy": 0.48344370860927155, + "college_biology": 0.5594405594405595 } } }, - "cmmlu": { + "c_eval": { "prompt_1": { - "accuracy": 0.2939068100358423 + "accuracy": 0.4227340267459138 }, "prompt_2": { - "accuracy": 0.27956989247311825 + "accuracy": 0.4279346210995542 }, "prompt_3": { - "accuracy": 0.3225806451612903 + "accuracy": 0.4323922734026746 }, "prompt_4": { - "accuracy": 0.3225806451612903 + "accuracy": 0.40936106983655274 }, "prompt_5": { - "accuracy": 0.25806451612903225 + "accuracy": 0.4323922734026746 } }, - "cmmlu_full": { + "c_eval_full": { "prompt_1": { - "accuracy": 0.2894146088758418, + "accuracy": 0.4246575342465753, "category_acc": { - "agronomy": 0.25443786982248523, - "anatomy": 0.28378378378378377, - "ancient_chinese": 0.23780487804878048, - "arts": 0.29375, - "astronomy": 0.2545454545454545, - "business_ethics": 0.27751196172248804, - "chinese_civil_service_exam": 0.29375, - "chinese_driving_rule": 0.4198473282442748, - "chinese_food_culture": 0.3382352941176471, - "chinese_foreign_policy": 0.3177570093457944, - "chinese_history": 0.26006191950464397, - "chinese_literature": 0.2549019607843137, - "chinese_teacher_qualification": 0.24022346368715083, - "clinical_knowledge": 0.2489451476793249, - "college_actuarial_science": 0.19811320754716982, - "college_education": 0.35514018691588783, - "college_engineering_hydrology": 0.20754716981132076, - "college_law": 0.2777777777777778, - "college_mathematics": 0.2, - "college_medical_statistics": 0.32075471698113206, - "college_medicine": 0.2893772893772894, - "computer_science": 0.3235294117647059, - "computer_security": 0.2807017543859649, - "conceptual_physics": 0.2857142857142857, - "construction_project_management": 0.33093525179856115, - "economics": 0.3018867924528302, - "education": 0.32515337423312884, - "electrical_engineering": 0.26744186046511625, - "elementary_chinese": 0.2976190476190476, - "elementary_commonsense": 0.2828282828282828, - "elementary_information_and_technology": 0.25210084033613445, - "elementary_mathematics": 0.22608695652173913, - "ethnology": 0.2740740740740741, - "food_science": 0.2937062937062937, - "genetics": 0.23295454545454544, - "global_facts": 0.28187919463087246, - "high_school_biology": 0.25443786982248523, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.3389830508474576, - "high_school_mathematics": 0.25, - "high_school_physics": 0.37272727272727274, - "high_school_politics": 0.25874125874125875, - "human_sexuality": 0.30952380952380953, - "international_law": 0.24324324324324326, - "journalism": 0.29069767441860467, - "jurisprudence": 0.2846715328467153, - "legal_and_moral_basis": 0.43457943925233644, - "logical": 0.25203252032520324, - "machine_learning": 0.30327868852459017, - "management": 0.3, - "marketing": 0.2777777777777778, - "marxist_theory": 0.2857142857142857, - "modern_chinese": 0.3275862068965517, - "nutrition": 0.2896551724137931, - "philosophy": 0.3238095238095238, - "professional_accounting": 0.32, - "professional_law": 0.2938388625592417, - "professional_medicine": 0.2553191489361702, - "professional_psychology": 0.33189655172413796, - "public_relations": 0.28735632183908044, - "security_study": 0.37777777777777777, - "sociology": 0.27876106194690264, - "sports_science": 0.3575757575757576, - "traditional_chinese_medicine": 0.3081081081081081, - "virology": 0.3136094674556213, - "world_history": 0.3416149068322981, - "world_religions": 0.3375 + "computer_network": 0.5, + "operating_system": 0.5416666666666666, + "computer_architecture": 0.4230769230769231, + "college_programming": 0.42857142857142855, + "college_physics": 0.4166666666666667, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.375, + "probability_and_statistics": 0.21739130434782608, + "discrete_mathematics": 0.47619047619047616, + "electrical_engineer": 0.35714285714285715, + "metrology_engineer": 0.41379310344827586, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.25, + "middle_school_mathematics": 0.16666666666666666, + "middle_school_biology": 0.4230769230769231, + "middle_school_physics": 0.625, + "middle_school_chemistry": 0.56, + "veterinary_medicine": 0.39285714285714285, + "college_economics": 0.45, + "business_administration": 0.4473684210526316, + "marxism": 0.4166666666666667, + "mao_zedong_thought": 0.6206896551724138, + "education_science": 0.47058823529411764, + "teacher_qualification": 0.6122448979591837, + "high_school_politics": 0.375, + "high_school_geography": 0.3333333333333333, + "middle_school_politics": 0.5384615384615384, + "middle_school_geography": 0.47058823529411764, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.75, + "logic": 0.48148148148148145, + "law": 0.3793103448275862, + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.3684210526315789, + "professional_tour_guide": 0.5, + "legal_professional": 0.5, + "high_school_chinese": 0.375, + "high_school_history": 0.56, + "middle_school_history": 0.6296296296296297, + "civil_servant": 0.36538461538461536, + "sports_science": 0.375, + "plant_protection": 0.4444444444444444, + "basic_medicine": 0.4583333333333333, + "clinical_medicine": 0.37037037037037035, + "urban_and_rural_planner": 0.45098039215686275, + "accountant": 0.37037037037037035, + "fire_engineer": 0.3888888888888889, + "environmental_impact_assessment_engineer": 0.4166666666666667, + "tax_accountant": 0.2962962962962963, + "physician": 0.3888888888888889 } }, "prompt_2": { - "accuracy": 0.31505784838542567, + "accuracy": 0.4265255292652553, "category_acc": { - "agronomy": 0.2958579881656805, - "anatomy": 0.3108108108108108, - "ancient_chinese": 0.24390243902439024, - "arts": 0.43125, - "astronomy": 0.2545454545454545, - "business_ethics": 0.36363636363636365, - "chinese_civil_service_exam": 0.3, - "chinese_driving_rule": 0.3435114503816794, - "chinese_food_culture": 0.3088235294117647, - "chinese_foreign_policy": 0.35514018691588783, - "chinese_history": 0.29721362229102166, - "chinese_literature": 0.31862745098039214, - "chinese_teacher_qualification": 0.3016759776536313, - "clinical_knowledge": 0.29535864978902954, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.3925233644859813, - "college_engineering_hydrology": 0.3490566037735849, - "college_law": 0.3425925925925926, - "college_mathematics": 0.19047619047619047, - "college_medical_statistics": 0.29245283018867924, - "college_medicine": 0.326007326007326, - "computer_science": 0.31862745098039214, - "computer_security": 0.3333333333333333, - "conceptual_physics": 0.2925170068027211, - "construction_project_management": 0.3237410071942446, - "economics": 0.27672955974842767, - "education": 0.3619631901840491, - "electrical_engineering": 0.3313953488372093, - "elementary_chinese": 0.3412698412698413, - "elementary_commonsense": 0.3686868686868687, - "elementary_information_and_technology": 0.3235294117647059, - "elementary_mathematics": 0.25217391304347825, - "ethnology": 0.3925925925925926, - "food_science": 0.3356643356643357, - "genetics": 0.2897727272727273, - "global_facts": 0.3691275167785235, - "high_school_biology": 0.2485207100591716, - "high_school_chemistry": 0.2196969696969697, - "high_school_geography": 0.2966101694915254, - "high_school_mathematics": 0.22560975609756098, - "high_school_physics": 0.22727272727272727, - "high_school_politics": 0.34265734265734266, - "human_sexuality": 0.31746031746031744, - "international_law": 0.2702702702702703, - "journalism": 0.37209302325581395, - "jurisprudence": 0.2871046228710462, - "legal_and_moral_basis": 0.397196261682243, - "logical": 0.3252032520325203, - "machine_learning": 0.32786885245901637, - "management": 0.29523809523809524, - "marketing": 0.3888888888888889, - "marxist_theory": 0.37566137566137564, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.35172413793103446, - "philosophy": 0.3238095238095238, - "professional_accounting": 0.2857142857142857, - "professional_law": 0.3033175355450237, - "professional_medicine": 0.28191489361702127, - "professional_psychology": 0.34051724137931033, - "public_relations": 0.3620689655172414, - "security_study": 0.37777777777777777, - "sociology": 0.3141592920353982, - "sports_science": 0.2909090909090909, - "traditional_chinese_medicine": 0.24324324324324326, - "virology": 0.3609467455621302, - "world_history": 0.3416149068322981, - "world_religions": 0.3 + "computer_network": 0.5416666666666666, + "operating_system": 0.5833333333333334, + "computer_architecture": 0.4230769230769231, + "college_programming": 0.42857142857142855, + "college_physics": 0.375, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.4166666666666667, + "probability_and_statistics": 0.17391304347826086, + "discrete_mathematics": 0.42857142857142855, + "electrical_engineer": 0.35714285714285715, + "metrology_engineer": 0.5172413793103449, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.25, + "high_school_chemistry": 0.2916666666666667, + "high_school_biology": 0.2916666666666667, + "middle_school_mathematics": 0.20833333333333334, + "middle_school_biology": 0.46153846153846156, + "middle_school_physics": 0.625, + "middle_school_chemistry": 0.52, + "veterinary_medicine": 0.39285714285714285, + "college_economics": 0.4166666666666667, + "business_administration": 0.42105263157894735, + "marxism": 0.4583333333333333, + "mao_zedong_thought": 0.6896551724137931, + "education_science": 0.47058823529411764, + "teacher_qualification": 0.6326530612244898, + "high_school_politics": 0.375, + "high_school_geography": 0.3333333333333333, + "middle_school_politics": 0.6538461538461539, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.6666666666666666, + "logic": 0.5185185185185185, + "law": 0.3793103448275862, + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.3684210526315789, + "professional_tour_guide": 0.5294117647058824, + "legal_professional": 0.5357142857142857, + "high_school_chinese": 0.375, + "high_school_history": 0.52, + "middle_school_history": 0.6666666666666666, + "civil_servant": 0.34615384615384615, + "sports_science": 0.25, + "plant_protection": 0.4444444444444444, + "basic_medicine": 0.5, + "clinical_medicine": 0.37037037037037035, + "urban_and_rural_planner": 0.43137254901960786, + "accountant": 0.37037037037037035, + "fire_engineer": 0.3611111111111111, + "environmental_impact_assessment_engineer": 0.4444444444444444, + "tax_accountant": 0.2962962962962963, + "physician": 0.42592592592592593 } }, "prompt_3": { - "accuracy": 0.31324469003626315, + "accuracy": 0.4339975093399751, "category_acc": { - "agronomy": 0.2781065088757396, - "anatomy": 0.27702702702702703, - "ancient_chinese": 0.24390243902439024, - "arts": 0.425, - "astronomy": 0.26666666666666666, - "business_ethics": 0.3492822966507177, - "chinese_civil_service_exam": 0.2875, - "chinese_driving_rule": 0.40458015267175573, - "chinese_food_culture": 0.25735294117647056, - "chinese_foreign_policy": 0.27102803738317754, - "chinese_history": 0.2693498452012384, - "chinese_literature": 0.29901960784313725, - "chinese_teacher_qualification": 0.3743016759776536, - "clinical_knowledge": 0.3037974683544304, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.3364485981308411, - "college_engineering_hydrology": 0.3113207547169811, - "college_law": 0.25925925925925924, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.27358490566037735, - "college_medicine": 0.31868131868131866, - "computer_science": 0.35294117647058826, - "computer_security": 0.30994152046783624, - "conceptual_physics": 0.25170068027210885, - "construction_project_management": 0.31654676258992803, - "economics": 0.3584905660377358, - "education": 0.34355828220858897, - "electrical_engineering": 0.3081395348837209, - "elementary_chinese": 0.32142857142857145, - "elementary_commonsense": 0.3383838383838384, - "elementary_information_and_technology": 0.3067226890756303, - "elementary_mathematics": 0.27391304347826084, - "ethnology": 0.34814814814814815, - "food_science": 0.38461538461538464, - "genetics": 0.2897727272727273, - "global_facts": 0.3691275167785235, - "high_school_biology": 0.24260355029585798, - "high_school_chemistry": 0.29545454545454547, - "high_school_geography": 0.3305084745762712, - "high_school_mathematics": 0.21951219512195122, - "high_school_physics": 0.2909090909090909, - "high_school_politics": 0.2867132867132867, - "human_sexuality": 0.373015873015873, - "international_law": 0.3027027027027027, - "journalism": 0.36627906976744184, - "jurisprudence": 0.34306569343065696, - "legal_and_moral_basis": 0.38317757009345793, - "logical": 0.2926829268292683, - "machine_learning": 0.32786885245901637, - "management": 0.3142857142857143, - "marketing": 0.37777777777777777, - "marxist_theory": 0.3333333333333333, - "modern_chinese": 0.29310344827586204, - "nutrition": 0.32413793103448274, - "philosophy": 0.37142857142857144, - "professional_accounting": 0.3028571428571429, - "professional_law": 0.27014218009478674, - "professional_medicine": 0.2553191489361702, - "professional_psychology": 0.34051724137931033, - "public_relations": 0.3218390804597701, - "security_study": 0.362962962962963, - "sociology": 0.3407079646017699, - "sports_science": 0.3393939393939394, - "traditional_chinese_medicine": 0.23243243243243245, - "virology": 0.34911242603550297, - "world_history": 0.3167701863354037, - "world_religions": 0.3125 + "computer_network": 0.5416666666666666, + "operating_system": 0.5833333333333334, + "computer_architecture": 0.4230769230769231, + "college_programming": 0.47619047619047616, + "college_physics": 0.375, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.375, + "probability_and_statistics": 0.21739130434782608, + "discrete_mathematics": 0.47619047619047616, + "electrical_engineer": 0.38095238095238093, + "metrology_engineer": 0.4482758620689655, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.25, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.25, + "middle_school_mathematics": 0.16666666666666666, + "middle_school_biology": 0.46153846153846156, + "middle_school_physics": 0.6666666666666666, + "middle_school_chemistry": 0.52, + "veterinary_medicine": 0.42857142857142855, + "college_economics": 0.45, + "business_administration": 0.42105263157894735, + "marxism": 0.5, + "mao_zedong_thought": 0.6206896551724138, + "education_science": 0.5, + "teacher_qualification": 0.6122448979591837, + "high_school_politics": 0.4166666666666667, + "high_school_geography": 0.375, + "middle_school_politics": 0.5, + "middle_school_geography": 0.35294117647058826, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.7916666666666666, + "logic": 0.5185185185185185, + "law": 0.41379310344827586, + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.3684210526315789, + "professional_tour_guide": 0.5588235294117647, + "legal_professional": 0.5, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.52, + "middle_school_history": 0.6296296296296297, + "civil_servant": 0.40384615384615385, + "sports_science": 0.25, + "plant_protection": 0.5185185185185185, + "basic_medicine": 0.5, + "clinical_medicine": 0.4074074074074074, + "urban_and_rural_planner": 0.4117647058823529, + "accountant": 0.3888888888888889, + "fire_engineer": 0.3611111111111111, + "environmental_impact_assessment_engineer": 0.5, + "tax_accountant": 0.25925925925925924, + "physician": 0.4444444444444444 } }, "prompt_4": { - "accuracy": 0.29554481091348644, + "accuracy": 0.4178082191780822, "category_acc": { - "agronomy": 0.2603550295857988, - "anatomy": 0.25, - "ancient_chinese": 0.21951219512195122, - "arts": 0.35, - "astronomy": 0.2909090909090909, - "business_ethics": 0.291866028708134, - "chinese_civil_service_exam": 0.30625, - "chinese_driving_rule": 0.37404580152671757, - "chinese_food_culture": 0.34558823529411764, - "chinese_foreign_policy": 0.3177570093457944, - "chinese_history": 0.28173374613003094, - "chinese_literature": 0.24509803921568626, - "chinese_teacher_qualification": 0.3240223463687151, - "clinical_knowledge": 0.31645569620253167, - "college_actuarial_science": 0.19811320754716982, - "college_education": 0.34579439252336447, - "college_engineering_hydrology": 0.24528301886792453, - "college_law": 0.2222222222222222, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.3584905660377358, - "college_medicine": 0.29304029304029305, - "computer_science": 0.3333333333333333, - "computer_security": 0.30409356725146197, - "conceptual_physics": 0.30612244897959184, - "construction_project_management": 0.2589928057553957, - "economics": 0.32075471698113206, - "education": 0.31901840490797545, - "electrical_engineering": 0.3313953488372093, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.26262626262626265, - "elementary_information_and_technology": 0.2815126050420168, - "elementary_mathematics": 0.24347826086956523, - "ethnology": 0.26666666666666666, - "food_science": 0.34265734265734266, - "genetics": 0.2727272727272727, - "global_facts": 0.3422818791946309, - "high_school_biology": 0.20710059171597633, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.3474576271186441, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.3090909090909091, - "high_school_politics": 0.3146853146853147, - "human_sexuality": 0.2857142857142857, - "international_law": 0.22702702702702704, - "journalism": 0.3546511627906977, - "jurisprudence": 0.3309002433090024, - "legal_and_moral_basis": 0.3691588785046729, - "logical": 0.2601626016260163, - "machine_learning": 0.23770491803278687, - "management": 0.3142857142857143, - "marketing": 0.31666666666666665, - "marxist_theory": 0.3333333333333333, - "modern_chinese": 0.28448275862068967, - "nutrition": 0.30344827586206896, - "philosophy": 0.3047619047619048, - "professional_accounting": 0.34285714285714286, - "professional_law": 0.2559241706161137, - "professional_medicine": 0.2473404255319149, - "professional_psychology": 0.34051724137931033, - "public_relations": 0.23563218390804597, - "security_study": 0.37777777777777777, - "sociology": 0.336283185840708, - "sports_science": 0.3333333333333333, - "traditional_chinese_medicine": 0.2810810810810811, - "virology": 0.2781065088757396, - "world_history": 0.2732919254658385, - "world_religions": 0.28125 + "computer_network": 0.4583333333333333, + "operating_system": 0.5833333333333334, + "computer_architecture": 0.4230769230769231, + "college_programming": 0.38095238095238093, + "college_physics": 0.375, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.25, + "probability_and_statistics": 0.21739130434782608, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.41379310344827586, + "high_school_mathematics": 0.4782608695652174, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.375, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.4230769230769231, + "middle_school_physics": 0.625, + "middle_school_chemistry": 0.6, + "veterinary_medicine": 0.39285714285714285, + "college_economics": 0.36666666666666664, + "business_administration": 0.42105263157894735, + "marxism": 0.375, + "mao_zedong_thought": 0.6896551724137931, + "education_science": 0.5, + "teacher_qualification": 0.5102040816326531, + "high_school_politics": 0.375, + "high_school_geography": 0.3333333333333333, + "middle_school_politics": 0.5384615384615384, + "middle_school_geography": 0.5294117647058824, + "modern_chinese_history": 0.5, + "ideological_and_moral_cultivation": 0.625, + "logic": 0.48148148148148145, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.42857142857142855, + "art_studies": 0.39473684210526316, + "professional_tour_guide": 0.4411764705882353, + "legal_professional": 0.5, + "high_school_chinese": 0.3333333333333333, + "high_school_history": 0.48, + "middle_school_history": 0.6296296296296297, + "civil_servant": 0.34615384615384615, + "sports_science": 0.375, + "plant_protection": 0.4444444444444444, + "basic_medicine": 0.5, + "clinical_medicine": 0.4074074074074074, + "urban_and_rural_planner": 0.4117647058823529, + "accountant": 0.37037037037037035, + "fire_engineer": 0.3888888888888889, + "environmental_impact_assessment_engineer": 0.3611111111111111, + "tax_accountant": 0.35185185185185186, + "physician": 0.42592592592592593 } }, "prompt_5": { - "accuracy": 0.2839751338283543, + "accuracy": 0.4259028642590286, "category_acc": { - "agronomy": 0.25443786982248523, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.21951219512195122, - "arts": 0.2875, - "astronomy": 0.23636363636363636, - "business_ethics": 0.3157894736842105, - "chinese_civil_service_exam": 0.35625, - "chinese_driving_rule": 0.45038167938931295, - "chinese_food_culture": 0.33088235294117646, - "chinese_foreign_policy": 0.3177570093457944, - "chinese_history": 0.2786377708978328, - "chinese_literature": 0.24509803921568626, - "chinese_teacher_qualification": 0.2737430167597765, - "clinical_knowledge": 0.25316455696202533, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.27102803738317754, - "college_engineering_hydrology": 0.3018867924528302, - "college_law": 0.2222222222222222, - "college_mathematics": 0.21904761904761905, - "college_medical_statistics": 0.32075471698113206, - "college_medicine": 0.27472527472527475, - "computer_science": 0.3382352941176471, - "computer_security": 0.27485380116959063, - "conceptual_physics": 0.29931972789115646, - "construction_project_management": 0.302158273381295, - "economics": 0.2830188679245283, - "education": 0.294478527607362, - "electrical_engineering": 0.3313953488372093, - "elementary_chinese": 0.2857142857142857, - "elementary_commonsense": 0.25757575757575757, - "elementary_information_and_technology": 0.25630252100840334, - "elementary_mathematics": 0.26956521739130435, - "ethnology": 0.2740740740740741, - "food_science": 0.3146853146853147, - "genetics": 0.2727272727272727, - "global_facts": 0.28859060402684567, - "high_school_biology": 0.16568047337278108, - "high_school_chemistry": 0.25, - "high_school_geography": 0.3389830508474576, - "high_school_mathematics": 0.23170731707317074, - "high_school_physics": 0.32727272727272727, - "high_school_politics": 0.2867132867132867, - "human_sexuality": 0.35714285714285715, - "international_law": 0.24864864864864866, - "journalism": 0.27325581395348836, - "jurisprudence": 0.2798053527980535, - "legal_and_moral_basis": 0.3691588785046729, - "logical": 0.2682926829268293, - "machine_learning": 0.28688524590163933, - "management": 0.3142857142857143, - "marketing": 0.35, - "marxist_theory": 0.291005291005291, - "modern_chinese": 0.3017241379310345, - "nutrition": 0.2689655172413793, - "philosophy": 0.3047619047619048, - "professional_accounting": 0.2857142857142857, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.21010638297872342, - "professional_psychology": 0.34051724137931033, - "public_relations": 0.3218390804597701, - "security_study": 0.31851851851851853, - "sociology": 0.26991150442477874, - "sports_science": 0.2909090909090909, - "traditional_chinese_medicine": 0.2918918918918919, - "virology": 0.27218934911242604, - "world_history": 0.2484472049689441, - "world_religions": 0.3 + "computer_network": 0.5833333333333334, + "operating_system": 0.5416666666666666, + "computer_architecture": 0.4230769230769231, + "college_programming": 0.42857142857142855, + "college_physics": 0.4166666666666667, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.375, + "probability_and_statistics": 0.17391304347826086, + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.3333333333333333, + "metrology_engineer": 0.4482758620689655, + "high_school_mathematics": 0.43478260869565216, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.2916666666666667, + "high_school_biology": 0.3333333333333333, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.38461538461538464, + "middle_school_physics": 0.625, + "middle_school_chemistry": 0.44, + "veterinary_medicine": 0.39285714285714285, + "college_economics": 0.48333333333333334, + "business_administration": 0.42105263157894735, + "marxism": 0.4166666666666667, + "mao_zedong_thought": 0.6896551724137931, + "education_science": 0.5, + "teacher_qualification": 0.5714285714285714, + "high_school_politics": 0.4583333333333333, + "high_school_geography": 0.3333333333333333, + "middle_school_politics": 0.5, + "middle_school_geography": 0.5294117647058824, + "modern_chinese_history": 0.39285714285714285, + "ideological_and_moral_cultivation": 0.7916666666666666, + "logic": 0.48148148148148145, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.39473684210526316, + "professional_tour_guide": 0.47058823529411764, + "legal_professional": 0.5357142857142857, + "high_school_chinese": 0.3333333333333333, + "high_school_history": 0.52, + "middle_school_history": 0.6296296296296297, + "civil_servant": 0.36538461538461536, + "sports_science": 0.2916666666666667, + "plant_protection": 0.5555555555555556, + "basic_medicine": 0.4166666666666667, + "clinical_medicine": 0.37037037037037035, + "urban_and_rural_planner": 0.43137254901960786, + "accountant": 0.35185185185185186, + "fire_engineer": 0.3888888888888889, + "environmental_impact_assessment_engineer": 0.4444444444444444, + "tax_accountant": 0.2962962962962963, + "physician": 0.4444444444444444 } } }, - "zbench": { + "cmmlu": { "prompt_1": { - "accuracy": 0.15151515151515152 + "accuracy": 0.4157706093189964 }, "prompt_2": { - "accuracy": 0.15151515151515152 + "accuracy": 0.3978494623655914 }, "prompt_3": { - "accuracy": 0.2727272727272727 + "accuracy": 0.4121863799283154 }, "prompt_4": { - "accuracy": 0.15151515151515152 + "accuracy": 0.4014336917562724 }, "prompt_5": { - "accuracy": 0.21212121212121213 + "accuracy": 0.3942652329749104 } }, - "ind_emotion": { + "cmmlu_full": { "prompt_1": { - "accuracy": 0.18863636363636363 + "accuracy": 0.41504058021067175, + "category_acc": { + "agronomy": 0.34911242603550297, + "anatomy": 0.22297297297297297, + "ancient_chinese": 0.25609756097560976, + "arts": 0.43125, + "astronomy": 0.3090909090909091, + "business_ethics": 0.46411483253588515, + "chinese_civil_service_exam": 0.3375, + "chinese_driving_rule": 0.6183206106870229, + "chinese_food_culture": 0.3161764705882353, + "chinese_foreign_policy": 0.48598130841121495, + "chinese_history": 0.43653250773993807, + "chinese_literature": 0.3088235294117647, + "chinese_teacher_qualification": 0.4860335195530726, + "clinical_knowledge": 0.34177215189873417, + "college_actuarial_science": 0.2830188679245283, + "college_education": 0.4766355140186916, + "college_engineering_hydrology": 0.4528301886792453, + "college_law": 0.3888888888888889, + "college_mathematics": 0.2761904761904762, + "college_medical_statistics": 0.36792452830188677, + "college_medicine": 0.38095238095238093, + "computer_science": 0.47058823529411764, + "computer_security": 0.5263157894736842, + "conceptual_physics": 0.4421768707482993, + "construction_project_management": 0.3669064748201439, + "economics": 0.4968553459119497, + "education": 0.44785276073619634, + "electrical_engineering": 0.42441860465116277, + "elementary_chinese": 0.30158730158730157, + "elementary_commonsense": 0.37373737373737376, + "elementary_information_and_technology": 0.6176470588235294, + "elementary_mathematics": 0.30434782608695654, + "ethnology": 0.4888888888888889, + "food_science": 0.45454545454545453, + "genetics": 0.3409090909090909, + "global_facts": 0.4563758389261745, + "high_school_biology": 0.34911242603550297, + "high_school_chemistry": 0.32575757575757575, + "high_school_geography": 0.4152542372881356, + "high_school_mathematics": 0.2621951219512195, + "high_school_physics": 0.3090909090909091, + "high_school_politics": 0.4195804195804196, + "human_sexuality": 0.4365079365079365, + "international_law": 0.3783783783783784, + "journalism": 0.47674418604651164, + "jurisprudence": 0.49148418491484186, + "legal_and_moral_basis": 0.7757009345794392, + "logical": 0.37398373983739835, + "machine_learning": 0.3114754098360656, + "management": 0.5190476190476191, + "marketing": 0.5666666666666667, + "marxist_theory": 0.48148148148148145, + "modern_chinese": 0.3620689655172414, + "nutrition": 0.3793103448275862, + "philosophy": 0.47619047619047616, + "professional_accounting": 0.4342857142857143, + "professional_law": 0.3459715639810427, + "professional_medicine": 0.28191489361702127, + "professional_psychology": 0.4827586206896552, + "public_relations": 0.5574712643678161, + "security_study": 0.45925925925925926, + "sociology": 0.504424778761062, + "sports_science": 0.41818181818181815, + "traditional_chinese_medicine": 0.3027027027027027, + "virology": 0.39644970414201186, + "world_history": 0.391304347826087, + "world_religions": 0.41875 + } }, "prompt_2": { - "accuracy": 0.15681818181818183 + "accuracy": 0.41875323778276635, + "category_acc": { + "agronomy": 0.33727810650887574, + "anatomy": 0.22297297297297297, + "ancient_chinese": 0.2804878048780488, + "arts": 0.4125, + "astronomy": 0.3090909090909091, + "business_ethics": 0.44019138755980863, + "chinese_civil_service_exam": 0.33125, + "chinese_driving_rule": 0.6335877862595419, + "chinese_food_culture": 0.3088235294117647, + "chinese_foreign_policy": 0.4766355140186916, + "chinese_history": 0.4086687306501548, + "chinese_literature": 0.3137254901960784, + "chinese_teacher_qualification": 0.4972067039106145, + "clinical_knowledge": 0.3459915611814346, + "college_actuarial_science": 0.25471698113207547, + "college_education": 0.5233644859813084, + "college_engineering_hydrology": 0.4811320754716981, + "college_law": 0.37037037037037035, + "college_mathematics": 0.3047619047619048, + "college_medical_statistics": 0.3490566037735849, + "college_medicine": 0.34798534798534797, + "computer_science": 0.4803921568627451, + "computer_security": 0.4853801169590643, + "conceptual_physics": 0.4421768707482993, + "construction_project_management": 0.38848920863309355, + "economics": 0.4779874213836478, + "education": 0.48466257668711654, + "electrical_engineering": 0.43023255813953487, + "elementary_chinese": 0.30158730158730157, + "elementary_commonsense": 0.3888888888888889, + "elementary_information_and_technology": 0.6428571428571429, + "elementary_mathematics": 0.30434782608695654, + "ethnology": 0.4962962962962963, + "food_science": 0.46153846153846156, + "genetics": 0.3693181818181818, + "global_facts": 0.4429530201342282, + "high_school_biology": 0.35502958579881655, + "high_school_chemistry": 0.29545454545454547, + "high_school_geography": 0.3983050847457627, + "high_school_mathematics": 0.2621951219512195, + "high_school_physics": 0.33636363636363636, + "high_school_politics": 0.43356643356643354, + "human_sexuality": 0.4444444444444444, + "international_law": 0.3837837837837838, + "journalism": 0.5232558139534884, + "jurisprudence": 0.49391727493917276, + "legal_and_moral_basis": 0.7757009345794392, + "logical": 0.3983739837398374, + "machine_learning": 0.36065573770491804, + "management": 0.5428571428571428, + "marketing": 0.5555555555555556, + "marxist_theory": 0.4708994708994709, + "modern_chinese": 0.35344827586206895, + "nutrition": 0.42758620689655175, + "philosophy": 0.45714285714285713, + "professional_accounting": 0.44571428571428573, + "professional_law": 0.33175355450236965, + "professional_medicine": 0.3058510638297872, + "professional_psychology": 0.4698275862068966, + "public_relations": 0.5517241379310345, + "security_study": 0.5185185185185185, + "sociology": 0.4823008849557522, + "sports_science": 0.41818181818181815, + "traditional_chinese_medicine": 0.3081081081081081, + "virology": 0.4319526627218935, + "world_history": 0.40993788819875776, + "world_religions": 0.43125 + } }, "prompt_3": { - "accuracy": 0.20681818181818182 + "accuracy": 0.41745812467622173, + "category_acc": { + "agronomy": 0.3254437869822485, + "anatomy": 0.20270270270270271, + "ancient_chinese": 0.2621951219512195, + "arts": 0.41875, + "astronomy": 0.2545454545454545, + "business_ethics": 0.45933014354066987, + "chinese_civil_service_exam": 0.325, + "chinese_driving_rule": 0.5954198473282443, + "chinese_food_culture": 0.3161764705882353, + "chinese_foreign_policy": 0.48598130841121495, + "chinese_history": 0.3931888544891641, + "chinese_literature": 0.3137254901960784, + "chinese_teacher_qualification": 0.5027932960893855, + "clinical_knowledge": 0.34177215189873417, + "college_actuarial_science": 0.3018867924528302, + "college_education": 0.5233644859813084, + "college_engineering_hydrology": 0.49056603773584906, + "college_law": 0.37037037037037035, + "college_mathematics": 0.2857142857142857, + "college_medical_statistics": 0.3490566037735849, + "college_medicine": 0.34798534798534797, + "computer_science": 0.49019607843137253, + "computer_security": 0.49707602339181284, + "conceptual_physics": 0.41496598639455784, + "construction_project_management": 0.381294964028777, + "economics": 0.5031446540880503, + "education": 0.49693251533742333, + "electrical_engineering": 0.436046511627907, + "elementary_chinese": 0.30158730158730157, + "elementary_commonsense": 0.3888888888888889, + "elementary_information_and_technology": 0.6596638655462185, + "elementary_mathematics": 0.3, + "ethnology": 0.4740740740740741, + "food_science": 0.46153846153846156, + "genetics": 0.375, + "global_facts": 0.42953020134228187, + "high_school_biology": 0.3609467455621302, + "high_school_chemistry": 0.30303030303030304, + "high_school_geography": 0.423728813559322, + "high_school_mathematics": 0.27439024390243905, + "high_school_physics": 0.34545454545454546, + "high_school_politics": 0.4195804195804196, + "human_sexuality": 0.46825396825396826, + "international_law": 0.3837837837837838, + "journalism": 0.4883720930232558, + "jurisprudence": 0.48418491484184917, + "legal_and_moral_basis": 0.7757009345794392, + "logical": 0.3821138211382114, + "machine_learning": 0.36885245901639346, + "management": 0.5142857142857142, + "marketing": 0.5555555555555556, + "marxist_theory": 0.4656084656084656, + "modern_chinese": 0.3620689655172414, + "nutrition": 0.4206896551724138, + "philosophy": 0.4380952380952381, + "professional_accounting": 0.42857142857142855, + "professional_law": 0.3459715639810427, + "professional_medicine": 0.324468085106383, + "professional_psychology": 0.47844827586206895, + "public_relations": 0.5517241379310345, + "security_study": 0.5111111111111111, + "sociology": 0.4823008849557522, + "sports_science": 0.4, + "traditional_chinese_medicine": 0.34054054054054056, + "virology": 0.4319526627218935, + "world_history": 0.40372670807453415, + "world_religions": 0.41875 + } }, "prompt_4": { - "accuracy": 0.2 - }, - "prompt_5": { - "accuracy": 0.20909090909090908 - } + "accuracy": 0.40986012778449316, + "category_acc": { + "agronomy": 0.33136094674556216, + "anatomy": 0.22297297297297297, + "ancient_chinese": 0.24390243902439024, + "arts": 0.4, + "astronomy": 0.3090909090909091, + "business_ethics": 0.45933014354066987, + "chinese_civil_service_exam": 0.36875, + "chinese_driving_rule": 0.6106870229007634, + "chinese_food_culture": 0.3014705882352941, + "chinese_foreign_policy": 0.45794392523364486, + "chinese_history": 0.4117647058823529, + "chinese_literature": 0.30392156862745096, + "chinese_teacher_qualification": 0.5139664804469274, + "clinical_knowledge": 0.350210970464135, + "college_actuarial_science": 0.24528301886792453, + "college_education": 0.514018691588785, + "college_engineering_hydrology": 0.44339622641509435, + "college_law": 0.39814814814814814, + "college_mathematics": 0.3333333333333333, + "college_medical_statistics": 0.39622641509433965, + "college_medicine": 0.31868131868131866, + "computer_science": 0.49019607843137253, + "computer_security": 0.5087719298245614, + "conceptual_physics": 0.41496598639455784, + "construction_project_management": 0.381294964028777, + "economics": 0.48427672955974843, + "education": 0.48466257668711654, + "electrical_engineering": 0.4127906976744186, + "elementary_chinese": 0.2857142857142857, + "elementary_commonsense": 0.37373737373737376, + "elementary_information_and_technology": 0.6386554621848739, + "elementary_mathematics": 0.2782608695652174, + "ethnology": 0.4888888888888889, + "food_science": 0.44755244755244755, + "genetics": 0.3465909090909091, + "global_facts": 0.4228187919463087, + "high_school_biology": 0.33727810650887574, + "high_school_chemistry": 0.3409090909090909, + "high_school_geography": 0.3898305084745763, + "high_school_mathematics": 0.23170731707317074, + "high_school_physics": 0.3181818181818182, + "high_school_politics": 0.45454545454545453, + "human_sexuality": 0.42857142857142855, + "international_law": 0.3621621621621622, + "journalism": 0.47674418604651164, + "jurisprudence": 0.44768856447688565, + "legal_and_moral_basis": 0.7523364485981309, + "logical": 0.37398373983739835, + "machine_learning": 0.3524590163934426, + "management": 0.5142857142857142, + "marketing": 0.55, + "marxist_theory": 0.48148148148148145, + "modern_chinese": 0.3879310344827586, + "nutrition": 0.43448275862068964, + "philosophy": 0.44761904761904764, + "professional_accounting": 0.41714285714285715, + "professional_law": 0.35071090047393366, + "professional_medicine": 0.2872340425531915, + "professional_psychology": 0.47844827586206895, + "public_relations": 0.5344827586206896, + "security_study": 0.45925925925925926, + "sociology": 0.49557522123893805, + "sports_science": 0.41818181818181815, + "traditional_chinese_medicine": 0.31351351351351353, + "virology": 0.40236686390532544, + "world_history": 0.37888198757763975, + "world_religions": 0.4 + } + }, + "prompt_5": { + "accuracy": 0.41875323778276635, + "category_acc": { + "agronomy": 0.33136094674556216, + "anatomy": 0.24324324324324326, + "ancient_chinese": 0.2804878048780488, + "arts": 0.4125, + "astronomy": 0.296969696969697, + "business_ethics": 0.47368421052631576, + "chinese_civil_service_exam": 0.35625, + "chinese_driving_rule": 0.6259541984732825, + "chinese_food_culture": 0.3235294117647059, + "chinese_foreign_policy": 0.4953271028037383, + "chinese_history": 0.43962848297213625, + "chinese_literature": 0.30392156862745096, + "chinese_teacher_qualification": 0.5083798882681564, + "clinical_knowledge": 0.35864978902953587, + "college_actuarial_science": 0.24528301886792453, + "college_education": 0.5420560747663551, + "college_engineering_hydrology": 0.4811320754716981, + "college_law": 0.35185185185185186, + "college_mathematics": 0.26666666666666666, + "college_medical_statistics": 0.41509433962264153, + "college_medicine": 0.3772893772893773, + "computer_science": 0.47058823529411764, + "computer_security": 0.5029239766081871, + "conceptual_physics": 0.4421768707482993, + "construction_project_management": 0.381294964028777, + "economics": 0.48427672955974843, + "education": 0.44785276073619634, + "electrical_engineering": 0.4127906976744186, + "elementary_chinese": 0.30158730158730157, + "elementary_commonsense": 0.40404040404040403, + "elementary_information_and_technology": 0.6596638655462185, + "elementary_mathematics": 0.3347826086956522, + "ethnology": 0.5037037037037037, + "food_science": 0.45454545454545453, + "genetics": 0.35795454545454547, + "global_facts": 0.44966442953020136, + "high_school_biology": 0.35502958579881655, + "high_school_chemistry": 0.25, + "high_school_geography": 0.4152542372881356, + "high_school_mathematics": 0.23780487804878048, + "high_school_physics": 0.32727272727272727, + "high_school_politics": 0.4195804195804196, + "human_sexuality": 0.4444444444444444, + "international_law": 0.3837837837837838, + "journalism": 0.5058139534883721, + "jurisprudence": 0.48905109489051096, + "legal_and_moral_basis": 0.7616822429906542, + "logical": 0.3902439024390244, + "machine_learning": 0.36885245901639346, + "management": 0.5190476190476191, + "marketing": 0.5388888888888889, + "marxist_theory": 0.48148148148148145, + "modern_chinese": 0.3793103448275862, + "nutrition": 0.36551724137931035, + "philosophy": 0.47619047619047616, + "professional_accounting": 0.44, + "professional_law": 0.35071090047393366, + "professional_medicine": 0.3058510638297872, + "professional_psychology": 0.4698275862068966, + "public_relations": 0.5574712643678161, + "security_study": 0.4666666666666667, + "sociology": 0.48672566371681414, + "sports_science": 0.41818181818181815, + "traditional_chinese_medicine": 0.31351351351351353, + "virology": 0.40828402366863903, + "world_history": 0.391304347826087, + "world_religions": 0.4 + } + } + }, + "zbench": { + "prompt_1": { + "accuracy": 0.30303030303030304 + }, + "prompt_2": { + "accuracy": 0.3939393939393939 + }, + "prompt_3": { + "accuracy": 0.30303030303030304 + }, + "prompt_4": { + "accuracy": 0.3333333333333333 + }, + "prompt_5": { + "accuracy": 0.3939393939393939 + } + }, + "ind_emotion": { + "prompt_1": { + "accuracy": 0.7090909090909091 + }, + "prompt_2": { + "accuracy": 0.6545454545454545 + }, + "prompt_3": { + "accuracy": 0.6613636363636364 + }, + "prompt_4": { + "accuracy": 0.6909090909090909 + }, + "prompt_5": { + "accuracy": 0.6818181818181818 + } }, "ocnli": { "prompt_1": { - "accuracy": 0.3159322033898305 + "accuracy": 0.4105084745762712 }, "prompt_2": { - "accuracy": 0.31457627118644066 + "accuracy": 0.42915254237288136 }, "prompt_3": { - "accuracy": 0.33152542372881355 + "accuracy": 0.4105084745762712 }, "prompt_4": { - "accuracy": 0.32033898305084746 + "accuracy": 0.41966101694915253 }, "prompt_5": { - "accuracy": 0.3271186440677966 + "accuracy": 0.408135593220339 } }, "c3": { "prompt_1": { - "accuracy": 0.3754674644727001 + "accuracy": 0.7857142857142857 }, "prompt_2": { - "accuracy": 0.3889304412864622 + "accuracy": 0.7756170531039641 }, "prompt_3": { - "accuracy": 0.37808526551982047 + "accuracy": 0.7729992520568437 }, "prompt_4": { - "accuracy": 0.3414360508601346 + "accuracy": 0.7789827973074046 }, "prompt_5": { - "accuracy": 0.3519072550486163 + "accuracy": 0.7756170531039641 } }, "dream": { "prompt_1": { - "accuracy": 0.4723174914257717 + "accuracy": 0.8574228319451249 }, "prompt_2": { - "accuracy": 0.47868691817736403 + "accuracy": 0.849093581577658 }, "prompt_3": { - "accuracy": 0.46251837334639884 + "accuracy": 0.8417442430181283 }, "prompt_4": { - "accuracy": 0.43900048995590396 + "accuracy": 0.8564429201371877 }, "prompt_5": { - "accuracy": 0.4463498285154336 + "accuracy": 0.854483096521313 } }, "samsum": { "prompt_1": { - "rouge1": 0.25472636821142525, - "rouge2": 0.09281876165708257, - "rougeL": 0.19737867647053448, - "avg_rouge": 0.18164126877968076 + "rouge1": 0.337934193303483, + "rouge2": 0.13154283775211653, + "rougeL": 0.2614570898741684, + "avg_rouge": 0.24364470697658933 }, "prompt_2": { - "rouge1": 0.20198569189921137, - "rouge2": 0.07484389996747501, - "rougeL": 0.16183729802925795, - "avg_rouge": 0.14622229663198144 + "rouge1": 0.370426617540352, + "rouge2": 0.14460103670266045, + "rougeL": 0.2857311872813845, + "avg_rouge": 0.2669196138414656 }, "prompt_3": { - "rouge1": 0.19090301860476078, - "rouge2": 0.06297299630767098, - "rougeL": 0.15086416246604725, - "avg_rouge": 0.134913392459493 + "rouge1": 0.3526840141075432, + "rouge2": 0.13318361789311367, + "rougeL": 0.27212472005604, + "avg_rouge": 0.2526641173522323 }, "prompt_4": { - "rouge1": 0.22180997435942065, - "rouge2": 0.0842989045943594, - "rougeL": 0.1734975551986423, - "avg_rouge": 0.15986881138414077 + "rouge1": 0.34499225565885405, + "rouge2": 0.131082047202366, + "rougeL": 0.26542039826170644, + "avg_rouge": 0.24716490037430883 }, "prompt_5": { - "rouge1": 0.2612154491902073, - "rouge2": 0.08794414006317752, - "rougeL": 0.19862465935780443, - "avg_rouge": 0.1825947495370631 + "rouge1": 0.36434660312828493, + "rouge2": 0.1342337732311142, + "rougeL": 0.2812806022953663, + "avg_rouge": 0.2599536595515885 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.2170439451100966, - "rouge2": 0.06177085085251368, - "rougeL": 0.16165354764691592, - "avg_rouge": 0.1468227812031754 + "rouge1": 0.23171463702568543, + "rouge2": 0.06482582508272836, + "rougeL": 0.17113911650235575, + "avg_rouge": 0.15589319287025652 }, "prompt_2": { - "rouge1": 0.21807849746765204, - "rouge2": 0.060215118698180706, - "rougeL": 0.1613356679463527, - "avg_rouge": 0.14654309470406182 + "rouge1": 0.23232726794331263, + "rouge2": 0.06413743246594411, + "rougeL": 0.17150409279989268, + "avg_rouge": 0.15598959773638313 }, "prompt_3": { - "rouge1": 0.21690989670134653, - "rouge2": 0.0578274417616976, - "rougeL": 0.16117634469978795, - "avg_rouge": 0.14530456105427736 + "rouge1": 0.23218474445664966, + "rouge2": 0.06416276534806573, + "rougeL": 0.17133699125529367, + "avg_rouge": 0.15589483368666968 }, "prompt_4": { - "rouge1": 0.21832817735185772, - "rouge2": 0.060118892162594806, - "rougeL": 0.16082806810758785, - "avg_rouge": 0.14642504587401348 + "rouge1": 0.23040880796958316, + "rouge2": 0.06366423137632587, + "rougeL": 0.16981402032766788, + "avg_rouge": 0.1546290198911923 }, "prompt_5": { - "rouge1": 0.21285118413216184, - "rouge2": 0.059616397210976564, - "rougeL": 0.15739937120296282, - "avg_rouge": 0.14328898418203376 + "rouge1": 0.23125728609443794, + "rouge2": 0.0640530946503918, + "rougeL": 0.17086375161470452, + "avg_rouge": 0.15539137745317808 } }, "sst2": { "prompt_1": { - "accuracy": 0.4988532110091743 + "accuracy": 0.9151376146788991 }, "prompt_2": { - "accuracy": 0.5034403669724771 + "accuracy": 0.9002293577981652 }, "prompt_3": { - "accuracy": 0.5 + "accuracy": 0.9139908256880734 }, "prompt_4": { - "accuracy": 0.5057339449541285 + "accuracy": 0.9094036697247706 }, "prompt_5": { - "accuracy": 0.5149082568807339 + "accuracy": 0.9220183486238532 } }, "cola": { "prompt_1": { - "accuracy": 0.49856184084372 + "accuracy": 0.7804410354745925 }, "prompt_2": { - "accuracy": 0.5349952061361457 + "accuracy": 0.7813998082454459 }, "prompt_3": { - "accuracy": 0.45445829338446786 + "accuracy": 0.7746883988494727 }, "prompt_4": { - "accuracy": 0.5158197507190796 + "accuracy": 0.7813998082454459 }, "prompt_5": { - "accuracy": 0.4956855225311601 + "accuracy": 0.7785234899328859 } }, "qqp": { "prompt_1": { - "accuracy": 0.5035 + "accuracy": 0.6935 }, "prompt_2": { - "accuracy": 0.5975 + "accuracy": 0.6935 }, "prompt_3": { - "accuracy": 0.602 + "accuracy": 0.6855 }, "prompt_4": { - "accuracy": 0.497 + "accuracy": 0.6905 }, "prompt_5": { - "accuracy": 0.495 + "accuracy": 0.688 } }, "mnli": { "prompt_1": { - "accuracy": 0.329 + "accuracy": 0.6 }, "prompt_2": { - "accuracy": 0.3225 + "accuracy": 0.5 }, "prompt_3": { - "accuracy": 0.3195 + "accuracy": 0.6 }, "prompt_4": { - "accuracy": 0.3165 + "accuracy": 0.4 }, "prompt_5": { - "accuracy": 0.335 + "accuracy": 0.7 } }, "qnli": { "prompt_1": { - "accuracy": 0.4765 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.492 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.481 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.5065 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.504 + "accuracy": 0.9 } }, "wnli": { "prompt_1": { - "accuracy": 0.4647887323943662 + "accuracy": 0.6 }, "prompt_2": { - "accuracy": 0.5211267605633803 + "accuracy": 0.7 }, "prompt_3": { - "accuracy": 0.49295774647887325 + "accuracy": 0.6 }, "prompt_4": { - "accuracy": 0.5070422535211268 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.4788732394366197 + "accuracy": 0.8 } }, "rte": { "prompt_1": { - "accuracy": 0.5090252707581228 + "accuracy": 0.8 }, "prompt_2": { - "accuracy": 0.5270758122743683 + "accuracy": 0.5 }, "prompt_3": { - "accuracy": 0.5234657039711191 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.5234657039711191 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.5306859205776173 + "accuracy": 0.6 } }, "mrpc": { "prompt_1": { - "accuracy": 0.5024509803921569 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.35784313725490197 + "accuracy": 0.8 }, "prompt_3": { - "accuracy": 0.3284313725490196 + "accuracy": 0.6 }, "prompt_4": { - "accuracy": 0.4803921568627451 + "accuracy": 0.7 }, "prompt_5": { - "accuracy": 0.5294117647058824 + "accuracy": 0.5 } } }, "five_shot": { + "cross_xquad": { + "prompt_1": -1 + }, + "cross_mmlu": { + "prompt_1": -1 + }, + "cross_logiqa": { + "prompt_1": -1 + }, + "sg_eval": { + "prompt_1": -1 + }, + "cn_eval": { + "prompt_1": -1 + }, + "us_eval": { + "prompt_1": -1 + }, + "ph_eval": { + "prompt_1": -1 + }, + "sing2eng": { + "prompt_1": -1 + }, + "indommlu": { + "prompt_1": -1 + }, + "flores_ind2eng": { + "prompt_1": -1 + }, + "flores_vie2eng": { + "prompt_1": -1 + }, + "flores_zho2eng": { + "prompt_1": -1 + }, + "flores_zsm2eng": { + "prompt_1": -1 + }, + "mmlu": { + "prompt_1": -1 + }, + "mmlu_full": { + "prompt_1": -1 + }, + "c_eval": { + "prompt_1": -1 + }, + "c_eval_full": { + "prompt_1": -1 + }, + "cmmlu": { + "prompt_1": -1 + }, + "cmmlu_full": { + "prompt_1": -1 + }, + "zbench": { + "prompt_1": -1 + }, + "ind_emotion": { + "prompt_1": -1 + }, + "ocnli": { + "prompt_1": -1 + }, + "c3": { + "prompt_1": -1 + }, + "dream": { + "prompt_1": -1 + }, + "samsum": { + "prompt_1": -1 + }, + "dialogsum": { + "prompt_1": -1 + }, + "sst2": { + "prompt_1": -1 + }, + "cola": { + "prompt_1": -1 + }, + "qqp": { + "prompt_1": -1 + }, + "mnli": { + "prompt_1": -1 + }, + "qnli": { + "prompt_1": -1 + }, + "wnli": { + "prompt_1": -1 + }, + "rte": { + "prompt_1": -1 + }, + "mrpc": { + "prompt_1": -1 + } + } + }, + "meta_llama_3_8b_instruct": { + "model_size": "8B", + "model_link": "https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct", + "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.49369747899159666, + "overall_acc": 0.9378151260504202, "language_acc": { - "English": 0.5235294117647059, - "Chinese": 0.4756302521008403, - "Spanish": 0.492436974789916, - "Vietnamese": 0.4831932773109244 - }, - "consistency_score_2": 0.5973389355742298, - "consistency_score_3": 0.4409663865546219, - "consistency_score_4": 0.35378151260504204, + "Spanish": 0.9394957983193277, + "English": 0.9478991596638655, + "Chinese": 0.9294117647058824, + "Vietnamese": 0.934453781512605 + }, + "consistency_score_2": 0.9329131652661066, + "consistency_score_3": 0.901890756302521, + "consistency_score_4": 0.8798319327731092, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.5941176470588235, - "English,Spanish": 0.626890756302521, - "English,Vietnamese": 0.6084033613445378, - "Chinese,Spanish": 0.565546218487395, - "Chinese,Vietnamese": 0.5798319327731093, - "Spanish,Vietnamese": 0.6092436974789915 + "Spanish,English": 0.9445378151260504, + "Spanish,Chinese": 0.926890756302521, + "Spanish,Vietnamese": 0.9302521008403362, + "English,Chinese": 0.9336134453781513, + "English,Vietnamese": 0.9428571428571428, + "Chinese,Vietnamese": 0.9193277310924369 }, "3_combine": { - "English,Chinese,Spanish": 0.4411764705882353, - "English,Chinese,Vietnamese": 0.4319327731092437, - "English,Spanish,Vietnamese": 0.46638655462184875, - "Chinese,Spanish,Vietnamese": 0.42436974789915966 + "Spanish,English,Chinese": 0.9042016806722689, + "Spanish,English,Vietnamese": 0.9117647058823529, + "Spanish,Chinese,Vietnamese": 0.8915966386554622, + "English,Chinese,Vietnamese": 0.9 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.35378151260504204 + "Spanish,English,Chinese,Vietnamese": 0.8798319327731092 } }, - "AC3_2": 0.5405955706562688, - "AC3_3": 0.46584446315688494, - "AC3_4": 0.41218966510845195 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.2619047619047619, + "AC3_2": 0.9353577231845365, + "AC3_3": 0.9195021893149736, + "AC3_4": 0.9078986934080167 + }, + "prompt_2": { + "overall_acc": 0.9336134453781513, "language_acc": { - "Spanish": 0.25333333333333335, - "English": 0.29333333333333333, - "Chinese": 0.25333333333333335, - "Vietnamese": 0.22666666666666666, - "Filipino": 0.28, - "Indonesian": 0.26666666666666666, - "Malay": 0.26 + "Spanish": 0.9310924369747899, + "English": 0.9470588235294117, + "Chinese": 0.9252100840336135, + "Vietnamese": 0.9310924369747899 }, - "consistency_score_2": 0.4438095238095238, - "consistency_score_3": 0.2394285714285714, - "consistency_score_4": 0.1428571428571428, - "consistency_score_5": 0.09015873015873016, - "consistency_score_6": 0.05904761904761906, - "consistency_score_7": 0.04, + "consistency_score_2": 0.922689075630252, + "consistency_score_3": 0.8876050420168067, + "consistency_score_4": 0.865546218487395, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5, - "Spanish,Chinese": 0.43333333333333335, - "Spanish,Vietnamese": 0.4266666666666667, - "Spanish,Filipino": 0.4866666666666667, - "Spanish,Indonesian": 0.4266666666666667, - "Spanish,Malay": 0.43333333333333335, - "English,Chinese": 0.38, - "English,Vietnamese": 0.44, - "English,Filipino": 0.48, - "English,Indonesian": 0.49333333333333335, - "English,Malay": 0.48, - "Chinese,Vietnamese": 0.4066666666666667, - "Chinese,Filipino": 0.3933333333333333, - "Chinese,Indonesian": 0.4533333333333333, - "Chinese,Malay": 0.4266666666666667, - "Vietnamese,Filipino": 0.3333333333333333, - "Vietnamese,Indonesian": 0.4533333333333333, - "Vietnamese,Malay": 0.4066666666666667, - "Filipino,Indonesian": 0.47333333333333333, - "Filipino,Malay": 0.4666666666666667, - "Indonesian,Malay": 0.5266666666666666 + "Spanish,English": 0.9361344537815126, + "Spanish,Chinese": 0.9168067226890756, + "Spanish,Vietnamese": 0.9310924369747899, + "English,Chinese": 0.9218487394957983, + "English,Vietnamese": 0.9277310924369748, + "Chinese,Vietnamese": 0.9025210084033614 }, "3_combine": { - "Spanish,English,Chinese": 0.23333333333333334, - "Spanish,English,Vietnamese": 0.26, - "Spanish,English,Filipino": 0.3, - "Spanish,English,Indonesian": 0.3, - "Spanish,English,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese": 0.20666666666666667, - "Spanish,Chinese,Filipino": 0.22666666666666666, - "Spanish,Chinese,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,Vietnamese,Indonesian": 0.23333333333333334, - "Spanish,Vietnamese,Malay": 0.21333333333333335, - "Spanish,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Filipino,Malay": 0.28, - "Spanish,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese": 0.18666666666666668, - "English,Chinese,Filipino": 0.19333333333333333, - "English,Chinese,Indonesian": 0.22666666666666666, - "English,Chinese,Malay": 0.22, - "English,Vietnamese,Filipino": 0.19333333333333333, - "English,Vietnamese,Indonesian": 0.25333333333333335, - "English,Vietnamese,Malay": 0.24, - "English,Filipino,Indonesian": 0.28, - "English,Filipino,Malay": 0.29333333333333333, - "English,Indonesian,Malay": 0.3, - "Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "Chinese,Vietnamese,Malay": 0.22666666666666666, - "Chinese,Filipino,Indonesian": 0.22666666666666666, - "Chinese,Filipino,Malay": 0.21333333333333335, - "Chinese,Indonesian,Malay": 0.26, - "Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Vietnamese,Filipino,Malay": 0.18666666666666668, - "Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Filipino,Indonesian,Malay": 0.30666666666666664 + "Spanish,English,Chinese": 0.8890756302521008, + "Spanish,English,Vietnamese": 0.9, + "Spanish,Chinese,Vietnamese": 0.880672268907563, + "English,Chinese,Vietnamese": 0.880672268907563 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.13333333333333333, - "Spanish,English,Chinese,Filipino": 0.14, - "Spanish,English,Chinese,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,English,Vietnamese,Malay": 0.16666666666666666, - "Spanish,English,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Filipino,Malay": 0.2, - "Spanish,English,Indonesian,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Filipino": 0.08, - "Spanish,Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.14, - "Spanish,Chinese,Indonesian,Malay": 0.14, - "Spanish,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,Filipino,Indonesian,Malay": 0.18, - "English,Chinese,Vietnamese,Filipino": 0.07333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Filipino,Malay": 0.13333333333333333, - "English,Chinese,Indonesian,Malay": 0.14, - "English,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "English,Vietnamese,Filipino,Malay": 0.12666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.18, - "English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.1, - "Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Vietnamese,Filipino,Indonesian,Malay": 0.14 + "Spanish,English,Chinese,Vietnamese": 0.865546218487395 + } + }, + "AC3_2": 0.9281191154096758, + "AC3_3": 0.9100281016278929, + "AC3_4": 0.8982922453863125 + }, + "prompt_3": { + "overall_acc": 0.9218487394957983, + "language_acc": { + "Spanish": 0.9193277310924369, + "English": 0.934453781512605, + "Chinese": 0.9126050420168067, + "Vietnamese": 0.9210084033613445 + }, + "consistency_score_2": 0.903781512605042, + "consistency_score_3": 0.859873949579832, + "consistency_score_4": 0.8285714285714286, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.9218487394957983, + "Spanish,Chinese": 0.8949579831932774, + "Spanish,Vietnamese": 0.9067226890756303, + "English,Chinese": 0.9008403361344538, + "English,Vietnamese": 0.9159663865546218, + "Chinese,Vietnamese": 0.8823529411764706 }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.06, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.1, - "Spanish,English,Chinese,Vietnamese,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.1, - "Spanish,English,Chinese,Filipino,Malay": 0.1, - "Spanish,English,Chinese,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,English,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.05333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.06, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "English,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 + "3_combine": { + "Spanish,English,Chinese": 0.8621848739495799, + "Spanish,English,Vietnamese": 0.8756302521008403, + "Spanish,Chinese,Vietnamese": 0.8478991596638655, + "English,Chinese,Vietnamese": 0.853781512605042 }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.05333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.06666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334 + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.8285714285714286 + } + }, + "AC3_2": 0.9127257254530945, + "AC3_3": 0.8897834902828757, + "AC3_4": 0.8727247787718962 + }, + "prompt_4": { + "overall_acc": 0.9304621848739496, + "language_acc": { + "Spanish": 0.9260504201680673, + "English": 0.9436974789915966, + "Chinese": 0.9193277310924369, + "Vietnamese": 0.9327731092436975 + }, + "consistency_score_2": 0.9184873949579831, + "consistency_score_3": 0.8808823529411764, + "consistency_score_4": 0.8546218487394958, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.9252100840336135, + "Spanish,Chinese": 0.9109243697478991, + "Spanish,Vietnamese": 0.9226890756302522, + "English,Chinese": 0.9168067226890756, + "English,Vietnamese": 0.9277310924369748, + "Chinese,Vietnamese": 0.907563025210084 }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04 + "3_combine": { + "Spanish,English,Chinese": 0.8798319327731092, + "Spanish,English,Vietnamese": 0.8907563025210085, + "Spanish,Chinese,Vietnamese": 0.8739495798319328, + "English,Chinese,Vietnamese": 0.8789915966386554 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.8546218487394958 + } + }, + "AC3_2": 0.9244360122824432, + "AC3_3": 0.904993723257036, + "AC3_4": 0.8909309563034448 + }, + "prompt_5": { + "overall_acc": 0.8934873949579832, + "language_acc": { + "Spanish": 0.9058823529411765, + "English": 0.915126050420168, + "Chinese": 0.8697478991596639, + "Vietnamese": 0.8831932773109243 + }, + "consistency_score_2": 0.8563025210084034, + "consistency_score_3": 0.7915966386554621, + "consistency_score_4": 0.746218487394958, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.8966386554621849, + "Spanish,Chinese": 0.8436974789915966, + "Spanish,Vietnamese": 0.8495798319327731, + "English,Chinese": 0.8554621848739495, + "English,Vietnamese": 0.8672268907563025, + "Chinese,Vietnamese": 0.8252100840336134 + }, + "3_combine": { + "Spanish,English,Chinese": 0.8050420168067227, + "Spanish,English,Vietnamese": 0.8142857142857143, + "Spanish,Chinese,Vietnamese": 0.7647058823529411, + "English,Chinese,Vietnamese": 0.7823529411764706 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.746218487394958 } }, - "AC3_2": 0.32941327673501986, - "AC3_3": 0.25016283103998327, - "AC3_4": 0.18487394953415714, - "AC3_5": 0.1341405813522547, - "AC3_6": 0.0963685177034771, - "AC3_7": 0.06940063089183891 + "AC3_2": 0.8744998491152861, + "AC3_3": 0.8394615394593914, + "AC3_4": 0.8132395199736593 } }, - "cross_logiqa": { + "cross_mmlu": { "prompt_1": { - "overall_acc": 0.30194805194805197, + "overall_acc": 0.6076190476190476, "language_acc": { - "English": 0.3409090909090909, - "Filipino": 0.26704545454545453, - "Spanish": 0.2897727272727273, - "Malay": 0.26704545454545453, - "Chinese": 0.30113636363636365, - "Indonesian": 0.3522727272727273, - "Vietnamese": 0.29545454545454547 + "Filipino": 0.5466666666666666, + "Vietnamese": 0.5933333333333334, + "Chinese": 0.64, + "Spanish": 0.6133333333333333, + "Malay": 0.6066666666666667, + "Indonesian": 0.52, + "English": 0.7333333333333333 }, - "consistency_score_2": 0.43777056277056275, - "consistency_score_3": 0.2332792207792207, - "consistency_score_4": 0.13993506493506494, - "consistency_score_5": 0.09253246753246754, - "consistency_score_6": 0.06655844155844155, - "consistency_score_7": 0.05113636363636364, + "consistency_score_2": 0.5895238095238096, + "consistency_score_3": 0.431047619047619, + "consistency_score_4": 0.34742857142857136, + "consistency_score_5": 0.29365079365079366, + "consistency_score_6": 0.2552380952380952, + "consistency_score_7": 0.22666666666666666, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.42045454545454547, - "English,Spanish": 0.5284090909090909, - "English,Malay": 0.3806818181818182, - "English,Chinese": 0.5056818181818182, - "English,Indonesian": 0.4602272727272727, - "English,Vietnamese": 0.44886363636363635, - "Filipino,Spanish": 0.3409090909090909, - "Filipino,Malay": 0.4090909090909091, - "Filipino,Chinese": 0.36363636363636365, - "Filipino,Indonesian": 0.45454545454545453, - "Filipino,Vietnamese": 0.375, - "Spanish,Malay": 0.3465909090909091, - "Spanish,Chinese": 0.5397727272727273, - "Spanish,Indonesian": 0.4715909090909091, - "Spanish,Vietnamese": 0.4431818181818182, - "Malay,Chinese": 0.4147727272727273, - "Malay,Indonesian": 0.4943181818181818, - "Malay,Vietnamese": 0.4772727272727273, - "Chinese,Indonesian": 0.4431818181818182, - "Chinese,Vietnamese": 0.4147727272727273, - "Indonesian,Vietnamese": 0.4602272727272727 + "Filipino,Vietnamese": 0.5266666666666666, + "Filipino,Chinese": 0.5933333333333334, + "Filipino,Spanish": 0.5333333333333333, + "Filipino,Malay": 0.5133333333333333, + "Filipino,Indonesian": 0.4866666666666667, + "Filipino,English": 0.58, + "Vietnamese,Chinese": 0.66, + "Vietnamese,Spanish": 0.58, + "Vietnamese,Malay": 0.6, + "Vietnamese,Indonesian": 0.6333333333333333, + "Vietnamese,English": 0.5933333333333334, + "Chinese,Spanish": 0.5533333333333333, + "Chinese,Malay": 0.6466666666666666, + "Chinese,Indonesian": 0.5933333333333334, + "Chinese,English": 0.64, + "Spanish,Malay": 0.5533333333333333, + "Spanish,Indonesian": 0.5733333333333334, + "Spanish,English": 0.6733333333333333, + "Malay,Indonesian": 0.6466666666666666, + "Malay,English": 0.6466666666666666, + "Indonesian,English": 0.5533333333333333 }, "3_combine": { - "English,Filipino,Spanish": 0.2159090909090909, - "English,Filipino,Malay": 0.2159090909090909, - "English,Filipino,Chinese": 0.2159090909090909, - "English,Filipino,Indonesian": 0.24431818181818182, - "English,Filipino,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay": 0.19318181818181818, - "English,Spanish,Chinese": 0.3409090909090909, - "English,Spanish,Indonesian": 0.2784090909090909, - "English,Spanish,Vietnamese": 0.29545454545454547, - "English,Malay,Chinese": 0.22727272727272727, - "English,Malay,Indonesian": 0.23295454545454544, - "English,Malay,Vietnamese": 0.23295454545454544, - "English,Chinese,Indonesian": 0.26704545454545453, - "English,Chinese,Vietnamese": 0.2727272727272727, - "English,Indonesian,Vietnamese": 0.25, - "Filipino,Spanish,Malay": 0.14772727272727273, - "Filipino,Spanish,Chinese": 0.1875, - "Filipino,Spanish,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Vietnamese": 0.1875, - "Filipino,Malay,Chinese": 0.1875, - "Filipino,Malay,Indonesian": 0.2556818181818182, - "Filipino,Malay,Vietnamese": 0.23295454545454544, - "Filipino,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Chinese,Vietnamese": 0.18181818181818182, - "Filipino,Indonesian,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Chinese": 0.21022727272727273, - "Spanish,Malay,Indonesian": 0.22727272727272727, - "Spanish,Malay,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Indonesian": 0.2784090909090909, - "Spanish,Chinese,Vietnamese": 0.26704545454545453, - "Spanish,Indonesian,Vietnamese": 0.2727272727272727, - "Malay,Chinese,Indonesian": 0.23863636363636365, - "Malay,Chinese,Vietnamese": 0.23863636363636365, - "Malay,Indonesian,Vietnamese": 0.2727272727272727, - "Chinese,Indonesian,Vietnamese": 0.23295454545454544 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.10795454545454546, - "English,Filipino,Spanish,Chinese": 0.14204545454545456, - "English,Filipino,Spanish,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Chinese": 0.13068181818181818, - "English,Filipino,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese": 0.125, - "English,Spanish,Malay,Indonesian": 0.125, - "English,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Spanish,Chinese,Indonesian": 0.18181818181818182, - "English,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "English,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay,Chinese": 0.09090909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.125, - "Filipino,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 + "Filipino,Vietnamese,Chinese": 0.43333333333333335, + "Filipino,Vietnamese,Spanish": 0.38, + "Filipino,Vietnamese,Malay": 0.38666666666666666, + "Filipino,Vietnamese,Indonesian": 0.38, + "Filipino,Vietnamese,English": 0.3933333333333333, + "Filipino,Chinese,Spanish": 0.38, + "Filipino,Chinese,Malay": 0.43333333333333335, + "Filipino,Chinese,Indonesian": 0.38, + "Filipino,Chinese,English": 0.4533333333333333, + "Filipino,Spanish,Malay": 0.36666666666666664, + "Filipino,Spanish,Indonesian": 0.35333333333333333, + "Filipino,Spanish,English": 0.4266666666666667, + "Filipino,Malay,Indonesian": 0.38, + "Filipino,Malay,English": 0.42, + "Filipino,Indonesian,English": 0.36666666666666664, + "Vietnamese,Chinese,Spanish": 0.44666666666666666, + "Vietnamese,Chinese,Malay": 0.48, + "Vietnamese,Chinese,Indonesian": 0.4666666666666667, + "Vietnamese,Chinese,English": 0.4866666666666667, + "Vietnamese,Spanish,Malay": 0.42, + "Vietnamese,Spanish,Indonesian": 0.4533333333333333, + "Vietnamese,Spanish,English": 0.44666666666666666, + "Vietnamese,Malay,Indonesian": 0.48, + "Vietnamese,Malay,English": 0.46, + "Vietnamese,Indonesian,English": 0.44, + "Chinese,Spanish,Malay": 0.44666666666666666, + "Chinese,Spanish,Indonesian": 0.4066666666666667, + "Chinese,Spanish,English": 0.4666666666666667, + "Chinese,Malay,Indonesian": 0.4866666666666667, + "Chinese,Malay,English": 0.5, + "Chinese,Indonesian,English": 0.44, + "Spanish,Malay,Indonesian": 0.44, + "Spanish,Malay,English": 0.48, + "Spanish,Indonesian,English": 0.44, + "Malay,Indonesian,English": 0.4666666666666667 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.31333333333333335, + "Filipino,Vietnamese,Chinese,Malay": 0.3466666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.32666666666666666, + "Filipino,Vietnamese,Chinese,English": 0.34, + "Filipino,Vietnamese,Spanish,Malay": 0.30666666666666664, + "Filipino,Vietnamese,Spanish,Indonesian": 0.31333333333333335, + "Filipino,Vietnamese,Spanish,English": 0.32666666666666666, + "Filipino,Vietnamese,Malay,Indonesian": 0.3333333333333333, + "Filipino,Vietnamese,Malay,English": 0.32666666666666666, + "Filipino,Vietnamese,Indonesian,English": 0.31333333333333335, + "Filipino,Chinese,Spanish,Malay": 0.32, + "Filipino,Chinese,Spanish,Indonesian": 0.2866666666666667, + "Filipino,Chinese,Spanish,English": 0.3333333333333333, + "Filipino,Chinese,Malay,Indonesian": 0.32666666666666666, + "Filipino,Chinese,Malay,English": 0.37333333333333335, + "Filipino,Chinese,Indonesian,English": 0.31333333333333335, + "Filipino,Spanish,Malay,Indonesian": 0.30666666666666664, + "Filipino,Spanish,Malay,English": 0.34, + "Filipino,Spanish,Indonesian,English": 0.3, + "Filipino,Malay,Indonesian,English": 0.31333333333333335, + "Vietnamese,Chinese,Spanish,Malay": 0.37333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian": 0.36666666666666664, + "Vietnamese,Chinese,Spanish,English": 0.38, + "Vietnamese,Chinese,Malay,Indonesian": 0.3933333333333333, + "Vietnamese,Chinese,Malay,English": 0.4066666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.38666666666666666, + "Vietnamese,Spanish,Malay,Indonesian": 0.37333333333333335, + "Vietnamese,Spanish,Malay,English": 0.36666666666666664, + "Vietnamese,Spanish,Indonesian,English": 0.36, + "Vietnamese,Malay,Indonesian,English": 0.38666666666666666, + "Chinese,Spanish,Malay,Indonesian": 0.36666666666666664, + "Chinese,Spanish,Malay,English": 0.4, + "Chinese,Spanish,Indonesian,English": 0.36, + "Chinese,Malay,Indonesian,English": 0.38666666666666666, + "Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.3, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.28, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.28, + "Filipino,Vietnamese,Spanish,Malay,English": 0.28, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2733333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.28, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Filipino,Chinese,Spanish,Malay,English": 0.3, + "Filipino,Chinese,Spanish,Indonesian,English": 0.26, + "Filipino,Chinese,Malay,Indonesian,English": 0.28, + "Filipino,Spanish,Malay,Indonesian,English": 0.28, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Malay,English": 0.3333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.32, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.34, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.32666666666666666, + "Chinese,Spanish,Malay,Indonesian,English": 0.3333333333333333 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0625, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.24, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.25333333333333335, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.29333333333333333 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666 } }, - "AC3_2": 0.3573898668514282, - "AC3_3": 0.26320858390553975, - "AC3_4": 0.1912411614403516, - "AC3_5": 0.1416546416187325, - "AC3_6": 0.10907374560802341, - "AC3_7": 0.08746081502225143 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.13592233009708737 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.2761904761904762 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.29906542056074764 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.31, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.2, - "history": 0.3333333333333333, - "literature": 0.3, - "politics": 0.5, - "culture": 0.3, - "film": 0.4, - "law": 0.3, - "geography": 0.1 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.07632049597846921 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.38820401046241154 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.3114307439995897 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.21752827177801026 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.384346973442308 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.323220536756126 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.27808366106542726, - "category_acc": { - "high_school_european_history": 0.3719512195121951, - "business_ethics": 0.29292929292929293, - "clinical_knowledge": 0.2878787878787879, - "medical_genetics": 0.3333333333333333, - "high_school_us_history": 0.3891625615763547, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.3474576271186441, - "virology": 0.2545454545454545, - "high_school_microeconomics": 0.2742616033755274, - "econometrics": 0.19469026548672566, - "college_computer_science": 0.35353535353535354, - "high_school_biology": 0.26537216828478966, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.24199288256227758, - "philosophy": 0.3, - "professional_medicine": 0.2767527675276753, - "nutrition": 0.32459016393442625, - "global_facts": 0.2222222222222222, - "machine_learning": 0.2882882882882883, - "security_studies": 0.29508196721311475, - "public_relations": 0.28440366972477066, - "professional_psychology": 0.27823240589198034, - "prehistory": 0.2755417956656347, - "anatomy": 0.20149253731343283, - "human_sexuality": 0.23846153846153847, - "college_medicine": 0.22674418604651161, - "high_school_government_and_politics": 0.328125, - "college_chemistry": 0.2828282828282828, - "logical_fallacies": 0.3148148148148148, - "high_school_geography": 0.29949238578680204, - "elementary_mathematics": 0.2440318302387268, - "human_aging": 0.30180180180180183, - "college_mathematics": 0.2222222222222222, - "high_school_psychology": 0.25183823529411764, - "formal_logic": 0.272, - "high_school_statistics": 0.2558139534883721, - "international_law": 0.3416666666666667, - "high_school_mathematics": 0.25650557620817843, - "high_school_computer_science": 0.25252525252525254, - "conceptual_physics": 0.2777777777777778, - "miscellaneous": 0.2864450127877238, - "high_school_chemistry": 0.23267326732673269, - "marketing": 0.33476394849785407, - "professional_law": 0.2576647097195042, - "management": 0.24509803921568626, - "college_physics": 0.16831683168316833, - "jurisprudence": 0.29906542056074764, - "world_religions": 0.35294117647058826, - "sociology": 0.35, - "us_foreign_policy": 0.3434343434343434, - "high_school_macroeconomics": 0.2519280205655527, - "computer_security": 0.3333333333333333, - "moral_scenarios": 0.2494407158836689, - "moral_disputes": 0.2985507246376812, - "electrical_engineering": 0.3055555555555556, - "astronomy": 0.2185430463576159, - "college_biology": 0.2727272727272727 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.2964338781575037 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.25653798256537985, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.16666666666666666, - "computer_architecture": 0.15384615384615385, - "college_programming": 0.23809523809523808, - "college_physics": 0.375, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.14285714285714285, - "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.25, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.08333333333333333, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.5416666666666666, - "middle_school_chemistry": 0.24, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.26666666666666666, - "business_administration": 0.2631578947368421, - "marxism": 0.375, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.32653061224489793, - "high_school_politics": 0.375, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.15384615384615385, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.25, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.3333333333333333, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.4117647058823529, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.24, - "middle_school_history": 0.18518518518518517, - "civil_servant": 0.3269230769230769, - "sports_science": 0.2916666666666667, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.18518518518518517, - "urban_and_rural_planner": 0.21568627450980393, - "accountant": 0.18518518518518517, - "fire_engineer": 0.08333333333333333, - "environmental_impact_assessment_engineer": 0.2222222222222222, - "tax_accountant": 0.12962962962962962, - "physician": 0.3148148148148148 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.35125448028673834 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.28337074771196685, - "category_acc": { - "agronomy": 0.23668639053254437, - "anatomy": 0.3108108108108108, - "ancient_chinese": 0.25, - "arts": 0.325, - "astronomy": 0.28484848484848485, - "business_ethics": 0.291866028708134, - "chinese_civil_service_exam": 0.3125, - "chinese_driving_rule": 0.3053435114503817, - "chinese_food_culture": 0.3088235294117647, - "chinese_foreign_policy": 0.308411214953271, - "chinese_history": 0.30959752321981426, - "chinese_literature": 0.28431372549019607, - "chinese_teacher_qualification": 0.2737430167597765, - "clinical_knowledge": 0.25316455696202533, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.3364485981308411, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.2037037037037037, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.330188679245283, - "college_medicine": 0.30036630036630035, - "computer_science": 0.29901960784313725, - "computer_security": 0.28654970760233917, - "conceptual_physics": 0.3129251700680272, - "construction_project_management": 0.3381294964028777, - "economics": 0.3081761006289308, - "education": 0.2822085889570552, - "electrical_engineering": 0.29069767441860467, - "elementary_chinese": 0.34523809523809523, - "elementary_commonsense": 0.2474747474747475, - "elementary_information_and_technology": 0.3235294117647059, - "elementary_mathematics": 0.2608695652173913, - "ethnology": 0.28888888888888886, - "food_science": 0.3356643356643357, - "genetics": 0.2784090909090909, - "global_facts": 0.28187919463087246, - "high_school_biology": 0.2958579881656805, - "high_school_chemistry": 0.20454545454545456, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.23170731707317074, - "high_school_physics": 0.2818181818181818, - "high_school_politics": 0.26573426573426573, - "human_sexuality": 0.2619047619047619, - "international_law": 0.24324324324324326, - "journalism": 0.2558139534883721, - "jurisprudence": 0.25304136253041365, - "legal_and_moral_basis": 0.40654205607476634, - "logical": 0.3008130081300813, - "machine_learning": 0.30327868852459017, - "management": 0.28095238095238095, - "marketing": 0.3, - "marxist_theory": 0.3386243386243386, - "modern_chinese": 0.21551724137931033, - "nutrition": 0.2689655172413793, - "philosophy": 0.29523809523809524, - "professional_accounting": 0.2914285714285714, - "professional_law": 0.23696682464454977, - "professional_medicine": 0.2127659574468085, - "professional_psychology": 0.2672413793103448, - "public_relations": 0.27586206896551724, - "security_study": 0.31851851851851853, - "sociology": 0.24778761061946902, - "sports_science": 0.28484848484848485, - "traditional_chinese_medicine": 0.2594594594594595, - "virology": 0.30177514792899407, - "world_history": 0.3105590062111801, - "world_religions": 0.29375 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.15151515151515152 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.19318181818181818 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3176271186440678 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.39902767389678384 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.5115139637432631 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.18376476259273392, - "rouge2": 0.06681490972966986, - "rougeL": 0.13970451344113952, - "avg_rouge": 0.13009472858784776 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.16834282356285027, - "rouge2": 0.05476954922469402, - "rougeL": 0.12844967535808846, - "avg_rouge": 0.11718734938187758 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.5 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.3652924256951103 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.5215 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.336 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.4965 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.5492957746478874 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.4657039711191336 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.4852941176470588 - } - } - } - }, - "sailor_7b": { - "model_size": "7B", - "model_link": "https://huggingface.co/sail/Sailor-7B", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.6567226890756303, - "language_acc": { - "English": 0.6596638655462185, - "Chinese": 0.6655462184873949, - "Spanish": 0.6420168067226891, - "Vietnamese": 0.6596638655462185 - }, - "consistency_score_2": 0.7439775910364146, - "consistency_score_3": 0.6350840336134453, - "consistency_score_4": 0.5689075630252101, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.7352941176470589, - "English,Spanish": 0.7739495798319328, - "English,Vietnamese": 0.7529411764705882, - "Chinese,Spanish": 0.719327731092437, - "Chinese,Vietnamese": 0.738655462184874, - "Spanish,Vietnamese": 0.7436974789915967 - }, - "3_combine": { - "English,Chinese,Spanish": 0.634453781512605, - "English,Chinese,Vietnamese": 0.6319327731092437, - "English,Spanish,Vietnamese": 0.653781512605042, - "Chinese,Spanish,Vietnamese": 0.6201680672268908 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5689075630252101 - } - }, - "AC3_2": 0.6976324215819805, - "AC3_3": 0.6457221300770689, - "AC3_4": 0.6096691950194396 + "AC3_2": 0.5984346705565198, + "AC3_3": 0.5043249237592222, + "AC3_4": 0.44208103021184547, + "AC3_5": 0.39594760055319295, + "AC3_6": 0.3594744034062524, + "AC3_7": 0.3301674276620994 }, "prompt_2": { - "overall_acc": 0.6348739495798319, - "language_acc": { - "English": 0.692436974789916, - "Chinese": 0.5042016806722689, - "Spanish": 0.6722689075630253, - "Vietnamese": 0.6705882352941176 - }, - "consistency_score_2": 0.7148459383753502, - "consistency_score_3": 0.5920168067226891, - "consistency_score_4": 0.5126050420168067, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.6336134453781512, - "English,Spanish": 0.7621848739495798, - "English,Vietnamese": 0.7848739495798319, - "Chinese,Spanish": 0.6453781512605042, - "Chinese,Vietnamese": 0.673109243697479, - "Spanish,Vietnamese": 0.7899159663865546 - }, - "3_combine": { - "English,Chinese,Spanish": 0.5462184873949579, - "English,Chinese,Vietnamese": 0.5672268907563025, - "English,Spanish,Vietnamese": 0.6798319327731093, - "Chinese,Spanish,Vietnamese": 0.5747899159663865 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5126050420168067 - } - }, - "AC3_2": 0.6724907416032754, - "AC3_3": 0.6126968458116933, - "AC3_4": 0.567225352193259 - }, - "prompt_3": { - "overall_acc": 0.6195378151260504, + "overall_acc": 0.5771428571428573, "language_acc": { - "English": 0.6823529411764706, - "Chinese": 0.5067226890756302, - "Spanish": 0.6218487394957983, - "Vietnamese": 0.6672268907563025 + "Filipino": 0.52, + "Vietnamese": 0.56, + "Chinese": 0.6266666666666667, + "Spanish": 0.5533333333333333, + "Malay": 0.5666666666666667, + "Indonesian": 0.5466666666666666, + "English": 0.6666666666666666 }, - "consistency_score_2": 0.6834733893557422, - "consistency_score_3": 0.548109243697479, - "consistency_score_4": 0.4579831932773109, + "consistency_score_2": 0.5517460317460318, + "consistency_score_3": 0.3826666666666666, + "consistency_score_4": 0.30019047619047623, + "consistency_score_5": 0.2526984126984127, + "consistency_score_6": 0.22190476190476188, + "consistency_score_7": 0.2, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.6445378151260505, - "English,Spanish": 0.7352941176470589, - "English,Vietnamese": 0.7680672268907563, - "Chinese,Spanish": 0.6016806722689075, - "Chinese,Vietnamese": 0.6252100840336134, - "Spanish,Vietnamese": 0.7260504201680672 + "Filipino,Vietnamese": 0.5266666666666666, + "Filipino,Chinese": 0.5, + "Filipino,Spanish": 0.5133333333333333, + "Filipino,Malay": 0.49333333333333335, + "Filipino,Indonesian": 0.49333333333333335, + "Filipino,English": 0.5333333333333333, + "Vietnamese,Chinese": 0.56, + "Vietnamese,Spanish": 0.5666666666666667, + "Vietnamese,Malay": 0.52, + "Vietnamese,Indonesian": 0.54, + "Vietnamese,English": 0.6133333333333333, + "Chinese,Spanish": 0.5666666666666667, + "Chinese,Malay": 0.5266666666666666, + "Chinese,Indonesian": 0.56, + "Chinese,English": 0.5666666666666667, + "Spanish,Malay": 0.5333333333333333, + "Spanish,Indonesian": 0.5133333333333333, + "Spanish,English": 0.6666666666666666, + "Malay,Indonesian": 0.6466666666666666, + "Malay,English": 0.6, + "Indonesian,English": 0.5466666666666666 }, "3_combine": { - "English,Chinese,Spanish": 0.5176470588235295, - "English,Chinese,Vietnamese": 0.5378151260504201, - "English,Spanish,Vietnamese": 0.6319327731092437, - "Chinese,Spanish,Vietnamese": 0.5050420168067227 + "Filipino,Vietnamese,Chinese": 0.36666666666666664, + "Filipino,Vietnamese,Spanish": 0.36, + "Filipino,Vietnamese,Malay": 0.35333333333333333, + "Filipino,Vietnamese,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,English": 0.37333333333333335, + "Filipino,Chinese,Spanish": 0.3333333333333333, + "Filipino,Chinese,Malay": 0.32666666666666666, + "Filipino,Chinese,Indonesian": 0.34, + "Filipino,Chinese,English": 0.36, + "Filipino,Spanish,Malay": 0.3333333333333333, + "Filipino,Spanish,Indonesian": 0.3, + "Filipino,Spanish,English": 0.3933333333333333, + "Filipino,Malay,Indonesian": 0.36666666666666664, + "Filipino,Malay,English": 0.36, + "Filipino,Indonesian,English": 0.34, + "Vietnamese,Chinese,Spanish": 0.4066666666666667, + "Vietnamese,Chinese,Malay": 0.36666666666666664, + "Vietnamese,Chinese,Indonesian": 0.38666666666666666, + "Vietnamese,Chinese,English": 0.41333333333333333, + "Vietnamese,Spanish,Malay": 0.38666666666666666, + "Vietnamese,Spanish,Indonesian": 0.3933333333333333, + "Vietnamese,Spanish,English": 0.4533333333333333, + "Vietnamese,Malay,Indonesian": 0.4066666666666667, + "Vietnamese,Malay,English": 0.42, + "Vietnamese,Indonesian,English": 0.4266666666666667, + "Chinese,Spanish,Malay": 0.38, + "Chinese,Spanish,Indonesian": 0.36666666666666664, + "Chinese,Spanish,English": 0.44, + "Chinese,Malay,Indonesian": 0.4066666666666667, + "Chinese,Malay,English": 0.4, + "Chinese,Indonesian,English": 0.4, + "Spanish,Malay,Indonesian": 0.4066666666666667, + "Spanish,Malay,English": 0.43333333333333335, + "Spanish,Indonesian,English": 0.4, + "Malay,Indonesian,English": 0.44 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.4579831932773109 - } - }, - "AC3_2": 0.649937021031461, - "AC3_3": 0.581638776367555, - "AC3_4": 0.5266494197112292 - }, - "prompt_4": { - "overall_acc": 0.6640756302521008, - "language_acc": { - "English": 0.6957983193277311, - "Chinese": 0.673109243697479, - "Spanish": 0.6302521008403361, - "Vietnamese": 0.6571428571428571 - }, - "consistency_score_2": 0.7564425770308123, - "consistency_score_3": 0.65, - "consistency_score_4": 0.5789915966386555, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.7546218487394958, - "English,Spanish": 0.7890756302521008, - "English,Vietnamese": 0.7924369747899159, - "Chinese,Spanish": 0.7142857142857143, - "Chinese,Vietnamese": 0.7394957983193278, - "Spanish,Vietnamese": 0.7487394957983193 - }, - "3_combine": { - "English,Chinese,Spanish": 0.6420168067226891, - "English,Chinese,Vietnamese": 0.6588235294117647, - "English,Spanish,Vietnamese": 0.6789915966386555, - "Chinese,Spanish,Vietnamese": 0.6201680672268908 + "Filipino,Vietnamese,Chinese,Spanish": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.28, + "Filipino,Vietnamese,Chinese,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,English": 0.28, + "Filipino,Vietnamese,Spanish,Malay": 0.28, + "Filipino,Vietnamese,Spanish,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Spanish,English": 0.30666666666666664, + "Filipino,Vietnamese,Malay,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,Malay,English": 0.29333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.29333333333333333, + "Filipino,Chinese,Spanish,Malay": 0.2733333333333333, + "Filipino,Chinese,Spanish,Indonesian": 0.26, + "Filipino,Chinese,Spanish,English": 0.2866666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.2733333333333333, + "Filipino,Chinese,Malay,English": 0.28, + "Filipino,Chinese,Indonesian,English": 0.29333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.25333333333333335, + "Filipino,Spanish,Malay,English": 0.2866666666666667, + "Filipino,Spanish,Indonesian,English": 0.26, + "Filipino,Malay,Indonesian,English": 0.2733333333333333, + "Vietnamese,Chinese,Spanish,Malay": 0.30666666666666664, + "Vietnamese,Chinese,Spanish,Indonesian": 0.32, + "Vietnamese,Chinese,Spanish,English": 0.35333333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.31333333333333335, + "Vietnamese,Chinese,Malay,English": 0.31333333333333335, + "Vietnamese,Chinese,Indonesian,English": 0.3333333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.32, + "Vietnamese,Spanish,Malay,English": 0.34, + "Vietnamese,Spanish,Indonesian,English": 0.34, + "Vietnamese,Malay,Indonesian,English": 0.3466666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, + "Chinese,Spanish,Malay,English": 0.3333333333333333, + "Chinese,Spanish,Indonesian,English": 0.32, + "Chinese,Malay,Indonesian,English": 0.32666666666666666, + "Spanish,Malay,Indonesian,English": 0.34 }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5789915966386555 - } - }, - "AC3_2": 0.7072560963744848, - "AC3_3": 0.6569624300059609, - "AC3_4": 0.6186217464348683 - }, - "prompt_5": { - "overall_acc": 0.6378151260504201, - "language_acc": { - "English": 0.6495798319327731, - "Chinese": 0.6420168067226891, - "Spanish": 0.6058823529411764, - "Vietnamese": 0.653781512605042 - }, - "consistency_score_2": 0.7410364145658264, - "consistency_score_3": 0.6277310924369748, - "consistency_score_4": 0.5588235294117647, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.7420168067226891, - "English,Spanish": 0.7756302521008404, - "English,Vietnamese": 0.7605042016806722, - "Chinese,Spanish": 0.7100840336134454, - "Chinese,Vietnamese": 0.726890756302521, - "Spanish,Vietnamese": 0.7310924369747899 + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.24, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,Malay,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.24666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, + "Filipino,Chinese,Spanish,Malay,English": 0.24666666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.24, + "Filipino,Chinese,Malay,Indonesian,English": 0.24, + "Filipino,Spanish,Malay,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,Malay,English": 0.28, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.29333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.28, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.2866666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666 }, - "3_combine": { - "English,Chinese,Spanish": 0.6285714285714286, - "English,Chinese,Vietnamese": 0.6319327731092437, - "English,Spanish,Vietnamese": 0.6495798319327731, - "Chinese,Spanish,Vietnamese": 0.6008403361344538 + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.22, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.22, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.24666666666666667 }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5588235294117647 + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2 } }, - "AC3_2": 0.6855621801296146, - "AC3_3": 0.6327329337609777, - "AC3_4": 0.5957121612192199 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.37047619047619046, + "AC3_2": 0.5641587658185775, + "AC3_3": 0.4602024210673534, + "AC3_4": 0.3949531666319877, + "AC3_5": 0.35149633835528205, + "AC3_6": 0.3205584879557959, + "AC3_7": 0.2970588234911873 + }, + "prompt_3": { + "overall_acc": 0.5466666666666666, "language_acc": { - "Spanish": 0.38666666666666666, - "English": 0.47333333333333333, - "Chinese": 0.32, - "Vietnamese": 0.35333333333333333, - "Filipino": 0.31333333333333335, - "Indonesian": 0.38, - "Malay": 0.36666666666666664 + "Filipino": 0.5066666666666667, + "Vietnamese": 0.5466666666666666, + "Chinese": 0.56, + "Spanish": 0.52, + "Malay": 0.49333333333333335, + "Indonesian": 0.5466666666666666, + "English": 0.6533333333333333 }, - "consistency_score_2": 0.5006349206349205, - "consistency_score_3": 0.31409523809523815, - "consistency_score_4": 0.2188571428571429, - "consistency_score_5": 0.1619047619047619, - "consistency_score_6": 0.12476190476190477, - "consistency_score_7": 0.1, + "consistency_score_2": 0.5396825396825398, + "consistency_score_3": 0.3613333333333334, + "consistency_score_4": 0.26647619047619053, + "consistency_score_5": 0.20952380952380953, + "consistency_score_6": 0.17238095238095238, + "consistency_score_7": 0.14666666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.56, - "Spanish,Chinese": 0.5466666666666666, - "Spanish,Vietnamese": 0.5, - "Spanish,Filipino": 0.58, - "Spanish,Indonesian": 0.48, - "Spanish,Malay": 0.54, - "English,Chinese": 0.5266666666666666, - "English,Vietnamese": 0.5, - "English,Filipino": 0.5733333333333334, - "English,Indonesian": 0.5, - "English,Malay": 0.44666666666666666, - "Chinese,Vietnamese": 0.4066666666666667, - "Chinese,Filipino": 0.5666666666666667, - "Chinese,Indonesian": 0.46, - "Chinese,Malay": 0.43333333333333335, - "Vietnamese,Filipino": 0.44, - "Vietnamese,Indonesian": 0.4666666666666667, - "Vietnamese,Malay": 0.5133333333333333, - "Filipino,Indonesian": 0.43333333333333335, - "Filipino,Malay": 0.4066666666666667, - "Indonesian,Malay": 0.6333333333333333 + "Filipino,Vietnamese": 0.5133333333333333, + "Filipino,Chinese": 0.5133333333333333, + "Filipino,Spanish": 0.5, + "Filipino,Malay": 0.48, + "Filipino,Indonesian": 0.5533333333333333, + "Filipino,English": 0.52, + "Vietnamese,Chinese": 0.6066666666666667, + "Vietnamese,Spanish": 0.54, + "Vietnamese,Malay": 0.5733333333333334, + "Vietnamese,Indonesian": 0.54, + "Vietnamese,English": 0.58, + "Chinese,Spanish": 0.5733333333333334, + "Chinese,Malay": 0.48, + "Chinese,Indonesian": 0.44, + "Chinese,English": 0.5533333333333333, + "Spanish,Malay": 0.5733333333333334, + "Spanish,Indonesian": 0.5133333333333333, + "Spanish,English": 0.5933333333333334, + "Malay,Indonesian": 0.5533333333333333, + "Malay,English": 0.5466666666666666, + "Indonesian,English": 0.5866666666666667 }, "3_combine": { - "Spanish,English,Chinese": 0.37333333333333335, - "Spanish,English,Vietnamese": 0.3466666666666667, - "Spanish,English,Filipino": 0.4066666666666667, - "Spanish,English,Indonesian": 0.34, - "Spanish,English,Malay": 0.3333333333333333, - "Spanish,Chinese,Vietnamese": 0.2733333333333333, - "Spanish,Chinese,Filipino": 0.3933333333333333, - "Spanish,Chinese,Indonesian": 0.29333333333333333, - "Spanish,Chinese,Malay": 0.30666666666666664, - "Spanish,Vietnamese,Filipino": 0.31333333333333335, - "Spanish,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,Vietnamese,Malay": 0.3333333333333333, - "Spanish,Filipino,Indonesian": 0.31333333333333335, - "Spanish,Filipino,Malay": 0.31333333333333335, - "Spanish,Indonesian,Malay": 0.36, - "English,Chinese,Vietnamese": 0.2866666666666667, - "English,Chinese,Filipino": 0.3933333333333333, - "English,Chinese,Indonesian": 0.31333333333333335, - "English,Chinese,Malay": 0.2733333333333333, - "English,Vietnamese,Filipino": 0.31333333333333335, - "English,Vietnamese,Indonesian": 0.3, - "English,Vietnamese,Malay": 0.30666666666666664, - "English,Filipino,Indonesian": 0.32, - "English,Filipino,Malay": 0.29333333333333333, - "English,Indonesian,Malay": 0.35333333333333333, - "Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Chinese,Vietnamese,Indonesian": 0.26, - "Chinese,Vietnamese,Malay": 0.26666666666666666, - "Chinese,Filipino,Indonesian": 0.2733333333333333, - "Chinese,Filipino,Malay": 0.26666666666666666, - "Chinese,Indonesian,Malay": 0.3333333333333333, - "Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Vietnamese,Filipino,Malay": 0.26, - "Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Filipino,Indonesian,Malay": 0.32666666666666666 + "Filipino,Vietnamese,Chinese": 0.38, + "Filipino,Vietnamese,Spanish": 0.34, + "Filipino,Vietnamese,Malay": 0.3466666666666667, + "Filipino,Vietnamese,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,English": 0.36, + "Filipino,Chinese,Spanish": 0.35333333333333333, + "Filipino,Chinese,Malay": 0.3, + "Filipino,Chinese,Indonesian": 0.30666666666666664, + "Filipino,Chinese,English": 0.3466666666666667, + "Filipino,Spanish,Malay": 0.3333333333333333, + "Filipino,Spanish,Indonesian": 0.3466666666666667, + "Filipino,Spanish,English": 0.35333333333333333, + "Filipino,Malay,Indonesian": 0.3466666666666667, + "Filipino,Malay,English": 0.3333333333333333, + "Filipino,Indonesian,English": 0.36, + "Vietnamese,Chinese,Spanish": 0.41333333333333333, + "Vietnamese,Chinese,Malay": 0.37333333333333335, + "Vietnamese,Chinese,Indonesian": 0.35333333333333333, + "Vietnamese,Chinese,English": 0.4066666666666667, + "Vietnamese,Spanish,Malay": 0.4, + "Vietnamese,Spanish,Indonesian": 0.35333333333333333, + "Vietnamese,Spanish,English": 0.38666666666666666, + "Vietnamese,Malay,Indonesian": 0.38666666666666666, + "Vietnamese,Malay,English": 0.4, + "Vietnamese,Indonesian,English": 0.41333333333333333, + "Chinese,Spanish,Malay": 0.37333333333333335, + "Chinese,Spanish,Indonesian": 0.32, + "Chinese,Spanish,English": 0.38666666666666666, + "Chinese,Malay,Indonesian": 0.3, + "Chinese,Malay,English": 0.3333333333333333, + "Chinese,Indonesian,English": 0.3466666666666667, + "Spanish,Malay,Indonesian": 0.37333333333333335, + "Spanish,Malay,English": 0.38666666666666666, + "Spanish,Indonesian,English": 0.38666666666666666, + "Malay,Indonesian,English": 0.3933333333333333 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.22, - "Spanish,English,Chinese,Filipino": 0.30666666666666664, - "Spanish,English,Chinese,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Malay": 0.22, - "Spanish,English,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,English,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Malay": 0.24, - "Spanish,English,Filipino,Indonesian": 0.24, - "Spanish,English,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Malay": 0.2, - "Spanish,Chinese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.22, - "Spanish,Chinese,Indonesian,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.22, - "Spanish,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.20666666666666667, - "English,Chinese,Vietnamese,Malay": 0.20666666666666667, - "English,Chinese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Filipino,Malay": 0.2, - "English,Chinese,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.24, - "English,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.18, - "Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Chinese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Vietnamese,Filipino,Indonesian,Malay": 0.2 + "Filipino,Vietnamese,Chinese,Spanish": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.26, + "Filipino,Vietnamese,Chinese,English": 0.2733333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.26, + "Filipino,Vietnamese,Spanish,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,English": 0.26, + "Filipino,Vietnamese,Malay,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Malay,English": 0.2733333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.2733333333333333, + "Filipino,Chinese,Spanish,Malay": 0.25333333333333335, + "Filipino,Chinese,Spanish,Indonesian": 0.24666666666666667, + "Filipino,Chinese,Spanish,English": 0.26666666666666666, + "Filipino,Chinese,Malay,Indonesian": 0.22, + "Filipino,Chinese,Malay,English": 0.23333333333333334, + "Filipino,Chinese,Indonesian,English": 0.22666666666666666, + "Filipino,Spanish,Malay,Indonesian": 0.26, + "Filipino,Spanish,Malay,English": 0.24, + "Filipino,Spanish,Indonesian,English": 0.26, + "Filipino,Malay,Indonesian,English": 0.26, + "Vietnamese,Chinese,Spanish,Malay": 0.31333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian": 0.2733333333333333, + "Vietnamese,Chinese,Spanish,English": 0.30666666666666664, + "Vietnamese,Chinese,Malay,Indonesian": 0.26, + "Vietnamese,Chinese,Malay,English": 0.2866666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.29333333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.28, + "Vietnamese,Spanish,Malay,English": 0.3, + "Vietnamese,Spanish,Indonesian,English": 0.2866666666666667, + "Vietnamese,Malay,Indonesian,English": 0.3, + "Chinese,Spanish,Malay,Indonesian": 0.25333333333333335, + "Chinese,Spanish,Malay,English": 0.26, + "Chinese,Spanish,Indonesian,English": 0.26, + "Chinese,Malay,Indonesian,English": 0.24, + "Spanish,Malay,Indonesian,English": 0.2866666666666667 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.16, - "Spanish,English,Chinese,Vietnamese,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.16, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.22, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.22, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2, + "Filipino,Vietnamese,Chinese,Malay,English": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.22, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2, + "Filipino,Chinese,Spanish,Malay,English": 0.19333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.19333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.17333333333333334, + "Filipino,Spanish,Malay,Indonesian,English": 0.19333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, + "Vietnamese,Chinese,Spanish,Malay,English": 0.24, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.22666666666666666, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.22, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.22666666666666666, + "Chinese,Spanish,Malay,Indonesian,English": 0.2 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.18, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 } }, - "AC3_2": 0.42583159789641684, - "AC3_3": 0.33996396104952564, - "AC3_4": 0.2751629882255414, - "AC3_5": 0.22533435552458791, - "AC3_6": 0.18666300362531207, - "AC3_7": 0.15748987850903554 + "AC3_2": 0.5431521526871927, + "AC3_3": 0.4350856582976534, + "AC3_4": 0.3582978058433421, + "AC3_5": 0.3029387069288725, + "AC3_6": 0.2621103752394857, + "AC3_7": 0.23128205124869328 }, - "prompt_2": { - "overall_acc": 0.3514285714285715, + "prompt_4": { + "overall_acc": 0.602857142857143, "language_acc": { - "Spanish": 0.35333333333333333, - "English": 0.37333333333333335, - "Chinese": 0.32, - "Vietnamese": 0.38, - "Filipino": 0.2733333333333333, - "Indonesian": 0.3933333333333333, - "Malay": 0.36666666666666664 + "Filipino": 0.5666666666666667, + "Vietnamese": 0.5866666666666667, + "Chinese": 0.64, + "Spanish": 0.6066666666666667, + "Malay": 0.5866666666666667, + "Indonesian": 0.5466666666666666, + "English": 0.6866666666666666 }, - "consistency_score_2": 0.5126984126984127, - "consistency_score_3": 0.32095238095238093, - "consistency_score_4": 0.2217142857142857, - "consistency_score_5": 0.16317460317460317, - "consistency_score_6": 0.12571428571428572, - "consistency_score_7": 0.1, + "consistency_score_2": 0.5930158730158731, + "consistency_score_3": 0.43466666666666665, + "consistency_score_4": 0.3512380952380953, + "consistency_score_5": 0.299047619047619, + "consistency_score_6": 0.2619047619047619, + "consistency_score_7": 0.23333333333333334, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5333333333333333, - "Spanish,Chinese": 0.5133333333333333, - "Spanish,Vietnamese": 0.5133333333333333, - "Spanish,Filipino": 0.5133333333333333, - "Spanish,Indonesian": 0.5466666666666666, - "Spanish,Malay": 0.5333333333333333, - "English,Chinese": 0.4533333333333333, - "English,Vietnamese": 0.48, - "English,Filipino": 0.41333333333333333, - "English,Indonesian": 0.52, - "English,Malay": 0.5266666666666666, - "Chinese,Vietnamese": 0.5666666666666667, - "Chinese,Filipino": 0.56, - "Chinese,Indonesian": 0.48, - "Chinese,Malay": 0.54, - "Vietnamese,Filipino": 0.4866666666666667, - "Vietnamese,Indonesian": 0.5, - "Vietnamese,Malay": 0.54, - "Filipino,Indonesian": 0.5, - "Filipino,Malay": 0.4666666666666667, - "Indonesian,Malay": 0.58 + "Filipino,Vietnamese": 0.5466666666666666, + "Filipino,Chinese": 0.6066666666666667, + "Filipino,Spanish": 0.5733333333333334, + "Filipino,Malay": 0.5733333333333334, + "Filipino,Indonesian": 0.58, + "Filipino,English": 0.58, + "Vietnamese,Chinese": 0.6133333333333333, + "Vietnamese,Spanish": 0.6333333333333333, + "Vietnamese,Malay": 0.5666666666666667, + "Vietnamese,Indonesian": 0.56, + "Vietnamese,English": 0.5933333333333334, + "Chinese,Spanish": 0.5866666666666667, + "Chinese,Malay": 0.5933333333333334, + "Chinese,Indonesian": 0.5733333333333334, + "Chinese,English": 0.6133333333333333, + "Spanish,Malay": 0.5933333333333334, + "Spanish,Indonesian": 0.56, + "Spanish,English": 0.66, + "Malay,Indonesian": 0.6666666666666666, + "Malay,English": 0.6133333333333333, + "Indonesian,English": 0.5666666666666667 }, "3_combine": { - "Spanish,English,Chinese": 0.30666666666666664, - "Spanish,English,Vietnamese": 0.31333333333333335, - "Spanish,English,Filipino": 0.2733333333333333, - "Spanish,English,Indonesian": 0.35333333333333333, - "Spanish,English,Malay": 0.36, - "Spanish,Chinese,Vietnamese": 0.34, - "Spanish,Chinese,Filipino": 0.3333333333333333, - "Spanish,Chinese,Indonesian": 0.31333333333333335, - "Spanish,Chinese,Malay": 0.3466666666666667, - "Spanish,Vietnamese,Filipino": 0.31333333333333335, - "Spanish,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,Vietnamese,Malay": 0.3466666666666667, - "Spanish,Filipino,Indonesian": 0.34, - "Spanish,Filipino,Malay": 0.31333333333333335, - "Spanish,Indonesian,Malay": 0.38666666666666666, - "English,Chinese,Vietnamese": 0.3, - "English,Chinese,Filipino": 0.26, - "English,Chinese,Indonesian": 0.2733333333333333, - "English,Chinese,Malay": 0.32666666666666666, - "English,Vietnamese,Filipino": 0.26666666666666666, - "English,Vietnamese,Indonesian": 0.31333333333333335, - "English,Vietnamese,Malay": 0.32666666666666666, - "English,Filipino,Indonesian": 0.28, - "English,Filipino,Malay": 0.2866666666666667, - "English,Indonesian,Malay": 0.35333333333333333, - "Chinese,Vietnamese,Filipino": 0.34, - "Chinese,Vietnamese,Indonesian": 0.3, - "Chinese,Vietnamese,Malay": 0.36, - "Chinese,Filipino,Indonesian": 0.30666666666666664, - "Chinese,Filipino,Malay": 0.32666666666666666, - "Chinese,Indonesian,Malay": 0.34, - "Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Vietnamese,Filipino,Malay": 0.31333333333333335, - "Vietnamese,Indonesian,Malay": 0.36, - "Filipino,Indonesian,Malay": 0.3333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,English,Chinese,Filipino": 0.20666666666666667, - "Spanish,English,Chinese,Indonesian": 0.22, - "Spanish,English,Chinese,Malay": 0.24, - "Spanish,English,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,English,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Malay": 0.22666666666666666, - "Spanish,English,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Filipino,Malay": 0.21333333333333335, - "Spanish,English,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.24, - "Spanish,Chinese,Filipino,Indonesian": 0.22, - "Spanish,Chinese,Filipino,Malay": 0.23333333333333334, - "Spanish,Chinese,Indonesian,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.22, - "Spanish,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.26, - "English,Chinese,Vietnamese,Filipino": 0.18, - "English,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "English,Chinese,Vietnamese,Malay": 0.23333333333333334, - "English,Chinese,Filipino,Indonesian": 0.17333333333333334, - "English,Chinese,Filipino,Malay": 0.21333333333333335, - "English,Chinese,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "English,Vietnamese,Filipino,Malay": 0.20666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.24, - "Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.14, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Filipino,Indonesian,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "English,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16 + "Filipino,Vietnamese,Chinese": 0.4266666666666667, + "Filipino,Vietnamese,Spanish": 0.43333333333333335, + "Filipino,Vietnamese,Malay": 0.4066666666666667, + "Filipino,Vietnamese,Indonesian": 0.4, + "Filipino,Vietnamese,English": 0.4066666666666667, + "Filipino,Chinese,Spanish": 0.43333333333333335, + "Filipino,Chinese,Malay": 0.4266666666666667, + "Filipino,Chinese,Indonesian": 0.41333333333333333, + "Filipino,Chinese,English": 0.43333333333333335, + "Filipino,Spanish,Malay": 0.44, + "Filipino,Spanish,Indonesian": 0.41333333333333333, + "Filipino,Spanish,English": 0.44666666666666666, + "Filipino,Malay,Indonesian": 0.44, + "Filipino,Malay,English": 0.43333333333333335, + "Filipino,Indonesian,English": 0.3933333333333333, + "Vietnamese,Chinese,Spanish": 0.4533333333333333, + "Vietnamese,Chinese,Malay": 0.43333333333333335, + "Vietnamese,Chinese,Indonesian": 0.42, + "Vietnamese,Chinese,English": 0.44, + "Vietnamese,Spanish,Malay": 0.44666666666666666, + "Vietnamese,Spanish,Indonesian": 0.43333333333333335, + "Vietnamese,Spanish,English": 0.4866666666666667, + "Vietnamese,Malay,Indonesian": 0.4533333333333333, + "Vietnamese,Malay,English": 0.43333333333333335, + "Vietnamese,Indonesian,English": 0.42, + "Chinese,Spanish,Malay": 0.4266666666666667, + "Chinese,Spanish,Indonesian": 0.41333333333333333, + "Chinese,Spanish,English": 0.4666666666666667, + "Chinese,Malay,Indonesian": 0.4533333333333333, + "Chinese,Malay,English": 0.44666666666666666, + "Chinese,Indonesian,English": 0.41333333333333333, + "Spanish,Malay,Indonesian": 0.4533333333333333, + "Spanish,Malay,English": 0.47333333333333333, + "Spanish,Indonesian,English": 0.4266666666666667, + "Malay,Indonesian,English": 0.47333333333333333 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.36, + "Filipino,Vietnamese,Chinese,Malay": 0.3333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.3333333333333333, + "Filipino,Vietnamese,Chinese,English": 0.32666666666666666, + "Filipino,Vietnamese,Spanish,Malay": 0.37333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,Spanish,English": 0.37333333333333335, + "Filipino,Vietnamese,Malay,Indonesian": 0.34, + "Filipino,Vietnamese,Malay,English": 0.34, + "Filipino,Vietnamese,Indonesian,English": 0.32666666666666666, + "Filipino,Chinese,Spanish,Malay": 0.36, + "Filipino,Chinese,Spanish,Indonesian": 0.34, + "Filipino,Chinese,Spanish,English": 0.35333333333333333, + "Filipino,Chinese,Malay,Indonesian": 0.34, + "Filipino,Chinese,Malay,English": 0.3466666666666667, + "Filipino,Chinese,Indonesian,English": 0.32, + "Filipino,Spanish,Malay,Indonesian": 0.35333333333333333, + "Filipino,Spanish,Malay,English": 0.38, + "Filipino,Spanish,Indonesian,English": 0.3333333333333333, + "Filipino,Malay,Indonesian,English": 0.34, + "Vietnamese,Chinese,Spanish,Malay": 0.3466666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.35333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.38, + "Vietnamese,Chinese,Malay,Indonesian": 0.35333333333333333, + "Vietnamese,Chinese,Malay,English": 0.3466666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.3466666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.36666666666666664, + "Vietnamese,Spanish,Malay,English": 0.37333333333333335, + "Vietnamese,Spanish,Indonesian,English": 0.35333333333333333, + "Vietnamese,Malay,Indonesian,English": 0.36, + "Chinese,Spanish,Malay,Indonesian": 0.34, + "Chinese,Spanish,Malay,English": 0.36666666666666664, + "Chinese,Spanish,Indonesian,English": 0.34, + "Chinese,Malay,Indonesian,English": 0.35333333333333333, + "Spanish,Malay,Indonesian,English": 0.38666666666666666 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.28, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.28, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.31333333333333335, + "Filipino,Vietnamese,Spanish,Malay,English": 0.32666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.3, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.2866666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, + "Filipino,Chinese,Spanish,Malay,English": 0.30666666666666664, + "Filipino,Chinese,Spanish,Indonesian,English": 0.28, + "Filipino,Chinese,Malay,Indonesian,English": 0.28, + "Filipino,Spanish,Malay,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.3, + "Vietnamese,Chinese,Spanish,Malay,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.29333333333333333, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.31333333333333335, + "Chinese,Spanish,Malay,Indonesian,English": 0.3 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.26, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.24666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.2733333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.25333333333333335, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334 } }, - "AC3_2": 0.4170147999937275, - "AC3_3": 0.33549979760287485, - "AC3_4": 0.27189289270288153, - "AC3_5": 0.2228677182963406, - "AC3_6": 0.1851839178397177, - "AC3_7": 0.1556962024971559 + "AC3_2": 0.5978960142091715, + "AC3_3": 0.5051294289948243, + "AC3_4": 0.4438684652873053, + "AC3_5": 0.3997827726212339, + "AC3_6": 0.3651667715122094, + "AC3_7": 0.33644646920805593 }, - "prompt_3": { - "overall_acc": 0.39999999999999997, + "prompt_5": { + "overall_acc": 0.5390476190476191, "language_acc": { - "Spanish": 0.3933333333333333, - "English": 0.4266666666666667, - "Chinese": 0.38666666666666666, - "Vietnamese": 0.4066666666666667, - "Filipino": 0.30666666666666664, - "Indonesian": 0.44666666666666666, - "Malay": 0.43333333333333335 + "Filipino": 0.4266666666666667, + "Vietnamese": 0.49333333333333335, + "Chinese": 0.56, + "Spanish": 0.6, + "Malay": 0.5333333333333333, + "Indonesian": 0.4866666666666667, + "English": 0.6733333333333333 }, - "consistency_score_2": 0.5292063492063492, - "consistency_score_3": 0.35523809523809535, - "consistency_score_4": 0.26361904761904753, - "consistency_score_5": 0.20507936507936508, - "consistency_score_6": 0.1638095238095238, - "consistency_score_7": 0.13333333333333333, + "consistency_score_2": 0.5266666666666666, + "consistency_score_3": 0.3462857142857143, + "consistency_score_4": 0.24952380952380956, + "consistency_score_5": 0.1892063492063492, + "consistency_score_6": 0.14857142857142858, + "consistency_score_7": 0.12, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5866666666666667, - "Spanish,Chinese": 0.5466666666666666, - "Spanish,Vietnamese": 0.54, - "Spanish,Filipino": 0.5866666666666667, - "Spanish,Indonesian": 0.6, - "Spanish,Malay": 0.5333333333333333, - "English,Chinese": 0.4266666666666667, - "English,Vietnamese": 0.5133333333333333, - "English,Filipino": 0.5666666666666667, - "English,Indonesian": 0.5933333333333334, - "English,Malay": 0.49333333333333335, - "Chinese,Vietnamese": 0.5466666666666666, - "Chinese,Filipino": 0.48, - "Chinese,Indonesian": 0.5133333333333333, - "Chinese,Malay": 0.43333333333333335, - "Vietnamese,Filipino": 0.5133333333333333, + "Filipino,Vietnamese": 0.4533333333333333, + "Filipino,Chinese": 0.4666666666666667, + "Filipino,Spanish": 0.47333333333333333, + "Filipino,Malay": 0.44666666666666666, + "Filipino,Indonesian": 0.43333333333333335, + "Filipino,English": 0.47333333333333333, + "Vietnamese,Chinese": 0.54, + "Vietnamese,Spanish": 0.5333333333333333, + "Vietnamese,Malay": 0.6133333333333333, "Vietnamese,Indonesian": 0.5333333333333333, - "Vietnamese,Malay": 0.5, - "Filipino,Indonesian": 0.5066666666666667, - "Filipino,Malay": 0.48, - "Indonesian,Malay": 0.62 + "Vietnamese,English": 0.4866666666666667, + "Chinese,Spanish": 0.5333333333333333, + "Chinese,Malay": 0.5866666666666667, + "Chinese,Indonesian": 0.5266666666666666, + "Chinese,English": 0.5866666666666667, + "Spanish,Malay": 0.5933333333333334, + "Spanish,Indonesian": 0.5533333333333333, + "Spanish,English": 0.5933333333333334, + "Malay,Indonesian": 0.6133333333333333, + "Malay,English": 0.5, + "Indonesian,English": 0.52 }, "3_combine": { - "Spanish,English,Chinese": 0.31333333333333335, - "Spanish,English,Vietnamese": 0.4066666666666667, - "Spanish,English,Filipino": 0.42, - "Spanish,English,Indonesian": 0.43333333333333335, - "Spanish,English,Malay": 0.37333333333333335, - "Spanish,Chinese,Vietnamese": 0.36, - "Spanish,Chinese,Filipino": 0.35333333333333333, - "Spanish,Chinese,Indonesian": 0.37333333333333335, - "Spanish,Chinese,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Filipino": 0.37333333333333335, - "Spanish,Vietnamese,Indonesian": 0.4, - "Spanish,Vietnamese,Malay": 0.36, - "Spanish,Filipino,Indonesian": 0.42, - "Spanish,Filipino,Malay": 0.38666666666666666, - "Spanish,Indonesian,Malay": 0.4, - "English,Chinese,Vietnamese": 0.3333333333333333, - "English,Chinese,Filipino": 0.3, - "English,Chinese,Indonesian": 0.3333333333333333, - "English,Chinese,Malay": 0.2733333333333333, - "English,Vietnamese,Filipino": 0.35333333333333333, - "English,Vietnamese,Indonesian": 0.38, - "English,Vietnamese,Malay": 0.3333333333333333, - "English,Filipino,Indonesian": 0.38666666666666666, - "English,Filipino,Malay": 0.3466666666666667, - "English,Indonesian,Malay": 0.38, - "Chinese,Vietnamese,Filipino": 0.3333333333333333, - "Chinese,Vietnamese,Indonesian": 0.35333333333333333, - "Chinese,Vietnamese,Malay": 0.32, - "Chinese,Filipino,Indonesian": 0.32, - "Chinese,Filipino,Malay": 0.2733333333333333, - "Chinese,Indonesian,Malay": 0.35333333333333333, - "Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "Vietnamese,Filipino,Malay": 0.30666666666666664, - "Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Filipino,Indonesian,Malay": 0.3466666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.26666666666666666, - "Spanish,English,Chinese,Filipino": 0.24666666666666667, - "Spanish,English,Chinese,Indonesian": 0.26, - "Spanish,English,Chinese,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino": 0.3, - "Spanish,English,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,English,Vietnamese,Malay": 0.2866666666666667, - "Spanish,English,Filipino,Indonesian": 0.32, - "Spanish,English,Filipino,Malay": 0.2866666666666667, - "Spanish,English,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian": 0.28, - "Spanish,Chinese,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.28, - "Spanish,Chinese,Filipino,Malay": 0.24, - "Spanish,Chinese,Indonesian,Malay": 0.28, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.26, - "Spanish,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.25333333333333335, - "English,Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "English,Chinese,Vietnamese,Malay": 0.22666666666666666, - "English,Chinese,Filipino,Indonesian": 0.25333333333333335, - "English,Chinese,Filipino,Malay": 0.21333333333333335, - "English,Chinese,Indonesian,Malay": 0.24, - "English,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "English,Vietnamese,Filipino,Malay": 0.24666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.26, - "English,Filipino,Indonesian,Malay": 0.2733333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian,Malay": 0.26, - "Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.22, - "Spanish,English,Chinese,Vietnamese,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Chinese,Filipino,Malay": 0.18, - "Spanish,English,Chinese,Indonesian,Malay": 0.2, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,English,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.2, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 - } - }, - "AC3_2": 0.4556200887791184, - "AC3_3": 0.376292559849293, - "AC3_4": 0.3177956371507339, - "AC3_5": 0.2711437565134258, - "AC3_6": 0.23243243239120706, - "AC3_7": 0.19999999996249998 - }, - "prompt_4": { - "overall_acc": 0.41619047619047617, - "language_acc": { - "Spanish": 0.4266666666666667, - "English": 0.47333333333333333, - "Chinese": 0.4066666666666667, - "Vietnamese": 0.46, - "Filipino": 0.3, - "Indonesian": 0.42, - "Malay": 0.4266666666666667 - }, - "consistency_score_2": 0.5492063492063491, - "consistency_score_3": 0.37714285714285706, - "consistency_score_4": 0.28057142857142864, - "consistency_score_5": 0.2149206349206349, - "consistency_score_6": 0.1657142857142857, - "consistency_score_7": 0.12666666666666668, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5866666666666667, - "Spanish,Chinese": 0.6133333333333333, - "Spanish,Vietnamese": 0.54, - "Spanish,Filipino": 0.64, - "Spanish,Indonesian": 0.6066666666666667, - "Spanish,Malay": 0.5133333333333333, - "English,Chinese": 0.5533333333333333, - "English,Vietnamese": 0.5666666666666667, - "English,Filipino": 0.5333333333333333, - "English,Indonesian": 0.58, - "English,Malay": 0.5, - "Chinese,Vietnamese": 0.54, - "Chinese,Filipino": 0.5133333333333333, - "Chinese,Indonesian": 0.6133333333333333, - "Chinese,Malay": 0.5266666666666666, - "Vietnamese,Filipino": 0.47333333333333333, - "Vietnamese,Indonesian": 0.58, - "Vietnamese,Malay": 0.4866666666666667, - "Filipino,Indonesian": 0.48, - "Filipino,Malay": 0.42, - "Indonesian,Malay": 0.6666666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.42, - "Spanish,English,Vietnamese": 0.4066666666666667, - "Spanish,English,Filipino": 0.42, - "Spanish,English,Indonesian": 0.4266666666666667, - "Spanish,English,Malay": 0.35333333333333333, - "Spanish,Chinese,Vietnamese": 0.38, - "Spanish,Chinese,Filipino": 0.41333333333333333, - "Spanish,Chinese,Indonesian": 0.46, - "Spanish,Chinese,Malay": 0.38, - "Spanish,Vietnamese,Filipino": 0.36, - "Spanish,Vietnamese,Indonesian": 0.41333333333333333, - "Spanish,Vietnamese,Malay": 0.34, - "Spanish,Filipino,Indonesian": 0.4, - "Spanish,Filipino,Malay": 0.32666666666666666, - "Spanish,Indonesian,Malay": 0.44, - "English,Chinese,Vietnamese": 0.38666666666666666, - "English,Chinese,Filipino": 0.34, - "English,Chinese,Indonesian": 0.43333333333333335, - "English,Chinese,Malay": 0.36, - "English,Vietnamese,Filipino": 0.34, - "English,Vietnamese,Indonesian": 0.4066666666666667, - "English,Vietnamese,Malay": 0.32666666666666666, - "English,Filipino,Indonesian": 0.34, - "English,Filipino,Malay": 0.28, - "English,Indonesian,Malay": 0.43333333333333335, - "Chinese,Vietnamese,Filipino": 0.32666666666666666, - "Chinese,Vietnamese,Indonesian": 0.43333333333333335, - "Chinese,Vietnamese,Malay": 0.35333333333333333, - "Chinese,Filipino,Indonesian": 0.36666666666666664, - "Chinese,Filipino,Malay": 0.31333333333333335, - "Chinese,Indonesian,Malay": 0.44, - "Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "Vietnamese,Filipino,Malay": 0.28, - "Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Filipino,Indonesian,Malay": 0.34 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.30666666666666664, - "Spanish,English,Chinese,Filipino": 0.29333333333333333, - "Spanish,English,Chinese,Indonesian": 0.34, - "Spanish,English,Chinese,Malay": 0.28, - "Spanish,English,Vietnamese,Filipino": 0.2733333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.32, - "Spanish,English,Vietnamese,Malay": 0.26, - "Spanish,English,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Filipino,Malay": 0.24, - "Spanish,English,Indonesian,Malay": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.26, - "Spanish,Chinese,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.28, - "Spanish,Chinese,Filipino,Indonesian": 0.32, - "Spanish,Chinese,Filipino,Malay": 0.26, - "Spanish,Chinese,Indonesian,Malay": 0.3466666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino": 0.24666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.3333333333333333, - "English,Chinese,Vietnamese,Malay": 0.26, - "English,Chinese,Filipino,Indonesian": 0.2733333333333333, - "English,Chinese,Filipino,Malay": 0.22, - "English,Chinese,Indonesian,Malay": 0.32, - "English,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "English,Vietnamese,Filipino,Malay": 0.2, - "English,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "English,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.2, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.22, - "Spanish,English,Chinese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.26, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.26, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.2, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - } - }, - "AC3_2": 0.47353470814530885, - "AC3_3": 0.39570571080589195, - "AC3_4": 0.3351823790817857, - "AC3_5": 0.28346172267234115, - "AC3_6": 0.23704465742945335, - "AC3_7": 0.19422222218644444 - }, - "prompt_5": { - "overall_acc": 0.41619047619047617, - "language_acc": { - "Spanish": 0.41333333333333333, - "English": 0.5, - "Chinese": 0.38666666666666666, - "Vietnamese": 0.43333333333333335, - "Filipino": 0.32, - "Indonesian": 0.44666666666666666, - "Malay": 0.41333333333333333 - }, - "consistency_score_2": 0.6123809523809524, - "consistency_score_3": 0.45028571428571423, - "consistency_score_4": 0.35238095238095235, - "consistency_score_5": 0.2844444444444444, - "consistency_score_6": 0.2333333333333333, - "consistency_score_7": 0.19333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.7133333333333334, - "Spanish,Chinese": 0.62, - "Spanish,Vietnamese": 0.6533333333333333, - "Spanish,Filipino": 0.6133333333333333, - "Spanish,Indonesian": 0.68, - "Spanish,Malay": 0.62, - "English,Chinese": 0.6933333333333334, - "English,Vietnamese": 0.6666666666666666, - "English,Filipino": 0.5733333333333334, - "English,Indonesian": 0.7133333333333334, - "English,Malay": 0.5533333333333333, - "Chinese,Vietnamese": 0.6333333333333333, - "Chinese,Filipino": 0.58, - "Chinese,Indonesian": 0.62, - "Chinese,Malay": 0.5666666666666667, - "Vietnamese,Filipino": 0.5466666666666666, - "Vietnamese,Indonesian": 0.6333333333333333, - "Vietnamese,Malay": 0.5533333333333333, - "Filipino,Indonesian": 0.5133333333333333, - "Filipino,Malay": 0.4533333333333333, - "Indonesian,Malay": 0.66 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5466666666666666, - "Spanish,English,Vietnamese": 0.5533333333333333, - "Spanish,English,Filipino": 0.48, - "Spanish,English,Indonesian": 0.5733333333333334, - "Spanish,English,Malay": 0.4666666666666667, - "Spanish,Chinese,Vietnamese": 0.49333333333333335, - "Spanish,Chinese,Filipino": 0.44, - "Spanish,Chinese,Indonesian": 0.4866666666666667, - "Spanish,Chinese,Malay": 0.43333333333333335, - "Spanish,Vietnamese,Filipino": 0.44, - "Spanish,Vietnamese,Indonesian": 0.5133333333333333, - "Spanish,Vietnamese,Malay": 0.4533333333333333, - "Spanish,Filipino,Indonesian": 0.4266666666666667, - "Spanish,Filipino,Malay": 0.36666666666666664, - "Spanish,Indonesian,Malay": 0.5, - "English,Chinese,Vietnamese": 0.5333333333333333, - "English,Chinese,Filipino": 0.4533333333333333, - "English,Chinese,Indonesian": 0.5333333333333333, - "English,Chinese,Malay": 0.44, - "English,Vietnamese,Filipino": 0.42, - "English,Vietnamese,Indonesian": 0.5533333333333333, - "English,Vietnamese,Malay": 0.43333333333333335, - "English,Filipino,Indonesian": 0.4266666666666667, - "English,Filipino,Malay": 0.32, - "English,Indonesian,Malay": 0.48, - "Chinese,Vietnamese,Filipino": 0.41333333333333333, - "Chinese,Vietnamese,Indonesian": 0.4866666666666667, - "Chinese,Vietnamese,Malay": 0.4266666666666667, - "Chinese,Filipino,Indonesian": 0.3933333333333333, - "Chinese,Filipino,Malay": 0.3333333333333333, - "Chinese,Indonesian,Malay": 0.4533333333333333, - "Vietnamese,Filipino,Indonesian": 0.3933333333333333, - "Vietnamese,Filipino,Malay": 0.32, - "Vietnamese,Indonesian,Malay": 0.44666666666666666, - "Filipino,Indonesian,Malay": 0.32666666666666666 + "Filipino,Vietnamese,Chinese": 0.30666666666666664, + "Filipino,Vietnamese,Spanish": 0.3, + "Filipino,Vietnamese,Malay": 0.32, + "Filipino,Vietnamese,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,English": 0.2733333333333333, + "Filipino,Chinese,Spanish": 0.30666666666666664, + "Filipino,Chinese,Malay": 0.32, + "Filipino,Chinese,Indonesian": 0.28, + "Filipino,Chinese,English": 0.31333333333333335, + "Filipino,Spanish,Malay": 0.32, + "Filipino,Spanish,Indonesian": 0.3, + "Filipino,Spanish,English": 0.32666666666666666, + "Filipino,Malay,Indonesian": 0.2866666666666667, + "Filipino,Malay,English": 0.2733333333333333, + "Filipino,Indonesian,English": 0.26666666666666666, + "Vietnamese,Chinese,Spanish": 0.36666666666666664, + "Vietnamese,Chinese,Malay": 0.4066666666666667, + "Vietnamese,Chinese,Indonesian": 0.35333333333333333, + "Vietnamese,Chinese,English": 0.36, + "Vietnamese,Spanish,Malay": 0.44, + "Vietnamese,Spanish,Indonesian": 0.38, + "Vietnamese,Spanish,English": 0.34, + "Vietnamese,Malay,Indonesian": 0.44, + "Vietnamese,Malay,English": 0.36666666666666664, + "Vietnamese,Indonesian,English": 0.35333333333333333, + "Chinese,Spanish,Malay": 0.4066666666666667, + "Chinese,Spanish,Indonesian": 0.36, + "Chinese,Spanish,English": 0.38666666666666666, + "Chinese,Malay,Indonesian": 0.41333333333333333, + "Chinese,Malay,English": 0.38, + "Chinese,Indonesian,English": 0.36666666666666664, + "Spanish,Malay,Indonesian": 0.42, + "Spanish,Malay,English": 0.38, + "Spanish,Indonesian,English": 0.37333333333333335, + "Malay,Indonesian,English": 0.36666666666666664 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.44666666666666666, - "Spanish,English,Chinese,Filipino": 0.38, - "Spanish,English,Chinese,Indonesian": 0.44, - "Spanish,English,Chinese,Malay": 0.38, - "Spanish,English,Vietnamese,Filipino": 0.37333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.47333333333333333, - "Spanish,English,Vietnamese,Malay": 0.3933333333333333, - "Spanish,English,Filipino,Indonesian": 0.38666666666666666, - "Spanish,English,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Indonesian,Malay": 0.42, - "Spanish,Chinese,Vietnamese,Filipino": 0.35333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.4, - "Spanish,Chinese,Vietnamese,Malay": 0.36666666666666664, - "Spanish,Chinese,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.2866666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.36666666666666664, - "Spanish,Vietnamese,Filipino,Indonesian": 0.34, - "Spanish,Vietnamese,Filipino,Malay": 0.2866666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.3466666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.44666666666666666, - "English,Chinese,Vietnamese,Malay": 0.37333333333333335, - "English,Chinese,Filipino,Indonesian": 0.3466666666666667, - "English,Chinese,Filipino,Malay": 0.26666666666666666, - "English,Chinese,Indonesian,Malay": 0.38666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "English,Vietnamese,Filipino,Malay": 0.26, - "English,Vietnamese,Indonesian,Malay": 0.3933333333333333, - "English,Filipino,Indonesian,Malay": 0.28, - "Chinese,Vietnamese,Filipino,Indonesian": 0.32, - "Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.36, - "Chinese,Filipino,Indonesian,Malay": 0.26, - "Vietnamese,Filipino,Indonesian,Malay": 0.26 + "Filipino,Vietnamese,Chinese,Spanish": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Malay": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,English": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Malay": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,English": 0.21333333333333335, + "Filipino,Vietnamese,Malay,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Malay,English": 0.22, + "Filipino,Vietnamese,Indonesian,English": 0.19333333333333333, + "Filipino,Chinese,Spanish,Malay": 0.24, + "Filipino,Chinese,Spanish,Indonesian": 0.2, + "Filipino,Chinese,Spanish,English": 0.23333333333333334, + "Filipino,Chinese,Malay,Indonesian": 0.22666666666666666, + "Filipino,Chinese,Malay,English": 0.21333333333333335, + "Filipino,Chinese,Indonesian,English": 0.20666666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.22, + "Filipino,Spanish,Malay,English": 0.23333333333333334, + "Filipino,Spanish,Indonesian,English": 0.21333333333333335, + "Filipino,Malay,Indonesian,English": 0.2, + "Vietnamese,Chinese,Spanish,Malay": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,English": 0.26666666666666666, + "Vietnamese,Chinese,Malay,Indonesian": 0.32, + "Vietnamese,Chinese,Malay,English": 0.29333333333333333, + "Vietnamese,Chinese,Indonesian,English": 0.26, + "Vietnamese,Spanish,Malay,Indonesian": 0.3333333333333333, + "Vietnamese,Spanish,Malay,English": 0.2866666666666667, + "Vietnamese,Spanish,Indonesian,English": 0.26, + "Vietnamese,Malay,Indonesian,English": 0.2866666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.30666666666666664, + "Chinese,Spanish,Malay,English": 0.2866666666666667, + "Chinese,Spanish,Indonesian,English": 0.26, + "Chinese,Malay,Indonesian,English": 0.2733333333333333, + "Spanish,Malay,Indonesian,English": 0.29333333333333333 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.38, - "Spanish,English,Chinese,Vietnamese,Malay": 0.3333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.32, - "Spanish,English,Chinese,Filipino,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.3333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.32, - "Spanish,English,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.36, - "Spanish,English,Filipino,Indonesian,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.24, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.22666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.18, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.18, + "Filipino,Vietnamese,Spanish,Malay,English": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.15333333333333332, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, + "Filipino,Chinese,Spanish,Malay,English": 0.18, + "Filipino,Chinese,Spanish,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Malay,Indonesian,English": 0.16, + "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.25333333333333335, + "Vietnamese,Chinese,Spanish,Malay,English": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.19333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.22666666666666666, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.22666666666666666, + "Chinese,Spanish,Malay,Indonesian,English": 0.22 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.3, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.22, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.16, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 } }, - "AC3_2": 0.49557495586010836, - "AC3_3": 0.43256728318344106, - "AC3_4": 0.3816368678325657, - "AC3_5": 0.33793082611095837, - "AC3_6": 0.2990224828474134, - "AC3_7": 0.2640208332900174 + "AC3_2": 0.532785224853194, + "AC3_3": 0.42168182931061965, + "AC3_4": 0.3411364158757646, + "AC3_5": 0.2800979781240327, + "AC3_6": 0.23294024531634128, + "AC3_7": 0.1963005780048966 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.32224025974025977, + "overall_acc": 0.4935064935064935, "language_acc": { - "English": 0.3465909090909091, - "Filipino": 0.2215909090909091, - "Spanish": 0.3465909090909091, - "Malay": 0.38636363636363635, - "Chinese": 0.2897727272727273, - "Indonesian": 0.39204545454545453, - "Vietnamese": 0.2727272727272727 + "Indonesian": 0.4715909090909091, + "English": 0.5681818181818182, + "Filipino": 0.4318181818181818, + "Spanish": 0.5340909090909091, + "Chinese": 0.5568181818181818, + "Malay": 0.42045454545454547, + "Vietnamese": 0.4715909090909091 }, - "consistency_score_2": 0.4978354978354978, - "consistency_score_3": 0.31298701298701304, - "consistency_score_4": 0.21866883116883115, - "consistency_score_5": 0.16233766233766236, - "consistency_score_6": 0.1258116883116883, - "consistency_score_7": 0.10227272727272728, + "consistency_score_2": 0.6071428571428572, + "consistency_score_3": 0.45, + "consistency_score_4": 0.3626623376623376, + "consistency_score_5": 0.3057359307359307, + "consistency_score_6": 0.26461038961038963, + "consistency_score_7": 0.23295454545454544, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.6534090909090909, - "English,Spanish": 0.6193181818181818, - "English,Malay": 0.4715909090909091, - "English,Chinese": 0.6136363636363636, - "English,Indonesian": 0.4090909090909091, - "English,Vietnamese": 0.5511363636363636, - "Filipino,Spanish": 0.6022727272727273, - "Filipino,Malay": 0.3977272727272727, - "Filipino,Chinese": 0.6647727272727273, - "Filipino,Indonesian": 0.3068181818181818, - "Filipino,Vietnamese": 0.4715909090909091, - "Spanish,Malay": 0.5397727272727273, - "Spanish,Chinese": 0.5170454545454546, - "Spanish,Indonesian": 0.4318181818181818, - "Spanish,Vietnamese": 0.5454545454545454, - "Malay,Chinese": 0.3806818181818182, - "Malay,Indonesian": 0.5511363636363636, - "Malay,Vietnamese": 0.48295454545454547, - "Chinese,Indonesian": 0.3465909090909091, - "Chinese,Vietnamese": 0.4318181818181818, - "Indonesian,Vietnamese": 0.4659090909090909 - }, - "3_combine": { - "English,Filipino,Spanish": 0.4659090909090909, - "English,Filipino,Malay": 0.30113636363636365, - "English,Filipino,Chinese": 0.5056818181818182, - "English,Filipino,Indonesian": 0.23863636363636365, - "English,Filipino,Vietnamese": 0.3806818181818182, - "English,Spanish,Malay": 0.35795454545454547, - "English,Spanish,Chinese": 0.42045454545454547, - "English,Spanish,Indonesian": 0.29545454545454547, - "English,Spanish,Vietnamese": 0.42045454545454547, - "English,Malay,Chinese": 0.2784090909090909, - "English,Malay,Indonesian": 0.29545454545454547, - "English,Malay,Vietnamese": 0.32954545454545453, - "English,Chinese,Indonesian": 0.25, - "English,Chinese,Vietnamese": 0.36363636363636365, - "English,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Malay": 0.3181818181818182, - "Filipino,Spanish,Chinese": 0.4318181818181818, - "Filipino,Spanish,Indonesian": 0.2556818181818182, - "Filipino,Spanish,Vietnamese": 0.375, - "Filipino,Malay,Chinese": 0.2784090909090909, - "Filipino,Malay,Indonesian": 0.2159090909090909, - "Filipino,Malay,Vietnamese": 0.26136363636363635, - "Filipino,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Chinese,Vietnamese": 0.3522727272727273, - "Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese": 0.2840909090909091, - "Spanish,Malay,Indonesian": 0.3409090909090909, - "Spanish,Malay,Vietnamese": 0.3409090909090909, - "Spanish,Chinese,Indonesian": 0.22727272727272727, - "Spanish,Chinese,Vietnamese": 0.3352272727272727, - "Spanish,Indonesian,Vietnamese": 0.2897727272727273, - "Malay,Chinese,Indonesian": 0.22727272727272727, - "Malay,Chinese,Vietnamese": 0.24431818181818182, - "Malay,Indonesian,Vietnamese": 0.32386363636363635, - "Chinese,Indonesian,Vietnamese": 0.2159090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.2556818181818182, - "English,Filipino,Spanish,Chinese": 0.36363636363636365, - "English,Filipino,Spanish,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Vietnamese": 0.32386363636363635, - "English,Filipino,Malay,Chinese": 0.23295454545454544, - "English,Filipino,Malay,Indonesian": 0.17045454545454544, - "English,Filipino,Malay,Vietnamese": 0.20454545454545456, - "English,Filipino,Chinese,Indonesian": 0.19318181818181818, - "English,Filipino,Chinese,Vietnamese": 0.3125, - "English,Filipino,Indonesian,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Chinese": 0.2159090909090909, - "English,Spanish,Malay,Indonesian": 0.23295454545454544, - "English,Spanish,Malay,Vietnamese": 0.26704545454545453, - "English,Spanish,Chinese,Indonesian": 0.1875, - "English,Spanish,Chinese,Vietnamese": 0.2897727272727273, - "English,Spanish,Indonesian,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese,Indonesian": 0.16477272727272727, - "English,Malay,Chinese,Vietnamese": 0.19886363636363635, - "English,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay,Chinese": 0.23863636363636365, - "Filipino,Spanish,Malay,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.3068181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Chinese,Indonesian": 0.17045454545454544, - "Spanish,Malay,Chinese,Vietnamese": 0.21022727272727273, - "Spanish,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727 + "Indonesian,English": 0.5909090909090909, + "Indonesian,Filipino": 0.5965909090909091, + "Indonesian,Spanish": 0.6477272727272727, + "Indonesian,Chinese": 0.6022727272727273, + "Indonesian,Malay": 0.6647727272727273, + "Indonesian,Vietnamese": 0.6477272727272727, + "English,Filipino": 0.5454545454545454, + "English,Spanish": 0.7840909090909091, + "English,Chinese": 0.5965909090909091, + "English,Malay": 0.5738636363636364, + "English,Vietnamese": 0.7272727272727273, + "Filipino,Spanish": 0.5738636363636364, + "Filipino,Chinese": 0.4943181818181818, + "Filipino,Malay": 0.6363636363636364, + "Filipino,Vietnamese": 0.5284090909090909, + "Spanish,Chinese": 0.5625, + "Spanish,Malay": 0.6079545454545454, + "Spanish,Vietnamese": 0.6477272727272727, + "Chinese,Malay": 0.5397727272727273, + "Chinese,Vietnamese": 0.6022727272727273, + "Malay,Vietnamese": 0.5795454545454546 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.42045454545454547, + "Indonesian,English,Spanish": 0.5340909090909091, + "Indonesian,English,Chinese": 0.4431818181818182, + "Indonesian,English,Malay": 0.45454545454545453, + "Indonesian,English,Vietnamese": 0.5113636363636364, + "Indonesian,Filipino,Spanish": 0.4602272727272727, + "Indonesian,Filipino,Chinese": 0.375, + "Indonesian,Filipino,Malay": 0.5, + "Indonesian,Filipino,Vietnamese": 0.4318181818181818, + "Indonesian,Spanish,Chinese": 0.4431818181818182, + "Indonesian,Spanish,Malay": 0.4943181818181818, + "Indonesian,Spanish,Vietnamese": 0.5, + "Indonesian,Chinese,Malay": 0.4431818181818182, + "Indonesian,Chinese,Vietnamese": 0.4659090909090909, + "Indonesian,Malay,Vietnamese": 0.48295454545454547, + "English,Filipino,Spanish": 0.4772727272727273, + "English,Filipino,Chinese": 0.3806818181818182, + "English,Filipino,Malay": 0.42045454545454547, + "English,Filipino,Vietnamese": 0.4431818181818182, + "English,Spanish,Chinese": 0.4943181818181818, + "English,Spanish,Malay": 0.5056818181818182, + "English,Spanish,Vietnamese": 0.6022727272727273, + "English,Chinese,Malay": 0.38636363636363635, + "English,Chinese,Vietnamese": 0.48863636363636365, + "English,Malay,Vietnamese": 0.4772727272727273, + "Filipino,Spanish,Chinese": 0.3693181818181818, + "Filipino,Spanish,Malay": 0.45454545454545453, + "Filipino,Spanish,Vietnamese": 0.42045454545454547, + "Filipino,Chinese,Malay": 0.375, + "Filipino,Chinese,Vietnamese": 0.3693181818181818, + "Filipino,Malay,Vietnamese": 0.4147727272727273, + "Spanish,Chinese,Malay": 0.39204545454545453, + "Spanish,Chinese,Vietnamese": 0.4431818181818182, + "Spanish,Malay,Vietnamese": 0.45454545454545453, + "Chinese,Malay,Vietnamese": 0.42045454545454547 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.38636363636363635, + "Indonesian,English,Filipino,Chinese": 0.3181818181818182, + "Indonesian,English,Filipino,Malay": 0.36363636363636365, + "Indonesian,English,Filipino,Vietnamese": 0.3693181818181818, + "Indonesian,English,Spanish,Chinese": 0.3977272727272727, + "Indonesian,English,Spanish,Malay": 0.4147727272727273, + "Indonesian,English,Spanish,Vietnamese": 0.4715909090909091, + "Indonesian,English,Chinese,Malay": 0.3409090909090909, + "Indonesian,English,Chinese,Vietnamese": 0.3977272727272727, + "Indonesian,English,Malay,Vietnamese": 0.4090909090909091, + "Indonesian,Filipino,Spanish,Chinese": 0.3125, + "Indonesian,Filipino,Spanish,Malay": 0.3977272727272727, + "Indonesian,Filipino,Spanish,Vietnamese": 0.35795454545454547, + "Indonesian,Filipino,Chinese,Malay": 0.32386363636363635, + "Indonesian,Filipino,Chinese,Vietnamese": 0.32386363636363635, + "Indonesian,Filipino,Malay,Vietnamese": 0.3693181818181818, + "Indonesian,Spanish,Chinese,Malay": 0.3465909090909091, + "Indonesian,Spanish,Chinese,Vietnamese": 0.39204545454545453, + "Indonesian,Spanish,Malay,Vietnamese": 0.39204545454545453, + "Indonesian,Chinese,Malay,Vietnamese": 0.3693181818181818, + "English,Filipino,Spanish,Chinese": 0.3352272727272727, + "English,Filipino,Spanish,Malay": 0.3806818181818182, + "English,Filipino,Spanish,Vietnamese": 0.3977272727272727, + "English,Filipino,Chinese,Malay": 0.30113636363636365, + "English,Filipino,Chinese,Vietnamese": 0.32386363636363635, + "English,Filipino,Malay,Vietnamese": 0.36363636363636365, + "English,Spanish,Chinese,Malay": 0.3522727272727273, + "English,Spanish,Chinese,Vietnamese": 0.4147727272727273, + "English,Spanish,Malay,Vietnamese": 0.42613636363636365, + "English,Chinese,Malay,Vietnamese": 0.3465909090909091, + "Filipino,Spanish,Chinese,Malay": 0.29545454545454547, + "Filipino,Spanish,Chinese,Vietnamese": 0.3125, + "Filipino,Spanish,Malay,Vietnamese": 0.3465909090909091, + "Filipino,Chinese,Malay,Vietnamese": 0.3068181818181818, + "Spanish,Chinese,Malay,Vietnamese": 0.3352272727272727 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.19318181818181818, - "English,Filipino,Spanish,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.26704545454545453, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 + "Indonesian,English,Filipino,Spanish,Chinese": 0.2897727272727273, + "Indonesian,English,Filipino,Spanish,Malay": 0.3352272727272727, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.3409090909090909, + "Indonesian,English,Filipino,Chinese,Malay": 0.2727272727272727, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2897727272727273, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.32386363636363635, + "Indonesian,English,Spanish,Chinese,Malay": 0.3125, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.3693181818181818, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.375, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.3181818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.26704545454545453, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.2784090909090909, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.3125, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.3068181818181818, + "English,Filipino,Spanish,Chinese,Malay": 0.2727272727272727, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.29545454545454547, + "English,Filipino,Spanish,Malay,Vietnamese": 0.32954545454545453, + "English,Filipino,Chinese,Malay,Vietnamese": 0.2727272727272727, + "English,Spanish,Chinese,Malay,Vietnamese": 0.3181818181818182, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.25, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.26704545454545453, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.30113636363636365, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.25, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.25 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544 } }, - "AC3_2": 0.39123858650595644, - "AC3_3": 0.31754624108429164, - "AC3_4": 0.2605387933912869, - "AC3_5": 0.2159063716407106, - "AC3_6": 0.1809682617702631, - "AC3_7": 0.15526681727613115 + "AC3_2": 0.5444584913116792, + "AC3_3": 0.4707501720079177, + "AC3_4": 0.4180862746927325, + "AC3_5": 0.3775642096090479, + "AC3_6": 0.34450346223103956, + "AC3_7": 0.31650584048461866 }, "prompt_2": { - "overall_acc": 0.2922077922077922, + "overall_acc": 0.48701298701298706, "language_acc": { - "English": 0.3125, - "Filipino": 0.22727272727272727, - "Spanish": 0.3181818181818182, - "Malay": 0.36363636363636365, - "Chinese": 0.19886363636363635, - "Indonesian": 0.3409090909090909, - "Vietnamese": 0.2840909090909091 + "Indonesian": 0.44886363636363635, + "English": 0.5511363636363636, + "Filipino": 0.4431818181818182, + "Spanish": 0.5227272727272727, + "Chinese": 0.5397727272727273, + "Malay": 0.4772727272727273, + "Vietnamese": 0.42613636363636365 }, - "consistency_score_2": 0.6396103896103899, - "consistency_score_3": 0.49253246753246743, - "consistency_score_4": 0.39918831168831154, - "consistency_score_5": 0.3292748917748918, - "consistency_score_6": 0.273538961038961, - "consistency_score_7": 0.22727272727272727, + "consistency_score_2": 0.5933441558441558, + "consistency_score_3": 0.436525974025974, + "consistency_score_4": 0.34902597402597413, + "consistency_score_5": 0.2919372294372295, + "consistency_score_6": 0.25162337662337664, + "consistency_score_7": 0.2215909090909091, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.6931818181818182, - "English,Spanish": 0.6988636363636364, + "Indonesian,English": 0.5852272727272727, + "Indonesian,Filipino": 0.5454545454545454, + "Indonesian,Spanish": 0.6193181818181818, + "Indonesian,Chinese": 0.5568181818181818, + "Indonesian,Malay": 0.6590909090909091, + "Indonesian,Vietnamese": 0.6079545454545454, + "English,Filipino": 0.5284090909090909, + "English,Spanish": 0.75, + "English,Chinese": 0.5909090909090909, "English,Malay": 0.6306818181818182, - "English,Chinese": 0.5511363636363636, - "English,Indonesian": 0.6704545454545454, - "English,Vietnamese": 0.6931818181818182, - "Filipino,Spanish": 0.7613636363636364, - "Filipino,Malay": 0.6079545454545454, - "Filipino,Chinese": 0.5568181818181818, - "Filipino,Indonesian": 0.7045454545454546, - "Filipino,Vietnamese": 0.7272727272727273, - "Spanish,Malay": 0.6534090909090909, - "Spanish,Chinese": 0.4943181818181818, - "Spanish,Indonesian": 0.7159090909090909, - "Spanish,Vietnamese": 0.7159090909090909, - "Malay,Chinese": 0.44886363636363635, - "Malay,Indonesian": 0.7159090909090909, - "Malay,Vietnamese": 0.6477272727272727, - "Chinese,Indonesian": 0.5227272727272727, - "Chinese,Vietnamese": 0.5113636363636364, - "Indonesian,Vietnamese": 0.7102272727272727 + "English,Vietnamese": 0.6079545454545454, + "Filipino,Spanish": 0.5681818181818182, + "Filipino,Chinese": 0.5284090909090909, + "Filipino,Malay": 0.5738636363636364, + "Filipino,Vietnamese": 0.5397727272727273, + "Spanish,Chinese": 0.5681818181818182, + "Spanish,Malay": 0.6647727272727273, + "Spanish,Vietnamese": 0.6306818181818182, + "Chinese,Malay": 0.5454545454545454, + "Chinese,Vietnamese": 0.5909090909090909, + "Malay,Vietnamese": 0.5681818181818182 }, "3_combine": { - "English,Filipino,Spanish": 0.5965909090909091, - "English,Filipino,Malay": 0.5056818181818182, - "English,Filipino,Chinese": 0.4375, - "English,Filipino,Indonesian": 0.5681818181818182, - "English,Filipino,Vietnamese": 0.5795454545454546, - "English,Spanish,Malay": 0.5284090909090909, - "English,Spanish,Chinese": 0.42045454545454547, - "English,Spanish,Indonesian": 0.5738636363636364, - "English,Spanish,Vietnamese": 0.5795454545454546, - "English,Malay,Chinese": 0.36363636363636365, - "English,Malay,Indonesian": 0.5397727272727273, - "English,Malay,Vietnamese": 0.5113636363636364, - "English,Chinese,Indonesian": 0.4147727272727273, - "English,Chinese,Vietnamese": 0.4090909090909091, - "English,Indonesian,Vietnamese": 0.5625, - "Filipino,Spanish,Malay": 0.5397727272727273, - "Filipino,Spanish,Chinese": 0.42613636363636365, - "Filipino,Spanish,Indonesian": 0.6136363636363636, - "Filipino,Spanish,Vietnamese": 0.625, - "Filipino,Malay,Chinese": 0.36363636363636365, - "Filipino,Malay,Indonesian": 0.5454545454545454, - "Filipino,Malay,Vietnamese": 0.5227272727272727, - "Filipino,Chinese,Indonesian": 0.4318181818181818, - "Filipino,Chinese,Vietnamese": 0.42045454545454547, - "Filipino,Indonesian,Vietnamese": 0.5965909090909091, - "Spanish,Malay,Chinese": 0.3522727272727273, - "Spanish,Malay,Indonesian": 0.5625, - "Spanish,Malay,Vietnamese": 0.5340909090909091, - "Spanish,Chinese,Indonesian": 0.4090909090909091, - "Spanish,Chinese,Vietnamese": 0.39204545454545453, - "Spanish,Indonesian,Vietnamese": 0.5965909090909091, - "Malay,Chinese,Indonesian": 0.38636363636363635, - "Malay,Chinese,Vietnamese": 0.36363636363636365, - "Malay,Indonesian,Vietnamese": 0.5625, - "Chinese,Indonesian,Vietnamese": 0.4034090909090909 + "Indonesian,English,Filipino": 0.3977272727272727, + "Indonesian,English,Spanish": 0.5056818181818182, + "Indonesian,English,Chinese": 0.42045454545454547, + "Indonesian,English,Malay": 0.4943181818181818, + "Indonesian,English,Vietnamese": 0.4375, + "Indonesian,Filipino,Spanish": 0.4090909090909091, + "Indonesian,Filipino,Chinese": 0.36363636363636365, + "Indonesian,Filipino,Malay": 0.44886363636363635, + "Indonesian,Filipino,Vietnamese": 0.39204545454545453, + "Indonesian,Spanish,Chinese": 0.4147727272727273, + "Indonesian,Spanish,Malay": 0.5170454545454546, + "Indonesian,Spanish,Vietnamese": 0.45454545454545453, + "Indonesian,Chinese,Malay": 0.4375, + "Indonesian,Chinese,Vietnamese": 0.42045454545454547, + "Indonesian,Malay,Vietnamese": 0.4715909090909091, + "English,Filipino,Spanish": 0.4659090909090909, + "English,Filipino,Chinese": 0.39204545454545453, + "English,Filipino,Malay": 0.42613636363636365, + "English,Filipino,Vietnamese": 0.38636363636363635, + "English,Spanish,Chinese": 0.48863636363636365, + "English,Spanish,Malay": 0.5454545454545454, + "English,Spanish,Vietnamese": 0.5113636363636364, + "English,Chinese,Malay": 0.4318181818181818, + "English,Chinese,Vietnamese": 0.4318181818181818, + "English,Malay,Vietnamese": 0.4431818181818182, + "Filipino,Spanish,Chinese": 0.38636363636363635, + "Filipino,Spanish,Malay": 0.44886363636363635, + "Filipino,Spanish,Vietnamese": 0.4034090909090909, + "Filipino,Chinese,Malay": 0.39204545454545453, + "Filipino,Chinese,Vietnamese": 0.39204545454545453, + "Filipino,Malay,Vietnamese": 0.39204545454545453, + "Spanish,Chinese,Malay": 0.4375, + "Spanish,Chinese,Vietnamese": 0.4318181818181818, + "Spanish,Malay,Vietnamese": 0.4772727272727273, + "Chinese,Malay,Vietnamese": 0.4090909090909091 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.4715909090909091, - "English,Filipino,Spanish,Chinese": 0.3693181818181818, - "English,Filipino,Spanish,Indonesian": 0.5113636363636364, - "English,Filipino,Spanish,Vietnamese": 0.5284090909090909, - "English,Filipino,Malay,Chinese": 0.3068181818181818, - "English,Filipino,Malay,Indonesian": 0.4602272727272727, - "English,Filipino,Malay,Vietnamese": 0.4375, - "English,Filipino,Chinese,Indonesian": 0.36363636363636365, - "English,Filipino,Chinese,Vietnamese": 0.3465909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.48863636363636365, - "English,Spanish,Malay,Chinese": 0.3125, - "English,Spanish,Malay,Indonesian": 0.48863636363636365, - "English,Spanish,Malay,Vietnamese": 0.4602272727272727, - "English,Spanish,Chinese,Indonesian": 0.35795454545454547, - "English,Spanish,Chinese,Vietnamese": 0.3409090909090909, - "English,Spanish,Indonesian,Vietnamese": 0.5056818181818182, - "English,Malay,Chinese,Indonesian": 0.3181818181818182, - "English,Malay,Chinese,Vietnamese": 0.30113636363636365, - "English,Malay,Indonesian,Vietnamese": 0.45454545454545453, - "English,Chinese,Indonesian,Vietnamese": 0.3352272727272727, - "Filipino,Spanish,Malay,Chinese": 0.3181818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.48295454545454547, - "Filipino,Spanish,Malay,Vietnamese": 0.4772727272727273, - "Filipino,Spanish,Chinese,Indonesian": 0.3693181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.35795454545454547, - "Filipino,Spanish,Indonesian,Vietnamese": 0.5397727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.32954545454545453, - "Filipino,Malay,Chinese,Vietnamese": 0.3068181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.48295454545454547, - "Filipino,Chinese,Indonesian,Vietnamese": 0.3522727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.32386363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.3068181818181818, - "Spanish,Malay,Indonesian,Vietnamese": 0.4943181818181818, - "Spanish,Chinese,Indonesian,Vietnamese": 0.3522727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.3181818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.2840909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.4318181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.42045454545454547, - "English,Filipino,Spanish,Chinese,Indonesian": 0.32386363636363635, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.3125, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.4602272727272727, - "English,Filipino,Malay,Chinese,Indonesian": 0.2784090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.2556818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.4034090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.29545454545454547, - "English,Spanish,Malay,Chinese,Indonesian": 0.2897727272727273, - "English,Spanish,Malay,Chinese,Vietnamese": 0.26704545454545453, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.4318181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.3068181818181818, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.2897727272727273, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.2784090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.4431818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.3181818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2840909090909091 + "Indonesian,English,Filipino,Spanish": 0.3465909090909091, + "Indonesian,English,Filipino,Chinese": 0.3125, + "Indonesian,English,Filipino,Malay": 0.36363636363636365, + "Indonesian,English,Filipino,Vietnamese": 0.3181818181818182, + "Indonesian,English,Spanish,Chinese": 0.3693181818181818, + "Indonesian,English,Spanish,Malay": 0.4375, + "Indonesian,English,Spanish,Vietnamese": 0.39204545454545453, + "Indonesian,English,Chinese,Malay": 0.36363636363636365, + "Indonesian,English,Chinese,Vietnamese": 0.3409090909090909, + "Indonesian,English,Malay,Vietnamese": 0.38636363636363635, + "Indonesian,Filipino,Spanish,Chinese": 0.2897727272727273, + "Indonesian,Filipino,Spanish,Malay": 0.36363636363636365, + "Indonesian,Filipino,Spanish,Vietnamese": 0.3181818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.3181818181818182, + "Indonesian,Filipino,Chinese,Vietnamese": 0.3125, + "Indonesian,Filipino,Malay,Vietnamese": 0.3352272727272727, + "Indonesian,Spanish,Chinese,Malay": 0.35795454545454547, + "Indonesian,Spanish,Chinese,Vietnamese": 0.3409090909090909, + "Indonesian,Spanish,Malay,Vietnamese": 0.3977272727272727, + "Indonesian,Chinese,Malay,Vietnamese": 0.3409090909090909, + "English,Filipino,Spanish,Chinese": 0.3465909090909091, + "English,Filipino,Spanish,Malay": 0.375, + "English,Filipino,Spanish,Vietnamese": 0.3465909090909091, + "English,Filipino,Chinese,Malay": 0.3352272727272727, + "English,Filipino,Chinese,Vietnamese": 0.3181818181818182, + "English,Filipino,Malay,Vietnamese": 0.32386363636363635, + "English,Spanish,Chinese,Malay": 0.38636363636363635, + "English,Spanish,Chinese,Vietnamese": 0.375, + "English,Spanish,Malay,Vietnamese": 0.3977272727272727, + "English,Chinese,Malay,Vietnamese": 0.3409090909090909, + "Filipino,Spanish,Chinese,Malay": 0.32386363636363635, + "Filipino,Spanish,Chinese,Vietnamese": 0.32386363636363635, + "Filipino,Spanish,Malay,Vietnamese": 0.3409090909090909, + "Filipino,Chinese,Malay,Vietnamese": 0.32954545454545453, + "Spanish,Chinese,Malay,Vietnamese": 0.3465909090909091 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.2727272727272727, + "Indonesian,English,Filipino,Spanish,Malay": 0.3125, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2840909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.2840909090909091, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.2897727272727273, + "Indonesian,English,Spanish,Chinese,Malay": 0.32386363636363635, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.3068181818181818, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.3465909090909091, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.30113636363636365, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.26136363636363635, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.26136363636363635, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2897727272727273, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547, + "English,Filipino,Spanish,Chinese,Malay": 0.29545454545454547, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.2897727272727273, + "English,Filipino,Spanish,Malay,Vietnamese": 0.2897727272727273, + "English,Filipino,Chinese,Malay,Vietnamese": 0.2840909090909091, + "English,Spanish,Chinese,Malay,Vietnamese": 0.3068181818181818, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2840909090909091 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.26136363636363635, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.24431818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.39204545454545453, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.25, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.24431818181818182, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.24431818181818182, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.25, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091 } }, - "AC3_2": 0.40114937323175864, - "AC3_3": 0.36680117560838504, - "AC3_4": 0.3374214420920569, - "AC3_5": 0.30963594519779053, - "AC3_6": 0.28256535425142015, - "AC3_7": 0.25568181813259944 + "AC3_2": 0.5349458955274844, + "AC3_3": 0.4603894961983203, + "AC3_4": 0.4066322027000697, + "AC3_5": 0.3650482896280321, + "AC3_6": 0.33181104605183825, + "AC3_7": 0.3045923148586084 }, "prompt_3": { - "overall_acc": 0.32142857142857134, + "overall_acc": 0.4715909090909091, "language_acc": { - "English": 0.32954545454545453, - "Filipino": 0.21022727272727273, - "Spanish": 0.35795454545454547, - "Malay": 0.3409090909090909, - "Chinese": 0.30113636363636365, - "Indonesian": 0.3806818181818182, - "Vietnamese": 0.32954545454545453 + "Indonesian": 0.4431818181818182, + "English": 0.5681818181818182, + "Filipino": 0.39204545454545453, + "Spanish": 0.5340909090909091, + "Chinese": 0.4772727272727273, + "Malay": 0.4659090909090909, + "Vietnamese": 0.42045454545454547 }, - "consistency_score_2": 0.5481601731601731, - "consistency_score_3": 0.3792207792207792, - "consistency_score_4": 0.2834415584415584, - "consistency_score_5": 0.21888528138528138, - "consistency_score_6": 0.17045454545454544, - "consistency_score_7": 0.13068181818181818, + "consistency_score_2": 0.554112554112554, + "consistency_score_3": 0.38165584415584414, + "consistency_score_4": 0.2931818181818182, + "consistency_score_5": 0.23971861471861478, + "consistency_score_6": 0.20454545454545456, + "consistency_score_7": 0.18181818181818182, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.625, - "English,Spanish": 0.6136363636363636, - "English,Malay": 0.5454545454545454, - "English,Chinese": 0.4943181818181818, - "English,Indonesian": 0.6193181818181818, - "English,Vietnamese": 0.6193181818181818, - "Filipino,Spanish": 0.6477272727272727, - "Filipino,Malay": 0.3977272727272727, - "Filipino,Chinese": 0.45454545454545453, - "Filipino,Indonesian": 0.5056818181818182, - "Filipino,Vietnamese": 0.6193181818181818, - "Spanish,Malay": 0.5170454545454546, - "Spanish,Chinese": 0.4715909090909091, - "Spanish,Indonesian": 0.5852272727272727, - "Spanish,Vietnamese": 0.6875, - "Malay,Chinese": 0.375, - "Malay,Indonesian": 0.6477272727272727, - "Malay,Vietnamese": 0.5681818181818182, - "Chinese,Indonesian": 0.4090909090909091, - "Chinese,Vietnamese": 0.44886363636363635, - "Indonesian,Vietnamese": 0.6590909090909091 + "Indonesian,English": 0.5227272727272727, + "Indonesian,Filipino": 0.5568181818181818, + "Indonesian,Spanish": 0.5965909090909091, + "Indonesian,Chinese": 0.5113636363636364, + "Indonesian,Malay": 0.6704545454545454, + "Indonesian,Vietnamese": 0.5397727272727273, + "English,Filipino": 0.4431818181818182, + "English,Spanish": 0.6818181818181818, + "English,Chinese": 0.5284090909090909, + "English,Malay": 0.5340909090909091, + "English,Vietnamese": 0.5681818181818182, + "Filipino,Spanish": 0.5340909090909091, + "Filipino,Chinese": 0.4602272727272727, + "Filipino,Malay": 0.6193181818181818, + "Filipino,Vietnamese": 0.5227272727272727, + "Spanish,Chinese": 0.48295454545454547, + "Spanish,Malay": 0.6306818181818182, + "Spanish,Vietnamese": 0.6022727272727273, + "Chinese,Malay": 0.5, + "Chinese,Vietnamese": 0.5625, + "Malay,Vietnamese": 0.5681818181818182 }, "3_combine": { - "English,Filipino,Spanish": 0.48863636363636365, - "English,Filipino,Malay": 0.3409090909090909, - "English,Filipino,Chinese": 0.36363636363636365, - "English,Filipino,Indonesian": 0.4318181818181818, - "English,Filipino,Vietnamese": 0.48295454545454547, - "English,Spanish,Malay": 0.375, + "Indonesian,English,Filipino": 0.3352272727272727, + "Indonesian,English,Spanish": 0.4318181818181818, + "Indonesian,English,Chinese": 0.3352272727272727, + "Indonesian,English,Malay": 0.4090909090909091, + "Indonesian,English,Vietnamese": 0.36363636363636365, + "Indonesian,Filipino,Spanish": 0.4034090909090909, + "Indonesian,Filipino,Chinese": 0.3125, + "Indonesian,Filipino,Malay": 0.4772727272727273, + "Indonesian,Filipino,Vietnamese": 0.36363636363636365, + "Indonesian,Spanish,Chinese": 0.3465909090909091, + "Indonesian,Spanish,Malay": 0.48863636363636365, + "Indonesian,Spanish,Vietnamese": 0.4147727272727273, + "Indonesian,Chinese,Malay": 0.3806818181818182, + "Indonesian,Chinese,Vietnamese": 0.36363636363636365, + "Indonesian,Malay,Vietnamese": 0.4431818181818182, + "English,Filipino,Spanish": 0.3806818181818182, + "English,Filipino,Chinese": 0.2897727272727273, + "English,Filipino,Malay": 0.35795454545454547, + "English,Filipino,Vietnamese": 0.32386363636363635, "English,Spanish,Chinese": 0.36363636363636365, - "English,Spanish,Indonesian": 0.44886363636363635, - "English,Spanish,Vietnamese": 0.48295454545454547, - "English,Malay,Chinese": 0.2897727272727273, - "English,Malay,Indonesian": 0.4431818181818182, - "English,Malay,Vietnamese": 0.4034090909090909, - "English,Chinese,Indonesian": 0.32386363636363635, - "English,Chinese,Vietnamese": 0.3465909090909091, - "English,Indonesian,Vietnamese": 0.4772727272727273, - "Filipino,Spanish,Malay": 0.3352272727272727, - "Filipino,Spanish,Chinese": 0.35795454545454547, - "Filipino,Spanish,Indonesian": 0.42045454545454547, - "Filipino,Spanish,Vietnamese": 0.5284090909090909, - "Filipino,Malay,Chinese": 0.23295454545454544, - "Filipino,Malay,Indonesian": 0.32386363636363635, - "Filipino,Malay,Vietnamese": 0.3352272727272727, - "Filipino,Chinese,Indonesian": 0.2897727272727273, - "Filipino,Chinese,Vietnamese": 0.3181818181818182, - "Filipino,Indonesian,Vietnamese": 0.42045454545454547, - "Spanish,Malay,Chinese": 0.26704545454545453, - "Spanish,Malay,Indonesian": 0.4147727272727273, - "Spanish,Malay,Vietnamese": 0.42045454545454547, - "Spanish,Chinese,Indonesian": 0.3181818181818182, - "Spanish,Chinese,Vietnamese": 0.3465909090909091, - "Spanish,Indonesian,Vietnamese": 0.4943181818181818, - "Malay,Chinese,Indonesian": 0.30113636363636365, - "Malay,Chinese,Vietnamese": 0.2840909090909091, - "Malay,Indonesian,Vietnamese": 0.4715909090909091, - "Chinese,Indonesian,Vietnamese": 0.32954545454545453 + "English,Spanish,Malay": 0.4602272727272727, + "English,Spanish,Vietnamese": 0.4431818181818182, + "English,Chinese,Malay": 0.3352272727272727, + "English,Chinese,Vietnamese": 0.3693181818181818, + "English,Malay,Vietnamese": 0.3693181818181818, + "Filipino,Spanish,Chinese": 0.3125, + "Filipino,Spanish,Malay": 0.42045454545454547, + "Filipino,Spanish,Vietnamese": 0.38636363636363635, + "Filipino,Chinese,Malay": 0.3693181818181818, + "Filipino,Chinese,Vietnamese": 0.3465909090909091, + "Filipino,Malay,Vietnamese": 0.4090909090909091, + "Spanish,Chinese,Malay": 0.3693181818181818, + "Spanish,Chinese,Vietnamese": 0.36363636363636365, + "Spanish,Malay,Vietnamese": 0.4431818181818182, + "Chinese,Malay,Vietnamese": 0.375 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.29545454545454547, - "English,Filipino,Spanish,Chinese": 0.29545454545454547, - "English,Filipino,Spanish,Indonesian": 0.375, - "English,Filipino,Spanish,Vietnamese": 0.42613636363636365, - "English,Filipino,Malay,Chinese": 0.19886363636363635, - "English,Filipino,Malay,Indonesian": 0.2897727272727273, - "English,Filipino,Malay,Vietnamese": 0.29545454545454547, - "English,Filipino,Chinese,Indonesian": 0.25, - "English,Filipino,Chinese,Vietnamese": 0.2727272727272727, - "English,Filipino,Indonesian,Vietnamese": 0.3693181818181818, - "English,Spanish,Malay,Chinese": 0.2215909090909091, - "English,Spanish,Malay,Indonesian": 0.32954545454545453, + "Indonesian,English,Filipino,Spanish": 0.29545454545454547, + "Indonesian,English,Filipino,Chinese": 0.23863636363636365, + "Indonesian,English,Filipino,Malay": 0.3181818181818182, + "Indonesian,English,Filipino,Vietnamese": 0.26704545454545453, + "Indonesian,English,Spanish,Chinese": 0.2784090909090909, + "Indonesian,English,Spanish,Malay": 0.36363636363636365, + "Indonesian,English,Spanish,Vietnamese": 0.32386363636363635, + "Indonesian,English,Chinese,Malay": 0.2784090909090909, + "Indonesian,English,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,English,Malay,Vietnamese": 0.3068181818181818, + "Indonesian,Filipino,Spanish,Chinese": 0.23863636363636365, + "Indonesian,Filipino,Spanish,Malay": 0.36363636363636365, + "Indonesian,Filipino,Spanish,Vietnamese": 0.30113636363636365, + "Indonesian,Filipino,Chinese,Malay": 0.30113636363636365, + "Indonesian,Filipino,Chinese,Vietnamese": 0.26136363636363635, + "Indonesian,Filipino,Malay,Vietnamese": 0.3409090909090909, + "Indonesian,Spanish,Chinese,Malay": 0.30113636363636365, + "Indonesian,Spanish,Chinese,Vietnamese": 0.2784090909090909, + "Indonesian,Spanish,Malay,Vietnamese": 0.36363636363636365, + "Indonesian,Chinese,Malay,Vietnamese": 0.3068181818181818, + "English,Filipino,Spanish,Chinese": 0.24431818181818182, + "English,Filipino,Spanish,Malay": 0.3125, + "English,Filipino,Spanish,Vietnamese": 0.2840909090909091, + "English,Filipino,Chinese,Malay": 0.2556818181818182, + "English,Filipino,Chinese,Vietnamese": 0.25, + "English,Filipino,Malay,Vietnamese": 0.2784090909090909, + "English,Spanish,Chinese,Malay": 0.2840909090909091, + "English,Spanish,Chinese,Vietnamese": 0.2897727272727273, "English,Spanish,Malay,Vietnamese": 0.32386363636363635, - "English,Spanish,Chinese,Indonesian": 0.26136363636363635, - "English,Spanish,Chinese,Vietnamese": 0.2840909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.375, - "English,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Malay,Chinese,Vietnamese": 0.23863636363636365, - "English,Malay,Indonesian,Vietnamese": 0.3522727272727273, - "English,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Malay,Chinese": 0.20454545454545456, - "Filipino,Spanish,Malay,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.3068181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.2556818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.2784090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.3693181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Chinese,Indonesian": 0.22727272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Indonesian,Vietnamese": 0.36363636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.25 + "English,Chinese,Malay,Vietnamese": 0.2727272727272727, + "Filipino,Spanish,Chinese,Malay": 0.2727272727272727, + "Filipino,Spanish,Chinese,Vietnamese": 0.26704545454545453, + "Filipino,Spanish,Malay,Vietnamese": 0.32386363636363635, + "Filipino,Chinese,Malay,Vietnamese": 0.3068181818181818, + "Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Indonesian": 0.26136363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.2727272727272727, - "English,Filipino,Spanish,Chinese,Indonesian": 0.2215909090909091, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.32954545454545453, - "English,Filipino,Malay,Chinese,Indonesian": 0.17045454545454544, - "English,Filipino,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.2556818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456 + "Indonesian,English,Filipino,Spanish,Chinese": 0.20454545454545456, + "Indonesian,English,Filipino,Spanish,Malay": 0.2784090909090909, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.23863636363636365, + "Indonesian,English,Filipino,Chinese,Malay": 0.23295454545454544, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2159090909090909, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,English,Spanish,Chinese,Malay": 0.24431818181818182, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.23863636363636365, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.23863636363636365, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.23295454545454544, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2840909090909091, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.25, + "English,Filipino,Spanish,Chinese,Malay": 0.2159090909090909, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.2159090909090909, + "English,Filipino,Spanish,Malay,Vietnamese": 0.24431818181818182, + "English,Filipino,Chinese,Malay,Vietnamese": 0.22727272727272727, + "English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.19886363636363635, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.1875, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 } }, - "AC3_2": 0.4052360209330417, - "AC3_3": 0.347941215361435, - "AC3_4": 0.30124223597504185, - "AC3_5": 0.26042635376823986, - "AC3_6": 0.22277227718243303, - "AC3_7": 0.18581687608098266 + "AC3_2": 0.5095321454624216, + "AC3_3": 0.421883648075642, + "AC3_4": 0.36157638790355184, + "AC3_5": 0.3178619592208479, + "AC3_6": 0.2853323147018791, + "AC3_7": 0.26245059284520916 }, "prompt_4": { - "overall_acc": 0.3425324675324676, + "overall_acc": 0.4813311688311689, "language_acc": { - "English": 0.38636363636363635, - "Filipino": 0.20454545454545456, - "Spanish": 0.32386363636363635, - "Malay": 0.4034090909090909, - "Chinese": 0.3522727272727273, - "Indonesian": 0.4090909090909091, - "Vietnamese": 0.3181818181818182 + "Indonesian": 0.4431818181818182, + "English": 0.5681818181818182, + "Filipino": 0.4090909090909091, + "Spanish": 0.5397727272727273, + "Chinese": 0.5056818181818182, + "Malay": 0.4318181818181818, + "Vietnamese": 0.4715909090909091 }, - "consistency_score_2": 0.5313852813852813, - "consistency_score_3": 0.35616883116883125, - "consistency_score_4": 0.26266233766233765, - "consistency_score_5": 0.2026515151515152, - "consistency_score_6": 0.16071428571428573, - "consistency_score_7": 0.13068181818181818, + "consistency_score_2": 0.5827922077922079, + "consistency_score_3": 0.4183441558441558, + "consistency_score_4": 0.32987012987012987, + "consistency_score_5": 0.2740800865800866, + "consistency_score_6": 0.2362012987012987, + "consistency_score_7": 0.21022727272727273, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.6079545454545454, - "English,Spanish": 0.625, + "Indonesian,English": 0.5511363636363636, + "Indonesian,Filipino": 0.5397727272727273, + "Indonesian,Spanish": 0.6477272727272727, + "Indonesian,Chinese": 0.5340909090909091, + "Indonesian,Malay": 0.6761363636363636, + "Indonesian,Vietnamese": 0.5852272727272727, + "English,Filipino": 0.48863636363636365, + "English,Spanish": 0.7556818181818182, + "English,Chinese": 0.5965909090909091, "English,Malay": 0.5738636363636364, - "English,Chinese": 0.6704545454545454, - "English,Indonesian": 0.45454545454545453, - "English,Vietnamese": 0.6647727272727273, - "Filipino,Spanish": 0.5454545454545454, - "Filipino,Malay": 0.38636363636363635, - "Filipino,Chinese": 0.5625, - "Filipino,Indonesian": 0.3181818181818182, - "Filipino,Vietnamese": 0.5397727272727273, - "Spanish,Malay": 0.5511363636363636, - "Spanish,Chinese": 0.5113636363636364, - "Spanish,Indonesian": 0.48863636363636365, + "English,Vietnamese": 0.6363636363636364, + "Filipino,Spanish": 0.5284090909090909, + "Filipino,Chinese": 0.4772727272727273, + "Filipino,Malay": 0.6136363636363636, + "Filipino,Vietnamese": 0.5227272727272727, + "Spanish,Chinese": 0.5738636363636364, + "Spanish,Malay": 0.6534090909090909, "Spanish,Vietnamese": 0.625, - "Malay,Chinese": 0.4431818181818182, - "Malay,Indonesian": 0.6022727272727273, - "Malay,Vietnamese": 0.5625, - "Chinese,Indonesian": 0.4375, - "Chinese,Vietnamese": 0.5, - "Indonesian,Vietnamese": 0.48863636363636365 + "Chinese,Malay": 0.5, + "Chinese,Vietnamese": 0.6079545454545454, + "Malay,Vietnamese": 0.5511363636363636 }, "3_combine": { - "English,Filipino,Spanish": 0.4375, - "English,Filipino,Malay": 0.3352272727272727, - "English,Filipino,Chinese": 0.44886363636363635, - "English,Filipino,Indonesian": 0.26704545454545453, - "English,Filipino,Vietnamese": 0.44886363636363635, - "English,Spanish,Malay": 0.4147727272727273, - "English,Spanish,Chinese": 0.4431818181818182, - "English,Spanish,Indonesian": 0.3465909090909091, - "English,Spanish,Vietnamese": 0.5, - "English,Malay,Chinese": 0.4034090909090909, - "English,Malay,Indonesian": 0.39204545454545453, - "English,Malay,Vietnamese": 0.4375, - "English,Chinese,Indonesian": 0.3465909090909091, + "Indonesian,English,Filipino": 0.36363636363636365, + "Indonesian,English,Spanish": 0.5113636363636364, + "Indonesian,English,Chinese": 0.38636363636363635, + "Indonesian,English,Malay": 0.44886363636363635, + "Indonesian,English,Vietnamese": 0.4375, + "Indonesian,Filipino,Spanish": 0.4090909090909091, + "Indonesian,Filipino,Chinese": 0.32386363636363635, + "Indonesian,Filipino,Malay": 0.45454545454545453, + "Indonesian,Filipino,Vietnamese": 0.375, + "Indonesian,Spanish,Chinese": 0.42045454545454547, + "Indonesian,Spanish,Malay": 0.5284090909090909, + "Indonesian,Spanish,Vietnamese": 0.4659090909090909, + "Indonesian,Chinese,Malay": 0.39204545454545453, + "Indonesian,Chinese,Vietnamese": 0.4034090909090909, + "Indonesian,Malay,Vietnamese": 0.4375, + "English,Filipino,Spanish": 0.4318181818181818, + "English,Filipino,Chinese": 0.3465909090909091, + "English,Filipino,Malay": 0.38636363636363635, + "English,Filipino,Vietnamese": 0.3806818181818182, + "English,Spanish,Chinese": 0.4943181818181818, + "English,Spanish,Malay": 0.5227272727272727, + "English,Spanish,Vietnamese": 0.5340909090909091, + "English,Chinese,Malay": 0.375, "English,Chinese,Vietnamese": 0.45454545454545453, - "English,Indonesian,Vietnamese": 0.3693181818181818, - "Filipino,Spanish,Malay": 0.2897727272727273, - "Filipino,Spanish,Chinese": 0.35795454545454547, - "Filipino,Spanish,Indonesian": 0.25, - "Filipino,Spanish,Vietnamese": 0.4090909090909091, - "Filipino,Malay,Chinese": 0.2727272727272727, - "Filipino,Malay,Indonesian": 0.23863636363636365, - "Filipino,Malay,Vietnamese": 0.3125, - "Filipino,Chinese,Indonesian": 0.21022727272727273, - "Filipino,Chinese,Vietnamese": 0.35795454545454547, - "Filipino,Indonesian,Vietnamese": 0.25, - "Spanish,Malay,Chinese": 0.32386363636363635, - "Spanish,Malay,Indonesian": 0.4034090909090909, - "Spanish,Malay,Vietnamese": 0.3977272727272727, - "Spanish,Chinese,Indonesian": 0.2784090909090909, - "Spanish,Chinese,Vietnamese": 0.39204545454545453, - "Spanish,Indonesian,Vietnamese": 0.3465909090909091, - "Malay,Chinese,Indonesian": 0.3181818181818182, - "Malay,Chinese,Vietnamese": 0.32954545454545453, - "Malay,Indonesian,Vietnamese": 0.38636363636363635, - "Chinese,Indonesian,Vietnamese": 0.29545454545454547 + "English,Malay,Vietnamese": 0.42613636363636365, + "Filipino,Spanish,Chinese": 0.3465909090909091, + "Filipino,Spanish,Malay": 0.4318181818181818, + "Filipino,Spanish,Vietnamese": 0.3806818181818182, + "Filipino,Chinese,Malay": 0.3465909090909091, + "Filipino,Chinese,Vietnamese": 0.3522727272727273, + "Filipino,Malay,Vietnamese": 0.39204545454545453, + "Spanish,Chinese,Malay": 0.4090909090909091, + "Spanish,Chinese,Vietnamese": 0.4375, + "Spanish,Malay,Vietnamese": 0.45454545454545453, + "Chinese,Malay,Vietnamese": 0.3806818181818182 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.2784090909090909, - "English,Filipino,Spanish,Chinese": 0.3181818181818182, - "English,Filipino,Spanish,Indonesian": 0.2215909090909091, - "English,Filipino,Spanish,Vietnamese": 0.36363636363636365, - "English,Filipino,Malay,Chinese": 0.26136363636363635, - "English,Filipino,Malay,Indonesian": 0.22727272727272727, - "English,Filipino,Malay,Vietnamese": 0.2784090909090909, - "English,Filipino,Chinese,Indonesian": 0.20454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.32954545454545453, - "English,Filipino,Indonesian,Vietnamese": 0.23863636363636365, - "English,Spanish,Malay,Chinese": 0.3068181818181818, - "English,Spanish,Malay,Indonesian": 0.3125, - "English,Spanish,Malay,Vietnamese": 0.3465909090909091, - "English,Spanish,Chinese,Indonesian": 0.2556818181818182, - "English,Spanish,Chinese,Vietnamese": 0.35795454545454547, - "English,Spanish,Indonesian,Vietnamese": 0.29545454545454547, - "English,Malay,Chinese,Indonesian": 0.29545454545454547, - "English,Malay,Chinese,Vietnamese": 0.3068181818181818, - "English,Malay,Indonesian,Vietnamese": 0.32386363636363635, - "English,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Malay,Chinese": 0.2215909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.20454545454545456, - "Filipino,Spanish,Malay,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Chinese,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1875, - "Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.29545454545454547, - "Spanish,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Malay,Chinese,Indonesian,Vietnamese": 0.25 + "Indonesian,English,Filipino,Spanish": 0.3465909090909091, + "Indonesian,English,Filipino,Chinese": 0.26704545454545453, + "Indonesian,English,Filipino,Malay": 0.32386363636363635, + "Indonesian,English,Filipino,Vietnamese": 0.30113636363636365, + "Indonesian,English,Spanish,Chinese": 0.36363636363636365, + "Indonesian,English,Spanish,Malay": 0.42613636363636365, + "Indonesian,English,Spanish,Vietnamese": 0.4147727272727273, + "Indonesian,English,Chinese,Malay": 0.3125, + "Indonesian,English,Chinese,Vietnamese": 0.32954545454545453, + "Indonesian,English,Malay,Vietnamese": 0.35795454545454547, + "Indonesian,Filipino,Spanish,Chinese": 0.2727272727272727, + "Indonesian,Filipino,Spanish,Malay": 0.375, + "Indonesian,Filipino,Spanish,Vietnamese": 0.3125, + "Indonesian,Filipino,Chinese,Malay": 0.2784090909090909, + "Indonesian,Filipino,Chinese,Vietnamese": 0.2784090909090909, + "Indonesian,Filipino,Malay,Vietnamese": 0.3181818181818182, + "Indonesian,Spanish,Chinese,Malay": 0.3465909090909091, + "Indonesian,Spanish,Chinese,Vietnamese": 0.3522727272727273, + "Indonesian,Spanish,Malay,Vietnamese": 0.38636363636363635, + "Indonesian,Chinese,Malay,Vietnamese": 0.3181818181818182, + "English,Filipino,Spanish,Chinese": 0.3125, + "English,Filipino,Spanish,Malay": 0.35795454545454547, + "English,Filipino,Spanish,Vietnamese": 0.3352272727272727, + "English,Filipino,Chinese,Malay": 0.2784090909090909, + "English,Filipino,Chinese,Vietnamese": 0.2840909090909091, + "English,Filipino,Malay,Vietnamese": 0.3125, + "English,Spanish,Chinese,Malay": 0.35795454545454547, + "English,Spanish,Chinese,Vietnamese": 0.3977272727272727, + "English,Spanish,Malay,Vietnamese": 0.39204545454545453, + "English,Chinese,Malay,Vietnamese": 0.3181818181818182, + "Filipino,Spanish,Chinese,Malay": 0.2840909090909091, + "Filipino,Spanish,Chinese,Vietnamese": 0.2840909090909091, + "Filipino,Spanish,Malay,Vietnamese": 0.32386363636363635, + "Filipino,Chinese,Malay,Vietnamese": 0.2897727272727273, + "Spanish,Chinese,Malay,Vietnamese": 0.3352272727272727 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.2159090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, - "English,Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.26704545454545453, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.17613636363636365, - "English,Filipino,Malay,Chinese,Vietnamese": 0.21022727272727273, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.23295454545454544, - "English,Spanish,Malay,Chinese,Vietnamese": 0.25, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1875, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635 + "Indonesian,English,Filipino,Spanish,Chinese": 0.2556818181818182, + "Indonesian,English,Filipino,Spanish,Malay": 0.3125, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2840909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.23863636363636365, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.23863636363636365, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,English,Spanish,Chinese,Malay": 0.3068181818181818, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.3181818181818182, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.3409090909090909, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.2727272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.25, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.24431818181818182, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2840909090909091, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.24431818181818182, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.29545454545454547, + "English,Filipino,Spanish,Chinese,Malay": 0.26136363636363635, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.26136363636363635, + "English,Filipino,Spanish,Malay,Vietnamese": 0.2840909090909091, + "English,Filipino,Chinese,Malay,Vietnamese": 0.24431818181818182, + "English,Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.25 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.18181818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.23295454545454544, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.22727272727272727, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.2159090909090909, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273 } }, - "AC3_2": 0.416553415576958, - "AC3_3": 0.34921758069641023, - "AC3_4": 0.2973270023476157, - "AC3_5": 0.2546469658905801, - "AC3_6": 0.21877880179984455, - "AC3_7": 0.1891860283398745 + "AC3_2": 0.5272246822094617, + "AC3_3": 0.447632775871545, + "AC3_4": 0.39146085048803886, + "AC3_5": 0.34927541114715466, + "AC3_6": 0.31689450251210277, + "AC3_7": 0.29264031152405184 }, "prompt_5": { - "overall_acc": 0.3490259740259741, + "overall_acc": 0.45454545454545453, "language_acc": { - "English": 0.38636363636363635, - "Filipino": 0.2215909090909091, - "Spanish": 0.35795454545454547, - "Malay": 0.4147727272727273, - "Chinese": 0.3125, - "Indonesian": 0.4034090909090909, - "Vietnamese": 0.3465909090909091 + "Indonesian": 0.4375, + "English": 0.5056818181818182, + "Filipino": 0.4147727272727273, + "Spanish": 0.48863636363636365, + "Chinese": 0.42613636363636365, + "Malay": 0.44886363636363635, + "Vietnamese": 0.4602272727272727 }, - "consistency_score_2": 0.5313852813852813, - "consistency_score_3": 0.3517857142857143, - "consistency_score_4": 0.25633116883116885, - "consistency_score_5": 0.19480519480519481, - "consistency_score_6": 0.15097402597402598, - "consistency_score_7": 0.11931818181818182, + "consistency_score_2": 0.5454545454545455, + "consistency_score_3": 0.3748376623376623, + "consistency_score_4": 0.2870129870129871, + "consistency_score_5": 0.23376623376623382, + "consistency_score_6": 0.19724025974025977, + "consistency_score_7": 0.17045454545454544, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.5568181818181818, - "English,Spanish": 0.6136363636363636, - "English,Malay": 0.5284090909090909, - "English,Chinese": 0.5681818181818182, - "English,Indonesian": 0.4602272727272727, - "English,Vietnamese": 0.6420454545454546, - "Filipino,Spanish": 0.6363636363636364, - "Filipino,Malay": 0.4034090909090909, - "Filipino,Chinese": 0.6193181818181818, - "Filipino,Indonesian": 0.3522727272727273, - "Filipino,Vietnamese": 0.5568181818181818, - "Spanish,Malay": 0.5738636363636364, + "Indonesian,English": 0.5113636363636364, + "Indonesian,Filipino": 0.5625, + "Indonesian,Spanish": 0.5852272727272727, + "Indonesian,Chinese": 0.5113636363636364, + "Indonesian,Malay": 0.6420454545454546, + "Indonesian,Vietnamese": 0.5681818181818182, + "English,Filipino": 0.4375, + "English,Spanish": 0.6534090909090909, + "English,Chinese": 0.5, + "English,Malay": 0.5113636363636364, + "English,Vietnamese": 0.5170454545454546, + "Filipino,Spanish": 0.5113636363636364, + "Filipino,Chinese": 0.4602272727272727, + "Filipino,Malay": 0.6079545454545454, + "Filipino,Vietnamese": 0.5227272727272727, "Spanish,Chinese": 0.5227272727272727, - "Spanish,Indonesian": 0.4659090909090909, - "Spanish,Vietnamese": 0.6193181818181818, - "Malay,Chinese": 0.44886363636363635, - "Malay,Indonesian": 0.6022727272727273, - "Malay,Vietnamese": 0.5681818181818182, - "Chinese,Indonesian": 0.4034090909090909, - "Chinese,Vietnamese": 0.5397727272727273, - "Indonesian,Vietnamese": 0.4772727272727273 + "Spanish,Malay": 0.6136363636363636, + "Spanish,Vietnamese": 0.6022727272727273, + "Chinese,Malay": 0.5284090909090909, + "Chinese,Vietnamese": 0.5, + "Malay,Vietnamese": 0.5852272727272727 }, "3_combine": { - "English,Filipino,Spanish": 0.4431818181818182, - "English,Filipino,Malay": 0.29545454545454547, - "English,Filipino,Chinese": 0.3977272727272727, - "English,Filipino,Indonesian": 0.25, - "English,Filipino,Vietnamese": 0.42045454545454547, - "English,Spanish,Malay": 0.4090909090909091, - "English,Spanish,Chinese": 0.3977272727272727, - "English,Spanish,Indonesian": 0.3352272727272727, - "English,Spanish,Vietnamese": 0.48295454545454547, - "English,Malay,Chinese": 0.3352272727272727, - "English,Malay,Indonesian": 0.3409090909090909, - "English,Malay,Vietnamese": 0.4147727272727273, - "English,Chinese,Indonesian": 0.2840909090909091, - "English,Chinese,Vietnamese": 0.4034090909090909, - "English,Indonesian,Vietnamese": 0.3522727272727273, - "Filipino,Spanish,Malay": 0.3409090909090909, - "Filipino,Spanish,Chinese": 0.42613636363636365, - "Filipino,Spanish,Indonesian": 0.2897727272727273, - "Filipino,Spanish,Vietnamese": 0.44886363636363635, - "Filipino,Malay,Chinese": 0.2840909090909091, - "Filipino,Malay,Indonesian": 0.26136363636363635, - "Filipino,Malay,Vietnamese": 0.3352272727272727, - "Filipino,Chinese,Indonesian": 0.23863636363636365, - "Filipino,Chinese,Vietnamese": 0.3977272727272727, - "Filipino,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Chinese": 0.3352272727272727, - "Spanish,Malay,Indonesian": 0.3693181818181818, - "Spanish,Malay,Vietnamese": 0.4375, - "Spanish,Chinese,Indonesian": 0.26136363636363635, - "Spanish,Chinese,Vietnamese": 0.3977272727272727, - "Spanish,Indonesian,Vietnamese": 0.3465909090909091, - "Malay,Chinese,Indonesian": 0.30113636363636365, - "Malay,Chinese,Vietnamese": 0.3352272727272727, - "Malay,Indonesian,Vietnamese": 0.38636363636363635, - "Chinese,Indonesian,Vietnamese": 0.2727272727272727 + "Indonesian,English,Filipino": 0.3352272727272727, + "Indonesian,English,Spanish": 0.4090909090909091, + "Indonesian,English,Chinese": 0.32386363636363635, + "Indonesian,English,Malay": 0.39204545454545453, + "Indonesian,English,Vietnamese": 0.3522727272727273, + "Indonesian,Filipino,Spanish": 0.3977272727272727, + "Indonesian,Filipino,Chinese": 0.32386363636363635, + "Indonesian,Filipino,Malay": 0.4602272727272727, + "Indonesian,Filipino,Vietnamese": 0.38636363636363635, + "Indonesian,Spanish,Chinese": 0.35795454545454547, + "Indonesian,Spanish,Malay": 0.4772727272727273, + "Indonesian,Spanish,Vietnamese": 0.42045454545454547, + "Indonesian,Chinese,Malay": 0.3977272727272727, + "Indonesian,Chinese,Vietnamese": 0.3465909090909091, + "Indonesian,Malay,Vietnamese": 0.4602272727272727, + "English,Filipino,Spanish": 0.35795454545454547, + "English,Filipino,Chinese": 0.29545454545454547, + "English,Filipino,Malay": 0.32954545454545453, + "English,Filipino,Vietnamese": 0.3125, + "English,Spanish,Chinese": 0.375, + "English,Spanish,Malay": 0.4318181818181818, + "English,Spanish,Vietnamese": 0.42613636363636365, + "English,Chinese,Malay": 0.32386363636363635, + "English,Chinese,Vietnamese": 0.3068181818181818, + "English,Malay,Vietnamese": 0.36363636363636365, + "Filipino,Spanish,Chinese": 0.32954545454545453, + "Filipino,Spanish,Malay": 0.42045454545454547, + "Filipino,Spanish,Vietnamese": 0.375, + "Filipino,Chinese,Malay": 0.3522727272727273, + "Filipino,Chinese,Vietnamese": 0.3181818181818182, + "Filipino,Malay,Vietnamese": 0.4090909090909091, + "Spanish,Chinese,Malay": 0.3806818181818182, + "Spanish,Chinese,Vietnamese": 0.36363636363636365, + "Spanish,Malay,Vietnamese": 0.4431818181818182, + "Chinese,Malay,Vietnamese": 0.36363636363636365 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.26136363636363635, - "English,Filipino,Spanish,Chinese": 0.3181818181818182, - "English,Filipino,Spanish,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Vietnamese": 0.35795454545454547, - "English,Filipino,Malay,Chinese": 0.22727272727272727, - "English,Filipino,Malay,Indonesian": 0.1875, - "English,Filipino,Malay,Vietnamese": 0.26136363636363635, - "English,Filipino,Chinese,Indonesian": 0.1875, - "English,Filipino,Chinese,Vietnamese": 0.3125, - "English,Filipino,Indonesian,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay,Chinese": 0.2784090909090909, - "English,Spanish,Malay,Indonesian": 0.26136363636363635, - "English,Spanish,Malay,Vietnamese": 0.3522727272727273, - "English,Spanish,Chinese,Indonesian": 0.2215909090909091, - "English,Spanish,Chinese,Vietnamese": 0.32386363636363635, - "English,Spanish,Indonesian,Vietnamese": 0.2840909090909091, - "English,Malay,Chinese,Indonesian": 0.2215909090909091, - "English,Malay,Chinese,Vietnamese": 0.2784090909090909, - "English,Malay,Indonesian,Vietnamese": 0.2897727272727273, - "English,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Malay,Chinese": 0.2556818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.32386363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.25, - "Filipino,Malay,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.25, - "Filipino,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese,Indonesian": 0.2215909090909091, - "Spanish,Malay,Chinese,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.29545454545454547, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182 + "Indonesian,English,Filipino,Spanish": 0.2897727272727273, + "Indonesian,English,Filipino,Chinese": 0.23863636363636365, + "Indonesian,English,Filipino,Malay": 0.30113636363636365, + "Indonesian,English,Filipino,Vietnamese": 0.2556818181818182, + "Indonesian,English,Spanish,Chinese": 0.2727272727272727, + "Indonesian,English,Spanish,Malay": 0.3522727272727273, + "Indonesian,English,Spanish,Vietnamese": 0.3181818181818182, + "Indonesian,English,Chinese,Malay": 0.2727272727272727, + "Indonesian,English,Chinese,Vietnamese": 0.23295454545454544, + "Indonesian,English,Malay,Vietnamese": 0.3181818181818182, + "Indonesian,Filipino,Spanish,Chinese": 0.25, + "Indonesian,Filipino,Spanish,Malay": 0.36363636363636365, + "Indonesian,Filipino,Spanish,Vietnamese": 0.3125, + "Indonesian,Filipino,Chinese,Malay": 0.2897727272727273, + "Indonesian,Filipino,Chinese,Vietnamese": 0.25, + "Indonesian,Filipino,Malay,Vietnamese": 0.35795454545454547, + "Indonesian,Spanish,Chinese,Malay": 0.30113636363636365, + "Indonesian,Spanish,Chinese,Vietnamese": 0.26704545454545453, + "Indonesian,Spanish,Malay,Vietnamese": 0.3806818181818182, + "Indonesian,Chinese,Malay,Vietnamese": 0.3068181818181818, + "English,Filipino,Spanish,Chinese": 0.24431818181818182, + "English,Filipino,Spanish,Malay": 0.29545454545454547, + "English,Filipino,Spanish,Vietnamese": 0.2727272727272727, + "English,Filipino,Chinese,Malay": 0.22727272727272727, + "English,Filipino,Chinese,Vietnamese": 0.2215909090909091, + "English,Filipino,Malay,Vietnamese": 0.2727272727272727, + "English,Spanish,Chinese,Malay": 0.2897727272727273, + "English,Spanish,Chinese,Vietnamese": 0.26136363636363635, + "English,Spanish,Malay,Vietnamese": 0.32954545454545453, + "English,Chinese,Malay,Vietnamese": 0.2556818181818182, + "Filipino,Spanish,Chinese,Malay": 0.2727272727272727, + "Filipino,Spanish,Chinese,Vietnamese": 0.2556818181818182, + "Filipino,Spanish,Malay,Vietnamese": 0.3409090909090909, + "Filipino,Chinese,Malay,Vietnamese": 0.2727272727272727, + "Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.20454545454545456, - "English,Filipino,Spanish,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, - "English,Filipino,Spanish,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.26136363636363635, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Malay,Chinese,Vietnamese": 0.20454545454545456, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Chinese,Indonesian": 0.18181818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635 + "Indonesian,English,Filipino,Spanish,Chinese": 0.20454545454545456, + "Indonesian,English,Filipino,Spanish,Malay": 0.2727272727272727, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.23863636363636365, + "Indonesian,English,Filipino,Chinese,Malay": 0.2159090909090909, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1875, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,English,Spanish,Chinese,Malay": 0.24431818181818182, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.21022727272727273, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.29545454545454547, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.22727272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.23295454545454544, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.3068181818181818, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.24431818181818182, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.25, + "English,Filipino,Spanish,Chinese,Malay": 0.20454545454545456, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, + "English,Filipino,Spanish,Malay,Vietnamese": 0.2556818181818182, + "English,Filipino,Chinese,Malay,Vietnamese": 0.19886363636363635, + "English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.18181818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1875, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 } }, - "AC3_2": 0.421319614572313, - "AC3_3": 0.3504004102878677, - "AC3_4": 0.29558166419735793, - "AC3_5": 0.25004845895770383, - "AC3_6": 0.21077542583067344, - "AC3_7": 0.1778399243357478 + "AC3_2": 0.4958677685454546, + "AC3_3": 0.41086140317827724, + "AC3_4": 0.35185480014360443, + "AC3_5": 0.30874785587281145, + "AC3_6": 0.2751047208906576, + "AC3_7": 0.2479338842578512 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.3786407766990291 + "accuracy": 0.6407766990291263 }, "prompt_2": { - "accuracy": 0.3106796116504854 + "accuracy": 0.6213592233009708 }, "prompt_3": { - "accuracy": 0.2621359223300971 + "accuracy": 0.6019417475728155 }, "prompt_4": { - "accuracy": 0.2621359223300971 + "accuracy": 0.6019417475728155 }, "prompt_5": { - "accuracy": 0.39805825242718446 + "accuracy": 0.6407766990291263 } }, "cn_eval": { @@ -43957,13614 +32000,12228 @@ "accuracy": 0.41904761904761906 }, "prompt_2": { - "accuracy": 0.41904761904761906 + "accuracy": 0.4857142857142857 }, "prompt_3": { - "accuracy": 0.38095238095238093 + "accuracy": 0.5238095238095238 }, "prompt_4": { - "accuracy": 0.45714285714285713 + "accuracy": 0.4666666666666667 }, "prompt_5": { - "accuracy": 0.44761904761904764 + "accuracy": 0.4857142857142857 } }, "us_eval": { "prompt_1": { - "accuracy": 0.3925233644859813 + "accuracy": 0.7009345794392523 }, "prompt_2": { - "accuracy": 0.3364485981308411 + "accuracy": 0.7009345794392523 }, "prompt_3": { - "accuracy": 0.3177570093457944 + "accuracy": 0.6915887850467289 }, "prompt_4": { - "accuracy": 0.27102803738317754 + "accuracy": 0.7009345794392523 }, "prompt_5": { - "accuracy": 0.37383177570093457 + "accuracy": 0.7009345794392523 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.34, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.2, - "history": 0.2, - "literature": 0.1, - "politics": 0.6, - "culture": 0.5, - "film": 0.2, - "law": 0.7, - "geography": 0.3 - } - }, - "prompt_2": { - "accuracy": 0.3, + "accuracy": 0.52, "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.2, - "history": 0.2, - "literature": 0.1, - "politics": 0.6, - "culture": 0.3, - "film": 0.1, + "brand": 0.5, + "demographics": 0.6, + "biology": 0.5, + "history": 0.4, + "literature": 0.2, + "politics": 0.7, + "culture": 0.8, + "film": 0.5, "law": 0.5, - "geography": 0.4 + "geography": 0.6 } }, - "prompt_3": { - "accuracy": 0.33, + "prompt_2": { + "accuracy": 0.52, "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.2, - "history": 0.2, + "brand": 0.6, + "demographics": 0.4, + "biology": 0.6, + "history": 0.4, "literature": 0.2, "politics": 0.6, - "culture": 0.3, - "film": 0.2, + "culture": 0.7, + "film": 0.6, + "law": 0.6, + "geography": 0.5 + } + }, + "prompt_3": { + "accuracy": 0.52, + "category_acc": { + "brand": 0.5, + "demographics": 0.2, + "biology": 0.5, + "history": 0.5333333333333333, + "literature": 0.3, + "politics": 0.7, + "culture": 0.8, + "film": 0.4, "law": 0.6, "geography": 0.5 } }, "prompt_4": { - "accuracy": 0.31, + "accuracy": 0.58, "category_acc": { - "brand": 0.4, - "demographics": 0.6, - "biology": 0.0, - "history": 0.2, + "brand": 0.6, + "demographics": 0.4, + "biology": 0.6, + "history": 0.4666666666666667, "literature": 0.3, - "politics": 0.5, - "culture": 0.2, - "film": 0.3, - "law": 0.5, - "geography": 0.3 + "politics": 0.7, + "culture": 0.8, + "film": 0.5, + "law": 0.7, + "geography": 0.7 } }, "prompt_5": { - "accuracy": 0.35, + "accuracy": 0.59, "category_acc": { - "brand": 0.3, - "demographics": 0.6, - "biology": 0.3, - "history": 0.2, + "brand": 0.5, + "demographics": 0.4, + "biology": 0.6, + "history": 0.6, "literature": 0.2, - "politics": 0.6, - "culture": 0.3, - "film": 0.2, - "law": 0.6, - "geography": 0.4 + "politics": 0.9, + "culture": 0.8, + "film": 0.5, + "law": 0.7, + "geography": 0.6 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.21069781448240776 + "bleu_score": 0.06094069384356101 }, "prompt_2": { - "bleu_score": 0.21040249436920064 + "bleu_score": 0.06207526753612465 }, "prompt_3": { - "bleu_score": 0.19143722930997414 + "bleu_score": 0.06192642106229149 }, "prompt_4": { - "bleu_score": 0.21380402709568694 + "bleu_score": 0.060092982025264 }, "prompt_5": { - "bleu_score": 0.16840887427398613 + "bleu_score": 0.05469202971301356 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "prompt_1": { + "accuracy": 0.5280726350223647, + "category_acc": { + "History": 0.5240963855421686, + "Geography": 0.5020408163265306, + "Lampungic": 0.3741496598639456, + "Social science": 0.7495826377295493, + "Balinese": 0.30148619957537154, + "Makassarese": 0.3763440860215054, + "Banjarese": 0.4444444444444444, + "Chemistry": 0.31386861313868614, + "Biology": 0.5029585798816568, + "Science": 0.6780185758513931, + "Christian religion": 0.6616915422885572, + "Art": 0.5940099833610649, + "Islam religion": 0.65149359886202, + "Hindu religion": 0.5466666666666666, + "Madurese": 0.288135593220339, + "Sport": 0.5337837837837838, + "Indonesian language": 0.587173100871731, + "Physics": 0.4686868686868687, + "Minangkabau culture": 0.457286432160804, + "Dayak language": 0.3486238532110092, + "Sociology": 0.5463709677419355, + "Economy": 0.5020491803278688, + "Sundanese": 0.4606741573033708, + "Javanese": 0.4092741935483871, + "Civic education": 0.6151645207439199 + } + }, + "prompt_2": { + "accuracy": 0.530676280125509, + "category_acc": { + "History": 0.5160642570281124, + "Geography": 0.5, + "Lampungic": 0.35374149659863946, + "Social science": 0.7612687813021702, + "Balinese": 0.28662420382165604, + "Makassarese": 0.3655913978494624, + "Banjarese": 0.4375, + "Chemistry": 0.345985401459854, + "Biology": 0.5076923076923077, + "Science": 0.6904024767801857, + "Christian religion": 0.6567164179104478, + "Art": 0.6073211314475874, + "Islam religion": 0.6486486486486487, + "Hindu religion": 0.58, + "Madurese": 0.3593220338983051, + "Sport": 0.5337837837837838, + "Indonesian language": 0.5840597758405978, + "Physics": 0.4909090909090909, + "Minangkabau culture": 0.4221105527638191, + "Dayak language": 0.24770642201834864, + "Sociology": 0.5504032258064516, + "Economy": 0.4979508196721312, + "Sundanese": 0.4520311149524633, + "Javanese": 0.42338709677419356, + "Civic education": 0.6065808297567954 + } + }, + "prompt_3": { + "accuracy": 0.5258695507043194, + "category_acc": { + "History": 0.5140562248995983, + "Geography": 0.5142857142857142, + "Lampungic": 0.3469387755102041, + "Social science": 0.7462437395659433, + "Balinese": 0.29723991507430997, + "Makassarese": 0.3602150537634409, + "Banjarese": 0.4097222222222222, + "Chemistry": 0.31386861313868614, + "Biology": 0.5183431952662721, + "Science": 0.6790505675954592, + "Christian religion": 0.6517412935323383, + "Art": 0.6073211314475874, + "Islam religion": 0.635846372688478, + "Hindu religion": 0.58, + "Madurese": 0.3423728813559322, + "Sport": 0.527027027027027, + "Indonesian language": 0.5874844333748444, + "Physics": 0.4585858585858586, + "Minangkabau culture": 0.46733668341708545, + "Dayak language": 0.26605504587155965, + "Sociology": 0.5362903225806451, + "Economy": 0.48565573770491804, + "Sundanese": 0.44511668107173724, + "Javanese": 0.41330645161290325, + "Civic education": 0.6022889842632332 + } + }, + "prompt_4": { + "accuracy": 0.5315441618265572, + "category_acc": { + "History": 0.5261044176706827, + "Geography": 0.5244897959183673, + "Lampungic": 0.36054421768707484, + "Social science": 0.7479131886477463, + "Balinese": 0.3205944798301486, + "Makassarese": 0.4032258064516129, + "Banjarese": 0.4166666666666667, + "Chemistry": 0.3284671532846715, + "Biology": 0.5360946745562131, + "Science": 0.6862745098039216, + "Christian religion": 0.6766169154228856, + "Art": 0.6023294509151415, + "Islam religion": 0.6273115220483642, + "Hindu religion": 0.56, + "Madurese": 0.3389830508474576, + "Sport": 0.527027027027027, + "Indonesian language": 0.5896637608966376, + "Physics": 0.45252525252525255, + "Minangkabau culture": 0.4623115577889447, + "Dayak language": 0.23853211009174313, + "Sociology": 0.5725806451612904, + "Economy": 0.5122950819672131, + "Sundanese": 0.452895419187554, + "Javanese": 0.39818548387096775, + "Civic education": 0.6051502145922747 + } + }, + "prompt_5": { + "accuracy": 0.5029040656919688, + "category_acc": { + "History": 0.4779116465863454, + "Geography": 0.4714285714285714, + "Lampungic": 0.38095238095238093, + "Social science": 0.6944908180300501, + "Balinese": 0.3057324840764331, + "Makassarese": 0.2903225806451613, + "Banjarese": 0.4027777777777778, + "Chemistry": 0.2934306569343066, + "Biology": 0.46153846153846156, + "Science": 0.6295149638802889, + "Christian religion": 0.6567164179104478, + "Art": 0.589018302828619, + "Islam religion": 0.6443812233285917, + "Hindu religion": 0.52, + "Madurese": 0.30847457627118646, + "Sport": 0.5135135135135135, + "Indonesian language": 0.5716064757160647, + "Physics": 0.4484848484848485, + "Minangkabau culture": 0.4271356783919598, + "Dayak language": 0.29357798165137616, + "Sociology": 0.48185483870967744, + "Economy": 0.4672131147540984, + "Sundanese": 0.4399308556611927, + "Javanese": 0.39818548387096775, + "Civic education": 0.5793991416309013 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.392668250609665 + "bleu_score": 0.07673773359783945 }, "prompt_2": { - "bleu_score": 0.2943468238807213 + "bleu_score": 0.08178489772334997 }, "prompt_3": { - "bleu_score": 0.2568274614648789 + "bleu_score": 0.08148612648063638 }, "prompt_4": { - "bleu_score": 0.3937874916781983 + "bleu_score": 0.07978071248841677 }, "prompt_5": { - "bleu_score": 0.36952569223201265 + "bleu_score": 0.08665723913254166 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.3323812043258411 + "bleu_score": 0.062421379207404384 }, "prompt_2": { - "bleu_score": 0.19971677891323225 + "bleu_score": 0.0654551871178475 }, "prompt_3": { - "bleu_score": 0.09956357193333733 + "bleu_score": 0.06586572647847766 }, "prompt_4": { - "bleu_score": 0.33200675697375076 + "bleu_score": 0.06481966663780118 }, "prompt_5": { - "bleu_score": 0.3177350450908323 + "bleu_score": 0.07179447292441708 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.19128728286038182 + "bleu_score": 0.050971705367587626 }, "prompt_2": { - "bleu_score": 0.224505016163668 + "bleu_score": 0.05134705353375384 }, "prompt_3": { - "bleu_score": 0.08608562591404366 + "bleu_score": 0.051760820535713056 }, "prompt_4": { - "bleu_score": 0.23640814308044278 + "bleu_score": 0.05136651836443981 }, "prompt_5": { - "bleu_score": 0.21617227141667897 + "bleu_score": 0.05595680934355571 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.37060213342807335 + "bleu_score": 0.07792654748439234 }, "prompt_2": { - "bleu_score": 0.28711242864582115 + "bleu_score": 0.08240154342677156 }, "prompt_3": { - "bleu_score": 0.17589584416647278 + "bleu_score": 0.08322691501291536 }, "prompt_4": { - "bleu_score": 0.3906242114282589 + "bleu_score": 0.07956020114017891 }, "prompt_5": { - "bleu_score": 0.35602597316783874 + "bleu_score": 0.08635159030424207 } }, "mmlu": { "prompt_1": { - "accuracy": 0.34655775962660446 + "accuracy": 0.6254375729288215 }, "prompt_2": { - "accuracy": 0.3617269544924154 + "accuracy": 0.646441073512252 }, "prompt_3": { - "accuracy": 0.39556592765460913 + "accuracy": 0.6277712952158693 }, "prompt_4": { - "accuracy": 0.3570595099183197 + "accuracy": 0.6394399066511085 }, "prompt_5": { - "accuracy": 0.4049008168028005 + "accuracy": 0.588098016336056 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.34036467643904184, + "accuracy": 0.619449410082231, "category_acc": { - "high_school_european_history": 0.4573170731707317, - "business_ethics": 0.37373737373737376, - "clinical_knowledge": 0.35984848484848486, - "medical_genetics": 0.3434343434343434, - "high_school_us_history": 0.5270935960591133, - "high_school_physics": 0.23333333333333334, - "high_school_world_history": 0.4830508474576271, - "virology": 0.2909090909090909, - "high_school_microeconomics": 0.350210970464135, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.29292929292929293, - "high_school_biology": 0.3818770226537217, - "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.28113879003558717, - "philosophy": 0.3935483870967742, - "professional_medicine": 0.3800738007380074, - "nutrition": 0.38688524590163936, - "global_facts": 0.2727272727272727, - "machine_learning": 0.32432432432432434, - "security_studies": 0.2540983606557377, - "public_relations": 0.3119266055045872, - "professional_psychology": 0.3322422258592471, - "prehistory": 0.33126934984520123, - "anatomy": 0.3880597014925373, - "human_sexuality": 0.3384615384615385, - "college_medicine": 0.28488372093023256, - "high_school_government_and_politics": 0.4010416666666667, - "college_chemistry": 0.2222222222222222, - "logical_fallacies": 0.41975308641975306, - "high_school_geography": 0.30456852791878175, - "elementary_mathematics": 0.28116710875331563, - "human_aging": 0.3918918918918919, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.40441176470588236, - "formal_logic": 0.264, - "high_school_statistics": 0.20465116279069767, - "international_law": 0.4083333333333333, - "high_school_mathematics": 0.2899628252788104, - "high_school_computer_science": 0.2828282828282828, - "conceptual_physics": 0.36324786324786323, - "miscellaneous": 0.4040920716112532, - "high_school_chemistry": 0.3069306930693069, - "marketing": 0.47639484978540775, - "professional_law": 0.30463144161774297, - "management": 0.3137254901960784, - "college_physics": 0.27722772277227725, - "jurisprudence": 0.4485981308411215, - "world_religions": 0.4647058823529412, - "sociology": 0.365, - "us_foreign_policy": 0.41414141414141414, - "high_school_macroeconomics": 0.30077120822622105, - "computer_security": 0.4444444444444444, - "moral_scenarios": 0.23042505592841164, - "moral_disputes": 0.3652173913043478, - "electrical_engineering": 0.3611111111111111, - "astronomy": 0.3576158940397351, - "college_biology": 0.4195804195804196 + "high_school_european_history": 0.7378048780487805, + "business_ethics": 0.5656565656565656, + "clinical_knowledge": 0.6856060606060606, + "medical_genetics": 0.7171717171717171, + "high_school_us_history": 0.7635467980295566, + "high_school_physics": 0.2866666666666667, + "high_school_world_history": 0.7796610169491526, + "virology": 0.4909090909090909, + "high_school_microeconomics": 0.6835443037974683, + "econometrics": 0.4336283185840708, + "college_computer_science": 0.4444444444444444, + "high_school_biology": 0.7216828478964401, + "abstract_algebra": 0.32323232323232326, + "professional_accounting": 0.47330960854092524, + "philosophy": 0.6483870967741936, + "professional_medicine": 0.7343173431734318, + "nutrition": 0.7475409836065574, + "global_facts": 0.5252525252525253, + "machine_learning": 0.3783783783783784, + "security_studies": 0.6352459016393442, + "public_relations": 0.6146788990825688, + "professional_psychology": 0.6415711947626841, + "prehistory": 0.7058823529411765, + "anatomy": 0.664179104477612, + "human_sexuality": 0.7230769230769231, + "college_medicine": 0.6046511627906976, + "high_school_government_and_politics": 0.8333333333333334, + "college_chemistry": 0.41414141414141414, + "logical_fallacies": 0.7469135802469136, + "high_school_geography": 0.7360406091370558, + "elementary_mathematics": 0.7029177718832891, + "human_aging": 0.6621621621621622, + "college_mathematics": 0.23232323232323232, + "high_school_psychology": 0.8051470588235294, + "formal_logic": 0.456, + "high_school_statistics": 0.4511627906976744, + "international_law": 0.75, + "high_school_mathematics": 0.3048327137546468, + "high_school_computer_science": 0.696969696969697, + "conceptual_physics": 0.5555555555555556, + "miscellaneous": 0.8414322250639387, + "high_school_chemistry": 0.4801980198019802, + "marketing": 0.8497854077253219, + "professional_law": 0.4592302674494455, + "management": 0.7647058823529411, + "college_physics": 0.3564356435643564, + "jurisprudence": 0.7383177570093458, + "world_religions": 0.8, + "sociology": 0.8, + "us_foreign_policy": 0.8585858585858586, + "high_school_macroeconomics": 0.5989717223650386, + "computer_security": 0.696969696969697, + "moral_scenarios": 0.4004474272930649, + "moral_disputes": 0.6608695652173913, + "electrical_engineering": 0.625, + "astronomy": 0.6887417218543046, + "college_biology": 0.6923076923076923 } }, "prompt_2": { - "accuracy": 0.34908830890239545, + "accuracy": 0.6249553092599214, "category_acc": { - "high_school_european_history": 0.49390243902439024, - "business_ethics": 0.3939393939393939, - "clinical_knowledge": 0.3484848484848485, - "medical_genetics": 0.4444444444444444, - "high_school_us_history": 0.5467980295566502, - "high_school_physics": 0.24, - "high_school_world_history": 0.5466101694915254, - "virology": 0.3939393939393939, - "high_school_microeconomics": 0.29957805907172996, - "econometrics": 0.22123893805309736, - "college_computer_science": 0.26262626262626265, - "high_school_biology": 0.3592233009708738, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.28825622775800713, - "philosophy": 0.4096774193548387, - "professional_medicine": 0.4833948339483395, - "nutrition": 0.32786885245901637, - "global_facts": 0.25252525252525254, - "machine_learning": 0.3153153153153153, - "security_studies": 0.3319672131147541, - "public_relations": 0.27522935779816515, - "professional_psychology": 0.353518821603928, - "prehistory": 0.35294117647058826, - "anatomy": 0.3283582089552239, - "human_sexuality": 0.33076923076923076, - "college_medicine": 0.3488372093023256, - "high_school_government_and_politics": 0.3802083333333333, - "college_chemistry": 0.29292929292929293, - "logical_fallacies": 0.48148148148148145, - "high_school_geography": 0.34517766497461927, - "elementary_mathematics": 0.26790450928381965, - "human_aging": 0.42342342342342343, - "college_mathematics": 0.20202020202020202, - "high_school_psychology": 0.41360294117647056, - "formal_logic": 0.28, - "high_school_statistics": 0.2930232558139535, - "international_law": 0.48333333333333334, - "high_school_mathematics": 0.27137546468401486, - "high_school_computer_science": 0.40404040404040403, - "conceptual_physics": 0.3247863247863248, - "miscellaneous": 0.34654731457800514, - "high_school_chemistry": 0.29207920792079206, - "marketing": 0.48068669527896996, - "professional_law": 0.33985649054142203, - "management": 0.3235294117647059, - "college_physics": 0.21782178217821782, - "jurisprudence": 0.4392523364485981, - "world_religions": 0.4235294117647059, - "sociology": 0.395, - "us_foreign_policy": 0.43434343434343436, - "high_school_macroeconomics": 0.2879177377892031, - "computer_security": 0.48484848484848486, - "moral_scenarios": 0.24272930648769575, - "moral_disputes": 0.34492753623188405, - "electrical_engineering": 0.3263888888888889, - "astronomy": 0.3708609271523179, - "college_biology": 0.3356643356643357 + "high_school_european_history": 0.7439024390243902, + "business_ethics": 0.6464646464646465, + "clinical_knowledge": 0.7424242424242424, + "medical_genetics": 0.7676767676767676, + "high_school_us_history": 0.7783251231527094, + "high_school_physics": 0.4066666666666667, + "high_school_world_history": 0.8220338983050848, + "virology": 0.49696969696969695, + "high_school_microeconomics": 0.7088607594936709, + "econometrics": 0.45132743362831856, + "college_computer_science": 0.47474747474747475, + "high_school_biology": 0.7734627831715211, + "abstract_algebra": 0.25252525252525254, + "professional_accounting": 0.5088967971530249, + "philosophy": 0.6516129032258065, + "professional_medicine": 0.7527675276752768, + "nutrition": 0.7344262295081967, + "global_facts": 0.45454545454545453, + "machine_learning": 0.4864864864864865, + "security_studies": 0.6721311475409836, + "public_relations": 0.6605504587155964, + "professional_psychology": 0.6612111292962357, + "prehistory": 0.7120743034055728, + "anatomy": 0.6417910447761194, + "human_sexuality": 0.7230769230769231, + "college_medicine": 0.6162790697674418, + "high_school_government_and_politics": 0.859375, + "college_chemistry": 0.37373737373737376, + "logical_fallacies": 0.7407407407407407, + "high_school_geography": 0.7766497461928934, + "elementary_mathematics": 0.7082228116710876, + "human_aging": 0.6666666666666666, + "college_mathematics": 0.3333333333333333, + "high_school_psychology": 0.8198529411764706, + "formal_logic": 0.504, + "high_school_statistics": 0.5209302325581395, + "international_law": 0.7583333333333333, + "high_school_mathematics": 0.35687732342007433, + "high_school_computer_science": 0.7070707070707071, + "conceptual_physics": 0.5769230769230769, + "miscellaneous": 0.8286445012787724, + "high_school_chemistry": 0.5148514851485149, + "marketing": 0.8755364806866953, + "professional_law": 0.4611872146118721, + "management": 0.8137254901960784, + "college_physics": 0.37623762376237624, + "jurisprudence": 0.7383177570093458, + "world_religions": 0.788235294117647, + "sociology": 0.795, + "us_foreign_policy": 0.8181818181818182, + "high_school_macroeconomics": 0.5938303341902313, + "computer_security": 0.6868686868686869, + "moral_scenarios": 0.25615212527964204, + "moral_disputes": 0.6840579710144927, + "electrical_engineering": 0.5625, + "astronomy": 0.7152317880794702, + "college_biology": 0.7202797202797203 } }, "prompt_3": { - "accuracy": 0.41344297461565965, + "accuracy": 0.6251698248122989, "category_acc": { - "high_school_european_history": 0.49390243902439024, - "business_ethics": 0.47474747474747475, - "clinical_knowledge": 0.4318181818181818, - "medical_genetics": 0.41414141414141414, - "high_school_us_history": 0.5911330049261084, - "high_school_physics": 0.23333333333333334, - "high_school_world_history": 0.559322033898305, - "virology": 0.3696969696969697, - "high_school_microeconomics": 0.39662447257383965, - "econometrics": 0.23893805309734514, - "college_computer_science": 0.35353535353535354, - "high_school_biology": 0.4627831715210356, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.33807829181494664, - "philosophy": 0.45806451612903226, - "professional_medicine": 0.4944649446494465, - "nutrition": 0.4459016393442623, - "global_facts": 0.3434343434343434, - "machine_learning": 0.36036036036036034, - "security_studies": 0.4385245901639344, - "public_relations": 0.44036697247706424, - "professional_psychology": 0.4075286415711948, - "prehistory": 0.43653250773993807, - "anatomy": 0.4253731343283582, - "human_sexuality": 0.4, - "college_medicine": 0.43023255813953487, - "high_school_government_and_politics": 0.5260416666666666, - "college_chemistry": 0.3333333333333333, - "logical_fallacies": 0.5, - "high_school_geography": 0.4619289340101523, - "elementary_mathematics": 0.3103448275862069, - "human_aging": 0.5135135135135135, - "college_mathematics": 0.18181818181818182, - "high_school_psychology": 0.5441176470588235, - "formal_logic": 0.288, - "high_school_statistics": 0.26976744186046514, - "international_law": 0.475, - "high_school_mathematics": 0.25650557620817843, - "high_school_computer_science": 0.3939393939393939, - "conceptual_physics": 0.3504273504273504, - "miscellaneous": 0.5831202046035806, - "high_school_chemistry": 0.30198019801980197, - "marketing": 0.6266094420600858, - "professional_law": 0.3594259621656882, - "management": 0.5294117647058824, - "college_physics": 0.22772277227722773, - "jurisprudence": 0.5233644859813084, - "world_religions": 0.5647058823529412, - "sociology": 0.56, - "us_foreign_policy": 0.5454545454545454, - "high_school_macroeconomics": 0.36246786632390743, - "computer_security": 0.494949494949495, - "moral_scenarios": 0.24272930648769575, - "moral_disputes": 0.4144927536231884, - "electrical_engineering": 0.3472222222222222, - "astronomy": 0.33774834437086093, - "college_biology": 0.4405594405594406 + "high_school_european_history": 0.7317073170731707, + "business_ethics": 0.696969696969697, + "clinical_knowledge": 0.7234848484848485, + "medical_genetics": 0.7474747474747475, + "high_school_us_history": 0.7684729064039408, + "high_school_physics": 0.36666666666666664, + "high_school_world_history": 0.809322033898305, + "virology": 0.503030303030303, + "high_school_microeconomics": 0.7215189873417721, + "econometrics": 0.46017699115044247, + "college_computer_science": 0.43434343434343436, + "high_school_biology": 0.7443365695792881, + "abstract_algebra": 0.2727272727272727, + "professional_accounting": 0.5088967971530249, + "philosophy": 0.6645161290322581, + "professional_medicine": 0.7490774907749077, + "nutrition": 0.7180327868852459, + "global_facts": 0.48484848484848486, + "machine_learning": 0.4864864864864865, + "security_studies": 0.6721311475409836, + "public_relations": 0.6330275229357798, + "professional_psychology": 0.6350245499181669, + "prehistory": 0.7151702786377709, + "anatomy": 0.6567164179104478, + "human_sexuality": 0.7461538461538462, + "college_medicine": 0.5988372093023255, + "high_school_government_and_politics": 0.859375, + "college_chemistry": 0.47474747474747475, + "logical_fallacies": 0.7592592592592593, + "high_school_geography": 0.7461928934010152, + "elementary_mathematics": 0.7294429708222812, + "human_aging": 0.6891891891891891, + "college_mathematics": 0.30303030303030304, + "high_school_psychology": 0.8051470588235294, + "formal_logic": 0.536, + "high_school_statistics": 0.5023255813953489, + "international_law": 0.7416666666666667, + "high_school_mathematics": 0.3345724907063197, + "high_school_computer_science": 0.6565656565656566, + "conceptual_physics": 0.6068376068376068, + "miscellaneous": 0.8350383631713555, + "high_school_chemistry": 0.5198019801980198, + "marketing": 0.8540772532188842, + "professional_law": 0.4683626875407697, + "management": 0.7647058823529411, + "college_physics": 0.37623762376237624, + "jurisprudence": 0.719626168224299, + "world_religions": 0.8058823529411765, + "sociology": 0.8, + "us_foreign_policy": 0.8686868686868687, + "high_school_macroeconomics": 0.583547557840617, + "computer_security": 0.6767676767676768, + "moral_scenarios": 0.2897091722595078, + "moral_disputes": 0.6753623188405797, + "electrical_engineering": 0.5763888888888888, + "astronomy": 0.695364238410596, + "college_biology": 0.7272727272727273 } }, "prompt_4": { - "accuracy": 0.3545942080800858, + "accuracy": 0.6171612441902038, "category_acc": { - "high_school_european_history": 0.4634146341463415, - "business_ethics": 0.43434343434343436, - "clinical_knowledge": 0.38257575757575757, - "medical_genetics": 0.40404040404040403, - "high_school_us_history": 0.5270935960591133, - "high_school_physics": 0.22666666666666666, - "high_school_world_history": 0.5254237288135594, - "virology": 0.3696969696969697, - "high_school_microeconomics": 0.350210970464135, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.2727272727272727, - "high_school_biology": 0.41423948220064727, - "abstract_algebra": 0.21212121212121213, - "professional_accounting": 0.28113879003558717, - "philosophy": 0.38064516129032255, - "professional_medicine": 0.3726937269372694, - "nutrition": 0.3540983606557377, - "global_facts": 0.25252525252525254, - "machine_learning": 0.3333333333333333, - "security_studies": 0.3360655737704918, - "public_relations": 0.3761467889908257, - "professional_psychology": 0.35515548281505727, - "prehistory": 0.3715170278637771, - "anatomy": 0.35074626865671643, - "human_sexuality": 0.35384615384615387, - "college_medicine": 0.29651162790697677, - "high_school_government_and_politics": 0.4635416666666667, - "college_chemistry": 0.2727272727272727, - "logical_fallacies": 0.41975308641975306, - "high_school_geography": 0.3147208121827411, - "elementary_mathematics": 0.27586206896551724, - "human_aging": 0.49099099099099097, - "college_mathematics": 0.23232323232323232, - "high_school_psychology": 0.4007352941176471, - "formal_logic": 0.256, - "high_school_statistics": 0.22790697674418606, - "international_law": 0.5583333333333333, - "high_school_mathematics": 0.2825278810408922, - "high_school_computer_science": 0.3838383838383838, - "conceptual_physics": 0.3333333333333333, - "miscellaneous": 0.38235294117647056, - "high_school_chemistry": 0.2722772277227723, - "marketing": 0.5493562231759657, - "professional_law": 0.32354859752120024, - "management": 0.38235294117647056, - "college_physics": 0.19801980198019803, - "jurisprudence": 0.45794392523364486, - "world_religions": 0.3941176470588235, - "sociology": 0.435, - "us_foreign_policy": 0.48484848484848486, - "high_school_macroeconomics": 0.32904884318766064, - "computer_security": 0.41414141414141414, - "moral_scenarios": 0.2516778523489933, - "moral_disputes": 0.4115942028985507, - "electrical_engineering": 0.2986111111111111, - "astronomy": 0.3509933774834437, - "college_biology": 0.3706293706293706 + "high_school_european_history": 0.7317073170731707, + "business_ethics": 0.6565656565656566, + "clinical_knowledge": 0.6590909090909091, + "medical_genetics": 0.7575757575757576, + "high_school_us_history": 0.7931034482758621, + "high_school_physics": 0.34, + "high_school_world_history": 0.8008474576271186, + "virology": 0.49696969696969695, + "high_school_microeconomics": 0.7088607594936709, + "econometrics": 0.3893805309734513, + "college_computer_science": 0.3939393939393939, + "high_school_biology": 0.7249190938511327, + "abstract_algebra": 0.30303030303030304, + "professional_accounting": 0.5480427046263345, + "philosophy": 0.6774193548387096, + "professional_medicine": 0.6678966789667896, + "nutrition": 0.7114754098360656, + "global_facts": 0.5050505050505051, + "machine_learning": 0.35135135135135137, + "security_studies": 0.6721311475409836, + "public_relations": 0.6513761467889908, + "professional_psychology": 0.630114566284779, + "prehistory": 0.7120743034055728, + "anatomy": 0.6119402985074627, + "human_sexuality": 0.6923076923076923, + "college_medicine": 0.6046511627906976, + "high_school_government_and_politics": 0.8229166666666666, + "college_chemistry": 0.3939393939393939, + "logical_fallacies": 0.7654320987654321, + "high_school_geography": 0.7309644670050761, + "elementary_mathematics": 0.6816976127320955, + "human_aging": 0.6486486486486487, + "college_mathematics": 0.35353535353535354, + "high_school_psychology": 0.7886029411764706, + "formal_logic": 0.488, + "high_school_statistics": 0.4744186046511628, + "international_law": 0.775, + "high_school_mathematics": 0.38661710037174724, + "high_school_computer_science": 0.6767676767676768, + "conceptual_physics": 0.594017094017094, + "miscellaneous": 0.7877237851662404, + "high_school_chemistry": 0.5, + "marketing": 0.8283261802575107, + "professional_law": 0.47619047619047616, + "management": 0.803921568627451, + "college_physics": 0.4158415841584158, + "jurisprudence": 0.7476635514018691, + "world_religions": 0.8, + "sociology": 0.8, + "us_foreign_policy": 0.8383838383838383, + "high_school_macroeconomics": 0.5732647814910026, + "computer_security": 0.6161616161616161, + "moral_scenarios": 0.3434004474272931, + "moral_disputes": 0.6695652173913044, + "electrical_engineering": 0.6180555555555556, + "astronomy": 0.7218543046357616, + "college_biology": 0.6503496503496503 } }, "prompt_5": { - "accuracy": 0.41723274937432964, + "accuracy": 0.5823382195209152, "category_acc": { - "high_school_european_history": 0.49390243902439024, - "business_ethics": 0.43434343434343436, - "clinical_knowledge": 0.45075757575757575, - "medical_genetics": 0.47474747474747475, - "high_school_us_history": 0.5320197044334976, - "high_school_physics": 0.22666666666666666, - "high_school_world_history": 0.5423728813559322, - "virology": 0.3696969696969697, - "high_school_microeconomics": 0.379746835443038, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.26262626262626265, - "high_school_biology": 0.4886731391585761, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.33451957295373663, - "philosophy": 0.45161290322580644, - "professional_medicine": 0.3763837638376384, - "nutrition": 0.47540983606557374, - "global_facts": 0.32323232323232326, - "machine_learning": 0.34234234234234234, - "security_studies": 0.38934426229508196, - "public_relations": 0.47706422018348627, - "professional_psychology": 0.4337152209492635, - "prehistory": 0.48297213622291024, - "anatomy": 0.48507462686567165, - "human_sexuality": 0.4230769230769231, - "college_medicine": 0.38372093023255816, - "high_school_government_and_politics": 0.5052083333333334, - "college_chemistry": 0.3333333333333333, - "logical_fallacies": 0.4382716049382716, - "high_school_geography": 0.5380710659898477, - "elementary_mathematics": 0.28116710875331563, - "human_aging": 0.5405405405405406, - "college_mathematics": 0.23232323232323232, - "high_school_psychology": 0.5275735294117647, - "formal_logic": 0.296, - "high_school_statistics": 0.24651162790697675, - "international_law": 0.5916666666666667, - "high_school_mathematics": 0.2379182156133829, - "high_school_computer_science": 0.40404040404040403, - "conceptual_physics": 0.41452991452991456, - "miscellaneous": 0.618925831202046, - "high_school_chemistry": 0.3712871287128713, - "marketing": 0.6952789699570815, - "professional_law": 0.3313763861709067, - "management": 0.5098039215686274, - "college_physics": 0.25742574257425743, - "jurisprudence": 0.4672897196261682, - "world_religions": 0.6411764705882353, - "sociology": 0.505, - "us_foreign_policy": 0.5858585858585859, - "high_school_macroeconomics": 0.35732647814910024, - "computer_security": 0.5151515151515151, - "moral_scenarios": 0.2505592841163311, - "moral_disputes": 0.47246376811594204, - "electrical_engineering": 0.3472222222222222, - "astronomy": 0.423841059602649, - "college_biology": 0.45454545454545453 + "high_school_european_history": 0.7134146341463414, + "business_ethics": 0.6161616161616161, + "clinical_knowledge": 0.6401515151515151, + "medical_genetics": 0.7373737373737373, + "high_school_us_history": 0.7783251231527094, + "high_school_physics": 0.32666666666666666, + "high_school_world_history": 0.7923728813559322, + "virology": 0.4909090909090909, + "high_school_microeconomics": 0.6118143459915611, + "econometrics": 0.35398230088495575, + "college_computer_science": 0.36363636363636365, + "high_school_biology": 0.6472491909385113, + "abstract_algebra": 0.29292929292929293, + "professional_accounting": 0.4483985765124555, + "philosophy": 0.6580645161290323, + "professional_medicine": 0.6457564575645757, + "nutrition": 0.6786885245901639, + "global_facts": 0.494949494949495, + "machine_learning": 0.36936936936936937, + "security_studies": 0.5983606557377049, + "public_relations": 0.5504587155963303, + "professional_psychology": 0.6104746317512275, + "prehistory": 0.6687306501547987, + "anatomy": 0.5895522388059702, + "human_sexuality": 0.5923076923076923, + "college_medicine": 0.5348837209302325, + "high_school_government_and_politics": 0.796875, + "college_chemistry": 0.31313131313131315, + "logical_fallacies": 0.7345679012345679, + "high_school_geography": 0.6700507614213198, + "elementary_mathematics": 0.726790450928382, + "human_aging": 0.6396396396396397, + "college_mathematics": 0.21212121212121213, + "high_school_psychology": 0.7408088235294118, + "formal_logic": 0.4, + "high_school_statistics": 0.40930232558139534, + "international_law": 0.7083333333333334, + "high_school_mathematics": 0.2936802973977695, + "high_school_computer_science": 0.6666666666666666, + "conceptual_physics": 0.5470085470085471, + "miscellaneous": 0.7902813299232737, + "high_school_chemistry": 0.41089108910891087, + "marketing": 0.8111587982832618, + "professional_law": 0.44879321591650356, + "management": 0.7549019607843137, + "college_physics": 0.3564356435643564, + "jurisprudence": 0.6915887850467289, + "world_religions": 0.8176470588235294, + "sociology": 0.775, + "us_foreign_policy": 0.8282828282828283, + "high_school_macroeconomics": 0.5475578406169666, + "computer_security": 0.6666666666666666, + "moral_scenarios": 0.30648769574944074, + "moral_disputes": 0.6347826086956522, + "electrical_engineering": 0.6111111111111112, + "astronomy": 0.6357615894039735, + "college_biology": 0.6013986013986014 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.43164933135215455 + "accuracy": 0.4903417533432392 }, "prompt_2": { - "accuracy": 0.3536404160475483 + "accuracy": 0.5044576523031203 }, "prompt_3": { - "accuracy": 0.35289747399702825 + "accuracy": 0.49257057949479943 }, "prompt_4": { - "accuracy": 0.4160475482912333 + "accuracy": 0.4390787518573551 }, "prompt_5": { - "accuracy": 0.40936106983655274 + "accuracy": 0.4576523031203566 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.4402241594022416, + "accuracy": 0.4919053549190536, "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.5416666666666666, + "computer_network": 0.7916666666666666, + "operating_system": 0.6666666666666666, "computer_architecture": 0.5, - "college_programming": 0.38095238095238093, - "college_physics": 0.375, + "college_programming": 0.6428571428571429, + "college_physics": 0.4583333333333333, "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.4583333333333333, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.42857142857142855, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.375, - "high_school_chemistry": 0.5, - "high_school_biology": 0.4583333333333333, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.46153846153846156, - "middle_school_physics": 0.5, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.391304347826087, + "discrete_mathematics": 0.3333333333333333, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.41379310344827586, + "high_school_mathematics": 0.17391304347826086, + "high_school_physics": 0.3333333333333333, + "high_school_chemistry": 0.20833333333333334, + "high_school_biology": 0.3333333333333333, + "middle_school_mathematics": 0.3333333333333333, + "middle_school_biology": 0.7692307692307693, + "middle_school_physics": 0.4166666666666667, "middle_school_chemistry": 0.56, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.36666666666666664, - "business_administration": 0.2631578947368421, - "marxism": 0.75, - "mao_zedong_thought": 0.7241379310344828, - "education_science": 0.6176470588235294, - "teacher_qualification": 0.6122448979591837, - "high_school_politics": 0.5, + "veterinary_medicine": 0.5, + "college_economics": 0.4, + "business_administration": 0.5, + "marxism": 0.5, + "mao_zedong_thought": 0.6551724137931034, + "education_science": 0.5588235294117647, + "teacher_qualification": 0.7142857142857143, + "high_school_politics": 0.625, "high_school_geography": 0.5416666666666666, - "middle_school_politics": 0.5384615384615384, - "middle_school_geography": 0.6470588235294118, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.25925925925925924, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.6428571428571429, - "art_studies": 0.5, - "professional_tour_guide": 0.6470588235294118, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.25, - "high_school_history": 0.68, - "middle_school_history": 0.6296296296296297, - "civil_servant": 0.19230769230769232, + "middle_school_politics": 0.7307692307692307, + "middle_school_geography": 0.7058823529411765, + "modern_chinese_history": 0.6071428571428571, + "ideological_and_moral_cultivation": 0.6666666666666666, + "logic": 0.4444444444444444, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.42857142857142855, + "art_studies": 0.631578947368421, + "professional_tour_guide": 0.5588235294117647, + "legal_professional": 0.42857142857142855, + "high_school_chinese": 0.3333333333333333, + "high_school_history": 0.52, + "middle_school_history": 0.7407407407407407, + "civil_servant": 0.4423076923076923, "sports_science": 0.4166666666666667, - "plant_protection": 0.5555555555555556, + "plant_protection": 0.7407407407407407, "basic_medicine": 0.5, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.45098039215686275, - "accountant": 0.3333333333333333, + "clinical_medicine": 0.48148148148148145, + "urban_and_rural_planner": 0.49019607843137253, + "accountant": 0.3888888888888889, "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.5, - "tax_accountant": 0.25925925925925924, - "physician": 0.48148148148148145 + "environmental_impact_assessment_engineer": 0.4166666666666667, + "tax_accountant": 0.3148148148148148, + "physician": 0.6296296296296297 } }, "prompt_2": { - "accuracy": 0.3779576587795766, + "accuracy": 0.48443337484433374, "category_acc": { - "computer_network": 0.25, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.38461538461538464, + "computer_network": 0.7083333333333334, + "operating_system": 0.7083333333333334, + "computer_architecture": 0.6153846153846154, "college_programming": 0.47619047619047616, - "college_physics": 0.25, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.35714285714285715, + "college_physics": 0.4166666666666667, + "college_chemistry": 0.41379310344827586, + "advanced_mathematics": 0.375, + "probability_and_statistics": 0.391304347826087, + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.25, - "high_school_chemistry": 0.2916666666666667, + "high_school_mathematics": 0.13043478260869565, + "high_school_physics": 0.4166666666666667, + "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.4166666666666667, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.46153846153846156, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.4, - "veterinary_medicine": 0.42857142857142855, - "college_economics": 0.36666666666666664, - "business_administration": 0.2894736842105263, - "marxism": 0.5, - "mao_zedong_thought": 0.5862068965517241, - "education_science": 0.4411764705882353, - "teacher_qualification": 0.4489795918367347, - "high_school_politics": 0.4166666666666667, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.7307692307692307, + "middle_school_physics": 0.4583333333333333, + "middle_school_chemistry": 0.44, + "veterinary_medicine": 0.5, + "college_economics": 0.4666666666666667, + "business_administration": 0.4473684210526316, + "marxism": 0.5833333333333334, + "mao_zedong_thought": 0.6206896551724138, + "education_science": 0.6176470588235294, + "teacher_qualification": 0.7142857142857143, + "high_school_politics": 0.625, "high_school_geography": 0.5416666666666666, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.35714285714285715, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.18518518518518517, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.6071428571428571, - "art_studies": 0.5, + "middle_school_politics": 0.7307692307692307, + "middle_school_geography": 0.47058823529411764, + "modern_chinese_history": 0.5357142857142857, + "ideological_and_moral_cultivation": 0.6666666666666666, + "logic": 0.48148148148148145, + "law": 0.4482758620689655, + "chinese_language_and_literature": 0.4642857142857143, + "art_studies": 0.6052631578947368, "professional_tour_guide": 0.5294117647058824, - "legal_professional": 0.2857142857142857, + "legal_professional": 0.32142857142857145, "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.52, - "middle_school_history": 0.5555555555555556, - "civil_servant": 0.21153846153846154, - "sports_science": 0.3333333333333333, - "plant_protection": 0.48148148148148145, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.43137254901960786, - "accountant": 0.3333333333333333, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.2962962962962963, - "physician": 0.42592592592592593 + "high_school_history": 0.4, + "middle_school_history": 0.6296296296296297, + "civil_servant": 0.4423076923076923, + "sports_science": 0.5416666666666666, + "plant_protection": 0.7037037037037037, + "basic_medicine": 0.5, + "clinical_medicine": 0.4074074074074074, + "urban_and_rural_planner": 0.5686274509803921, + "accountant": 0.42592592592592593, + "fire_engineer": 0.4166666666666667, + "environmental_impact_assessment_engineer": 0.3611111111111111, + "tax_accountant": 0.3148148148148148, + "physician": 0.5740740740740741 } }, "prompt_3": { - "accuracy": 0.3493150684931507, + "accuracy": 0.4919053549190536, "category_acc": { - "computer_network": 0.25, - "operating_system": 0.25, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.38095238095238093, - "college_physics": 0.125, - "college_chemistry": 0.13793103448275862, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.2857142857142857, + "computer_network": 0.5, + "operating_system": 0.625, + "computer_architecture": 0.5, + "college_programming": 0.6666666666666666, + "college_physics": 0.4166666666666667, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.3333333333333333, + "probability_and_statistics": 0.391304347826087, + "discrete_mathematics": 0.42857142857142855, "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.27586206896551724, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.35, - "business_administration": 0.21052631578947367, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.5517241379310345, - "education_science": 0.3235294117647059, - "teacher_qualification": 0.46938775510204084, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.5, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.4583333333333333, - "logic": 0.2222222222222222, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.5357142857142857, - "art_studies": 0.5, - "professional_tour_guide": 0.5294117647058824, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.20833333333333334, + "metrology_engineer": 0.5862068965517241, + "high_school_mathematics": 0.21739130434782608, + "high_school_physics": 0.3333333333333333, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.375, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.7692307692307693, + "middle_school_physics": 0.5, + "middle_school_chemistry": 0.52, + "veterinary_medicine": 0.4642857142857143, + "college_economics": 0.45, + "business_administration": 0.3684210526315789, + "marxism": 0.5833333333333334, + "mao_zedong_thought": 0.5862068965517241, + "education_science": 0.6470588235294118, + "teacher_qualification": 0.7346938775510204, + "high_school_politics": 0.5833333333333334, + "high_school_geography": 0.5833333333333334, + "middle_school_politics": 0.7307692307692307, + "middle_school_geography": 0.5294117647058824, + "modern_chinese_history": 0.5357142857142857, + "ideological_and_moral_cultivation": 0.75, + "logic": 0.48148148148148145, + "law": 0.3793103448275862, + "chinese_language_and_literature": 0.42857142857142855, + "art_studies": 0.5526315789473685, + "professional_tour_guide": 0.5882352941176471, + "legal_professional": 0.5, + "high_school_chinese": 0.25, "high_school_history": 0.56, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.21153846153846154, - "sports_science": 0.3333333333333333, - "plant_protection": 0.5925925925925926, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.39215686274509803, - "accountant": 0.35185185185185186, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.2222222222222222, - "physician": 0.35185185185185186 + "middle_school_history": 0.6666666666666666, + "civil_servant": 0.3076923076923077, + "sports_science": 0.5, + "plant_protection": 0.7407407407407407, + "basic_medicine": 0.5, + "clinical_medicine": 0.48148148148148145, + "urban_and_rural_planner": 0.5882352941176471, + "accountant": 0.3888888888888889, + "fire_engineer": 0.4722222222222222, + "environmental_impact_assessment_engineer": 0.3888888888888889, + "tax_accountant": 0.25925925925925924, + "physician": 0.6111111111111112 } }, "prompt_4": { - "accuracy": 0.43586550435865506, + "accuracy": 0.41594022415940224, "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.30952380952380953, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.20833333333333334, + "computer_network": 0.4583333333333333, + "operating_system": 0.3333333333333333, + "computer_architecture": 0.5384615384615384, + "college_programming": 0.35714285714285715, + "college_physics": 0.3333333333333333, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.2916666666666667, "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.047619047619047616, - "electrical_engineer": 0.40476190476190477, - "metrology_engineer": 0.5172413793103449, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.4583333333333333, - "high_school_chemistry": 0.4583333333333333, - "high_school_biology": 0.4166666666666667, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.48, + "discrete_mathematics": 0.14285714285714285, + "electrical_engineer": 0.38095238095238093, + "metrology_engineer": 0.3793103448275862, + "high_school_mathematics": 0.17391304347826086, + "high_school_physics": 0.3333333333333333, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.2916666666666667, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.5769230769230769, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.32, "veterinary_medicine": 0.4642857142857143, - "college_economics": 0.31666666666666665, - "business_administration": 0.3157894736842105, - "marxism": 0.6666666666666666, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.7058823529411765, - "teacher_qualification": 0.5714285714285714, - "high_school_politics": 0.5416666666666666, - "high_school_geography": 0.5833333333333334, - "middle_school_politics": 0.5769230769230769, - "middle_school_geography": 0.5882352941176471, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.2962962962962963, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.5714285714285714, - "art_studies": 0.5263157894736842, - "professional_tour_guide": 0.6176470588235294, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.25, + "college_economics": 0.36666666666666664, + "business_administration": 0.42105263157894735, + "marxism": 0.5416666666666666, + "mao_zedong_thought": 0.4482758620689655, + "education_science": 0.5, + "teacher_qualification": 0.5918367346938775, + "high_school_politics": 0.5833333333333334, + "high_school_geography": 0.4583333333333333, + "middle_school_politics": 0.6153846153846154, + "middle_school_geography": 0.5294117647058824, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.5, + "logic": 0.5185185185185185, + "law": 0.41379310344827586, + "chinese_language_and_literature": 0.4642857142857143, + "art_studies": 0.6052631578947368, + "professional_tour_guide": 0.47058823529411764, + "legal_professional": 0.35714285714285715, + "high_school_chinese": 0.2916666666666667, "high_school_history": 0.64, - "middle_school_history": 0.6296296296296297, - "civil_servant": 0.28846153846153844, - "sports_science": 0.4583333333333333, + "middle_school_history": 0.37037037037037035, + "civil_servant": 0.4230769230769231, + "sports_science": 0.375, "plant_protection": 0.5555555555555556, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.43137254901960786, - "accountant": 0.3148148148148148, - "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.4166666666666667, + "basic_medicine": 0.2916666666666667, + "clinical_medicine": 0.37037037037037035, + "urban_and_rural_planner": 0.49019607843137253, + "accountant": 0.3333333333333333, + "fire_engineer": 0.3333333333333333, + "environmental_impact_assessment_engineer": 0.4722222222222222, "tax_accountant": 0.25925925925925924, - "physician": 0.46296296296296297 + "physician": 0.5370370370370371 } }, "prompt_5": { - "accuracy": 0.41718555417185554, + "accuracy": 0.4701120797011208, "category_acc": { - "computer_network": 0.3333333333333333, + "computer_network": 0.5833333333333334, "operating_system": 0.4583333333333333, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.40476190476190477, + "computer_architecture": 0.46153846153846156, + "college_programming": 0.35714285714285715, "college_physics": 0.4166666666666667, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.47619047619047616, - "metrology_engineer": 0.4482758620689655, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.375, - "high_school_biology": 0.4166666666666667, + "college_chemistry": 0.3103448275862069, + "advanced_mathematics": 0.3333333333333333, + "probability_and_statistics": 0.30434782608695654, + "discrete_mathematics": 0.3333333333333333, + "electrical_engineer": 0.42857142857142855, + "metrology_engineer": 0.4827586206896552, + "high_school_mathematics": 0.17391304347826086, + "high_school_physics": 0.375, + "high_school_chemistry": 0.2916666666666667, + "high_school_biology": 0.375, "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.46153846153846156, + "middle_school_biology": 0.7307692307692307, "middle_school_physics": 0.4166666666666667, "middle_school_chemistry": 0.52, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.38333333333333336, - "business_administration": 0.3157894736842105, - "marxism": 0.625, - "mao_zedong_thought": 0.5172413793103449, - "education_science": 0.5588235294117647, - "teacher_qualification": 0.5510204081632653, - "high_school_politics": 0.5416666666666666, - "high_school_geography": 0.5416666666666666, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.5294117647058824, - "modern_chinese_history": 0.35714285714285715, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.37037037037037035, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.5357142857142857, + "veterinary_medicine": 0.42857142857142855, + "college_economics": 0.36666666666666664, + "business_administration": 0.47368421052631576, + "marxism": 0.5416666666666666, + "mao_zedong_thought": 0.5862068965517241, + "education_science": 0.6470588235294118, + "teacher_qualification": 0.673469387755102, + "high_school_politics": 0.6666666666666666, + "high_school_geography": 0.5, + "middle_school_politics": 0.7307692307692307, + "middle_school_geography": 0.5882352941176471, + "modern_chinese_history": 0.5, + "ideological_and_moral_cultivation": 0.6666666666666666, + "logic": 0.5185185185185185, + "law": 0.3793103448275862, + "chinese_language_and_literature": 0.4642857142857143, "art_studies": 0.5526315789473685, - "professional_tour_guide": 0.5882352941176471, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.48, - "middle_school_history": 0.48148148148148145, - "civil_servant": 0.3076923076923077, - "sports_science": 0.3333333333333333, - "plant_protection": 0.5185185185185185, - "basic_medicine": 0.5, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.47058823529411764, - "accountant": 0.2962962962962963, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.2222222222222222, - "physician": 0.5555555555555556 + "professional_tour_guide": 0.5294117647058824, + "legal_professional": 0.4642857142857143, + "high_school_chinese": 0.16666666666666666, + "high_school_history": 0.4, + "middle_school_history": 0.5925925925925926, + "civil_servant": 0.36538461538461536, + "sports_science": 0.5, + "plant_protection": 0.6296296296296297, + "basic_medicine": 0.5833333333333334, + "clinical_medicine": 0.5185185185185185, + "urban_and_rural_planner": 0.5490196078431373, + "accountant": 0.48148148148148145, + "fire_engineer": 0.3888888888888889, + "environmental_impact_assessment_engineer": 0.4444444444444444, + "tax_accountant": 0.25925925925925924, + "physician": 0.6296296296296297 } } }, "cmmlu": { "prompt_1": { - "accuracy": 0.5017921146953405 + "accuracy": 0.5806451612903226 }, "prompt_2": { - "accuracy": 0.43010752688172044 + "accuracy": 0.5448028673835126 }, "prompt_3": { - "accuracy": 0.3978494623655914 + "accuracy": 0.5232974910394266 }, "prompt_4": { - "accuracy": 0.4910394265232975 + "accuracy": 0.4731182795698925 }, "prompt_5": { - "accuracy": 0.4910394265232975 + "accuracy": 0.5161290322580645 } }, "cmmlu_full": { "prompt_1": { - "accuracy": 0.445173545156277, - "category_acc": { - "agronomy": 0.41420118343195267, - "anatomy": 0.3581081081081081, - "ancient_chinese": 0.27439024390243905, - "arts": 0.7, - "astronomy": 0.32727272727272727, - "business_ethics": 0.49760765550239233, - "chinese_civil_service_exam": 0.34375, - "chinese_driving_rule": 0.6183206106870229, - "chinese_food_culture": 0.41911764705882354, - "chinese_foreign_policy": 0.4672897196261682, - "chinese_history": 0.4613003095975232, - "chinese_literature": 0.4019607843137255, - "chinese_teacher_qualification": 0.547486033519553, - "clinical_knowledge": 0.4092827004219409, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.5981308411214953, - "college_engineering_hydrology": 0.42452830188679247, - "college_law": 0.48148148148148145, - "college_mathematics": 0.20952380952380953, - "college_medical_statistics": 0.4056603773584906, - "college_medicine": 0.43956043956043955, - "computer_science": 0.47058823529411764, - "computer_security": 0.49707602339181284, - "conceptual_physics": 0.3945578231292517, - "construction_project_management": 0.39568345323741005, - "economics": 0.5031446540880503, - "education": 0.5398773006134969, - "electrical_engineering": 0.4476744186046512, - "elementary_chinese": 0.4246031746031746, - "elementary_commonsense": 0.5151515151515151, - "elementary_information_and_technology": 0.6134453781512605, - "elementary_mathematics": 0.33043478260869563, - "ethnology": 0.5111111111111111, - "food_science": 0.44755244755244755, - "genetics": 0.32386363636363635, - "global_facts": 0.5033557046979866, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.3983050847457627, - "high_school_mathematics": 0.3170731707317073, - "high_school_physics": 0.33636363636363636, - "high_school_politics": 0.32167832167832167, - "human_sexuality": 0.3888888888888889, - "international_law": 0.3945945945945946, - "journalism": 0.4941860465116279, - "jurisprudence": 0.3771289537712895, - "legal_and_moral_basis": 0.7990654205607477, - "logical": 0.34959349593495936, - "machine_learning": 0.3360655737704918, - "management": 0.580952380952381, - "marketing": 0.5166666666666667, - "marxist_theory": 0.5343915343915344, - "modern_chinese": 0.33620689655172414, - "nutrition": 0.4, - "philosophy": 0.5142857142857142, - "professional_accounting": 0.4857142857142857, - "professional_law": 0.36492890995260663, - "professional_medicine": 0.3670212765957447, - "professional_psychology": 0.4956896551724138, - "public_relations": 0.5114942528735632, - "security_study": 0.5407407407407407, - "sociology": 0.5, - "sports_science": 0.4727272727272727, - "traditional_chinese_medicine": 0.41081081081081083, - "virology": 0.4970414201183432, - "world_history": 0.4720496894409938, - "world_religions": 0.5875 - } - }, - "prompt_2": { - "accuracy": 0.383094456915904, + "accuracy": 0.5118286997064411, "category_acc": { - "agronomy": 0.40828402366863903, - "anatomy": 0.3108108108108108, - "ancient_chinese": 0.3231707317073171, - "arts": 0.5875, - "astronomy": 0.3090909090909091, - "business_ethics": 0.44019138755980863, - "chinese_civil_service_exam": 0.2625, - "chinese_driving_rule": 0.549618320610687, - "chinese_food_culture": 0.4117647058823529, - "chinese_foreign_policy": 0.40186915887850466, - "chinese_history": 0.3931888544891641, - "chinese_literature": 0.36764705882352944, - "chinese_teacher_qualification": 0.41899441340782123, - "clinical_knowledge": 0.28270042194092826, - "college_actuarial_science": 0.1792452830188679, - "college_education": 0.4953271028037383, - "college_engineering_hydrology": 0.36792452830188677, - "college_law": 0.35185185185185186, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.3584905660377358, - "college_medicine": 0.3626373626373626, - "computer_science": 0.36764705882352944, - "computer_security": 0.4619883040935672, - "conceptual_physics": 0.3401360544217687, - "construction_project_management": 0.3381294964028777, - "economics": 0.39622641509433965, - "education": 0.4294478527607362, - "electrical_engineering": 0.436046511627907, - "elementary_chinese": 0.4246031746031746, - "elementary_commonsense": 0.46464646464646464, - "elementary_information_and_technology": 0.5252100840336135, - "elementary_mathematics": 0.3695652173913043, - "ethnology": 0.45925925925925926, - "food_science": 0.4195804195804196, - "genetics": 0.32954545454545453, - "global_facts": 0.436241610738255, - "high_school_biology": 0.28402366863905326, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.3050847457627119, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.19090909090909092, - "high_school_politics": 0.2937062937062937, - "human_sexuality": 0.3492063492063492, - "international_law": 0.3837837837837838, - "journalism": 0.42441860465116277, - "jurisprudence": 0.340632603406326, - "legal_and_moral_basis": 0.5514018691588785, - "logical": 0.3008130081300813, - "machine_learning": 0.3360655737704918, - "management": 0.5095238095238095, - "marketing": 0.4111111111111111, - "marxist_theory": 0.4603174603174603, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.32413793103448274, - "philosophy": 0.4095238095238095, - "professional_accounting": 0.3942857142857143, - "professional_law": 0.3033175355450237, - "professional_medicine": 0.3377659574468085, - "professional_psychology": 0.39655172413793105, - "public_relations": 0.42528735632183906, - "security_study": 0.4444444444444444, - "sociology": 0.39823008849557523, - "sports_science": 0.37575757575757573, - "traditional_chinese_medicine": 0.3675675675675676, - "virology": 0.40236686390532544, - "world_history": 0.36645962732919257, - "world_religions": 0.50625 + "agronomy": 0.4911242603550296, + "anatomy": 0.4189189189189189, + "ancient_chinese": 0.32926829268292684, + "arts": 0.6, + "astronomy": 0.3212121212121212, + "business_ethics": 0.5263157894736842, + "chinese_civil_service_exam": 0.43125, + "chinese_driving_rule": 0.6717557251908397, + "chinese_food_culture": 0.4411764705882353, + "chinese_foreign_policy": 0.5607476635514018, + "chinese_history": 0.5139318885448917, + "chinese_literature": 0.3382352941176471, + "chinese_teacher_qualification": 0.5865921787709497, + "clinical_knowledge": 0.5021097046413502, + "college_actuarial_science": 0.2641509433962264, + "college_education": 0.6448598130841121, + "college_engineering_hydrology": 0.5, + "college_law": 0.3888888888888889, + "college_mathematics": 0.29523809523809524, + "college_medical_statistics": 0.5283018867924528, + "college_medicine": 0.5384615384615384, + "computer_science": 0.6323529411764706, + "computer_security": 0.672514619883041, + "conceptual_physics": 0.5102040816326531, + "construction_project_management": 0.4028776978417266, + "economics": 0.6352201257861635, + "education": 0.5337423312883436, + "electrical_engineering": 0.5058139534883721, + "elementary_chinese": 0.3492063492063492, + "elementary_commonsense": 0.47474747474747475, + "elementary_information_and_technology": 0.7521008403361344, + "elementary_mathematics": 0.3826086956521739, + "ethnology": 0.48148148148148145, + "food_science": 0.5384615384615384, + "genetics": 0.42613636363636365, + "global_facts": 0.5771812080536913, + "high_school_biology": 0.42011834319526625, + "high_school_chemistry": 0.32575757575757575, + "high_school_geography": 0.4745762711864407, + "high_school_mathematics": 0.2865853658536585, + "high_school_physics": 0.4090909090909091, + "high_school_politics": 0.4965034965034965, + "human_sexuality": 0.5714285714285714, + "international_law": 0.4810810810810811, + "journalism": 0.5523255813953488, + "jurisprudence": 0.5036496350364964, + "legal_and_moral_basis": 0.8130841121495327, + "logical": 0.6097560975609756, + "machine_learning": 0.4426229508196721, + "management": 0.6285714285714286, + "marketing": 0.6055555555555555, + "marxist_theory": 0.6084656084656085, + "modern_chinese": 0.43103448275862066, + "nutrition": 0.5310344827586206, + "philosophy": 0.6476190476190476, + "professional_accounting": 0.5885714285714285, + "professional_law": 0.33175355450236965, + "professional_medicine": 0.4654255319148936, + "professional_psychology": 0.5732758620689655, + "public_relations": 0.5862068965517241, + "security_study": 0.5703703703703704, + "sociology": 0.6061946902654868, + "sports_science": 0.5636363636363636, + "traditional_chinese_medicine": 0.41621621621621624, + "virology": 0.6094674556213018, + "world_history": 0.5217391304347826, + "world_religions": 0.6375 + } + }, + "prompt_2": { + "accuracy": 0.49378345708858573, + "category_acc": { + "agronomy": 0.5384615384615384, + "anatomy": 0.36486486486486486, + "ancient_chinese": 0.36585365853658536, + "arts": 0.56875, + "astronomy": 0.2909090909090909, + "business_ethics": 0.5215311004784688, + "chinese_civil_service_exam": 0.3875, + "chinese_driving_rule": 0.6870229007633588, + "chinese_food_culture": 0.4117647058823529, + "chinese_foreign_policy": 0.5794392523364486, + "chinese_history": 0.4891640866873065, + "chinese_literature": 0.3627450980392157, + "chinese_teacher_qualification": 0.5754189944134078, + "clinical_knowledge": 0.5021097046413502, + "college_actuarial_science": 0.20754716981132076, + "college_education": 0.6261682242990654, + "college_engineering_hydrology": 0.49056603773584906, + "college_law": 0.3888888888888889, + "college_mathematics": 0.3333333333333333, + "college_medical_statistics": 0.6037735849056604, + "college_medicine": 0.5347985347985348, + "computer_science": 0.5980392156862745, + "computer_security": 0.6023391812865497, + "conceptual_physics": 0.3197278911564626, + "construction_project_management": 0.41007194244604317, + "economics": 0.5786163522012578, + "education": 0.5337423312883436, + "electrical_engineering": 0.5116279069767442, + "elementary_chinese": 0.3611111111111111, + "elementary_commonsense": 0.4444444444444444, + "elementary_information_and_technology": 0.7142857142857143, + "elementary_mathematics": 0.41304347826086957, + "ethnology": 0.4888888888888889, + "food_science": 0.48951048951048953, + "genetics": 0.4034090909090909, + "global_facts": 0.5704697986577181, + "high_school_biology": 0.31952662721893493, + "high_school_chemistry": 0.29545454545454547, + "high_school_geography": 0.5, + "high_school_mathematics": 0.2073170731707317, + "high_school_physics": 0.34545454545454546, + "high_school_politics": 0.4755244755244755, + "human_sexuality": 0.5079365079365079, + "international_law": 0.43243243243243246, + "journalism": 0.5465116279069767, + "jurisprudence": 0.5060827250608273, + "legal_and_moral_basis": 0.822429906542056, + "logical": 0.5203252032520326, + "machine_learning": 0.4098360655737705, + "management": 0.6095238095238096, + "marketing": 0.5944444444444444, + "marxist_theory": 0.6084656084656085, + "modern_chinese": 0.3879310344827586, + "nutrition": 0.4827586206896552, + "philosophy": 0.5714285714285714, + "professional_accounting": 0.5428571428571428, + "professional_law": 0.3459715639810427, + "professional_medicine": 0.4441489361702128, + "professional_psychology": 0.5732758620689655, + "public_relations": 0.5459770114942529, + "security_study": 0.5185185185185185, + "sociology": 0.584070796460177, + "sports_science": 0.5818181818181818, + "traditional_chinese_medicine": 0.4486486486486487, + "virology": 0.5798816568047337, + "world_history": 0.546583850931677, + "world_religions": 0.61875 } }, "prompt_3": { - "accuracy": 0.360386807114488, + "accuracy": 0.5107926092212053, "category_acc": { - "agronomy": 0.378698224852071, - "anatomy": 0.35135135135135137, - "ancient_chinese": 0.2926829268292683, - "arts": 0.53125, - "astronomy": 0.32727272727272727, - "business_ethics": 0.37799043062200954, - "chinese_civil_service_exam": 0.33125, - "chinese_driving_rule": 0.45038167938931295, - "chinese_food_culture": 0.39705882352941174, - "chinese_foreign_policy": 0.37383177570093457, - "chinese_history": 0.3622291021671827, - "chinese_literature": 0.3431372549019608, - "chinese_teacher_qualification": 0.4860335195530726, - "clinical_knowledge": 0.31223628691983124, + "agronomy": 0.5325443786982249, + "anatomy": 0.3918918918918919, + "ancient_chinese": 0.3353658536585366, + "arts": 0.575, + "astronomy": 0.3212121212121212, + "business_ethics": 0.5550239234449761, + "chinese_civil_service_exam": 0.44375, + "chinese_driving_rule": 0.648854961832061, + "chinese_food_culture": 0.375, + "chinese_foreign_policy": 0.6074766355140186, + "chinese_history": 0.5201238390092879, + "chinese_literature": 0.35294117647058826, + "chinese_teacher_qualification": 0.5977653631284916, + "clinical_knowledge": 0.5569620253164557, "college_actuarial_science": 0.29245283018867924, - "college_education": 0.4766355140186916, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.3055555555555556, + "college_education": 0.6261682242990654, + "college_engineering_hydrology": 0.5283018867924528, + "college_law": 0.3888888888888889, "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.3113207547169811, - "college_medicine": 0.31135531135531136, - "computer_science": 0.3333333333333333, - "computer_security": 0.36257309941520466, - "conceptual_physics": 0.32653061224489793, - "construction_project_management": 0.3381294964028777, - "economics": 0.3018867924528302, - "education": 0.44171779141104295, - "electrical_engineering": 0.38372093023255816, - "elementary_chinese": 0.3968253968253968, - "elementary_commonsense": 0.40404040404040403, - "elementary_information_and_technology": 0.47058823529411764, - "elementary_mathematics": 0.3521739130434783, - "ethnology": 0.43703703703703706, - "food_science": 0.43356643356643354, - "genetics": 0.29545454545454547, - "global_facts": 0.4228187919463087, - "high_school_biology": 0.22485207100591717, - "high_school_chemistry": 0.25, - "high_school_geography": 0.3728813559322034, - "high_school_mathematics": 0.20121951219512196, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.32167832167832167, - "human_sexuality": 0.29365079365079366, - "international_law": 0.35135135135135137, - "journalism": 0.42441860465116277, - "jurisprudence": 0.34549878345498786, - "legal_and_moral_basis": 0.5607476635514018, - "logical": 0.36585365853658536, - "machine_learning": 0.29508196721311475, - "management": 0.43333333333333335, - "marketing": 0.35555555555555557, - "marxist_theory": 0.47619047619047616, - "modern_chinese": 0.3103448275862069, - "nutrition": 0.30344827586206896, - "philosophy": 0.3619047619047619, - "professional_accounting": 0.32571428571428573, - "professional_law": 0.3127962085308057, - "professional_medicine": 0.2925531914893617, - "professional_psychology": 0.3793103448275862, - "public_relations": 0.3793103448275862, - "security_study": 0.34074074074074073, - "sociology": 0.3938053097345133, - "sports_science": 0.3515151515151515, - "traditional_chinese_medicine": 0.31891891891891894, - "virology": 0.3905325443786982, - "world_history": 0.30434782608695654, - "world_religions": 0.425 + "college_medical_statistics": 0.5471698113207547, + "college_medicine": 0.5347985347985348, + "computer_science": 0.6421568627450981, + "computer_security": 0.6608187134502924, + "conceptual_physics": 0.4421768707482993, + "construction_project_management": 0.37410071942446044, + "economics": 0.6226415094339622, + "education": 0.5398773006134969, + "electrical_engineering": 0.5058139534883721, + "elementary_chinese": 0.34523809523809523, + "elementary_commonsense": 0.45454545454545453, + "elementary_information_and_technology": 0.7605042016806722, + "elementary_mathematics": 0.4391304347826087, + "ethnology": 0.48148148148148145, + "food_science": 0.5314685314685315, + "genetics": 0.4318181818181818, + "global_facts": 0.5906040268456376, + "high_school_biology": 0.34911242603550297, + "high_school_chemistry": 0.30303030303030304, + "high_school_geography": 0.5084745762711864, + "high_school_mathematics": 0.25609756097560976, + "high_school_physics": 0.32727272727272727, + "high_school_politics": 0.48951048951048953, + "human_sexuality": 0.5555555555555556, + "international_law": 0.4486486486486487, + "journalism": 0.5581395348837209, + "jurisprudence": 0.48905109489051096, + "legal_and_moral_basis": 0.8364485981308412, + "logical": 0.5528455284552846, + "machine_learning": 0.4262295081967213, + "management": 0.6047619047619047, + "marketing": 0.6444444444444445, + "marxist_theory": 0.6031746031746031, + "modern_chinese": 0.3620689655172414, + "nutrition": 0.5103448275862069, + "philosophy": 0.6666666666666666, + "professional_accounting": 0.56, + "professional_law": 0.3744075829383886, + "professional_medicine": 0.4734042553191489, + "professional_psychology": 0.5732758620689655, + "public_relations": 0.5574712643678161, + "security_study": 0.6222222222222222, + "sociology": 0.5707964601769911, + "sports_science": 0.5757575757575758, + "traditional_chinese_medicine": 0.4486486486486487, + "virology": 0.6094674556213018, + "world_history": 0.5714285714285714, + "world_religions": 0.64375 } }, "prompt_4": { - "accuracy": 0.4177171472975307, + "accuracy": 0.4285960973925056, "category_acc": { - "agronomy": 0.3727810650887574, - "anatomy": 0.32432432432432434, + "agronomy": 0.4319526627218935, + "anatomy": 0.3310810810810811, "ancient_chinese": 0.3048780487804878, - "arts": 0.6875, - "astronomy": 0.2909090909090909, - "business_ethics": 0.4449760765550239, - "chinese_civil_service_exam": 0.3, + "arts": 0.5125, + "astronomy": 0.296969696969697, + "business_ethics": 0.40669856459330145, + "chinese_civil_service_exam": 0.4125, "chinese_driving_rule": 0.5725190839694656, - "chinese_food_culture": 0.47794117647058826, - "chinese_foreign_policy": 0.40186915887850466, - "chinese_history": 0.44891640866873067, - "chinese_literature": 0.3627450980392157, - "chinese_teacher_qualification": 0.5418994413407822, - "clinical_knowledge": 0.4177215189873418, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.4953271028037383, - "college_engineering_hydrology": 0.3867924528301887, - "college_law": 0.4166666666666667, - "college_mathematics": 0.2, - "college_medical_statistics": 0.41509433962264153, - "college_medicine": 0.41025641025641024, - "computer_science": 0.3627450980392157, - "computer_security": 0.42105263157894735, - "conceptual_physics": 0.4217687074829932, - "construction_project_management": 0.39568345323741005, - "economics": 0.4779874213836478, - "education": 0.5276073619631901, - "electrical_engineering": 0.42441860465116277, - "elementary_chinese": 0.4801587301587302, - "elementary_commonsense": 0.5050505050505051, - "elementary_information_and_technology": 0.5756302521008403, - "elementary_mathematics": 0.32608695652173914, - "ethnology": 0.48148148148148145, - "food_science": 0.46153846153846156, - "genetics": 0.2840909090909091, - "global_facts": 0.436241610738255, - "high_school_biology": 0.24260355029585798, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.3813559322033898, - "high_school_mathematics": 0.29878048780487804, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.3356643356643357, - "human_sexuality": 0.38095238095238093, - "international_law": 0.33513513513513515, + "chinese_food_culture": 0.3602941176470588, + "chinese_foreign_policy": 0.4953271028037383, + "chinese_history": 0.4582043343653251, + "chinese_literature": 0.3284313725490196, + "chinese_teacher_qualification": 0.48044692737430167, + "clinical_knowledge": 0.5274261603375527, + "college_actuarial_science": 0.2358490566037736, + "college_education": 0.5514018691588785, + "college_engineering_hydrology": 0.5094339622641509, + "college_law": 0.3425925925925926, + "college_mathematics": 0.2571428571428571, + "college_medical_statistics": 0.3584905660377358, + "college_medicine": 0.4652014652014652, + "computer_science": 0.4950980392156863, + "computer_security": 0.4853801169590643, + "conceptual_physics": 0.24489795918367346, + "construction_project_management": 0.3597122302158273, + "economics": 0.46540880503144655, + "education": 0.4294478527607362, + "electrical_engineering": 0.4127906976744186, + "elementary_chinese": 0.35714285714285715, + "elementary_commonsense": 0.398989898989899, + "elementary_information_and_technology": 0.5672268907563025, + "elementary_mathematics": 0.3173913043478261, + "ethnology": 0.4074074074074074, + "food_science": 0.44755244755244755, + "genetics": 0.30113636363636365, + "global_facts": 0.4966442953020134, + "high_school_biology": 0.3254437869822485, + "high_school_chemistry": 0.2803030303030303, + "high_school_geography": 0.3898305084745763, + "high_school_mathematics": 0.2804878048780488, + "high_school_physics": 0.3090909090909091, + "high_school_politics": 0.44755244755244755, + "human_sexuality": 0.48412698412698413, + "international_law": 0.43243243243243246, "journalism": 0.4418604651162791, - "jurisprudence": 0.36982968369829683, - "legal_and_moral_basis": 0.6869158878504673, - "logical": 0.3902439024390244, - "machine_learning": 0.3770491803278688, - "management": 0.5380952380952381, - "marketing": 0.49444444444444446, - "marxist_theory": 0.48677248677248675, - "modern_chinese": 0.31896551724137934, + "jurisprudence": 0.39659367396593675, + "legal_and_moral_basis": 0.7102803738317757, + "logical": 0.4065040650406504, + "machine_learning": 0.36885245901639346, + "management": 0.5333333333333333, + "marketing": 0.55, + "marxist_theory": 0.5026455026455027, + "modern_chinese": 0.3017241379310345, "nutrition": 0.4, - "philosophy": 0.4857142857142857, - "professional_accounting": 0.4685714285714286, - "professional_law": 0.33175355450236965, - "professional_medicine": 0.3377659574468085, - "professional_psychology": 0.4525862068965517, - "public_relations": 0.45977011494252873, - "security_study": 0.4444444444444444, - "sociology": 0.43805309734513276, - "sports_science": 0.4303030303030303, - "traditional_chinese_medicine": 0.4, - "virology": 0.41420118343195267, - "world_history": 0.4472049689440994, - "world_religions": 0.51875 + "philosophy": 0.5619047619047619, + "professional_accounting": 0.5085714285714286, + "professional_law": 0.3175355450236967, + "professional_medicine": 0.4095744680851064, + "professional_psychology": 0.49137931034482757, + "public_relations": 0.4827586206896552, + "security_study": 0.5259259259259259, + "sociology": 0.4823008849557522, + "sports_science": 0.41818181818181815, + "traditional_chinese_medicine": 0.372972972972973, + "virology": 0.4319526627218935, + "world_history": 0.5031055900621118, + "world_religions": 0.53125 } }, "prompt_5": { - "accuracy": 0.42263857710240027, + "accuracy": 0.4709894664134001, "category_acc": { - "agronomy": 0.40236686390532544, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.2804878048780488, - "arts": 0.69375, - "astronomy": 0.34545454545454546, - "business_ethics": 0.45454545454545453, - "chinese_civil_service_exam": 0.26875, - "chinese_driving_rule": 0.46564885496183206, - "chinese_food_culture": 0.5220588235294118, - "chinese_foreign_policy": 0.40186915887850466, - "chinese_history": 0.3715170278637771, - "chinese_literature": 0.4117647058823529, - "chinese_teacher_qualification": 0.547486033519553, - "clinical_knowledge": 0.3628691983122363, - "college_actuarial_science": 0.20754716981132076, - "college_education": 0.5794392523364486, - "college_engineering_hydrology": 0.3867924528301887, - "college_law": 0.46296296296296297, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.3867924528301887, - "college_medicine": 0.43223443223443225, - "computer_science": 0.4411764705882353, - "computer_security": 0.4502923976608187, - "conceptual_physics": 0.3129251700680272, - "construction_project_management": 0.38848920863309355, - "economics": 0.4716981132075472, - "education": 0.5521472392638037, - "electrical_engineering": 0.4069767441860465, - "elementary_chinese": 0.45634920634920634, - "elementary_commonsense": 0.5050505050505051, - "elementary_information_and_technology": 0.5882352941176471, - "elementary_mathematics": 0.3391304347826087, - "ethnology": 0.4740740740740741, - "food_science": 0.4405594405594406, - "genetics": 0.32386363636363635, - "global_facts": 0.5302013422818792, - "high_school_biology": 0.27218934911242604, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.4067796610169492, - "high_school_mathematics": 0.25, - "high_school_physics": 0.23636363636363636, - "high_school_politics": 0.2517482517482518, - "human_sexuality": 0.36507936507936506, - "international_law": 0.372972972972973, - "journalism": 0.5174418604651163, - "jurisprudence": 0.3746958637469586, - "legal_and_moral_basis": 0.6822429906542056, - "logical": 0.3821138211382114, - "machine_learning": 0.36065573770491804, - "management": 0.5571428571428572, - "marketing": 0.5166666666666667, - "marxist_theory": 0.4603174603174603, - "modern_chinese": 0.28448275862068967, - "nutrition": 0.45517241379310347, - "philosophy": 0.49523809523809526, - "professional_accounting": 0.4114285714285714, - "professional_law": 0.36492890995260663, - "professional_medicine": 0.3351063829787234, - "professional_psychology": 0.5086206896551724, - "public_relations": 0.4540229885057471, - "security_study": 0.5333333333333333, - "sociology": 0.4911504424778761, - "sports_science": 0.42424242424242425, - "traditional_chinese_medicine": 0.44324324324324327, - "virology": 0.4378698224852071, - "world_history": 0.4472049689440994, - "world_religions": 0.54375 + "agronomy": 0.47337278106508873, + "anatomy": 0.33783783783783783, + "ancient_chinese": 0.2926829268292683, + "arts": 0.54375, + "astronomy": 0.3212121212121212, + "business_ethics": 0.49760765550239233, + "chinese_civil_service_exam": 0.38125, + "chinese_driving_rule": 0.6793893129770993, + "chinese_food_culture": 0.36764705882352944, + "chinese_foreign_policy": 0.5327102803738317, + "chinese_history": 0.4953560371517028, + "chinese_literature": 0.3382352941176471, + "chinese_teacher_qualification": 0.5642458100558659, + "clinical_knowledge": 0.510548523206751, + "college_actuarial_science": 0.24528301886792453, + "college_education": 0.5607476635514018, + "college_engineering_hydrology": 0.5094339622641509, + "college_law": 0.37962962962962965, + "college_mathematics": 0.21904761904761905, + "college_medical_statistics": 0.5094339622641509, + "college_medicine": 0.47619047619047616, + "computer_science": 0.5735294117647058, + "computer_security": 0.5497076023391813, + "conceptual_physics": 0.3401360544217687, + "construction_project_management": 0.3597122302158273, + "economics": 0.559748427672956, + "education": 0.5460122699386503, + "electrical_engineering": 0.4883720930232558, + "elementary_chinese": 0.3531746031746032, + "elementary_commonsense": 0.4292929292929293, + "elementary_information_and_technology": 0.6848739495798319, + "elementary_mathematics": 0.3652173913043478, + "ethnology": 0.4888888888888889, + "food_science": 0.5524475524475524, + "genetics": 0.3977272727272727, + "global_facts": 0.5369127516778524, + "high_school_biology": 0.33136094674556216, + "high_school_chemistry": 0.25757575757575757, + "high_school_geography": 0.4661016949152542, + "high_school_mathematics": 0.29878048780487804, + "high_school_physics": 0.3181818181818182, + "high_school_politics": 0.4965034965034965, + "human_sexuality": 0.5158730158730159, + "international_law": 0.42162162162162165, + "journalism": 0.5116279069767442, + "jurisprudence": 0.44525547445255476, + "legal_and_moral_basis": 0.8084112149532711, + "logical": 0.5121951219512195, + "machine_learning": 0.4344262295081967, + "management": 0.6285714285714286, + "marketing": 0.5722222222222222, + "marxist_theory": 0.582010582010582, + "modern_chinese": 0.3275862068965517, + "nutrition": 0.46206896551724136, + "philosophy": 0.6095238095238096, + "professional_accounting": 0.4514285714285714, + "professional_law": 0.32701421800947866, + "professional_medicine": 0.4122340425531915, + "professional_psychology": 0.5172413793103449, + "public_relations": 0.5459770114942529, + "security_study": 0.5407407407407407, + "sociology": 0.5132743362831859, + "sports_science": 0.509090909090909, + "traditional_chinese_medicine": 0.4, + "virology": 0.5088757396449705, + "world_history": 0.5341614906832298, + "world_religions": 0.5875 } } }, "zbench": { "prompt_1": { - "accuracy": 0.3333333333333333 + "accuracy": 0.30303030303030304 }, "prompt_2": { - "accuracy": 0.18181818181818182 + "accuracy": 0.2727272727272727 }, "prompt_3": { - "accuracy": 0.24242424242424243 + "accuracy": 0.42424242424242425 }, "prompt_4": { - "accuracy": 0.30303030303030304 + "accuracy": 0.2727272727272727 }, "prompt_5": { - "accuracy": 0.2727272727272727 + "accuracy": 0.3939393939393939 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.25227272727272726 + "accuracy": 0.6863636363636364 }, "prompt_2": { - "accuracy": 0.19545454545454546 + "accuracy": 0.6477272727272727 }, "prompt_3": { - "accuracy": 0.2772727272727273 + "accuracy": 0.6795454545454546 }, "prompt_4": { - "accuracy": 0.25227272727272726 + "accuracy": 0.6590909090909091 }, "prompt_5": { - "accuracy": 0.2340909090909091 + "accuracy": 0.6363636363636364 } }, "ocnli": { "prompt_1": { - "accuracy": 0.3257627118644068 + "accuracy": 0.49627118644067797 }, "prompt_2": { - "accuracy": 0.32033898305084746 + "accuracy": 0.4505084745762712 }, "prompt_3": { - "accuracy": 0.31322033898305085 + "accuracy": 0.5345762711864407 }, "prompt_4": { - "accuracy": 0.336271186440678 + "accuracy": 0.48033898305084743 }, "prompt_5": { - "accuracy": 0.3305084745762712 + "accuracy": 0.42338983050847456 } }, "c3": { "prompt_1": { - "accuracy": 0.6929693343305908 + "accuracy": 0.8844427823485415 }, "prompt_2": { - "accuracy": 0.5568436798803291 + "accuracy": 0.8825729244577412 }, "prompt_3": { - "accuracy": 0.5336574420344053 + "accuracy": 0.8739715781600599 }, "prompt_4": { - "accuracy": 0.6952131637995512 + "accuracy": 0.8754674644727001 }, "prompt_5": { - "accuracy": 0.656320119670905 + "accuracy": 0.8694839192221391 } }, "dream": { "prompt_1": { - "accuracy": 0.6046055854973053 + "accuracy": 0.8931896129348359 }, "prompt_2": { - "accuracy": 0.6575208231259186 + "accuracy": 0.8990690837824595 }, "prompt_3": { - "accuracy": 0.6692797648211661 + "accuracy": 0.9118079372856442 }, "prompt_4": { - "accuracy": 0.5918667319941205 + "accuracy": 0.9024987751102401 }, "prompt_5": { - "accuracy": 0.6001959823615874 + "accuracy": 0.9064184223419892 } }, "samsum": { "prompt_1": { - "rouge1": 0.2722898259856251, - "rouge2": 0.09500595501291095, - "rougeL": 0.2083963108699263, - "avg_rouge": 0.19189736395615412 + "rouge1": 0.22783552386864417, + "rouge2": 0.0839403119084876, + "rougeL": 0.17007006250391818, + "avg_rouge": 0.16061529942701666 }, "prompt_2": { - "rouge1": 0.280285585162671, - "rouge2": 0.11414303446320374, - "rougeL": 0.2183636756557759, - "avg_rouge": 0.2042640984272169 + "rouge1": 0.21906277397329468, + "rouge2": 0.07981495779273613, + "rougeL": 0.1611019992992204, + "avg_rouge": 0.15332657702175043 }, "prompt_3": { - "rouge1": 0.24364593439232496, - "rouge2": 0.0914801141630061, - "rougeL": 0.18816056846940268, - "avg_rouge": 0.17442887234157792 + "rouge1": 0.22051601719774955, + "rouge2": 0.07733715420256924, + "rougeL": 0.16309052127918097, + "avg_rouge": 0.15364789755983324 }, "prompt_4": { - "rouge1": 0.23078035591580603, - "rouge2": 0.078927541972213, - "rougeL": 0.17639939121952822, - "avg_rouge": 0.16203576303584907 + "rouge1": 0.21865452524714854, + "rouge2": 0.07795006249760242, + "rougeL": 0.16086961877960224, + "avg_rouge": 0.1524914021747844 }, "prompt_5": { - "rouge1": 0.23603344399273724, - "rouge2": 0.0777379315672454, - "rougeL": 0.18021784099523483, - "avg_rouge": 0.16466307218507248 + "rouge1": 0.22042637368342233, + "rouge2": 0.08018824424560042, + "rougeL": 0.1712332946359059, + "avg_rouge": 0.15728263752164287 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.2246615757842708, - "rouge2": 0.0621634596765895, - "rougeL": 0.16560170323903578, - "avg_rouge": 0.15080891289996537 + "rouge1": 0.12705281041724492, + "rouge2": 0.034748531923445534, + "rougeL": 0.09916061070001848, + "avg_rouge": 0.08698731768023632 }, "prompt_2": { - "rouge1": 0.21721803053321123, - "rouge2": 0.05970700920829458, - "rougeL": 0.1599075453082249, - "avg_rouge": 0.14561086168324358 + "rouge1": 0.13768897790409756, + "rouge2": 0.037895101154647844, + "rougeL": 0.10580890122208217, + "avg_rouge": 0.0937976600936092 }, "prompt_3": { - "rouge1": 0.21433909125035078, - "rouge2": 0.05917455576452657, - "rougeL": 0.15710191600669401, - "avg_rouge": 0.14353852100719044 + "rouge1": 0.15013721929677626, + "rouge2": 0.043554485551585684, + "rougeL": 0.11271506493749443, + "avg_rouge": 0.1021355899286188 }, "prompt_4": { - "rouge1": 0.226006737311942, - "rouge2": 0.06315860156617512, - "rougeL": 0.1667275305746969, - "avg_rouge": 0.15196428981760468 + "rouge1": 0.16301602801162368, + "rouge2": 0.04606017775287866, + "rougeL": 0.12228466252459767, + "avg_rouge": 0.11045362276303333 }, "prompt_5": { - "rouge1": 0.21907442211986763, - "rouge2": 0.06192272034737452, - "rougeL": 0.1613661722224539, - "avg_rouge": 0.14745443822989868 + "rouge1": 0.19426426170381578, + "rouge2": 0.05026998096295629, + "rougeL": 0.1427788616737507, + "avg_rouge": 0.1291043681135076 } }, "sst2": { "prompt_1": { - "accuracy": 0.8268348623853211 + "accuracy": 0.9025229357798165 }, "prompt_2": { - "accuracy": 0.8176605504587156 + "accuracy": 0.911697247706422 }, "prompt_3": { - "accuracy": 0.8818807339449541 + "accuracy": 0.8692660550458715 }, "prompt_4": { - "accuracy": 0.8577981651376146 + "accuracy": 0.9128440366972477 }, "prompt_5": { - "accuracy": 0.7087155963302753 + "accuracy": 0.8990825688073395 } }, "cola": { "prompt_1": { - "accuracy": 0.6567593480345159 + "accuracy": 0.6663470757430489 }, "prompt_2": { - "accuracy": 0.50143815915628 + "accuracy": 0.6452540747842761 }, "prompt_3": { - "accuracy": 0.5397890699904123 + "accuracy": 0.6903163950143816 }, "prompt_4": { - "accuracy": 0.6577181208053692 + "accuracy": 0.7948226270373921 }, "prompt_5": { - "accuracy": 0.700862895493768 + "accuracy": 0.6586768935762224 } }, "qqp": { "prompt_1": { - "accuracy": 0.4635 + "accuracy": 0.639 }, "prompt_2": { - "accuracy": 0.5245 + "accuracy": 0.7385 }, "prompt_3": { - "accuracy": 0.5205 + "accuracy": 0.6695 }, "prompt_4": { - "accuracy": 0.452 + "accuracy": 0.5695 }, "prompt_5": { - "accuracy": 0.464 + "accuracy": 0.75 } }, "mnli": { "prompt_1": { - "accuracy": 0.363 + "accuracy": 0.7 }, "prompt_2": { - "accuracy": 0.37 + "accuracy": 0.7 }, "prompt_3": { - "accuracy": 0.366 + "accuracy": 0.7 }, "prompt_4": { - "accuracy": 0.3625 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.3715 + "accuracy": 0.6 } }, "qnli": { "prompt_1": { - "accuracy": 0.511 + "accuracy": 1.0 }, "prompt_2": { - "accuracy": 0.5025 + "accuracy": 1.0 }, "prompt_3": { - "accuracy": 0.4925 + "accuracy": 1.0 }, "prompt_4": { - "accuracy": 0.4715 + "accuracy": 1.0 }, "prompt_5": { - "accuracy": 0.5095 + "accuracy": 1.0 } }, "wnli": { "prompt_1": { - "accuracy": 0.5352112676056338 + "accuracy": 0.4 }, "prompt_2": { - "accuracy": 0.5492957746478874 + "accuracy": 0.6 }, "prompt_3": { - "accuracy": 0.6056338028169014 + "accuracy": 0.5 }, "prompt_4": { - "accuracy": 0.4084507042253521 + "accuracy": 0.3 }, "prompt_5": { - "accuracy": 0.4647887323943662 + "accuracy": 0.4 } }, "rte": { "prompt_1": { - "accuracy": 0.5415162454873647 + "accuracy": 0.8 }, "prompt_2": { - "accuracy": 0.5234657039711191 + "accuracy": 0.6 }, "prompt_3": { - "accuracy": 0.49458483754512633 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.5234657039711191 + "accuracy": 0.8 }, "prompt_5": { - "accuracy": 0.48736462093862815 + "accuracy": 0.7 } }, "mrpc": { "prompt_1": { - "accuracy": 0.6764705882352942 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.5122549019607843 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.4681372549019608 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.4411764705882353 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.6691176470588235 + "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.7840336134453781, - "language_acc": { - "English": 0.8159663865546218, - "Chinese": 0.7873949579831933, - "Spanish": 0.7630252100840336, - "Vietnamese": 0.7697478991596639 - }, - "consistency_score_2": 0.8127450980392156, - "consistency_score_3": 0.73109243697479, - "consistency_score_4": 0.6815126050420168, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8319327731092437, - "English,Spanish": 0.8294117647058824, - "English,Vietnamese": 0.8336134453781513, - "Chinese,Spanish": 0.7857142857142857, - "Chinese,Vietnamese": 0.7907563025210084, - "Spanish,Vietnamese": 0.8050420168067227 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7344537815126051, - "English,Chinese,Vietnamese": 0.7394957983193278, - "English,Spanish,Vietnamese": 0.7436974789915967, - "Chinese,Spanish,Vietnamese": 0.7067226890756303 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.6815126050420168 - } - }, - "AC3_2": 0.7981312268289792, - "AC3_3": 0.7566380961203096, - "AC3_4": 0.7291872252990997 - } + "prompt_1": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.3523809523809524, - "language_acc": { - "Spanish": 0.32, - "English": 0.4066666666666667, - "Chinese": 0.32, - "Vietnamese": 0.38666666666666666, - "Filipino": 0.28, - "Indonesian": 0.38, - "Malay": 0.37333333333333335 - }, - "consistency_score_2": 0.45206349206349217, - "consistency_score_3": 0.24304761904761898, - "consistency_score_4": 0.14666666666666664, - "consistency_score_5": 0.09936507936507938, - "consistency_score_6": 0.07428571428571429, - "consistency_score_7": 0.06, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.4, - "Spanish,Chinese": 0.4666666666666667, - "Spanish,Vietnamese": 0.46, - "Spanish,Filipino": 0.38, - "Spanish,Indonesian": 0.4533333333333333, - "Spanish,Malay": 0.43333333333333335, - "English,Chinese": 0.48, - "English,Vietnamese": 0.4066666666666667, - "English,Filipino": 0.38, - "English,Indonesian": 0.5066666666666667, - "English,Malay": 0.4866666666666667, - "Chinese,Vietnamese": 0.4266666666666667, - "Chinese,Filipino": 0.44, - "Chinese,Indonesian": 0.43333333333333335, - "Chinese,Malay": 0.47333333333333333, - "Vietnamese,Filipino": 0.43333333333333335, - "Vietnamese,Indonesian": 0.5933333333333334, - "Vietnamese,Malay": 0.5333333333333333, - "Filipino,Indonesian": 0.46, - "Filipino,Malay": 0.3466666666666667, - "Indonesian,Malay": 0.5 - }, - "3_combine": { - "Spanish,English,Chinese": 0.24666666666666667, - "Spanish,English,Vietnamese": 0.22, - "Spanish,English,Filipino": 0.19333333333333333, - "Spanish,English,Indonesian": 0.26, - "Spanish,English,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese": 0.24, - "Spanish,Chinese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Filipino": 0.22, - "Spanish,Vietnamese,Indonesian": 0.3, - "Spanish,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Filipino,Indonesian": 0.24666666666666667, - "Spanish,Filipino,Malay": 0.15333333333333332, - "Spanish,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese": 0.20666666666666667, - "English,Chinese,Filipino": 0.21333333333333335, - "English,Chinese,Indonesian": 0.26, - "English,Chinese,Malay": 0.28, - "English,Vietnamese,Filipino": 0.19333333333333333, - "English,Vietnamese,Indonesian": 0.31333333333333335, - "English,Vietnamese,Malay": 0.26666666666666666, - "English,Filipino,Indonesian": 0.25333333333333335, - "English,Filipino,Malay": 0.17333333333333334, - "English,Indonesian,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Filipino": 0.22, - "Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "Chinese,Vietnamese,Malay": 0.2733333333333333, - "Chinese,Filipino,Indonesian": 0.24, - "Chinese,Filipino,Malay": 0.21333333333333335, - "Chinese,Indonesian,Malay": 0.25333333333333335, - "Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Vietnamese,Filipino,Malay": 0.21333333333333335, - "Vietnamese,Indonesian,Malay": 0.36, - "Filipino,Indonesian,Malay": 0.21333333333333335 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.13333333333333333, - "Spanish,English,Chinese,Filipino": 0.11333333333333333, - "Spanish,English,Chinese,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Malay": 0.14666666666666667, - "Spanish,English,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Indonesian,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Malay": 0.16666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.14, - "Spanish,Chinese,Filipino,Malay": 0.1, - "Spanish,Chinese,Indonesian,Malay": 0.14, - "Spanish,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "English,Chinese,Vietnamese,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Filipino,Malay": 0.10666666666666667, - "English,Chinese,Indonesian,Malay": 0.18, - "English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Vietnamese,Filipino,Malay": 0.12, - "English,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "English,Filipino,Indonesian,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Vietnamese,Filipino,Indonesian,Malay": 0.16 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.08, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.08, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "English,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.06, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.06666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06 - } - }, - "AC3_2": 0.39604615321870484, - "AC3_3": 0.28767632448021757, - "AC3_4": 0.2071246818923382, - "AC3_5": 0.15501790312132505, - "AC3_6": 0.1227040816038943, - "AC3_7": 0.10254041567952253 - } + "prompt_1": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.4172077922077922, - "language_acc": { - "English": 0.4147727272727273, - "Filipino": 0.29545454545454547, - "Spanish": 0.4431818181818182, - "Malay": 0.44886363636363635, - "Chinese": 0.4375, - "Indonesian": 0.45454545454545453, - "Vietnamese": 0.42613636363636365 - }, - "consistency_score_2": 0.4772727272727273, - "consistency_score_3": 0.29577922077922075, - "consistency_score_4": 0.2076298701298701, - "consistency_score_5": 0.1547619047619048, - "consistency_score_6": 0.11850649350649352, - "consistency_score_7": 0.09090909090909091, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.44886363636363635, - "English,Spanish": 0.5113636363636364, - "English,Malay": 0.4772727272727273, - "English,Chinese": 0.5397727272727273, - "English,Indonesian": 0.5227272727272727, - "English,Vietnamese": 0.5056818181818182, - "Filipino,Spanish": 0.3465909090909091, - "Filipino,Malay": 0.3409090909090909, - "Filipino,Chinese": 0.4147727272727273, - "Filipino,Indonesian": 0.4034090909090909, - "Filipino,Vietnamese": 0.3181818181818182, - "Spanish,Malay": 0.5056818181818182, - "Spanish,Chinese": 0.4772727272727273, - "Spanish,Indonesian": 0.5113636363636364, - "Spanish,Vietnamese": 0.5568181818181818, - "Malay,Chinese": 0.4602272727272727, - "Malay,Indonesian": 0.6534090909090909, - "Malay,Vietnamese": 0.5397727272727273, - "Chinese,Indonesian": 0.5, - "Chinese,Vietnamese": 0.4375, - "Indonesian,Vietnamese": 0.5511363636363636 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2215909090909091, - "English,Filipino,Malay": 0.24431818181818182, - "English,Filipino,Chinese": 0.2784090909090909, - "English,Filipino,Indonesian": 0.2556818181818182, - "English,Filipino,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay": 0.32954545454545453, - "English,Spanish,Chinese": 0.32954545454545453, - "English,Spanish,Indonesian": 0.3522727272727273, - "English,Spanish,Vietnamese": 0.3465909090909091, - "English,Malay,Chinese": 0.3181818181818182, - "English,Malay,Indonesian": 0.39204545454545453, - "English,Malay,Vietnamese": 0.32386363636363635, - "English,Chinese,Indonesian": 0.3409090909090909, - "English,Chinese,Vietnamese": 0.3068181818181818, - "English,Indonesian,Vietnamese": 0.3522727272727273, - "Filipino,Spanish,Malay": 0.22727272727272727, - "Filipino,Spanish,Chinese": 0.2215909090909091, - "Filipino,Spanish,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese": 0.22727272727272727, - "Filipino,Malay,Indonesian": 0.26704545454545453, - "Filipino,Malay,Vietnamese": 0.19318181818181818, - "Filipino,Chinese,Indonesian": 0.2556818181818182, - "Filipino,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Indonesian,Vietnamese": 0.21022727272727273, - "Spanish,Malay,Chinese": 0.32386363636363635, - "Spanish,Malay,Indonesian": 0.38636363636363635, - "Spanish,Malay,Vietnamese": 0.36363636363636365, - "Spanish,Chinese,Indonesian": 0.32386363636363635, - "Spanish,Chinese,Vietnamese": 0.3125, - "Spanish,Indonesian,Vietnamese": 0.375, - "Malay,Chinese,Indonesian": 0.375, - "Malay,Chinese,Vietnamese": 0.3068181818181818, - "Malay,Indonesian,Vietnamese": 0.4090909090909091, - "Chinese,Indonesian,Vietnamese": 0.3181818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.17045454545454544, - "English,Filipino,Spanish,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese": 0.18181818181818182, - "English,Filipino,Malay,Indonesian": 0.20454545454545456, - "English,Filipino,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay,Chinese": 0.25, - "English,Spanish,Malay,Indonesian": 0.2784090909090909, - "English,Spanish,Malay,Vietnamese": 0.25, - "English,Spanish,Chinese,Indonesian": 0.25, - "English,Spanish,Chinese,Vietnamese": 0.23295454545454544, - "English,Spanish,Indonesian,Vietnamese": 0.2727272727272727, - "English,Malay,Chinese,Indonesian": 0.2727272727272727, - "English,Malay,Chinese,Vietnamese": 0.2159090909090909, - "English,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Malay,Chinese": 0.18181818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.26704545454545453, - "Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.2897727272727273, - "Spanish,Chinese,Indonesian,Vietnamese": 0.25, - "Malay,Chinese,Indonesian,Vietnamese": 0.26704545454545453 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.21022727272727273, - "English,Spanish,Malay,Chinese,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.125, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - } - }, - "AC3_2": 0.4452235604187951, - "AC3_3": 0.34615327745718677, - "AC3_4": 0.2772713775777431, - "AC3_5": 0.22577375317042023, - "AC3_6": 0.18458284136656777, - "AC3_7": 0.14928841123986122 - } + "prompt_1": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.32038834951456313 - } + "prompt_1": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.3904761904761905 - } + "prompt_1": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.37383177570093457 - } + "prompt_1": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.29, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.3, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.5, - "culture": 0.2, - "film": 0.3, - "law": 0.2, - "geography": 0.3 - } - } + "prompt_1": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.08626082502601325 - } + "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.402548524934417 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.3379214451296861 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.2468827962157602 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.4180135900463724 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.40373395565927656 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.38291026099392206, - "category_acc": { - "high_school_european_history": 0.5, - "business_ethics": 0.31313131313131315, - "clinical_knowledge": 0.4431818181818182, - "medical_genetics": 0.42424242424242425, - "high_school_us_history": 0.5714285714285714, - "high_school_physics": 0.30666666666666664, - "high_school_world_history": 0.538135593220339, - "virology": 0.44242424242424244, - "high_school_microeconomics": 0.4050632911392405, - "econometrics": 0.2920353982300885, - "college_computer_science": 0.32323232323232326, - "high_school_biology": 0.42071197411003236, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.31316725978647686, - "philosophy": 0.4161290322580645, - "professional_medicine": 0.4280442804428044, - "nutrition": 0.39672131147540984, - "global_facts": 0.2727272727272727, - "machine_learning": 0.3783783783783784, - "security_studies": 0.4672131147540984, - "public_relations": 0.3669724770642202, - "professional_psychology": 0.37315875613747956, - "prehistory": 0.43343653250773995, - "anatomy": 0.3358208955223881, - "human_sexuality": 0.38461538461538464, - "college_medicine": 0.3488372093023256, - "high_school_government_and_politics": 0.5104166666666666, - "college_chemistry": 0.35353535353535354, - "logical_fallacies": 0.3888888888888889, - "high_school_geography": 0.4263959390862944, - "elementary_mathematics": 0.29973474801061006, - "human_aging": 0.481981981981982, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.47058823529411764, - "formal_logic": 0.296, - "high_school_statistics": 0.33488372093023255, - "international_law": 0.5333333333333333, - "high_school_mathematics": 0.2899628252788104, - "high_school_computer_science": 0.3939393939393939, - "conceptual_physics": 0.38461538461538464, - "miscellaneous": 0.4322250639386189, - "high_school_chemistry": 0.38613861386138615, - "marketing": 0.5150214592274678, - "professional_law": 0.3020221787345075, - "management": 0.37254901960784315, - "college_physics": 0.25742574257425743, - "jurisprudence": 0.3925233644859813, - "world_religions": 0.43529411764705883, - "sociology": 0.53, - "us_foreign_policy": 0.48484848484848486, - "high_school_macroeconomics": 0.36503856041131105, - "computer_security": 0.45454545454545453, - "moral_scenarios": 0.24272930648769575, - "moral_disputes": 0.3507246376811594, - "electrical_engineering": 0.3958333333333333, - "astronomy": 0.4105960264900662, - "college_biology": 0.43356643356643354 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.4138187221396731 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.4396014943960149, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.42857142857142855, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.40476190476190477, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.5, - "high_school_biology": 0.5, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.6538461538461539, - "middle_school_physics": 0.6666666666666666, - "middle_school_chemistry": 0.64, - "veterinary_medicine": 0.4642857142857143, - "college_economics": 0.4166666666666667, - "business_administration": 0.42105263157894735, - "marxism": 0.5, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.5294117647058824, - "teacher_qualification": 0.6326530612244898, - "high_school_politics": 0.625, - "high_school_geography": 0.75, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.5294117647058824, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.6666666666666666, - "logic": 0.25925925925925924, - "law": 0.13793103448275862, - "chinese_language_and_literature": 0.42857142857142855, - "art_studies": 0.6052631578947368, - "professional_tour_guide": 0.47058823529411764, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.6, - "middle_school_history": 0.5925925925925926, - "civil_servant": 0.38461538461538464, - "sports_science": 0.375, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.48148148148148145, - "urban_and_rural_planner": 0.45098039215686275, - "accountant": 0.3888888888888889, - "fire_engineer": 0.3611111111111111, - "environmental_impact_assessment_engineer": 0.5, - "tax_accountant": 0.4074074074074074, - "physician": 0.4444444444444444 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.5555555555555556 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.46434121913313764, - "category_acc": { - "agronomy": 0.4911242603550296, - "anatomy": 0.2905405405405405, - "ancient_chinese": 0.29878048780487804, - "arts": 0.63125, - "astronomy": 0.2787878787878788, - "business_ethics": 0.48325358851674644, - "chinese_civil_service_exam": 0.3875, - "chinese_driving_rule": 0.5419847328244275, - "chinese_food_culture": 0.5, - "chinese_foreign_policy": 0.5514018691588785, - "chinese_history": 0.5232198142414861, - "chinese_literature": 0.4166666666666667, - "chinese_teacher_qualification": 0.5307262569832403, - "clinical_knowledge": 0.45147679324894513, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.6074766355140186, - "college_engineering_hydrology": 0.32075471698113206, - "college_law": 0.4166666666666667, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.4339622641509434, - "college_medicine": 0.4432234432234432, - "computer_science": 0.4068627450980392, - "computer_security": 0.5555555555555556, - "conceptual_physics": 0.4013605442176871, - "construction_project_management": 0.34532374100719426, - "economics": 0.5220125786163522, - "education": 0.5214723926380368, - "electrical_engineering": 0.46511627906976744, - "elementary_chinese": 0.45634920634920634, - "elementary_commonsense": 0.5454545454545454, - "elementary_information_and_technology": 0.6092436974789915, - "elementary_mathematics": 0.32608695652173914, - "ethnology": 0.4444444444444444, - "food_science": 0.5524475524475524, - "genetics": 0.3693181818181818, - "global_facts": 0.5167785234899329, - "high_school_biology": 0.4378698224852071, - "high_school_chemistry": 0.3181818181818182, - "high_school_geography": 0.4745762711864407, - "high_school_mathematics": 0.3048780487804878, - "high_school_physics": 0.42727272727272725, - "high_school_politics": 0.43356643356643354, - "human_sexuality": 0.48412698412698413, - "international_law": 0.4, - "journalism": 0.5116279069767442, - "jurisprudence": 0.46228710462287104, - "legal_and_moral_basis": 0.7289719626168224, - "logical": 0.3983739837398374, - "machine_learning": 0.4016393442622951, - "management": 0.5476190476190477, - "marketing": 0.4888888888888889, - "marxist_theory": 0.5925925925925926, - "modern_chinese": 0.28448275862068967, - "nutrition": 0.4827586206896552, - "philosophy": 0.49523809523809526, - "professional_accounting": 0.52, - "professional_law": 0.3886255924170616, - "professional_medicine": 0.3962765957446808, - "professional_psychology": 0.5905172413793104, - "public_relations": 0.4942528735632184, - "security_study": 0.5481481481481482, - "sociology": 0.4690265486725664, - "sports_science": 0.5212121212121212, - "traditional_chinese_medicine": 0.43783783783783786, - "virology": 0.4970414201183432, - "world_history": 0.5403726708074534, - "world_religions": 0.5375 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.42424242424242425 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.22954545454545455 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.3233898305084746 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.6570680628272252 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.6927976482116609 - } + "prompt_1": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.19038450939168935, - "rouge2": 0.07575042452552877, - "rougeL": 0.14687628393730315, - "avg_rouge": 0.13767040595150706 - } + "prompt_1": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.16000973694319584, - "rouge2": 0.05350448605342644, - "rougeL": 0.12315118642126766, - "avg_rouge": 0.11222180313929664 - } + "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.5091743119266054 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.43720038350910834 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.5555 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.3665 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.482 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.5070422535211268 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.555956678700361 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.4485294117647059 - } + "prompt_1": -1 } } }, - "qwen_1_5_7b_chat": { - "model_size": "7B", - "model_link": "https://huggingface.co/Qwen/Qwen1.5-7B-Chat", + "flan_t5_base": { + "model_size": "0.25B", + "model_link": "https://huggingface.co/google/flan-t5-base", "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.9260504201680672, + "overall_acc": 0.7214285714285714, "language_acc": { - "English": 0.9411764705882353, - "Chinese": 0.9226890756302522, - "Spanish": 0.9252100840336135, - "Vietnamese": 0.915126050420168 + "Spanish": 0.8546218487394958, + "English": 0.9495798319327731, + "Chinese": 0.3521008403361345, + "Vietnamese": 0.7294117647058823 }, - "consistency_score_2": 0.9180672268907563, - "consistency_score_3": 0.8815126050420168, - "consistency_score_4": 0.8579831932773109, + "consistency_score_2": 0.5616246498599441, + "consistency_score_3": 0.3859243697478991, + "consistency_score_4": 0.2605042016806723, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9235294117647059, - "English,Spanish": 0.9319327731092437, - "English,Vietnamese": 0.9252100840336135, - "Chinese,Spanish": 0.9050420168067227, - "Chinese,Vietnamese": 0.9058823529411765, - "Spanish,Vietnamese": 0.9168067226890756 + "Spanish,English": 0.8630252100840337, + "Spanish,Chinese": 0.35042016806722687, + "Spanish,Vietnamese": 0.7218487394957983, + "English,Chinese": 0.35042016806722687, + "English,Vietnamese": 0.7260504201680672, + "Chinese,Vietnamese": 0.35798319327731093 }, "3_combine": { - "English,Chinese,Spanish": 0.8831932773109243, - "English,Chinese,Vietnamese": 0.8823529411764706, - "English,Spanish,Vietnamese": 0.8915966386554622, - "Chinese,Spanish,Vietnamese": 0.8689075630252101 + "Spanish,English,Chinese": 0.3142857142857143, + "Spanish,English,Vietnamese": 0.6756302521008404, + "Spanish,Chinese,Vietnamese": 0.2773109243697479, + "English,Chinese,Vietnamese": 0.27647058823529413 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8579831932773109 + "Spanish,English,Chinese,Vietnamese": 0.2605042016806723 } }, - "AC3_2": 0.9220415438402843, - "AC3_3": 0.9032328132985218, - "AC3_4": 0.8907183032864803 + "AC3_2": 0.6315748436203689, + "AC3_3": 0.5028511802673522, + "AC3_4": 0.38278623383839344 }, "prompt_2": { - "overall_acc": 0.928361344537815, + "overall_acc": 0.7195378151260504, "language_acc": { - "English": 0.9420168067226891, - "Chinese": 0.9260504201680673, - "Spanish": 0.9302521008403362, - "Vietnamese": 0.915126050420168 + "Spanish": 0.8554621848739495, + "English": 0.946218487394958, + "Chinese": 0.34705882352941175, + "Vietnamese": 0.7294117647058823 }, - "consistency_score_2": 0.9168067226890756, - "consistency_score_3": 0.8796218487394958, - "consistency_score_4": 0.8546218487394958, + "consistency_score_2": 0.5628851540616246, + "consistency_score_3": 0.38886554621848735, + "consistency_score_4": 0.2689075630252101, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9252100840336135, - "English,Spanish": 0.9336134453781513, - "English,Vietnamese": 0.915126050420168, - "Chinese,Spanish": 0.9084033613445378, - "Chinese,Vietnamese": 0.9033613445378151, - "Spanish,Vietnamese": 0.915126050420168 + "Spanish,English": 0.8563025210084033, + "Spanish,Chinese": 0.34705882352941175, + "Spanish,Vietnamese": 0.726890756302521, + "English,Chinese": 0.346218487394958, + "English,Vietnamese": 0.7243697478991596, + "Chinese,Vietnamese": 0.3764705882352941 }, "3_combine": { - "English,Chinese,Spanish": 0.8873949579831932, - "English,Chinese,Vietnamese": 0.8764705882352941, - "English,Spanish,Vietnamese": 0.8865546218487395, - "Chinese,Spanish,Vietnamese": 0.8680672268907563 + "Spanish,English,Chinese": 0.31092436974789917, + "Spanish,English,Vietnamese": 0.673109243697479, + "Spanish,Chinese,Vietnamese": 0.2865546218487395, + "English,Chinese,Vietnamese": 0.2848739495798319 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8546218487394958 + "Spanish,English,Chinese,Vietnamese": 0.2689075630252101 } }, - "AC3_2": 0.922547855480692, - "AC3_3": 0.9033346384766067, - "AC3_4": 0.8899667607795498 + "AC3_2": 0.6316436364912575, + "AC3_3": 0.5048766094306893, + "AC3_4": 0.3915019780098478 }, "prompt_3": { - "overall_acc": 0.9235294117647058, + "overall_acc": 0.7153361344537815, "language_acc": { - "English": 0.934453781512605, - "Chinese": 0.9117647058823529, - "Spanish": 0.9327731092436975, - "Vietnamese": 0.915126050420168 + "Spanish": 0.8495798319327731, + "English": 0.9453781512605042, + "Chinese": 0.34789915966386553, + "Vietnamese": 0.7184873949579832 }, - "consistency_score_2": 0.9084033613445378, - "consistency_score_3": 0.8672268907563025, - "consistency_score_4": 0.8369747899159664, + "consistency_score_2": 0.5570028011204482, + "consistency_score_3": 0.3779411764705882, + "consistency_score_4": 0.25630252100840334, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.907563025210084, - "English,Spanish": 0.9235294117647059, - "English,Vietnamese": 0.9109243697478991, - "Chinese,Spanish": 0.9042016806722689, - "Chinese,Vietnamese": 0.8941176470588236, - "Spanish,Vietnamese": 0.9100840336134454 + "Spanish,English": 0.8579831932773109, + "Spanish,Chinese": 0.34369747899159664, + "Spanish,Vietnamese": 0.7151260504201681, + "English,Chinese": 0.3445378151260504, + "English,Vietnamese": 0.7126050420168067, + "Chinese,Vietnamese": 0.3680672268907563 }, "3_combine": { - "English,Chinese,Spanish": 0.8697478991596639, - "English,Chinese,Vietnamese": 0.8621848739495799, - "English,Spanish,Vietnamese": 0.8764705882352941, - "Chinese,Spanish,Vietnamese": 0.8605042016806723 + "Spanish,English,Chinese": 0.3050420168067227, + "Spanish,English,Vietnamese": 0.6638655462184874, + "Spanish,Chinese,Vietnamese": 0.2680672268907563, + "English,Chinese,Vietnamese": 0.2747899159663866 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8369747899159664 + "Spanish,English,Chinese,Vietnamese": 0.25630252100840334 } }, - "AC3_2": 0.9159039395074778, - "AC3_3": 0.8944930576142226, - "AC3_4": 0.8781243857426241 + "AC3_2": 0.6263177514445691, + "AC3_3": 0.49457713519010127, + "AC3_4": 0.37738814440812757 }, "prompt_4": { - "overall_acc": 0.9237394957983193, + "overall_acc": 0.7256302521008403, "language_acc": { - "English": 0.938655462184874, - "Chinese": 0.9235294117647059, - "Spanish": 0.9184873949579831, - "Vietnamese": 0.9142857142857143 + "Spanish": 0.8588235294117647, + "English": 0.9487394957983193, + "Chinese": 0.3680672268907563, + "Vietnamese": 0.726890756302521 }, - "consistency_score_2": 0.9144257703081232, - "consistency_score_3": 0.8756302521008403, - "consistency_score_4": 0.8487394957983193, + "consistency_score_2": 0.5682072829131654, + "consistency_score_3": 0.39768907563025213, + "consistency_score_4": 0.2747899159663866, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9176470588235294, - "English,Spanish": 0.9277310924369748, - "English,Vietnamese": 0.9168067226890756, - "Chinese,Spanish": 0.9067226890756303, - "Chinese,Vietnamese": 0.9, - "Spanish,Vietnamese": 0.9176470588235294 + "Spanish,English": 0.8638655462184874, + "Spanish,Chinese": 0.36722689075630255, + "Spanish,Vietnamese": 0.7243697478991596, + "English,Chinese": 0.36554621848739494, + "English,Vietnamese": 0.726890756302521, + "Chinese,Vietnamese": 0.36134453781512604 }, "3_combine": { - "English,Chinese,Spanish": 0.8798319327731092, - "English,Chinese,Vietnamese": 0.8722689075630252, - "English,Spanish,Vietnamese": 0.8848739495798319, - "Chinese,Spanish,Vietnamese": 0.865546218487395 + "Spanish,English,Chinese": 0.3319327731092437, + "Spanish,English,Vietnamese": 0.6773109243697479, + "Spanish,Chinese,Vietnamese": 0.2915966386554622, + "English,Chinese,Vietnamese": 0.28991596638655465 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8487394957983193 + "Spanish,English,Chinese,Vietnamese": 0.2747899159663866 } }, - "AC3_2": 0.919059037333098, - "AC3_3": 0.8990417322296377, - "AC3_4": 0.8846527350514787 + "AC3_2": 0.6373418342813549, + "AC3_3": 0.5137901878079704, + "AC3_4": 0.3986242627690158 }, "prompt_5": { - "overall_acc": 0.9105042016806723, + "overall_acc": 0.7016806722689075, "language_acc": { - "English": 0.926890756302521, - "Chinese": 0.9142857142857143, - "Spanish": 0.9067226890756303, - "Vietnamese": 0.8941176470588236 + "Spanish": 0.8428571428571429, + "English": 0.9394957983193277, + "Chinese": 0.3302521008403361, + "Vietnamese": 0.6941176470588235 }, - "consistency_score_2": 0.8969187675070028, - "consistency_score_3": 0.8521008403361345, - "consistency_score_4": 0.8235294117647058, + "consistency_score_2": 0.5323529411764707, + "consistency_score_3": 0.34936974789915964, + "consistency_score_4": 0.22941176470588234, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9117647058823529, - "English,Spanish": 0.9100840336134454, - "English,Vietnamese": 0.8974789915966387, - "Chinese,Spanish": 0.8915966386554622, - "Chinese,Vietnamese": 0.8831932773109243, - "Spanish,Vietnamese": 0.8873949579831932 + "Spanish,English": 0.8403361344537815, + "Spanish,Chinese": 0.32857142857142857, + "Spanish,Vietnamese": 0.6865546218487395, + "English,Chinese": 0.32521008403361346, + "English,Vietnamese": 0.6865546218487395, + "Chinese,Vietnamese": 0.326890756302521 }, "3_combine": { - "English,Chinese,Spanish": 0.8621848739495799, - "English,Chinese,Vietnamese": 0.8529411764705882, - "English,Spanish,Vietnamese": 0.853781512605042, - "Chinese,Spanish,Vietnamese": 0.8394957983193277 + "Spanish,English,Chinese": 0.2815126050420168, + "Spanish,English,Vietnamese": 0.6277310924369748, + "Spanish,Chinese,Vietnamese": 0.24369747899159663, + "English,Chinese,Vietnamese": 0.2445378151260504 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8235294117647058 + "Spanish,English,Chinese,Vietnamese": 0.22941176470588234 } }, - "AC3_2": 0.9036604273135093, - "AC3_3": 0.8803349324927979, - "AC3_4": 0.864835587686553 + "AC3_2": 0.605399667478831, + "AC3_3": 0.46647809629109394, + "AC3_4": 0.34577404965494263 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.5628571428571428, + "overall_acc": 0.31428571428571433, "language_acc": { - "Spanish": 0.6133333333333333, - "English": 0.68, - "Chinese": 0.66, - "Vietnamese": 0.54, - "Filipino": 0.44, - "Indonesian": 0.5666666666666667, - "Malay": 0.44 - }, - "consistency_score_2": 0.533015873015873, - "consistency_score_3": 0.3666666666666667, - "consistency_score_4": 0.2782857142857143, - "consistency_score_5": 0.22158730158730156, - "consistency_score_6": 0.18, - "consistency_score_7": 0.14666666666666667, + "Filipino": 0.2866666666666667, + "Vietnamese": 0.3, + "Chinese": 0.30666666666666664, + "Spanish": 0.34, + "Malay": 0.2866666666666667, + "Indonesian": 0.26666666666666666, + "English": 0.41333333333333333 + }, + "consistency_score_2": 0.4257142857142857, + "consistency_score_3": 0.23733333333333329, + "consistency_score_4": 0.15314285714285714, + "consistency_score_5": 0.10603174603174605, + "consistency_score_6": 0.07523809523809523, + "consistency_score_7": 0.05333333333333334, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6666666666666666, - "Spanish,Chinese": 0.5866666666666667, - "Spanish,Vietnamese": 0.5866666666666667, - "Spanish,Filipino": 0.4866666666666667, - "Spanish,Indonesian": 0.5666666666666667, - "Spanish,Malay": 0.54, - "English,Chinese": 0.6533333333333333, - "English,Vietnamese": 0.5333333333333333, - "English,Filipino": 0.5133333333333333, - "English,Indonesian": 0.56, - "English,Malay": 0.4533333333333333, - "Chinese,Vietnamese": 0.5, - "Chinese,Filipino": 0.47333333333333333, - "Chinese,Indonesian": 0.62, - "Chinese,Malay": 0.4666666666666667, - "Vietnamese,Filipino": 0.4066666666666667, - "Vietnamese,Indonesian": 0.5266666666666666, - "Vietnamese,Malay": 0.5266666666666666, - "Filipino,Indonesian": 0.4666666666666667, - "Filipino,Malay": 0.4866666666666667, - "Indonesian,Malay": 0.5733333333333334 + "Filipino,Vietnamese": 0.44, + "Filipino,Chinese": 0.38666666666666666, + "Filipino,Spanish": 0.43333333333333335, + "Filipino,Malay": 0.5933333333333334, + "Filipino,Indonesian": 0.5, + "Filipino,English": 0.52, + "Vietnamese,Chinese": 0.36666666666666664, + "Vietnamese,Spanish": 0.44, + "Vietnamese,Malay": 0.46, + "Vietnamese,Indonesian": 0.4066666666666667, + "Vietnamese,English": 0.36666666666666664, + "Chinese,Spanish": 0.25333333333333335, + "Chinese,Malay": 0.37333333333333335, + "Chinese,Indonesian": 0.32, + "Chinese,English": 0.32666666666666666, + "Spanish,Malay": 0.41333333333333333, + "Spanish,Indonesian": 0.44, + "Spanish,English": 0.48, + "Malay,Indonesian": 0.58, + "Malay,English": 0.42, + "Indonesian,English": 0.42 }, "3_combine": { - "Spanish,English,Chinese": 0.4866666666666667, - "Spanish,English,Vietnamese": 0.43333333333333335, - "Spanish,English,Filipino": 0.38, - "Spanish,English,Indonesian": 0.4533333333333333, - "Spanish,English,Malay": 0.38666666666666666, - "Spanish,Chinese,Vietnamese": 0.37333333333333335, - "Spanish,Chinese,Filipino": 0.3466666666666667, - "Spanish,Chinese,Indonesian": 0.43333333333333335, - "Spanish,Chinese,Malay": 0.36, - "Spanish,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian": 0.4066666666666667, - "Spanish,Vietnamese,Malay": 0.4, - "Spanish,Filipino,Indonesian": 0.3466666666666667, - "Spanish,Filipino,Malay": 0.3466666666666667, - "Spanish,Indonesian,Malay": 0.41333333333333333, - "English,Chinese,Vietnamese": 0.38666666666666666, - "English,Chinese,Filipino": 0.36, - "English,Chinese,Indonesian": 0.44666666666666666, - "English,Chinese,Malay": 0.35333333333333333, - "English,Vietnamese,Filipino": 0.29333333333333333, - "English,Vietnamese,Indonesian": 0.37333333333333335, - "English,Vietnamese,Malay": 0.3466666666666667, - "English,Filipino,Indonesian": 0.3333333333333333, - "English,Filipino,Malay": 0.32666666666666666, - "English,Indonesian,Malay": 0.38666666666666666, - "Chinese,Vietnamese,Filipino": 0.2733333333333333, - "Chinese,Vietnamese,Indonesian": 0.36666666666666664, - "Chinese,Vietnamese,Malay": 0.31333333333333335, - "Chinese,Filipino,Indonesian": 0.34, - "Chinese,Filipino,Malay": 0.29333333333333333, - "Chinese,Indonesian,Malay": 0.38666666666666666, - "Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Vietnamese,Filipino,Malay": 0.31333333333333335, - "Vietnamese,Indonesian,Malay": 0.38, - "Filipino,Indonesian,Malay": 0.35333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.3333333333333333, - "Spanish,English,Chinese,Filipino": 0.28, - "Spanish,English,Chinese,Indonesian": 0.37333333333333335, - "Spanish,English,Chinese,Malay": 0.30666666666666664, - "Spanish,English,Vietnamese,Filipino": 0.26, - "Spanish,English,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,English,Vietnamese,Malay": 0.3, - "Spanish,English,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Filipino,Malay": 0.2866666666666667, - "Spanish,English,Indonesian,Malay": 0.34, - "Spanish,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Chinese,Filipino,Malay": 0.25333333333333335, - "Spanish,Chinese,Indonesian,Malay": 0.30666666666666664, - "Spanish,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,Vietnamese,Filipino,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino": 0.22, - "English,Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "English,Chinese,Vietnamese,Malay": 0.26666666666666666, - "English,Chinese,Filipino,Indonesian": 0.2733333333333333, - "English,Chinese,Filipino,Malay": 0.24666666666666667, - "English,Chinese,Indonesian,Malay": 0.30666666666666664, - "English,Vietnamese,Filipino,Indonesian": 0.24, - "English,Vietnamese,Filipino,Malay": 0.26, - "English,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "English,Filipino,Indonesian,Malay": 0.2733333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.26 + "Filipino,Vietnamese,Chinese": 0.20666666666666667, + "Filipino,Vietnamese,Spanish": 0.28, + "Filipino,Vietnamese,Malay": 0.32666666666666666, + "Filipino,Vietnamese,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,English": 0.26666666666666666, + "Filipino,Chinese,Spanish": 0.14666666666666667, + "Filipino,Chinese,Malay": 0.25333333333333335, + "Filipino,Chinese,Indonesian": 0.19333333333333333, + "Filipino,Chinese,English": 0.19333333333333333, + "Filipino,Spanish,Malay": 0.30666666666666664, + "Filipino,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Spanish,English": 0.30666666666666664, + "Filipino,Malay,Indonesian": 0.38666666666666666, + "Filipino,Malay,English": 0.3333333333333333, + "Filipino,Indonesian,English": 0.29333333333333333, + "Vietnamese,Chinese,Spanish": 0.17333333333333334, + "Vietnamese,Chinese,Malay": 0.21333333333333335, + "Vietnamese,Chinese,Indonesian": 0.17333333333333334, + "Vietnamese,Chinese,English": 0.14666666666666667, + "Vietnamese,Spanish,Malay": 0.2733333333333333, + "Vietnamese,Spanish,Indonesian": 0.26666666666666666, + "Vietnamese,Spanish,English": 0.26, + "Vietnamese,Malay,Indonesian": 0.29333333333333333, + "Vietnamese,Malay,English": 0.24, + "Vietnamese,Indonesian,English": 0.21333333333333335, + "Chinese,Spanish,Malay": 0.14, + "Chinese,Spanish,Indonesian": 0.13333333333333333, + "Chinese,Spanish,English": 0.14, + "Chinese,Malay,Indonesian": 0.21333333333333335, + "Chinese,Malay,English": 0.16, + "Chinese,Indonesian,English": 0.14666666666666667, + "Spanish,Malay,Indonesian": 0.29333333333333333, + "Spanish,Malay,English": 0.26, + "Spanish,Indonesian,English": 0.26, + "Malay,Indonesian,English": 0.29333333333333333 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.12, + "Filipino,Vietnamese,Chinese,Malay": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,English": 0.12, + "Filipino,Vietnamese,Spanish,Malay": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian": 0.2, + "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.22, + "Filipino,Vietnamese,Malay,English": 0.20666666666666667, + "Filipino,Vietnamese,Indonesian,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Malay": 0.11333333333333333, + "Filipino,Chinese,Spanish,Indonesian": 0.08666666666666667, + "Filipino,Chinese,Spanish,English": 0.1, + "Filipino,Chinese,Malay,Indonesian": 0.14666666666666667, + "Filipino,Chinese,Malay,English": 0.14, + "Filipino,Chinese,Indonesian,English": 0.11333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.22, + "Filipino,Spanish,Malay,English": 0.22, + "Filipino,Spanish,Indonesian,English": 0.19333333333333333, + "Filipino,Malay,Indonesian,English": 0.24, + "Vietnamese,Chinese,Spanish,Malay": 0.12, + "Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.10666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.14, + "Vietnamese,Chinese,Malay,English": 0.11333333333333333, + "Vietnamese,Chinese,Indonesian,English": 0.09333333333333334, + "Vietnamese,Spanish,Malay,Indonesian": 0.21333333333333335, + "Vietnamese,Spanish,Malay,English": 0.2, + "Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, + "Vietnamese,Malay,Indonesian,English": 0.18, + "Chinese,Spanish,Malay,Indonesian": 0.1, + "Chinese,Spanish,Malay,English": 0.09333333333333334, + "Chinese,Spanish,Indonesian,English": 0.08666666666666667, + "Chinese,Malay,Indonesian,English": 0.10666666666666667, + "Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.2, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.26, - "Spanish,English,Chinese,Vietnamese,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.22, - "Spanish,English,Chinese,Indonesian,Malay": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.08, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.1, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.07333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Malay,English": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.15333333333333332, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, + "Filipino,Chinese,Spanish,Malay,English": 0.08, + "Filipino,Chinese,Spanish,Indonesian,English": 0.06666666666666667, + "Filipino,Chinese,Malay,Indonesian,English": 0.08666666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.16, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, + "Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.08, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.16, + "Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.06, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.06666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.13333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334 } }, - "AC3_2": 0.5475302101801315, - "AC3_3": 0.4440573770014078, - "AC3_4": 0.3724340061669029, - "AC3_5": 0.31798809037976733, - "AC3_6": 0.2727692307325118, - "AC3_7": 0.23269798654438487 + "AC3_2": 0.36161059013315255, + "AC3_3": 0.27044198890124926, + "AC3_4": 0.20593782740966227, + "AC3_5": 0.15856711260793696, + "AC3_6": 0.12141110719900902, + "AC3_7": 0.09119170981975355 }, "prompt_2": { - "overall_acc": 0.5266666666666666, + "overall_acc": 0.31238095238095237, "language_acc": { - "Spanish": 0.5866666666666667, - "English": 0.6133333333333333, - "Chinese": 0.6466666666666666, - "Vietnamese": 0.47333333333333333, - "Filipino": 0.4266666666666667, - "Indonesian": 0.5333333333333333, - "Malay": 0.4066666666666667 + "Filipino": 0.3, + "Vietnamese": 0.3, + "Chinese": 0.24, + "Spanish": 0.3333333333333333, + "Malay": 0.30666666666666664, + "Indonesian": 0.29333333333333333, + "English": 0.41333333333333333 }, - "consistency_score_2": 0.5095238095238096, - "consistency_score_3": 0.34, - "consistency_score_4": 0.2539047619047619, - "consistency_score_5": 0.19968253968253968, - "consistency_score_6": 0.1619047619047619, - "consistency_score_7": 0.13333333333333333, + "consistency_score_2": 0.4152380952380952, + "consistency_score_3": 0.22780952380952385, + "consistency_score_4": 0.1459047619047619, + "consistency_score_5": 0.10317460317460314, + "consistency_score_6": 0.07714285714285715, + "consistency_score_7": 0.06, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.62, - "Spanish,Chinese": 0.6, - "Spanish,Vietnamese": 0.4666666666666667, - "Spanish,Filipino": 0.4533333333333333, - "Spanish,Indonesian": 0.52, - "Spanish,Malay": 0.46, - "English,Chinese": 0.6533333333333333, - "English,Vietnamese": 0.5333333333333333, - "English,Filipino": 0.48, - "English,Indonesian": 0.58, - "English,Malay": 0.46, - "Chinese,Vietnamese": 0.5, - "Chinese,Filipino": 0.42, - "Chinese,Indonesian": 0.58, - "Chinese,Malay": 0.44666666666666666, - "Vietnamese,Filipino": 0.4533333333333333, - "Vietnamese,Indonesian": 0.5266666666666666, - "Vietnamese,Malay": 0.5, - "Filipino,Indonesian": 0.44666666666666666, - "Filipino,Malay": 0.44, - "Indonesian,Malay": 0.56 + "Filipino,Vietnamese": 0.4266666666666667, + "Filipino,Chinese": 0.29333333333333333, + "Filipino,Spanish": 0.47333333333333333, + "Filipino,Malay": 0.5333333333333333, + "Filipino,Indonesian": 0.47333333333333333, + "Filipino,English": 0.5, + "Vietnamese,Chinese": 0.26666666666666666, + "Vietnamese,Spanish": 0.44666666666666666, + "Vietnamese,Malay": 0.5133333333333333, + "Vietnamese,Indonesian": 0.44, + "Vietnamese,English": 0.36666666666666664, + "Chinese,Spanish": 0.29333333333333333, + "Chinese,Malay": 0.30666666666666664, + "Chinese,Indonesian": 0.32666666666666666, + "Chinese,English": 0.25333333333333335, + "Spanish,Malay": 0.44666666666666666, + "Spanish,Indonesian": 0.4866666666666667, + "Spanish,English": 0.47333333333333333, + "Malay,Indonesian": 0.6333333333333333, + "Malay,English": 0.38, + "Indonesian,English": 0.38666666666666666 }, "3_combine": { - "Spanish,English,Chinese": 0.5, - "Spanish,English,Vietnamese": 0.36666666666666664, - "Spanish,English,Filipino": 0.3466666666666667, - "Spanish,English,Indonesian": 0.43333333333333335, - "Spanish,English,Malay": 0.34, - "Spanish,Chinese,Vietnamese": 0.3466666666666667, - "Spanish,Chinese,Filipino": 0.32, - "Spanish,Chinese,Indonesian": 0.41333333333333333, - "Spanish,Chinese,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino": 0.28, - "Spanish,Vietnamese,Indonesian": 0.3466666666666667, - "Spanish,Vietnamese,Malay": 0.29333333333333333, - "Spanish,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Filipino,Malay": 0.26, - "Spanish,Indonesian,Malay": 0.34, - "English,Chinese,Vietnamese": 0.41333333333333333, - "English,Chinese,Filipino": 0.3466666666666667, - "English,Chinese,Indonesian": 0.47333333333333333, - "English,Chinese,Malay": 0.36, - "English,Vietnamese,Filipino": 0.29333333333333333, - "English,Vietnamese,Indonesian": 0.3933333333333333, - "English,Vietnamese,Malay": 0.32, - "English,Filipino,Indonesian": 0.32666666666666666, - "English,Filipino,Malay": 0.28, - "English,Indonesian,Malay": 0.38, - "Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Chinese,Vietnamese,Indonesian": 0.37333333333333335, - "Chinese,Vietnamese,Malay": 0.30666666666666664, - "Chinese,Filipino,Indonesian": 0.31333333333333335, - "Chinese,Filipino,Malay": 0.24, - "Chinese,Indonesian,Malay": 0.37333333333333335, - "Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Vietnamese,Filipino,Malay": 0.2733333333333333, - "Vietnamese,Indonesian,Malay": 0.38, - "Filipino,Indonesian,Malay": 0.29333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.31333333333333335, - "Spanish,English,Chinese,Filipino": 0.2866666666666667, - "Spanish,English,Chinese,Indonesian": 0.38666666666666666, - "Spanish,English,Chinese,Malay": 0.29333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.22666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,English,Vietnamese,Malay": 0.24666666666666667, - "Spanish,English,Filipino,Indonesian": 0.26, - "Spanish,English,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.22666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.2866666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Indonesian,Malay": 0.26, - "Spanish,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.24, - "English,Chinese,Vietnamese,Indonesian": 0.32666666666666666, - "English,Chinese,Vietnamese,Malay": 0.26, - "English,Chinese,Filipino,Indonesian": 0.28, - "English,Chinese,Filipino,Malay": 0.22, - "English,Chinese,Indonesian,Malay": 0.32666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "English,Filipino,Indonesian,Malay": 0.22, - "Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 + "Filipino,Vietnamese,Chinese": 0.12, + "Filipino,Vietnamese,Spanish": 0.26666666666666666, + "Filipino,Vietnamese,Malay": 0.32, + "Filipino,Vietnamese,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,English": 0.24, + "Filipino,Chinese,Spanish": 0.15333333333333332, + "Filipino,Chinese,Malay": 0.17333333333333334, + "Filipino,Chinese,Indonesian": 0.18, + "Filipino,Chinese,English": 0.14666666666666667, + "Filipino,Spanish,Malay": 0.29333333333333333, + "Filipino,Spanish,Indonesian": 0.29333333333333333, + "Filipino,Spanish,English": 0.31333333333333335, + "Filipino,Malay,Indonesian": 0.37333333333333335, + "Filipino,Malay,English": 0.29333333333333333, + "Filipino,Indonesian,English": 0.2733333333333333, + "Vietnamese,Chinese,Spanish": 0.13333333333333333, + "Vietnamese,Chinese,Malay": 0.18, + "Vietnamese,Chinese,Indonesian": 0.16666666666666666, + "Vietnamese,Chinese,English": 0.09333333333333334, + "Vietnamese,Spanish,Malay": 0.30666666666666664, + "Vietnamese,Spanish,Indonesian": 0.2866666666666667, + "Vietnamese,Spanish,English": 0.26666666666666666, + "Vietnamese,Malay,Indonesian": 0.3466666666666667, + "Vietnamese,Malay,English": 0.24, + "Vietnamese,Indonesian,English": 0.22, + "Chinese,Spanish,Malay": 0.14, + "Chinese,Spanish,Indonesian": 0.17333333333333334, + "Chinese,Spanish,English": 0.14, + "Chinese,Malay,Indonesian": 0.2, + "Chinese,Malay,English": 0.12666666666666668, + "Chinese,Indonesian,English": 0.12, + "Spanish,Malay,Indonesian": 0.34, + "Spanish,Malay,English": 0.25333333333333335, + "Spanish,Indonesian,English": 0.26, + "Malay,Indonesian,English": 0.2866666666666667 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.1, + "Filipino,Vietnamese,Chinese,Indonesian": 0.1, + "Filipino,Vietnamese,Chinese,English": 0.07333333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian": 0.2, + "Filipino,Vietnamese,Spanish,English": 0.19333333333333333, + "Filipino,Vietnamese,Malay,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Malay,English": 0.2, + "Filipino,Vietnamese,Indonesian,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Malay": 0.1, + "Filipino,Chinese,Spanish,Indonesian": 0.12, + "Filipino,Chinese,Spanish,English": 0.08666666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.14666666666666667, + "Filipino,Chinese,Malay,English": 0.10666666666666667, + "Filipino,Chinese,Indonesian,English": 0.09333333333333334, + "Filipino,Spanish,Malay,Indonesian": 0.24, + "Filipino,Spanish,Malay,English": 0.2, + "Filipino,Spanish,Indonesian,English": 0.2, + "Filipino,Malay,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish,Malay": 0.10666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.10666666666666667, + "Vietnamese,Chinese,Spanish,English": 0.07333333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, + "Vietnamese,Chinese,Malay,English": 0.08666666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.08666666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, + "Vietnamese,Spanish,Malay,English": 0.2, + "Vietnamese,Spanish,Indonesian,English": 0.18, + "Vietnamese,Malay,Indonesian,English": 0.19333333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.12, + "Chinese,Spanish,Malay,English": 0.08, + "Chinese,Spanish,Indonesian,English": 0.09333333333333334, + "Chinese,Malay,Indonesian,English": 0.1, + "Spanish,Malay,Indonesian,English": 0.22 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.2, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Chinese,Filipino,Indonesian": 0.24, - "Spanish,English,Chinese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.28, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.16, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.24, - "English,Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.08, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.06, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Malay,English": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.06666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Malay,English": 0.16, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14666666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.16, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, + "Filipino,Chinese,Spanish,Malay,English": 0.06666666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.07333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.08666666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, + "Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.07333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.08, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Chinese,Spanish,Malay,Indonesian,English": 0.08 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.06, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.06, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.06666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06 } }, - "AC3_2": 0.5179534313225627, - "AC3_3": 0.4132307691830887, - "AC3_4": 0.34262892463481043, - "AC3_5": 0.28957459203472496, - "AC3_6": 0.24767173809220072, - "AC3_7": 0.2127946127623712 + "AC3_2": 0.3565395162810914, + "AC3_3": 0.263475048644892, + "AC3_4": 0.1989059097852009, + "AC3_5": 0.15511659205405445, + "AC3_6": 0.12373035274504304, + "AC3_7": 0.10066496160979585 }, "prompt_3": { - "overall_acc": 0.47904761904761906, + "overall_acc": 0.32285714285714284, "language_acc": { - "Spanish": 0.5266666666666666, - "English": 0.5933333333333334, - "Chinese": 0.54, - "Vietnamese": 0.48, - "Filipino": 0.36, - "Indonesian": 0.44666666666666666, - "Malay": 0.4066666666666667 - }, - "consistency_score_2": 0.4752380952380952, - "consistency_score_3": 0.2967619047619048, - "consistency_score_4": 0.2116190476190476, - "consistency_score_5": 0.1606349206349206, - "consistency_score_6": 0.12571428571428572, - "consistency_score_7": 0.1, + "Filipino": 0.29333333333333333, + "Vietnamese": 0.32666666666666666, + "Chinese": 0.28, + "Spanish": 0.3466666666666667, + "Malay": 0.32666666666666666, + "Indonesian": 0.2866666666666667, + "English": 0.4 + }, + "consistency_score_2": 0.4403174603174603, + "consistency_score_3": 0.2582857142857144, + "consistency_score_4": 0.1746666666666667, + "consistency_score_5": 0.12730158730158725, + "consistency_score_6": 0.09619047619047619, + "consistency_score_7": 0.07333333333333333, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5533333333333333, - "Spanish,Chinese": 0.6, - "Spanish,Vietnamese": 0.5666666666666667, - "Spanish,Filipino": 0.42, - "Spanish,Indonesian": 0.49333333333333335, - "Spanish,Malay": 0.5333333333333333, - "English,Chinese": 0.52, - "English,Vietnamese": 0.4666666666666667, - "English,Filipino": 0.4533333333333333, - "English,Indonesian": 0.44, - "English,Malay": 0.49333333333333335, - "Chinese,Vietnamese": 0.49333333333333335, - "Chinese,Filipino": 0.41333333333333333, - "Chinese,Indonesian": 0.52, - "Chinese,Malay": 0.4533333333333333, - "Vietnamese,Filipino": 0.43333333333333335, - "Vietnamese,Indonesian": 0.44, - "Vietnamese,Malay": 0.44, - "Filipino,Indonesian": 0.36, - "Filipino,Malay": 0.38666666666666666, - "Indonesian,Malay": 0.5 + "Filipino,Vietnamese": 0.41333333333333333, + "Filipino,Chinese": 0.35333333333333333, + "Filipino,Spanish": 0.46, + "Filipino,Malay": 0.5266666666666666, + "Filipino,Indonesian": 0.4666666666666667, + "Filipino,English": 0.5466666666666666, + "Vietnamese,Chinese": 0.32666666666666666, + "Vietnamese,Spanish": 0.46, + "Vietnamese,Malay": 0.5266666666666666, + "Vietnamese,Indonesian": 0.4533333333333333, + "Vietnamese,English": 0.36666666666666664, + "Chinese,Spanish": 0.3466666666666667, + "Chinese,Malay": 0.36666666666666664, + "Chinese,Indonesian": 0.37333333333333335, + "Chinese,English": 0.35333333333333333, + "Spanish,Malay": 0.4266666666666667, + "Spanish,Indonesian": 0.44666666666666666, + "Spanish,English": 0.5066666666666667, + "Malay,Indonesian": 0.6466666666666666, + "Malay,English": 0.44666666666666666, + "Indonesian,English": 0.43333333333333335 }, "3_combine": { - "Spanish,English,Chinese": 0.4066666666666667, - "Spanish,English,Vietnamese": 0.35333333333333333, - "Spanish,English,Filipino": 0.32666666666666666, - "Spanish,English,Indonesian": 0.31333333333333335, - "Spanish,English,Malay": 0.35333333333333333, - "Spanish,Chinese,Vietnamese": 0.4, - "Spanish,Chinese,Filipino": 0.30666666666666664, - "Spanish,Chinese,Indonesian": 0.36, - "Spanish,Chinese,Malay": 0.36, - "Spanish,Vietnamese,Filipino": 0.28, - "Spanish,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,Vietnamese,Malay": 0.3466666666666667, - "Spanish,Filipino,Indonesian": 0.24666666666666667, - "Spanish,Filipino,Malay": 0.26, - "Spanish,Indonesian,Malay": 0.3333333333333333, - "English,Chinese,Vietnamese": 0.30666666666666664, - "English,Chinese,Filipino": 0.28, - "English,Chinese,Indonesian": 0.32666666666666666, - "English,Chinese,Malay": 0.31333333333333335, - "English,Vietnamese,Filipino": 0.2733333333333333, - "English,Vietnamese,Indonesian": 0.26666666666666666, - "English,Vietnamese,Malay": 0.2733333333333333, - "English,Filipino,Indonesian": 0.22666666666666666, - "English,Filipino,Malay": 0.25333333333333335, - "English,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino": 0.24, - "Chinese,Vietnamese,Indonesian": 0.31333333333333335, - "Chinese,Vietnamese,Malay": 0.26666666666666666, - "Chinese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Filipino,Malay": 0.23333333333333334, - "Chinese,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Vietnamese,Filipino,Malay": 0.22666666666666666, - "Vietnamese,Indonesian,Malay": 0.28, - "Filipino,Indonesian,Malay": 0.22 + "Filipino,Vietnamese,Chinese": 0.16666666666666666, + "Filipino,Vietnamese,Spanish": 0.28, + "Filipino,Vietnamese,Malay": 0.32, + "Filipino,Vietnamese,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,English": 0.28, + "Filipino,Chinese,Spanish": 0.20666666666666667, + "Filipino,Chinese,Malay": 0.21333333333333335, + "Filipino,Chinese,Indonesian": 0.20666666666666667, + "Filipino,Chinese,English": 0.24, + "Filipino,Spanish,Malay": 0.29333333333333333, + "Filipino,Spanish,Indonesian": 0.28, + "Filipino,Spanish,English": 0.3466666666666667, + "Filipino,Malay,Indonesian": 0.37333333333333335, + "Filipino,Malay,English": 0.34, + "Filipino,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish": 0.16666666666666666, + "Vietnamese,Chinese,Malay": 0.2, + "Vietnamese,Chinese,Indonesian": 0.17333333333333334, + "Vietnamese,Chinese,English": 0.16666666666666666, + "Vietnamese,Spanish,Malay": 0.30666666666666664, + "Vietnamese,Spanish,Indonesian": 0.29333333333333333, + "Vietnamese,Spanish,English": 0.28, + "Vietnamese,Malay,Indonesian": 0.36, + "Vietnamese,Malay,English": 0.2733333333333333, + "Vietnamese,Indonesian,English": 0.24666666666666667, + "Chinese,Spanish,Malay": 0.19333333333333333, + "Chinese,Spanish,Indonesian": 0.2, + "Chinese,Spanish,English": 0.22666666666666666, + "Chinese,Malay,Indonesian": 0.24, + "Chinese,Malay,English": 0.20666666666666667, + "Chinese,Indonesian,English": 0.19333333333333333, + "Spanish,Malay,Indonesian": 0.32666666666666666, + "Spanish,Malay,English": 0.26666666666666666, + "Spanish,Indonesian,English": 0.28, + "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.2733333333333333, - "Spanish,English,Chinese,Filipino": 0.24666666666666667, - "Spanish,English,Chinese,Indonesian": 0.26666666666666666, - "Spanish,English,Chinese,Malay": 0.28, - "Spanish,English,Vietnamese,Filipino": 0.22, - "Spanish,English,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Malay": 0.24666666666666667, - "Spanish,English,Filipino,Indonesian": 0.2, - "Spanish,English,Filipino,Malay": 0.21333333333333335, - "Spanish,English,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.22, - "Spanish,Chinese,Filipino,Malay": 0.21333333333333335, - "Spanish,Chinese,Indonesian,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.18, - "English,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.21333333333333335, - "English,Chinese,Vietnamese,Malay": 0.2, - "English,Chinese,Filipino,Indonesian": 0.19333333333333333, - "English,Chinese,Filipino,Malay": 0.18666666666666668, - "English,Chinese,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Vietnamese,Filipino,Malay": 0.16, - "English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "English,Filipino,Indonesian,Malay": 0.16, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 + "Filipino,Vietnamese,Chinese,Spanish": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Malay": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,English": 0.14, + "Filipino,Vietnamese,Spanish,Malay": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,English": 0.22, + "Filipino,Vietnamese,Malay,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Malay,English": 0.24, + "Filipino,Vietnamese,Indonesian,English": 0.18666666666666668, + "Filipino,Chinese,Spanish,Malay": 0.14, + "Filipino,Chinese,Spanish,Indonesian": 0.14, + "Filipino,Chinese,Spanish,English": 0.16666666666666666, + "Filipino,Chinese,Malay,Indonesian": 0.16666666666666666, + "Filipino,Chinese,Malay,English": 0.17333333333333334, + "Filipino,Chinese,Indonesian,English": 0.15333333333333332, + "Filipino,Spanish,Malay,Indonesian": 0.24, + "Filipino,Spanish,Malay,English": 0.23333333333333334, + "Filipino,Spanish,Indonesian,English": 0.21333333333333335, + "Filipino,Malay,Indonesian,English": 0.26, + "Vietnamese,Chinese,Spanish,Malay": 0.12, + "Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.12, + "Vietnamese,Chinese,Malay,Indonesian": 0.14, + "Vietnamese,Chinese,Malay,English": 0.13333333333333333, + "Vietnamese,Chinese,Indonesian,English": 0.12, + "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, + "Vietnamese,Spanish,Malay,English": 0.20666666666666667, + "Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, + "Vietnamese,Malay,Indonesian,English": 0.21333333333333335, + "Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Chinese,Spanish,Malay,English": 0.14, + "Chinese,Spanish,Indonesian,English": 0.13333333333333333, + "Chinese,Malay,Indonesian,English": 0.14666666666666667, + "Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.2, - "Spanish,English,Chinese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.18, - "Spanish,English,Chinese,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.1, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.12, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.1, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Spanish,Malay,English": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.18, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, + "Filipino,Chinese,Spanish,Malay,English": 0.12666666666666668, + "Filipino,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.13333333333333333, + "Filipino,Spanish,Malay,Indonesian,English": 0.18666666666666668, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, + "Vietnamese,Chinese,Spanish,Malay,English": 0.09333333333333334, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.09333333333333334, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.10666666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.09333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 } }, - "AC3_2": 0.4771352532529188, - "AC3_3": 0.36648965891479846, - "AC3_4": 0.2935586898427168, - "AC3_5": 0.24059364287861212, - "AC3_6": 0.19916310457899103, - "AC3_7": 0.1654605262872149 + "AC3_2": 0.3725481340134615, + "AC3_3": 0.28698412693474434, + "AC3_4": 0.22669218984723838, + "AC3_5": 0.18260326411418085, + "AC3_6": 0.14822077918540835, + "AC3_7": 0.11951923073906365 }, "prompt_4": { - "overall_acc": 0.5219047619047619, + "overall_acc": 0.3295238095238095, "language_acc": { - "Spanish": 0.54, - "English": 0.6933333333333334, - "Chinese": 0.64, - "Vietnamese": 0.5266666666666666, - "Filipino": 0.38666666666666666, - "Indonesian": 0.47333333333333333, - "Malay": 0.3933333333333333 + "Filipino": 0.30666666666666664, + "Vietnamese": 0.3333333333333333, + "Chinese": 0.30666666666666664, + "Spanish": 0.3466666666666667, + "Malay": 0.30666666666666664, + "Indonesian": 0.30666666666666664, + "English": 0.4 }, - "consistency_score_2": 0.49936507936507946, - "consistency_score_3": 0.3161904761904762, - "consistency_score_4": 0.21885714285714283, - "consistency_score_5": 0.16031746031746033, - "consistency_score_6": 0.1238095238095238, - "consistency_score_7": 0.1, + "consistency_score_2": 0.46126984126984116, + "consistency_score_3": 0.27999999999999997, + "consistency_score_4": 0.19980952380952377, + "consistency_score_5": 0.15746031746031747, + "consistency_score_6": 0.13142857142857142, + "consistency_score_7": 0.11333333333333333, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6466666666666666, - "Spanish,Chinese": 0.5866666666666667, - "Spanish,Vietnamese": 0.5, - "Spanish,Filipino": 0.44666666666666666, - "Spanish,Indonesian": 0.5066666666666667, - "Spanish,Malay": 0.46, - "English,Chinese": 0.68, - "English,Vietnamese": 0.5666666666666667, - "English,Filipino": 0.42, - "English,Indonesian": 0.58, - "English,Malay": 0.47333333333333333, - "Chinese,Vietnamese": 0.54, - "Chinese,Filipino": 0.44666666666666666, - "Chinese,Indonesian": 0.54, + "Filipino,Vietnamese": 0.44666666666666666, + "Filipino,Chinese": 0.38666666666666666, + "Filipino,Spanish": 0.47333333333333333, + "Filipino,Malay": 0.5933333333333334, + "Filipino,Indonesian": 0.5, + "Filipino,English": 0.52, + "Vietnamese,Chinese": 0.42, + "Vietnamese,Spanish": 0.4533333333333333, + "Vietnamese,Malay": 0.5333333333333333, + "Vietnamese,Indonesian": 0.4866666666666667, + "Vietnamese,English": 0.38666666666666666, + "Chinese,Spanish": 0.30666666666666664, "Chinese,Malay": 0.4533333333333333, - "Vietnamese,Filipino": 0.41333333333333333, - "Vietnamese,Indonesian": 0.5133333333333333, - "Vietnamese,Malay": 0.43333333333333335, - "Filipino,Indonesian": 0.41333333333333333, - "Filipino,Malay": 0.4, - "Indonesian,Malay": 0.4666666666666667 + "Chinese,Indonesian": 0.4066666666666667, + "Chinese,English": 0.36666666666666664, + "Spanish,Malay": 0.44, + "Spanish,Indonesian": 0.49333333333333335, + "Spanish,English": 0.5066666666666667, + "Malay,Indonesian": 0.6666666666666666, + "Malay,English": 0.42, + "Indonesian,English": 0.4266666666666667 }, "3_combine": { - "Spanish,English,Chinese": 0.47333333333333333, - "Spanish,English,Vietnamese": 0.38666666666666666, - "Spanish,English,Filipino": 0.32666666666666666, - "Spanish,English,Indonesian": 0.4066666666666667, - "Spanish,English,Malay": 0.3466666666666667, - "Spanish,Chinese,Vietnamese": 0.3466666666666667, - "Spanish,Chinese,Filipino": 0.30666666666666664, - "Spanish,Chinese,Indonesian": 0.3466666666666667, - "Spanish,Chinese,Malay": 0.32, - "Spanish,Vietnamese,Filipino": 0.28, - "Spanish,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,Vietnamese,Malay": 0.26, - "Spanish,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Filipino,Malay": 0.25333333333333335, - "Spanish,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese": 0.4266666666666667, - "English,Chinese,Filipino": 0.34, - "English,Chinese,Indonesian": 0.42, - "English,Chinese,Malay": 0.36, - "English,Vietnamese,Filipino": 0.28, - "English,Vietnamese,Indonesian": 0.38666666666666666, - "English,Vietnamese,Malay": 0.30666666666666664, - "English,Filipino,Indonesian": 0.26666666666666666, - "English,Filipino,Malay": 0.25333333333333335, - "English,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino": 0.29333333333333333, - "Chinese,Vietnamese,Indonesian": 0.3466666666666667, - "Chinese,Vietnamese,Malay": 0.2866666666666667, - "Chinese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Filipino,Malay": 0.24666666666666667, - "Chinese,Indonesian,Malay": 0.3, - "Vietnamese,Filipino,Indonesian": 0.24, - "Vietnamese,Filipino,Malay": 0.23333333333333334, - "Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Filipino,Indonesian,Malay": 0.24666666666666667 + "Filipino,Vietnamese,Chinese": 0.22666666666666666, + "Filipino,Vietnamese,Spanish": 0.2866666666666667, + "Filipino,Vietnamese,Malay": 0.36, + "Filipino,Vietnamese,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,English": 0.26, + "Filipino,Chinese,Spanish": 0.18666666666666668, + "Filipino,Chinese,Malay": 0.29333333333333333, + "Filipino,Chinese,Indonesian": 0.24, + "Filipino,Chinese,English": 0.22, + "Filipino,Spanish,Malay": 0.31333333333333335, + "Filipino,Spanish,Indonesian": 0.31333333333333335, + "Filipino,Spanish,English": 0.34, + "Filipino,Malay,Indonesian": 0.42, + "Filipino,Malay,English": 0.3333333333333333, + "Filipino,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish": 0.21333333333333335, + "Vietnamese,Chinese,Malay": 0.3, + "Vietnamese,Chinese,Indonesian": 0.25333333333333335, + "Vietnamese,Chinese,English": 0.18666666666666668, + "Vietnamese,Spanish,Malay": 0.32, + "Vietnamese,Spanish,Indonesian": 0.31333333333333335, + "Vietnamese,Spanish,English": 0.2866666666666667, + "Vietnamese,Malay,Indonesian": 0.4, + "Vietnamese,Malay,English": 0.26666666666666666, + "Vietnamese,Indonesian,English": 0.24666666666666667, + "Chinese,Spanish,Malay": 0.2, + "Chinese,Spanish,Indonesian": 0.20666666666666667, + "Chinese,Spanish,English": 0.19333333333333333, + "Chinese,Malay,Indonesian": 0.31333333333333335, + "Chinese,Malay,English": 0.2, + "Chinese,Indonesian,English": 0.21333333333333335, + "Spanish,Malay,Indonesian": 0.36, + "Spanish,Malay,English": 0.29333333333333333, + "Spanish,Indonesian,English": 0.29333333333333333, + "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.29333333333333333, - "Spanish,English,Chinese,Filipino": 0.26, - "Spanish,English,Chinese,Indonesian": 0.3, - "Spanish,English,Chinese,Malay": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,English,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Indonesian,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.2, - "Spanish,Chinese,Vietnamese,Indonesian": 0.24, - "Spanish,Chinese,Vietnamese,Malay": 0.19333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.18, - "Spanish,Chinese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Indonesian,Malay": 0.22, - "Spanish,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.3, - "English,Chinese,Vietnamese,Malay": 0.24666666666666667, - "English,Chinese,Filipino,Indonesian": 0.22, - "English,Chinese,Filipino,Malay": 0.2, - "English,Chinese,Indonesian,Malay": 0.26, - "English,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "English,Vietnamese,Filipino,Malay": 0.18, - "English,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "English,Filipino,Indonesian,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Chinese,Filipino,Indonesian,Malay": 0.18, - "Vietnamese,Filipino,Indonesian,Malay": 0.16 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.22, - "Spanish,English,Chinese,Vietnamese,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.16, - "Spanish,English,Chinese,Indonesian,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Malay": 0.14, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.2, - "English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 - } - }, - "AC3_2": 0.5103861923505159, - "AC3_3": 0.3938008657538782, - "AC3_4": 0.3083921683453939, - "AC3_5": 0.24528795227852204, - "AC3_6": 0.20014046913602263, - "AC3_7": 0.16784073504192454 - }, - "prompt_5": { - "overall_acc": 0.518095238095238, - "language_acc": { - "Spanish": 0.5933333333333334, - "English": 0.66, - "Chinese": 0.5533333333333333, - "Vietnamese": 0.5, - "Filipino": 0.4, - "Indonesian": 0.48, - "Malay": 0.44 + "Filipino,Vietnamese,Chinese,Spanish": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,English": 0.14, + "Filipino,Vietnamese,Spanish,Malay": 0.24, + "Filipino,Vietnamese,Spanish,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,English": 0.22, + "Filipino,Vietnamese,Malay,Indonesian": 0.28, + "Filipino,Vietnamese,Malay,English": 0.23333333333333334, + "Filipino,Vietnamese,Indonesian,English": 0.21333333333333335, + "Filipino,Chinese,Spanish,Malay": 0.15333333333333332, + "Filipino,Chinese,Spanish,Indonesian": 0.15333333333333332, + "Filipino,Chinese,Spanish,English": 0.14666666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.22666666666666666, + "Filipino,Chinese,Malay,English": 0.17333333333333334, + "Filipino,Chinese,Indonesian,English": 0.16666666666666666, + "Filipino,Spanish,Malay,Indonesian": 0.26666666666666666, + "Filipino,Spanish,Malay,English": 0.25333333333333335, + "Filipino,Spanish,Indonesian,English": 0.24, + "Filipino,Malay,Indonesian,English": 0.2733333333333333, + "Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Indonesian": 0.16, + "Vietnamese,Chinese,Spanish,English": 0.14666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.22666666666666666, + "Vietnamese,Chinese,Malay,English": 0.15333333333333332, + "Vietnamese,Chinese,Indonesian,English": 0.13333333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.2733333333333333, + "Vietnamese,Spanish,Malay,English": 0.22666666666666666, + "Vietnamese,Spanish,Indonesian,English": 0.22666666666666666, + "Vietnamese,Malay,Indonesian,English": 0.22, + "Chinese,Spanish,Malay,Indonesian": 0.18, + "Chinese,Spanish,Malay,English": 0.15333333333333332, + "Chinese,Spanish,Indonesian,English": 0.15333333333333332, + "Chinese,Malay,Indonesian,English": 0.18, + "Spanish,Malay,Indonesian,English": 0.25333333333333335 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.12, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Malay,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.12, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22, + "Filipino,Vietnamese,Spanish,Malay,English": 0.2, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.2, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.13333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.16, + "Filipino,Spanish,Malay,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.12666666666666668, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.13333333333333333, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.18666666666666668, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 + } }, - "consistency_score_2": 0.48349206349206353, - "consistency_score_3": 0.3099047619047619, - "consistency_score_4": 0.22076190476190474, - "consistency_score_5": 0.16634920634920633, - "consistency_score_6": 0.13047619047619047, - "consistency_score_7": 0.10666666666666667, + "AC3_2": 0.3844223967704124, + "AC3_3": 0.30274999995033003, + "AC3_4": 0.24877328256579573, + "AC3_5": 0.21309492762744078, + "AC3_6": 0.18791027150586953, + "AC3_7": 0.16865949816980086 + }, + "prompt_5": { + "overall_acc": 0.30190476190476195, + "language_acc": { + "Filipino": 0.3, + "Vietnamese": 0.26666666666666666, + "Chinese": 0.32, + "Spanish": 0.29333333333333333, + "Malay": 0.29333333333333333, + "Indonesian": 0.26666666666666666, + "English": 0.37333333333333335 + }, + "consistency_score_2": 0.4374603174603175, + "consistency_score_3": 0.2500952380952381, + "consistency_score_4": 0.1622857142857143, + "consistency_score_5": 0.11174603174603175, + "consistency_score_6": 0.07809523809523809, + "consistency_score_7": 0.05333333333333334, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.64, - "Spanish,Chinese": 0.5933333333333334, - "Spanish,Vietnamese": 0.54, - "Spanish,Filipino": 0.43333333333333335, - "Spanish,Indonesian": 0.52, - "Spanish,Malay": 0.44666666666666666, - "English,Chinese": 0.62, - "English,Vietnamese": 0.5133333333333333, - "English,Filipino": 0.43333333333333335, - "English,Indonesian": 0.5533333333333333, - "English,Malay": 0.43333333333333335, - "Chinese,Vietnamese": 0.48, - "Chinese,Filipino": 0.41333333333333333, - "Chinese,Indonesian": 0.44, - "Chinese,Malay": 0.4266666666666667, - "Vietnamese,Filipino": 0.4066666666666667, - "Vietnamese,Indonesian": 0.4666666666666667, - "Vietnamese,Malay": 0.5, - "Filipino,Indonesian": 0.4266666666666667, - "Filipino,Malay": 0.38666666666666666, - "Indonesian,Malay": 0.48 + "Filipino,Vietnamese": 0.4533333333333333, + "Filipino,Chinese": 0.36, + "Filipino,Spanish": 0.4533333333333333, + "Filipino,Malay": 0.5133333333333333, + "Filipino,Indonesian": 0.49333333333333335, + "Filipino,English": 0.4666666666666667, + "Vietnamese,Chinese": 0.43333333333333335, + "Vietnamese,Spanish": 0.3933333333333333, + "Vietnamese,Malay": 0.5266666666666666, + "Vietnamese,Indonesian": 0.49333333333333335, + "Vietnamese,English": 0.32, + "Chinese,Spanish": 0.30666666666666664, + "Chinese,Malay": 0.4, + "Chinese,Indonesian": 0.4, + "Chinese,English": 0.24666666666666667, + "Spanish,Malay": 0.4533333333333333, + "Spanish,Indonesian": 0.49333333333333335, + "Spanish,English": 0.5133333333333333, + "Malay,Indonesian": 0.6466666666666666, + "Malay,English": 0.43333333333333335, + "Indonesian,English": 0.38666666666666666 }, "3_combine": { - "Spanish,English,Chinese": 0.48, - "Spanish,English,Vietnamese": 0.4, - "Spanish,English,Filipino": 0.3333333333333333, - "Spanish,English,Indonesian": 0.4266666666666667, - "Spanish,English,Malay": 0.3333333333333333, - "Spanish,Chinese,Vietnamese": 0.34, - "Spanish,Chinese,Filipino": 0.31333333333333335, - "Spanish,Chinese,Indonesian": 0.3333333333333333, - "Spanish,Chinese,Malay": 0.30666666666666664, - "Spanish,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,Vietnamese,Indonesian": 0.32, - "Spanish,Vietnamese,Malay": 0.3333333333333333, - "Spanish,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Filipino,Malay": 0.25333333333333335, - "Spanish,Indonesian,Malay": 0.3, - "English,Chinese,Vietnamese": 0.36666666666666664, - "English,Chinese,Filipino": 0.31333333333333335, - "English,Chinese,Indonesian": 0.37333333333333335, - "English,Chinese,Malay": 0.30666666666666664, - "English,Vietnamese,Filipino": 0.28, - "English,Vietnamese,Indonesian": 0.3333333333333333, - "English,Vietnamese,Malay": 0.32, - "English,Filipino,Indonesian": 0.2866666666666667, - "English,Filipino,Malay": 0.24, - "English,Indonesian,Malay": 0.32, - "Chinese,Vietnamese,Filipino": 0.26, - "Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Chinese,Vietnamese,Malay": 0.29333333333333333, - "Chinese,Filipino,Indonesian": 0.24666666666666667, - "Chinese,Filipino,Malay": 0.24666666666666667, - "Chinese,Indonesian,Malay": 0.2733333333333333, - "Vietnamese,Filipino,Indonesian": 0.24, - "Vietnamese,Filipino,Malay": 0.24666666666666667, - "Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Filipino,Indonesian,Malay": 0.24666666666666667 + "Filipino,Vietnamese,Chinese": 0.21333333333333335, + "Filipino,Vietnamese,Spanish": 0.26, + "Filipino,Vietnamese,Malay": 0.34, + "Filipino,Vietnamese,Indonesian": 0.31333333333333335, + "Filipino,Vietnamese,English": 0.24, + "Filipino,Chinese,Spanish": 0.18, + "Filipino,Chinese,Malay": 0.22, + "Filipino,Chinese,Indonesian": 0.22, + "Filipino,Chinese,English": 0.16666666666666666, + "Filipino,Spanish,Malay": 0.29333333333333333, + "Filipino,Spanish,Indonesian": 0.30666666666666664, + "Filipino,Spanish,English": 0.29333333333333333, + "Filipino,Malay,Indonesian": 0.36, + "Filipino,Malay,English": 0.3, + "Filipino,Indonesian,English": 0.26, + "Vietnamese,Chinese,Spanish": 0.18, + "Vietnamese,Chinese,Malay": 0.2733333333333333, + "Vietnamese,Chinese,Indonesian": 0.25333333333333335, + "Vietnamese,Chinese,English": 0.12, + "Vietnamese,Spanish,Malay": 0.2733333333333333, + "Vietnamese,Spanish,Indonesian": 0.26666666666666666, + "Vietnamese,Spanish,English": 0.22666666666666666, + "Vietnamese,Malay,Indonesian": 0.38666666666666666, + "Vietnamese,Malay,English": 0.23333333333333334, + "Vietnamese,Indonesian,English": 0.22, + "Chinese,Spanish,Malay": 0.17333333333333334, + "Chinese,Spanish,Indonesian": 0.2, + "Chinese,Spanish,English": 0.14, + "Chinese,Malay,Indonesian": 0.28, + "Chinese,Malay,English": 0.15333333333333332, + "Chinese,Indonesian,English": 0.14666666666666667, + "Spanish,Malay,Indonesian": 0.36, + "Spanish,Malay,English": 0.30666666666666664, + "Spanish,Indonesian,English": 0.2733333333333333, + "Malay,Indonesian,English": 0.32 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.3, - "Spanish,English,Chinese,Filipino": 0.26666666666666666, - "Spanish,English,Chinese,Indonesian": 0.31333333333333335, - "Spanish,English,Chinese,Malay": 0.26, - "Spanish,English,Vietnamese,Filipino": 0.24, - "Spanish,English,Vietnamese,Indonesian": 0.26, - "Spanish,English,Vietnamese,Malay": 0.26, - "Spanish,English,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Filipino,Malay": 0.2, - "Spanish,English,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.2, - "Spanish,Chinese,Vietnamese,Malay": 0.22666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.22, - "Spanish,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.22, - "English,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "English,Chinese,Vietnamese,Malay": 0.24, - "English,Chinese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Filipino,Malay": 0.2, - "English,Chinese,Indonesian,Malay": 0.24, - "English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Chinese,Vietnamese,Filipino,Malay": 0.18, - "Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 + "Filipino,Vietnamese,Chinese,Spanish": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Malay": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,English": 0.1, + "Filipino,Vietnamese,Spanish,Malay": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,English": 0.18, + "Filipino,Vietnamese,Malay,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Malay,English": 0.2, + "Filipino,Vietnamese,Indonesian,English": 0.18, + "Filipino,Chinese,Spanish,Malay": 0.12, + "Filipino,Chinese,Spanish,Indonesian": 0.14, + "Filipino,Chinese,Spanish,English": 0.11333333333333333, + "Filipino,Chinese,Malay,Indonesian": 0.16, + "Filipino,Chinese,Malay,English": 0.12, + "Filipino,Chinese,Indonesian,English": 0.11333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.22666666666666666, + "Filipino,Spanish,Malay,English": 0.23333333333333334, + "Filipino,Spanish,Indonesian,English": 0.19333333333333333, + "Filipino,Malay,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish,Malay": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,Indonesian": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,English": 0.08666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.2, + "Vietnamese,Chinese,Malay,English": 0.10666666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.09333333333333334, + "Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, + "Vietnamese,Spanish,Malay,English": 0.18666666666666668, + "Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, + "Vietnamese,Malay,Indonesian,English": 0.2, + "Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Chinese,Spanish,Malay,English": 0.1, + "Chinese,Spanish,Indonesian,English": 0.1, + "Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Spanish,Malay,Indonesian,English": 0.24 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Vietnamese,Malay": 0.2, - "Spanish,English,Chinese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.2, - "Spanish,English,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "English,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.08, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Malay,English": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.07333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Malay,English": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14666666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.1, + "Filipino,Chinese,Spanish,Malay,English": 0.08666666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.08666666666666667, + "Filipino,Chinese,Malay,Indonesian,English": 0.09333333333333334, + "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.06666666666666667, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.08666666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.16, + "Chinese,Spanish,Malay,Indonesian,English": 0.08 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.06666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.06, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.06666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334 } }, - "AC3_2": 0.5001959096927205, - "AC3_3": 0.3878265250881748, - "AC3_4": 0.30960164004697627, - "AC3_5": 0.2518385015976254, - "AC3_6": 0.20845535274039267, - "AC3_7": 0.17691056907737457 + "AC3_2": 0.35725612868054457, + "AC3_3": 0.2735686351339825, + "AC3_4": 0.21109795411361834, + "AC3_5": 0.16311661729053772, + "AC3_6": 0.12409118029912626, + "AC3_7": 0.09065236816036149 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.4724025974025974, + "overall_acc": 0.275974025974026, "language_acc": { - "English": 0.5625, - "Filipino": 0.3693181818181818, - "Spanish": 0.4943181818181818, - "Malay": 0.4090909090909091, - "Chinese": 0.6022727272727273, - "Indonesian": 0.45454545454545453, - "Vietnamese": 0.4147727272727273 + "Indonesian": 0.25, + "English": 0.42045454545454547, + "Filipino": 0.26704545454545453, + "Spanish": 0.2897727272727273, + "Chinese": 0.23295454545454544, + "Malay": 0.24431818181818182, + "Vietnamese": 0.22727272727272727 }, - "consistency_score_2": 0.5403138528138528, - "consistency_score_3": 0.3676948051948051, - "consistency_score_4": 0.2699675324675325, - "consistency_score_5": 0.20454545454545453, - "consistency_score_6": 0.15665584415584416, - "consistency_score_7": 0.11931818181818182, + "consistency_score_2": 0.3533549783549783, + "consistency_score_3": 0.15454545454545449, + "consistency_score_4": 0.07581168831168832, + "consistency_score_5": 0.03923160173160173, + "consistency_score_6": 0.022727272727272724, + "consistency_score_7": 0.017045454545454544, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4318181818181818, - "English,Spanish": 0.625, - "English,Malay": 0.5, - "English,Chinese": 0.6590909090909091, - "English,Indonesian": 0.5568181818181818, - "English,Vietnamese": 0.5170454545454546, - "Filipino,Spanish": 0.4715909090909091, - "Filipino,Malay": 0.4943181818181818, - "Filipino,Chinese": 0.48295454545454547, - "Filipino,Indonesian": 0.4772727272727273, - "Filipino,Vietnamese": 0.5170454545454546, - "Spanish,Malay": 0.5284090909090909, - "Spanish,Chinese": 0.5795454545454546, - "Spanish,Indonesian": 0.5852272727272727, - "Spanish,Vietnamese": 0.5738636363636364, - "Malay,Chinese": 0.5, - "Malay,Indonesian": 0.6022727272727273, - "Malay,Vietnamese": 0.5397727272727273, - "Chinese,Indonesian": 0.5681818181818182, - "Chinese,Vietnamese": 0.5454545454545454, - "Indonesian,Vietnamese": 0.5909090909090909 + "Indonesian,English": 0.26704545454545453, + "Indonesian,Filipino": 0.4943181818181818, + "Indonesian,Spanish": 0.4318181818181818, + "Indonesian,Chinese": 0.26704545454545453, + "Indonesian,Malay": 0.5397727272727273, + "Indonesian,Vietnamese": 0.4375, + "English,Filipino": 0.3522727272727273, + "English,Spanish": 0.3693181818181818, + "English,Chinese": 0.25, + "English,Malay": 0.29545454545454547, + "English,Vietnamese": 0.24431818181818182, + "Filipino,Spanish": 0.4431818181818182, + "Filipino,Chinese": 0.26136363636363635, + "Filipino,Malay": 0.4772727272727273, + "Filipino,Vietnamese": 0.45454545454545453, + "Spanish,Chinese": 0.2556818181818182, + "Spanish,Malay": 0.3693181818181818, + "Spanish,Vietnamese": 0.3181818181818182, + "Chinese,Malay": 0.24431818181818182, + "Chinese,Vietnamese": 0.18181818181818182, + "Malay,Vietnamese": 0.4659090909090909 }, "3_combine": { - "English,Filipino,Spanish": 0.32954545454545453, - "English,Filipino,Malay": 0.29545454545454547, - "English,Filipino,Chinese": 0.3352272727272727, - "English,Filipino,Indonesian": 0.3068181818181818, - "English,Filipino,Vietnamese": 0.3125, - "English,Spanish,Malay": 0.3806818181818182, - "English,Spanish,Chinese": 0.4715909090909091, - "English,Spanish,Indonesian": 0.4375, - "English,Spanish,Vietnamese": 0.4090909090909091, - "English,Malay,Chinese": 0.375, - "English,Malay,Indonesian": 0.3693181818181818, - "English,Malay,Vietnamese": 0.3465909090909091, - "English,Chinese,Indonesian": 0.4318181818181818, - "English,Chinese,Vietnamese": 0.4034090909090909, - "English,Indonesian,Vietnamese": 0.3977272727272727, - "Filipino,Spanish,Malay": 0.32386363636363635, - "Filipino,Spanish,Chinese": 0.32954545454545453, - "Filipino,Spanish,Indonesian": 0.32386363636363635, - "Filipino,Spanish,Vietnamese": 0.3465909090909091, - "Filipino,Malay,Chinese": 0.3068181818181818, - "Filipino,Malay,Indonesian": 0.3522727272727273, - "Filipino,Malay,Vietnamese": 0.32386363636363635, - "Filipino,Chinese,Indonesian": 0.3352272727272727, - "Filipino,Chinese,Vietnamese": 0.3522727272727273, - "Filipino,Indonesian,Vietnamese": 0.3522727272727273, - "Spanish,Malay,Chinese": 0.36363636363636365, - "Spanish,Malay,Indonesian": 0.4034090909090909, - "Spanish,Malay,Vietnamese": 0.36363636363636365, - "Spanish,Chinese,Indonesian": 0.42613636363636365, - "Spanish,Chinese,Vietnamese": 0.39204545454545453, - "Spanish,Indonesian,Vietnamese": 0.4147727272727273, - "Malay,Chinese,Indonesian": 0.38636363636363635, - "Malay,Chinese,Vietnamese": 0.3465909090909091, - "Malay,Indonesian,Vietnamese": 0.4090909090909091, - "Chinese,Indonesian,Vietnamese": 0.4147727272727273 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.24431818181818182, - "English,Filipino,Spanish,Chinese": 0.26704545454545453, - "English,Filipino,Spanish,Indonesian": 0.24431818181818182, - "English,Filipino,Spanish,Vietnamese": 0.25, - "English,Filipino,Malay,Chinese": 0.2215909090909091, - "English,Filipino,Malay,Indonesian": 0.23295454545454544, - "English,Filipino,Malay,Vietnamese": 0.2159090909090909, - "English,Filipino,Chinese,Indonesian": 0.2556818181818182, - "English,Filipino,Chinese,Vietnamese": 0.2556818181818182, - "English,Filipino,Indonesian,Vietnamese": 0.25, - "English,Spanish,Malay,Chinese": 0.30113636363636365, - "English,Spanish,Malay,Indonesian": 0.29545454545454547, - "English,Spanish,Malay,Vietnamese": 0.2840909090909091, - "English,Spanish,Chinese,Indonesian": 0.35795454545454547, - "English,Spanish,Chinese,Vietnamese": 0.3181818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.32954545454545453, - "English,Malay,Chinese,Indonesian": 0.29545454545454547, - "English,Malay,Chinese,Vietnamese": 0.26704545454545453, - "English,Malay,Indonesian,Vietnamese": 0.2897727272727273, - "English,Chinese,Indonesian,Vietnamese": 0.32386363636363635, - "Filipino,Spanish,Malay,Chinese": 0.2215909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.25, - "Filipino,Spanish,Chinese,Vietnamese": 0.2556818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.24431818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "Spanish,Malay,Chinese,Indonesian": 0.29545454545454547, - "Spanish,Malay,Chinese,Vietnamese": 0.26704545454545453, - "Spanish,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.32954545454545453, - "Malay,Chinese,Indonesian,Vietnamese": 0.30113636363636365 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.1875, - "English,Filipino,Spanish,Malay,Indonesian": 0.1875, - "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Chinese,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.19886363636363635, - "English,Filipino,Malay,Chinese,Indonesian": 0.1875, - "English,Filipino,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365 + "Indonesian,English,Filipino": 0.1590909090909091, + "Indonesian,English,Spanish": 0.1590909090909091, + "Indonesian,English,Chinese": 0.07386363636363637, + "Indonesian,English,Malay": 0.14772727272727273, + "Indonesian,English,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Spanish": 0.2727272727272727, + "Indonesian,Filipino,Chinese": 0.13636363636363635, + "Indonesian,Filipino,Malay": 0.3125, + "Indonesian,Filipino,Vietnamese": 0.2727272727272727, + "Indonesian,Spanish,Chinese": 0.11931818181818182, + "Indonesian,Spanish,Malay": 0.24431818181818182, + "Indonesian,Spanish,Vietnamese": 0.19318181818181818, + "Indonesian,Chinese,Malay": 0.1534090909090909, + "Indonesian,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,Malay,Vietnamese": 0.2840909090909091, + "English,Filipino,Spanish": 0.19886363636363635, + "English,Filipino,Chinese": 0.10227272727272728, + "English,Filipino,Malay": 0.17613636363636365, + "English,Filipino,Vietnamese": 0.13636363636363635, + "English,Spanish,Chinese": 0.08522727272727272, + "English,Spanish,Malay": 0.14204545454545456, + "English,Spanish,Vietnamese": 0.10227272727272728, + "English,Chinese,Malay": 0.09659090909090909, + "English,Chinese,Vietnamese": 0.05113636363636364, + "English,Malay,Vietnamese": 0.13068181818181818, + "Filipino,Spanish,Chinese": 0.11363636363636363, + "Filipino,Spanish,Malay": 0.2215909090909091, + "Filipino,Spanish,Vietnamese": 0.20454545454545456, + "Filipino,Chinese,Malay": 0.13068181818181818, + "Filipino,Chinese,Vietnamese": 0.09090909090909091, + "Filipino,Malay,Vietnamese": 0.2784090909090909, + "Spanish,Chinese,Malay": 0.10227272727272728, + "Spanish,Chinese,Vietnamese": 0.07954545454545454, + "Spanish,Malay,Vietnamese": 0.17613636363636365, + "Chinese,Malay,Vietnamese": 0.07954545454545454 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.10795454545454546, + "Indonesian,English,Filipino,Chinese": 0.045454545454545456, + "Indonesian,English,Filipino,Malay": 0.09090909090909091, + "Indonesian,English,Filipino,Vietnamese": 0.07954545454545454, + "Indonesian,English,Spanish,Chinese": 0.045454545454545456, + "Indonesian,English,Spanish,Malay": 0.07954545454545454, + "Indonesian,English,Spanish,Vietnamese": 0.06818181818181818, + "Indonesian,English,Chinese,Malay": 0.05113636363636364, + "Indonesian,English,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,Filipino,Spanish,Chinese": 0.07386363636363637, + "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Chinese,Malay": 0.09090909090909091, + "Indonesian,Filipino,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,Filipino,Malay,Vietnamese": 0.19318181818181818, + "Indonesian,Spanish,Chinese,Malay": 0.09090909090909091, + "Indonesian,Spanish,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,Spanish,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,Chinese,Malay,Vietnamese": 0.0625, + "English,Filipino,Spanish,Chinese": 0.045454545454545456, + "English,Filipino,Spanish,Malay": 0.10227272727272728, + "English,Filipino,Spanish,Vietnamese": 0.09090909090909091, + "English,Filipino,Chinese,Malay": 0.0625, + "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, + "English,Filipino,Malay,Vietnamese": 0.08522727272727272, + "English,Spanish,Chinese,Malay": 0.045454545454545456, + "English,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "English,Spanish,Malay,Vietnamese": 0.056818181818181816, + "English,Chinese,Malay,Vietnamese": 0.028409090909090908, + "Filipino,Spanish,Chinese,Malay": 0.0625, + "Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, + "Filipino,Chinese,Malay,Vietnamese": 0.0625, + "Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.028409090909090908, + "Indonesian,English,Filipino,Spanish,Malay": 0.056818181818181816, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Chinese,Malay": 0.03409090909090909, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,English,Spanish,Chinese,Malay": 0.03409090909090909, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.05113636363636364, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "English,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, + "English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, + "English,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.022727272727272728, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 } }, - "AC3_2": 0.5040812112849762, - "AC3_3": 0.4135234330394271, - "AC3_4": 0.3435843076206847, - "AC3_5": 0.2854807063018439, - "AC3_6": 0.23528697105601967, - "AC3_7": 0.19051627381741018 + "AC3_2": 0.309907203667074, + "AC3_3": 0.19813519808917573, + "AC3_4": 0.1189477342807884, + "AC3_5": 0.0686973970017828, + "AC3_6": 0.041996047416770495, + "AC3_7": 0.03210778140430308 }, "prompt_2": { - "overall_acc": 0.46022727272727265, + "overall_acc": 0.2775974025974026, "language_acc": { - "English": 0.5681818181818182, - "Filipino": 0.3352272727272727, - "Spanish": 0.44886363636363635, - "Malay": 0.3977272727272727, - "Chinese": 0.5738636363636364, - "Indonesian": 0.4772727272727273, - "Vietnamese": 0.42045454545454547 + "Indonesian": 0.22727272727272727, + "English": 0.4147727272727273, + "Filipino": 0.2556818181818182, + "Spanish": 0.2840909090909091, + "Chinese": 0.2897727272727273, + "Malay": 0.24431818181818182, + "Vietnamese": 0.22727272727272727 }, - "consistency_score_2": 0.5081168831168831, - "consistency_score_3": 0.33035714285714274, - "consistency_score_4": 0.237987012987013, - "consistency_score_5": 0.17965367965367965, - "consistency_score_6": 0.13879870129870128, - "consistency_score_7": 0.10795454545454546, + "consistency_score_2": 0.35633116883116883, + "consistency_score_3": 0.15665584415584413, + "consistency_score_4": 0.07499999999999998, + "consistency_score_5": 0.03733766233766234, + "consistency_score_6": 0.01948051948051948, + "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4090909090909091, - "English,Spanish": 0.6022727272727273, - "English,Malay": 0.4431818181818182, - "English,Chinese": 0.5852272727272727, - "English,Indonesian": 0.5681818181818182, - "English,Vietnamese": 0.5511363636363636, - "Filipino,Spanish": 0.4318181818181818, - "Filipino,Malay": 0.4659090909090909, - "Filipino,Chinese": 0.4602272727272727, - "Filipino,Indonesian": 0.42613636363636365, - "Filipino,Vietnamese": 0.4772727272727273, - "Spanish,Malay": 0.5340909090909091, - "Spanish,Chinese": 0.5284090909090909, - "Spanish,Indonesian": 0.5681818181818182, - "Spanish,Vietnamese": 0.5113636363636364, - "Malay,Chinese": 0.5, - "Malay,Indonesian": 0.5284090909090909, - "Malay,Vietnamese": 0.5340909090909091, - "Chinese,Indonesian": 0.48295454545454547, - "Chinese,Vietnamese": 0.5170454545454546, - "Indonesian,Vietnamese": 0.5454545454545454 + "Indonesian,English": 0.29545454545454547, + "Indonesian,Filipino": 0.48295454545454547, + "Indonesian,Spanish": 0.45454545454545453, + "Indonesian,Chinese": 0.19886363636363635, + "Indonesian,Malay": 0.6079545454545454, + "Indonesian,Vietnamese": 0.48295454545454547, + "English,Filipino": 0.36363636363636365, + "English,Spanish": 0.375, + "English,Chinese": 0.24431818181818182, + "English,Malay": 0.26704545454545453, + "English,Vietnamese": 0.24431818181818182, + "Filipino,Spanish": 0.4090909090909091, + "Filipino,Chinese": 0.19318181818181818, + "Filipino,Malay": 0.48295454545454547, + "Filipino,Vietnamese": 0.5738636363636364, + "Spanish,Chinese": 0.2159090909090909, + "Spanish,Malay": 0.375, + "Spanish,Vietnamese": 0.3352272727272727, + "Chinese,Malay": 0.18181818181818182, + "Chinese,Vietnamese": 0.19318181818181818, + "Malay,Vietnamese": 0.5056818181818182 }, "3_combine": { - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Malay": 0.23863636363636365, - "English,Filipino,Chinese": 0.2897727272727273, - "English,Filipino,Indonesian": 0.2784090909090909, - "English,Filipino,Vietnamese": 0.2840909090909091, - "English,Spanish,Malay": 0.3522727272727273, - "English,Spanish,Chinese": 0.42045454545454547, - "English,Spanish,Indonesian": 0.4090909090909091, - "English,Spanish,Vietnamese": 0.4090909090909091, - "English,Malay,Chinese": 0.3409090909090909, - "English,Malay,Indonesian": 0.32954545454545453, - "English,Malay,Vietnamese": 0.3352272727272727, - "English,Chinese,Indonesian": 0.375, - "English,Chinese,Vietnamese": 0.39204545454545453, - "English,Indonesian,Vietnamese": 0.39204545454545453, - "Filipino,Spanish,Malay": 0.30113636363636365, - "Filipino,Spanish,Chinese": 0.29545454545454547, - "Filipino,Spanish,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Vietnamese": 0.2784090909090909, - "Filipino,Malay,Chinese": 0.2784090909090909, - "Filipino,Malay,Indonesian": 0.30113636363636365, - "Filipino,Malay,Vietnamese": 0.29545454545454547, - "Filipino,Chinese,Indonesian": 0.26704545454545453, - "Filipino,Chinese,Vietnamese": 0.2897727272727273, - "Filipino,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Chinese": 0.3522727272727273, - "Spanish,Malay,Indonesian": 0.3693181818181818, - "Spanish,Malay,Vietnamese": 0.35795454545454547, - "Spanish,Chinese,Indonesian": 0.36363636363636365, - "Spanish,Chinese,Vietnamese": 0.3522727272727273, - "Spanish,Indonesian,Vietnamese": 0.375, - "Malay,Chinese,Indonesian": 0.3465909090909091, - "Malay,Chinese,Vietnamese": 0.3352272727272727, - "Malay,Indonesian,Vietnamese": 0.35795454545454547, - "Chinese,Indonesian,Vietnamese": 0.3465909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.20454545454545456, - "English,Filipino,Spanish,Chinese": 0.23863636363636365, - "English,Filipino,Spanish,Indonesian": 0.20454545454545456, - "English,Filipino,Spanish,Vietnamese": 0.22727272727272727, - "English,Filipino,Malay,Chinese": 0.18181818181818182, - "English,Filipino,Malay,Indonesian": 0.18181818181818182, - "English,Filipino,Malay,Vietnamese": 0.19318181818181818, - "English,Filipino,Chinese,Indonesian": 0.19318181818181818, - "English,Filipino,Chinese,Vietnamese": 0.2159090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Chinese": 0.2840909090909091, - "English,Spanish,Malay,Indonesian": 0.26704545454545453, - "English,Spanish,Malay,Vietnamese": 0.2840909090909091, - "English,Spanish,Chinese,Indonesian": 0.3125, - "English,Spanish,Chinese,Vietnamese": 0.3068181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.3181818181818182, - "English,Malay,Chinese,Indonesian": 0.26136363636363635, - "English,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "English,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Malay,Chinese": 0.2159090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.21022727272727273, - "Filipino,Spanish,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.2784090909090909, - "Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.17045454545454544, - "English,Filipino,Spanish,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.1875, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.22727272727272727, - "English,Spanish,Malay,Chinese,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.25, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727 + "Indonesian,English,Filipino": 0.18181818181818182, + "Indonesian,English,Spanish": 0.18181818181818182, + "Indonesian,English,Chinese": 0.056818181818181816, + "Indonesian,English,Malay": 0.17045454545454544, + "Indonesian,English,Vietnamese": 0.125, + "Indonesian,Filipino,Spanish": 0.25, + "Indonesian,Filipino,Chinese": 0.08522727272727272, + "Indonesian,Filipino,Malay": 0.3409090909090909, + "Indonesian,Filipino,Vietnamese": 0.3181818181818182, + "Indonesian,Spanish,Chinese": 0.09090909090909091, + "Indonesian,Spanish,Malay": 0.2784090909090909, + "Indonesian,Spanish,Vietnamese": 0.20454545454545456, + "Indonesian,Chinese,Malay": 0.10795454545454546, + "Indonesian,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,Malay,Vietnamese": 0.3409090909090909, + "English,Filipino,Spanish": 0.17613636363636365, + "English,Filipino,Chinese": 0.09090909090909091, + "English,Filipino,Malay": 0.16477272727272727, + "English,Filipino,Vietnamese": 0.17045454545454544, + "English,Spanish,Chinese": 0.08522727272727272, + "English,Spanish,Malay": 0.13636363636363635, + "English,Spanish,Vietnamese": 0.11931818181818182, + "English,Chinese,Malay": 0.0625, + "English,Chinese,Vietnamese": 0.05113636363636364, + "English,Malay,Vietnamese": 0.11931818181818182, + "Filipino,Spanish,Chinese": 0.08522727272727272, + "Filipino,Spanish,Malay": 0.2159090909090909, + "Filipino,Spanish,Vietnamese": 0.23863636363636365, + "Filipino,Chinese,Malay": 0.07386363636363637, + "Filipino,Chinese,Vietnamese": 0.10227272727272728, + "Filipino,Malay,Vietnamese": 0.3409090909090909, + "Spanish,Chinese,Malay": 0.06818181818181818, + "Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Spanish,Malay,Vietnamese": 0.19318181818181818, + "Chinese,Malay,Vietnamese": 0.07954545454545454 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.11363636363636363, + "Indonesian,English,Filipino,Chinese": 0.03977272727272727, + "Indonesian,English,Filipino,Malay": 0.11931818181818182, + "Indonesian,English,Filipino,Vietnamese": 0.10795454545454546, + "Indonesian,English,Spanish,Chinese": 0.03409090909090909, + "Indonesian,English,Spanish,Malay": 0.10227272727272728, + "Indonesian,English,Spanish,Vietnamese": 0.08522727272727272, + "Indonesian,English,Chinese,Malay": 0.028409090909090908, + "Indonesian,English,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,Filipino,Spanish,Chinese": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Vietnamese": 0.1590909090909091, + "Indonesian,Filipino,Chinese,Malay": 0.05113636363636364, + "Indonesian,Filipino,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,Filipino,Malay,Vietnamese": 0.23863636363636365, + "Indonesian,Spanish,Chinese,Malay": 0.045454545454545456, + "Indonesian,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,Spanish,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,Chinese,Malay,Vietnamese": 0.056818181818181816, + "English,Filipino,Spanish,Chinese": 0.045454545454545456, + "English,Filipino,Spanish,Malay": 0.09659090909090909, + "English,Filipino,Spanish,Vietnamese": 0.10227272727272728, + "English,Filipino,Chinese,Malay": 0.03977272727272727, + "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, + "English,Filipino,Malay,Vietnamese": 0.10227272727272728, + "English,Spanish,Chinese,Malay": 0.017045454545454544, + "English,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "English,Spanish,Malay,Vietnamese": 0.06818181818181818, + "English,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Filipino,Spanish,Chinese,Malay": 0.028409090909090908, + "Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, + "Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.022727272727272728, + "Indonesian,English,Filipino,Spanish,Malay": 0.07386363636363637, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Chinese,Malay": 0.022727272727272728, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Spanish,Chinese,Malay": 0.011363636363636364, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.05113636363636364, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.022727272727272728, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, + "English,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, + "English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.011363636363636364, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.05113636363636364, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, - "AC3_2": 0.48298788381814023, - "AC3_3": 0.38462525662486047, - "AC3_4": 0.31373781991236943, - "AC3_5": 0.2584278300576335, - "AC3_6": 0.2132760531794738, - "AC3_7": 0.17488636360558363 + "AC3_2": 0.3120749288630882, + "AC3_3": 0.2002852287437799, + "AC3_4": 0.11809392261844114, + "AC3_5": 0.06582206451250455, + "AC3_6": 0.036406216721830816, + "AC3_7": 0.021833503567520746 }, "prompt_3": { - "overall_acc": 0.41314935064935066, + "overall_acc": 0.27272727272727276, "language_acc": { - "English": 0.5170454545454546, - "Filipino": 0.32386363636363635, - "Spanish": 0.4772727272727273, - "Malay": 0.375, - "Chinese": 0.48863636363636365, - "Indonesian": 0.3409090909090909, - "Vietnamese": 0.3693181818181818 + "Indonesian": 0.2215909090909091, + "English": 0.4034090909090909, + "Filipino": 0.26136363636363635, + "Spanish": 0.30113636363636365, + "Chinese": 0.23295454545454544, + "Malay": 0.23863636363636365, + "Vietnamese": 0.25 }, - "consistency_score_2": 0.44182900432900435, - "consistency_score_3": 0.24870129870129867, - "consistency_score_4": 0.1608766233766234, - "consistency_score_5": 0.11390692640692639, - "consistency_score_6": 0.08603896103896104, - "consistency_score_7": 0.06818181818181818, + "consistency_score_2": 0.3566017316017316, + "consistency_score_3": 0.15584415584415584, + "consistency_score_4": 0.07483766233766233, + "consistency_score_5": 0.036525974025974024, + "consistency_score_6": 0.01704545454545454, + "consistency_score_7": 0.005681818181818182, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.29545454545454547, - "English,Spanish": 0.5113636363636364, - "English,Malay": 0.36363636363636365, - "English,Chinese": 0.5, - "English,Indonesian": 0.45454545454545453, - "English,Vietnamese": 0.4431818181818182, - "Filipino,Spanish": 0.42045454545454547, - "Filipino,Malay": 0.39204545454545453, - "Filipino,Chinese": 0.375, - "Filipino,Indonesian": 0.3977272727272727, - "Filipino,Vietnamese": 0.4375, - "Spanish,Malay": 0.45454545454545453, - "Spanish,Chinese": 0.48863636363636365, - "Spanish,Indonesian": 0.5, - "Spanish,Vietnamese": 0.5170454545454546, - "Malay,Chinese": 0.375, - "Malay,Indonesian": 0.4431818181818182, - "Malay,Vietnamese": 0.4715909090909091, - "Chinese,Indonesian": 0.44886363636363635, - "Chinese,Vietnamese": 0.4659090909090909, - "Indonesian,Vietnamese": 0.5227272727272727 + "Indonesian,English": 0.29545454545454547, + "Indonesian,Filipino": 0.4772727272727273, + "Indonesian,Spanish": 0.4318181818181818, + "Indonesian,Chinese": 0.17613636363636365, + "Indonesian,Malay": 0.5738636363636364, + "Indonesian,Vietnamese": 0.5, + "English,Filipino": 0.3465909090909091, + "English,Spanish": 0.4147727272727273, + "English,Chinese": 0.22727272727272727, + "English,Malay": 0.2784090909090909, + "English,Vietnamese": 0.23863636363636365, + "Filipino,Spanish": 0.4147727272727273, + "Filipino,Chinese": 0.24431818181818182, + "Filipino,Malay": 0.48863636363636365, + "Filipino,Vietnamese": 0.5340909090909091, + "Spanish,Chinese": 0.2556818181818182, + "Spanish,Malay": 0.36363636363636365, + "Spanish,Vietnamese": 0.3125, + "Chinese,Malay": 0.17613636363636365, + "Chinese,Vietnamese": 0.19886363636363635, + "Malay,Vietnamese": 0.5397727272727273 }, "3_combine": { - "English,Filipino,Spanish": 0.19886363636363635, - "English,Filipino,Malay": 0.1590909090909091, - "English,Filipino,Chinese": 0.1875, - "English,Filipino,Indonesian": 0.1875, - "English,Filipino,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay": 0.24431818181818182, - "English,Spanish,Chinese": 0.3125, - "English,Spanish,Indonesian": 0.3068181818181818, - "English,Spanish,Vietnamese": 0.30113636363636365, - "English,Malay,Chinese": 0.19318181818181818, - "English,Malay,Indonesian": 0.2215909090909091, - "English,Malay,Vietnamese": 0.21022727272727273, - "English,Chinese,Indonesian": 0.2727272727272727, - "English,Chinese,Vietnamese": 0.2840909090909091, - "English,Indonesian,Vietnamese": 0.2840909090909091, - "Filipino,Spanish,Malay": 0.23295454545454544, - "Filipino,Spanish,Chinese": 0.24431818181818182, - "Filipino,Spanish,Indonesian": 0.25, - "Filipino,Spanish,Vietnamese": 0.26704545454545453, - "Filipino,Malay,Chinese": 0.17613636363636365, - "Filipino,Malay,Indonesian": 0.2215909090909091, - "Filipino,Malay,Vietnamese": 0.23863636363636365, - "Filipino,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Chinese,Vietnamese": 0.24431818181818182, - "Filipino,Indonesian,Vietnamese": 0.26704545454545453, - "Spanish,Malay,Chinese": 0.23295454545454544, - "Spanish,Malay,Indonesian": 0.2840909090909091, - "Spanish,Malay,Vietnamese": 0.2784090909090909, - "Spanish,Chinese,Indonesian": 0.29545454545454547, - "Spanish,Chinese,Vietnamese": 0.32386363636363635, - "Spanish,Indonesian,Vietnamese": 0.32954545454545453, - "Malay,Chinese,Indonesian": 0.21022727272727273, - "Malay,Chinese,Vietnamese": 0.22727272727272727, - "Malay,Indonesian,Vietnamese": 0.29545454545454547, - "Chinese,Indonesian,Vietnamese": 0.30113636363636365 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.125, - "English,Filipino,Spanish,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese": 0.10227272727272728, - "English,Filipino,Malay,Indonesian": 0.11931818181818182, - "English,Filipino,Malay,Vietnamese": 0.125, - "English,Filipino,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese": 0.14204545454545456, - "English,Spanish,Malay,Indonesian": 0.17045454545454544, - "English,Spanish,Malay,Vietnamese": 0.16477272727272727, - "English,Spanish,Chinese,Indonesian": 0.2159090909090909, - "English,Spanish,Chinese,Vietnamese": 0.2215909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.2159090909090909, - "English,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.125, - "Filipino,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.17045454545454544, - "Spanish,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 + "Indonesian,English,Filipino": 0.18181818181818182, + "Indonesian,English,Spanish": 0.18181818181818182, + "Indonesian,English,Chinese": 0.03977272727272727, + "Indonesian,English,Malay": 0.17613636363636365, + "Indonesian,English,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Spanish": 0.24431818181818182, + "Indonesian,Filipino,Chinese": 0.08522727272727272, + "Indonesian,Filipino,Malay": 0.32954545454545453, + "Indonesian,Filipino,Vietnamese": 0.3125, + "Indonesian,Spanish,Chinese": 0.07954545454545454, + "Indonesian,Spanish,Malay": 0.2556818181818182, + "Indonesian,Spanish,Vietnamese": 0.20454545454545456, + "Indonesian,Chinese,Malay": 0.06818181818181818, + "Indonesian,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,Malay,Vietnamese": 0.3522727272727273, + "English,Filipino,Spanish": 0.19318181818181818, + "English,Filipino,Chinese": 0.07954545454545454, + "English,Filipino,Malay": 0.17045454545454544, + "English,Filipino,Vietnamese": 0.16477272727272727, + "English,Spanish,Chinese": 0.09090909090909091, + "English,Spanish,Malay": 0.1590909090909091, + "English,Spanish,Vietnamese": 0.125, + "English,Chinese,Malay": 0.056818181818181816, + "English,Chinese,Vietnamese": 0.045454545454545456, + "English,Malay,Vietnamese": 0.13636363636363635, + "Filipino,Spanish,Chinese": 0.125, + "Filipino,Spanish,Malay": 0.2159090909090909, + "Filipino,Spanish,Vietnamese": 0.21022727272727273, + "Filipino,Chinese,Malay": 0.09090909090909091, + "Filipino,Chinese,Vietnamese": 0.10795454545454546, + "Filipino,Malay,Vietnamese": 0.3409090909090909, + "Spanish,Chinese,Malay": 0.0625, + "Spanish,Chinese,Vietnamese": 0.07954545454545454, + "Spanish,Malay,Vietnamese": 0.19886363636363635, + "Chinese,Malay,Vietnamese": 0.07954545454545454 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.125, + "Indonesian,English,Filipino,Chinese": 0.028409090909090908, + "Indonesian,English,Filipino,Malay": 0.11363636363636363, + "Indonesian,English,Filipino,Vietnamese": 0.10795454545454546, + "Indonesian,English,Spanish,Chinese": 0.022727272727272728, + "Indonesian,English,Spanish,Malay": 0.10795454545454546, + "Indonesian,English,Spanish,Vietnamese": 0.08522727272727272, + "Indonesian,English,Chinese,Malay": 0.011363636363636364, + "Indonesian,English,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Chinese": 0.05113636363636364, + "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Chinese,Malay": 0.05113636363636364, + "Indonesian,Filipino,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,Filipino,Malay,Vietnamese": 0.23863636363636365, + "Indonesian,Spanish,Chinese,Malay": 0.03409090909090909, + "Indonesian,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "Indonesian,Spanish,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Chinese,Malay,Vietnamese": 0.03977272727272727, + "English,Filipino,Spanish,Chinese": 0.045454545454545456, + "English,Filipino,Spanish,Malay": 0.11363636363636363, + "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, + "English,Filipino,Chinese,Malay": 0.03409090909090909, + "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, + "English,Filipino,Malay,Vietnamese": 0.11363636363636363, + "English,Spanish,Chinese,Malay": 0.022727272727272728, + "English,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "English,Spanish,Malay,Vietnamese": 0.08522727272727272, + "English,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, + "Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.017045454545454544, + "Indonesian,English,Filipino,Spanish,Malay": 0.07954545454545454, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Chinese,Malay": 0.011363636363636364, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Spanish,Chinese,Malay": 0.005681818181818182, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.056818181818181816, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, + "English,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, + "English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.005681818181818182, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 } }, - "AC3_2": 0.4270081579313156, - "AC3_3": 0.3104953667398443, - "AC3_4": 0.23157862354195718, - "AC3_5": 0.17857892873755157, - "AC3_6": 0.14241896312214036, - "AC3_7": 0.11704737081731985 + "AC3_2": 0.3090752754921998, + "AC3_3": 0.19834710739173553, + "AC3_4": 0.11744724212153103, + "AC3_5": 0.06442376519033474, + "AC3_6": 0.03208556148625353, + "AC3_7": 0.01113172541344137 }, "prompt_4": { - "overall_acc": 0.4586038961038961, + "overall_acc": 0.27516233766233766, "language_acc": { - "English": 0.5852272727272727, - "Filipino": 0.32386363636363635, - "Spanish": 0.45454545454545453, - "Malay": 0.4034090909090909, - "Chinese": 0.5454545454545454, - "Indonesian": 0.4715909090909091, - "Vietnamese": 0.42613636363636365 + "Indonesian": 0.25, + "English": 0.4090909090909091, + "Filipino": 0.2556818181818182, + "Spanish": 0.2784090909090909, + "Chinese": 0.24431818181818182, + "Malay": 0.2556818181818182, + "Vietnamese": 0.23295454545454544 }, - "consistency_score_2": 0.5062229437229437, - "consistency_score_3": 0.32694805194805193, - "consistency_score_4": 0.23538961038961037, - "consistency_score_5": 0.1793831168831169, - "consistency_score_6": 0.14285714285714288, - "consistency_score_7": 0.11931818181818182, + "consistency_score_2": 0.36093073593073594, + "consistency_score_3": 0.15941558441558443, + "consistency_score_4": 0.07499999999999997, + "consistency_score_5": 0.034902597402597414, + "consistency_score_6": 0.01542207792207792, + "consistency_score_7": 0.005681818181818182, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4090909090909091, - "English,Spanish": 0.5909090909090909, - "English,Malay": 0.4715909090909091, - "English,Chinese": 0.6590909090909091, - "English,Indonesian": 0.5454545454545454, - "English,Vietnamese": 0.5454545454545454, - "Filipino,Spanish": 0.4090909090909091, - "Filipino,Malay": 0.4772727272727273, - "Filipino,Chinese": 0.44886363636363635, - "Filipino,Indonesian": 0.45454545454545453, - "Filipino,Vietnamese": 0.4659090909090909, - "Spanish,Malay": 0.4715909090909091, - "Spanish,Chinese": 0.5227272727272727, - "Spanish,Indonesian": 0.5681818181818182, - "Spanish,Vietnamese": 0.4602272727272727, - "Malay,Chinese": 0.4602272727272727, - "Malay,Indonesian": 0.5568181818181818, - "Malay,Vietnamese": 0.4659090909090909, - "Chinese,Indonesian": 0.5511363636363636, - "Chinese,Vietnamese": 0.5511363636363636, - "Indonesian,Vietnamese": 0.5454545454545454 + "Indonesian,English": 0.30113636363636365, + "Indonesian,Filipino": 0.4943181818181818, + "Indonesian,Spanish": 0.48863636363636365, + "Indonesian,Chinese": 0.26704545454545453, + "Indonesian,Malay": 0.5568181818181818, + "Indonesian,Vietnamese": 0.48863636363636365, + "English,Filipino": 0.35795454545454547, + "English,Spanish": 0.39204545454545453, + "English,Chinese": 0.26704545454545453, + "English,Malay": 0.2727272727272727, + "English,Vietnamese": 0.25, + "Filipino,Spanish": 0.3977272727272727, + "Filipino,Chinese": 0.17613636363636365, + "Filipino,Malay": 0.5, + "Filipino,Vietnamese": 0.5113636363636364, + "Spanish,Chinese": 0.2556818181818182, + "Spanish,Malay": 0.4318181818181818, + "Spanish,Vietnamese": 0.3522727272727273, + "Chinese,Malay": 0.19886363636363635, + "Chinese,Vietnamese": 0.11931818181818182, + "Malay,Vietnamese": 0.5 }, "3_combine": { - "English,Filipino,Spanish": 0.2840909090909091, - "English,Filipino,Malay": 0.2784090909090909, - "English,Filipino,Chinese": 0.3068181818181818, - "English,Filipino,Indonesian": 0.2840909090909091, - "English,Filipino,Vietnamese": 0.3068181818181818, - "English,Spanish,Malay": 0.32954545454545453, - "English,Spanish,Chinese": 0.4318181818181818, - "English,Spanish,Indonesian": 0.4034090909090909, - "English,Spanish,Vietnamese": 0.3409090909090909, - "English,Malay,Chinese": 0.3352272727272727, - "English,Malay,Indonesian": 0.3522727272727273, - "English,Malay,Vietnamese": 0.30113636363636365, - "English,Chinese,Indonesian": 0.4090909090909091, - "English,Chinese,Vietnamese": 0.42045454545454547, - "English,Indonesian,Vietnamese": 0.36363636363636365, - "Filipino,Spanish,Malay": 0.2727272727272727, - "Filipino,Spanish,Chinese": 0.2840909090909091, - "Filipino,Spanish,Indonesian": 0.29545454545454547, - "Filipino,Spanish,Vietnamese": 0.24431818181818182, - "Filipino,Malay,Chinese": 0.29545454545454547, - "Filipino,Malay,Indonesian": 0.3352272727272727, - "Filipino,Malay,Vietnamese": 0.30113636363636365, - "Filipino,Chinese,Indonesian": 0.3125, - "Filipino,Chinese,Vietnamese": 0.3181818181818182, - "Filipino,Indonesian,Vietnamese": 0.3125, - "Spanish,Malay,Chinese": 0.30113636363636365, - "Spanish,Malay,Indonesian": 0.3522727272727273, - "Spanish,Malay,Vietnamese": 0.26136363636363635, - "Spanish,Chinese,Indonesian": 0.3806818181818182, - "Spanish,Chinese,Vietnamese": 0.32386363636363635, - "Spanish,Indonesian,Vietnamese": 0.3409090909090909, - "Malay,Chinese,Indonesian": 0.3522727272727273, - "Malay,Chinese,Vietnamese": 0.30113636363636365, - "Malay,Indonesian,Vietnamese": 0.32954545454545453, - "Chinese,Indonesian,Vietnamese": 0.3806818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.21022727272727273, - "English,Filipino,Spanish,Chinese": 0.23295454545454544, - "English,Filipino,Spanish,Indonesian": 0.2215909090909091, - "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, - "English,Filipino,Malay,Chinese": 0.2159090909090909, - "English,Filipino,Malay,Indonesian": 0.22727272727272727, - "English,Filipino,Malay,Vietnamese": 0.2159090909090909, - "English,Filipino,Chinese,Indonesian": 0.22727272727272727, - "English,Filipino,Chinese,Vietnamese": 0.24431818181818182, - "English,Filipino,Indonesian,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay,Chinese": 0.2556818181818182, - "English,Spanish,Malay,Indonesian": 0.2727272727272727, - "English,Spanish,Malay,Vietnamese": 0.2215909090909091, - "English,Spanish,Chinese,Indonesian": 0.3181818181818182, - "English,Spanish,Chinese,Vietnamese": 0.2727272727272727, - "English,Spanish,Indonesian,Vietnamese": 0.2784090909090909, - "English,Malay,Chinese,Indonesian": 0.2727272727272727, - "English,Malay,Chinese,Vietnamese": 0.22727272727272727, - "English,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Malay,Chinese": 0.20454545454545456, - "Filipino,Spanish,Malay,Indonesian": 0.23295454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.23295454545454544, - "Filipino,Malay,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Spanish,Malay,Chinese,Indonesian": 0.26136363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Spanish,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Indonesian": 0.1875, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.2215909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365 + "Indonesian,English,Filipino": 0.17045454545454544, + "Indonesian,English,Spanish": 0.19318181818181818, + "Indonesian,English,Chinese": 0.10227272727272728, + "Indonesian,English,Malay": 0.1534090909090909, + "Indonesian,English,Vietnamese": 0.125, + "Indonesian,Filipino,Spanish": 0.26704545454545453, + "Indonesian,Filipino,Chinese": 0.09659090909090909, + "Indonesian,Filipino,Malay": 0.3352272727272727, + "Indonesian,Filipino,Vietnamese": 0.3068181818181818, + "Indonesian,Spanish,Chinese": 0.14772727272727273, + "Indonesian,Spanish,Malay": 0.30113636363636365, + "Indonesian,Spanish,Vietnamese": 0.23295454545454544, + "Indonesian,Chinese,Malay": 0.13068181818181818, + "Indonesian,Chinese,Vietnamese": 0.06818181818181818, + "Indonesian,Malay,Vietnamese": 0.32386363636363635, + "English,Filipino,Spanish": 0.17613636363636365, + "English,Filipino,Chinese": 0.08522727272727272, + "English,Filipino,Malay": 0.16477272727272727, + "English,Filipino,Vietnamese": 0.14204545454545456, + "English,Spanish,Chinese": 0.10227272727272728, + "English,Spanish,Malay": 0.1590909090909091, + "English,Spanish,Vietnamese": 0.125, + "English,Chinese,Malay": 0.07386363636363637, + "English,Chinese,Vietnamese": 0.03977272727272727, + "English,Malay,Vietnamese": 0.11363636363636363, + "Filipino,Spanish,Chinese": 0.07386363636363637, + "Filipino,Spanish,Malay": 0.23863636363636365, + "Filipino,Spanish,Vietnamese": 0.2215909090909091, + "Filipino,Chinese,Malay": 0.07386363636363637, + "Filipino,Chinese,Vietnamese": 0.0625, + "Filipino,Malay,Vietnamese": 0.3125, + "Spanish,Chinese,Malay": 0.11363636363636363, + "Spanish,Chinese,Vietnamese": 0.07954545454545454, + "Spanish,Malay,Vietnamese": 0.2215909090909091, + "Chinese,Malay,Vietnamese": 0.045454545454545456 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.11363636363636363, + "Indonesian,English,Filipino,Chinese": 0.05113636363636364, + "Indonesian,English,Filipino,Malay": 0.10795454545454546, + "Indonesian,English,Filipino,Vietnamese": 0.08522727272727272, + "Indonesian,English,Spanish,Chinese": 0.06818181818181818, + "Indonesian,English,Spanish,Malay": 0.10227272727272728, + "Indonesian,English,Spanish,Vietnamese": 0.07954545454545454, + "Indonesian,English,Chinese,Malay": 0.045454545454545456, + "Indonesian,English,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,Filipino,Spanish,Chinese": 0.056818181818181816, + "Indonesian,Filipino,Spanish,Malay": 0.18181818181818182, + "Indonesian,Filipino,Spanish,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Chinese,Malay": 0.0625, + "Indonesian,Filipino,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Malay,Vietnamese": 0.2159090909090909, + "Indonesian,Spanish,Chinese,Malay": 0.09090909090909091, + "Indonesian,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,Spanish,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,Chinese,Malay,Vietnamese": 0.045454545454545456, + "English,Filipino,Spanish,Chinese": 0.03409090909090909, + "English,Filipino,Spanish,Malay": 0.10227272727272728, + "English,Filipino,Spanish,Vietnamese": 0.08522727272727272, + "English,Filipino,Chinese,Malay": 0.03977272727272727, + "English,Filipino,Chinese,Vietnamese": 0.017045454545454544, + "English,Filipino,Malay,Vietnamese": 0.07954545454545454, + "English,Spanish,Chinese,Malay": 0.03977272727272727, + "English,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "English,Spanish,Malay,Vietnamese": 0.06818181818181818, + "English,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, + "Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.028409090909090908, + "Indonesian,English,Filipino,Spanish,Malay": 0.07386363636363637, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.056818181818181816, + "Indonesian,English,Filipino,Chinese,Malay": 0.028409090909090908, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.05113636363636364, + "Indonesian,English,Spanish,Chinese,Malay": 0.03409090909090909, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, + "English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "English,Filipino,Spanish,Malay,Vietnamese": 0.05113636363636364, + "English,Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, + "English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.005681818181818182, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 } }, - "AC3_2": 0.48123830034854564, - "AC3_3": 0.38174343733650334, - "AC3_4": 0.31109971894962835, - "AC3_5": 0.25789175832843486, - "AC3_6": 0.21785232307926053, - "AC3_7": 0.18936733908878567 + "AC3_2": 0.312264192585157, + "AC3_3": 0.20187479680406284, + "AC3_4": 0.11787204447259654, + "AC3_5": 0.06194754196683012, + "AC3_6": 0.029207175496006994, + "AC3_7": 0.01113373620202616 }, "prompt_5": { - "overall_acc": 0.46185064935064934, + "overall_acc": 0.2824675324675325, "language_acc": { - "English": 0.5681818181818182, - "Filipino": 0.3409090909090909, - "Spanish": 0.4602272727272727, - "Malay": 0.38636363636363635, - "Chinese": 0.5625, - "Indonesian": 0.44886363636363635, - "Vietnamese": 0.4659090909090909 + "Indonesian": 0.25, + "English": 0.4034090909090909, + "Filipino": 0.2840909090909091, + "Spanish": 0.26704545454545453, + "Chinese": 0.2727272727272727, + "Malay": 0.26704545454545453, + "Vietnamese": 0.23295454545454544 }, - "consistency_score_2": 0.48051948051948057, - "consistency_score_3": 0.299025974025974, - "consistency_score_4": 0.21298701298701297, - "consistency_score_5": 0.16450216450216448, - "consistency_score_6": 0.13555194805194806, - "consistency_score_7": 0.11931818181818182, + "consistency_score_2": 0.36688311688311687, + "consistency_score_3": 0.16818181818181818, + "consistency_score_4": 0.08571428571428572, + "consistency_score_5": 0.04518398268398267, + "consistency_score_6": 0.023538961038961036, + "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4090909090909091, - "English,Spanish": 0.5568181818181818, - "English,Malay": 0.4659090909090909, - "English,Chinese": 0.5625, - "English,Indonesian": 0.4715909090909091, - "English,Vietnamese": 0.5, - "Filipino,Spanish": 0.4034090909090909, - "Filipino,Malay": 0.4034090909090909, - "Filipino,Chinese": 0.42613636363636365, - "Filipino,Indonesian": 0.4318181818181818, - "Filipino,Vietnamese": 0.4147727272727273, - "Spanish,Malay": 0.44886363636363635, - "Spanish,Chinese": 0.5113636363636364, - "Spanish,Indonesian": 0.5113636363636364, - "Spanish,Vietnamese": 0.4943181818181818, - "Malay,Chinese": 0.4772727272727273, - "Malay,Indonesian": 0.5511363636363636, - "Malay,Vietnamese": 0.44886363636363635, - "Chinese,Indonesian": 0.5681818181818182, - "Chinese,Vietnamese": 0.48863636363636365, - "Indonesian,Vietnamese": 0.5454545454545454 + "Indonesian,English": 0.2840909090909091, + "Indonesian,Filipino": 0.45454545454545453, + "Indonesian,Spanish": 0.44886363636363635, + "Indonesian,Chinese": 0.26136363636363635, + "Indonesian,Malay": 0.5397727272727273, + "Indonesian,Vietnamese": 0.3977272727272727, + "English,Filipino": 0.3125, + "English,Spanish": 0.4090909090909091, + "English,Chinese": 0.29545454545454547, + "English,Malay": 0.26136363636363635, + "English,Vietnamese": 0.26704545454545453, + "Filipino,Spanish": 0.4943181818181818, + "Filipino,Chinese": 0.22727272727272727, + "Filipino,Malay": 0.5511363636363636, + "Filipino,Vietnamese": 0.44886363636363635, + "Spanish,Chinese": 0.2159090909090909, + "Spanish,Malay": 0.42045454545454547, + "Spanish,Vietnamese": 0.3465909090909091, + "Chinese,Malay": 0.26704545454545453, + "Chinese,Vietnamese": 0.30113636363636365, + "Malay,Vietnamese": 0.5 }, "3_combine": { - "English,Filipino,Spanish": 0.2784090909090909, - "English,Filipino,Malay": 0.23863636363636365, - "English,Filipino,Chinese": 0.2840909090909091, - "English,Filipino,Indonesian": 0.2556818181818182, - "English,Filipino,Vietnamese": 0.26136363636363635, - "English,Spanish,Malay": 0.30113636363636365, - "English,Spanish,Chinese": 0.3693181818181818, - "English,Spanish,Indonesian": 0.3352272727272727, - "English,Spanish,Vietnamese": 0.32386363636363635, - "English,Malay,Chinese": 0.3068181818181818, - "English,Malay,Indonesian": 0.3181818181818182, - "English,Malay,Vietnamese": 0.2784090909090909, - "English,Chinese,Indonesian": 0.3409090909090909, - "English,Chinese,Vietnamese": 0.3522727272727273, - "English,Indonesian,Vietnamese": 0.32386363636363635, - "Filipino,Spanish,Malay": 0.24431818181818182, - "Filipino,Spanish,Chinese": 0.26136363636363635, - "Filipino,Spanish,Indonesian": 0.2556818181818182, - "Filipino,Spanish,Vietnamese": 0.2556818181818182, - "Filipino,Malay,Chinese": 0.25, - "Filipino,Malay,Indonesian": 0.29545454545454547, - "Filipino,Malay,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian": 0.30113636363636365, - "Filipino,Chinese,Vietnamese": 0.26136363636363635, - "Filipino,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Chinese": 0.30113636363636365, - "Spanish,Malay,Indonesian": 0.32954545454545453, - "Spanish,Malay,Vietnamese": 0.2556818181818182, - "Spanish,Chinese,Indonesian": 0.35795454545454547, - "Spanish,Chinese,Vietnamese": 0.3125, - "Spanish,Indonesian,Vietnamese": 0.32954545454545453, - "Malay,Chinese,Indonesian": 0.3522727272727273, - "Malay,Chinese,Vietnamese": 0.30113636363636365, - "Malay,Indonesian,Vietnamese": 0.3409090909090909, - "Chinese,Indonesian,Vietnamese": 0.36363636363636365 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.19318181818181818, - "English,Filipino,Spanish,Chinese": 0.21022727272727273, - "English,Filipino,Spanish,Indonesian": 0.19886363636363635, - "English,Filipino,Spanish,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese": 0.19318181818181818, - "English,Filipino,Malay,Indonesian": 0.19318181818181818, - "English,Filipino,Malay,Vietnamese": 0.16477272727272727, - "English,Filipino,Chinese,Indonesian": 0.19318181818181818, - "English,Filipino,Chinese,Vietnamese": 0.19886363636363635, - "English,Filipino,Indonesian,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay,Chinese": 0.22727272727272727, - "English,Spanish,Malay,Indonesian": 0.23295454545454544, - "English,Spanish,Malay,Vietnamese": 0.19318181818181818, - "English,Spanish,Chinese,Indonesian": 0.25, - "English,Spanish,Chinese,Vietnamese": 0.23863636363636365, - "English,Spanish,Indonesian,Vietnamese": 0.24431818181818182, - "English,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Malay,Chinese,Vietnamese": 0.23295454545454544, - "English,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Spanish,Malay,Chinese": 0.1875, - "Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.1875, - "Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.2159090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.1875, - "Filipino,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.2556818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.2784090909090909 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.1875, - "English,Spanish,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456 + "Indonesian,English,Filipino": 0.1534090909090909, + "Indonesian,English,Spanish": 0.1875, + "Indonesian,English,Chinese": 0.09090909090909091, + "Indonesian,English,Malay": 0.14204545454545456, + "Indonesian,English,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Spanish": 0.2727272727272727, + "Indonesian,Filipino,Chinese": 0.10227272727272728, + "Indonesian,Filipino,Malay": 0.3125, + "Indonesian,Filipino,Vietnamese": 0.23863636363636365, + "Indonesian,Spanish,Chinese": 0.07954545454545454, + "Indonesian,Spanish,Malay": 0.26136363636363635, + "Indonesian,Spanish,Vietnamese": 0.19318181818181818, + "Indonesian,Chinese,Malay": 0.125, + "Indonesian,Chinese,Vietnamese": 0.125, + "Indonesian,Malay,Vietnamese": 0.2840909090909091, + "English,Filipino,Spanish": 0.20454545454545456, + "English,Filipino,Chinese": 0.07954545454545454, + "English,Filipino,Malay": 0.17613636363636365, + "English,Filipino,Vietnamese": 0.14204545454545456, + "English,Spanish,Chinese": 0.11931818181818182, + "English,Spanish,Malay": 0.17613636363636365, + "English,Spanish,Vietnamese": 0.14204545454545456, + "English,Chinese,Malay": 0.10227272727272728, + "English,Chinese,Vietnamese": 0.10227272727272728, + "English,Malay,Vietnamese": 0.14204545454545456, + "Filipino,Spanish,Chinese": 0.10227272727272728, + "Filipino,Spanish,Malay": 0.30113636363636365, + "Filipino,Spanish,Vietnamese": 0.23295454545454544, + "Filipino,Chinese,Malay": 0.14204545454545456, + "Filipino,Chinese,Vietnamese": 0.13068181818181818, + "Filipino,Malay,Vietnamese": 0.32954545454545453, + "Spanish,Chinese,Malay": 0.09659090909090909, + "Spanish,Chinese,Vietnamese": 0.10795454545454546, + "Spanish,Malay,Vietnamese": 0.2159090909090909, + "Chinese,Malay,Vietnamese": 0.16477272727272727 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.10227272727272728, + "Indonesian,English,Filipino,Chinese": 0.03409090909090909, + "Indonesian,English,Filipino,Malay": 0.09090909090909091, + "Indonesian,English,Filipino,Vietnamese": 0.06818181818181818, + "Indonesian,English,Spanish,Chinese": 0.05113636363636364, + "Indonesian,English,Spanish,Malay": 0.10227272727272728, + "Indonesian,English,Spanish,Vietnamese": 0.07386363636363637, + "Indonesian,English,Chinese,Malay": 0.045454545454545456, + "Indonesian,English,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,English,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,Filipino,Spanish,Chinese": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Malay": 0.18181818181818182, + "Indonesian,Filipino,Spanish,Vietnamese": 0.1534090909090909, + "Indonesian,Filipino,Chinese,Malay": 0.08522727272727272, + "Indonesian,Filipino,Chinese,Vietnamese": 0.06818181818181818, + "Indonesian,Filipino,Malay,Vietnamese": 0.19886363636363635, + "Indonesian,Spanish,Chinese,Malay": 0.05113636363636364, + "Indonesian,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,Spanish,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,Chinese,Malay,Vietnamese": 0.09659090909090909, + "English,Filipino,Spanish,Chinese": 0.045454545454545456, + "English,Filipino,Spanish,Malay": 0.13068181818181818, + "English,Filipino,Spanish,Vietnamese": 0.08522727272727272, + "English,Filipino,Chinese,Malay": 0.06818181818181818, + "English,Filipino,Chinese,Vietnamese": 0.056818181818181816, + "English,Filipino,Malay,Vietnamese": 0.10795454545454546, + "English,Spanish,Chinese,Malay": 0.056818181818181816, + "English,Spanish,Chinese,Vietnamese": 0.06818181818181818, + "English,Spanish,Malay,Vietnamese": 0.08522727272727272, + "English,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Filipino,Spanish,Chinese,Malay": 0.06818181818181818, + "Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, + "Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, + "Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.017045454545454544, + "Indonesian,English,Filipino,Spanish,Malay": 0.06818181818181818, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.045454545454545456, + "Indonesian,English,Filipino,Chinese,Malay": 0.028409090909090908, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.056818181818181816, + "Indonesian,English,Spanish,Chinese,Malay": 0.028409090909090908, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.05113636363636364, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.125, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, + "English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, + "English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, + "English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, - "AC3_2": 0.4710001453687697, - "AC3_3": 0.3630163840375878, - "AC3_4": 0.29153141778121733, - "AC3_5": 0.242596280570508, - "AC3_6": 0.20958983268654272, - "AC3_7": 0.18964258503593145 + "AC3_2": 0.3191883116391567, + "AC3_3": 0.21083311496500506, + "AC3_4": 0.13151927434069596, + "AC3_5": 0.0779059915898941, + "AC3_6": 0.04345654344234227, + "AC3_7": 0.021848317420992257 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.5922330097087378 + "accuracy": 0.4174757281553398 }, "prompt_2": { - "accuracy": 0.6601941747572816 + "accuracy": 0.42718446601941745 }, "prompt_3": { - "accuracy": 0.6213592233009708 + "accuracy": 0.39805825242718446 }, "prompt_4": { - "accuracy": 0.6601941747572816 + "accuracy": 0.3883495145631068 }, "prompt_5": { - "accuracy": 0.5728155339805825 + "accuracy": 0.4077669902912621 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.6190476190476191 + "accuracy": 0.2571428571428571 }, "prompt_2": { - "accuracy": 0.638095238095238 + "accuracy": 0.24761904761904763 }, "prompt_3": { - "accuracy": 0.5904761904761905 + "accuracy": 0.24761904761904763 }, "prompt_4": { - "accuracy": 0.6285714285714286 + "accuracy": 0.23809523809523808 }, "prompt_5": { - "accuracy": 0.6666666666666666 + "accuracy": 0.24761904761904763 } }, "us_eval": { "prompt_1": { - "accuracy": 0.6448598130841121 + "accuracy": 0.2616822429906542 }, "prompt_2": { - "accuracy": 0.6915887850467289 + "accuracy": 0.2897196261682243 }, "prompt_3": { - "accuracy": 0.6261682242990654 + "accuracy": 0.2897196261682243 }, "prompt_4": { - "accuracy": 0.6728971962616822 + "accuracy": 0.29906542056074764 }, "prompt_5": { - "accuracy": 0.6448598130841121 + "accuracy": 0.3177570093457944 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.4, + "accuracy": 0.18, "category_acc": { - "brand": 0.2, - "demographics": 0.6, - "biology": 0.4, - "history": 0.2, - "literature": 0.5, - "politics": 0.5, - "culture": 0.5, - "film": 0.3, - "law": 0.5, - "geography": 0.5 + "brand": 0.1, + "demographics": 0.0, + "biology": 0.1, + "history": 0.3333333333333333, + "literature": 0.2, + "politics": 0.2, + "culture": 0.2, + "film": 0.2, + "law": 0.1, + "geography": 0.2 } }, "prompt_2": { - "accuracy": 0.43, + "accuracy": 0.2, "category_acc": { - "brand": 0.5, - "demographics": 0.2, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.4, - "politics": 0.4, - "culture": 0.5, + "brand": 0.1, + "demographics": 0.0, + "biology": 0.2, + "history": 0.2, + "literature": 0.2, + "politics": 0.2, + "culture": 0.2, "film": 0.3, - "law": 0.6, - "geography": 0.6 + "law": 0.1, + "geography": 0.4 } }, "prompt_3": { - "accuracy": 0.42, + "accuracy": 0.22, "category_acc": { - "brand": 0.2, + "brand": 0.1, "demographics": 0.2, - "biology": 0.4, - "history": 0.2, - "literature": 0.4, - "politics": 0.5, - "culture": 0.5, - "film": 0.4, - "law": 0.6, - "geography": 0.8 + "biology": 0.2, + "history": 0.26666666666666666, + "literature": 0.2, + "politics": 0.2, + "culture": 0.2, + "film": 0.2, + "law": 0.2, + "geography": 0.4 } }, "prompt_4": { - "accuracy": 0.45, + "accuracy": 0.2, "category_acc": { - "brand": 0.4, - "demographics": 0.4, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.5, - "politics": 0.6, - "culture": 0.5, - "film": 0.4, - "law": 0.5, - "geography": 0.6 + "brand": 0.1, + "demographics": 0.0, + "biology": 0.1, + "history": 0.3333333333333333, + "literature": 0.2, + "politics": 0.3, + "culture": 0.3, + "film": 0.2, + "law": 0.1, + "geography": 0.2 } }, "prompt_5": { - "accuracy": 0.41, + "accuracy": 0.22, "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.5, - "history": 0.2, - "literature": 0.4, - "politics": 0.6, - "culture": 0.4, - "film": 0.3, - "law": 0.5, - "geography": 0.6 + "brand": 0.1, + "demographics": 0.4, + "biology": 0.1, + "history": 0.26666666666666666, + "literature": 0.2, + "politics": 0.2, + "culture": 0.2, + "film": 0.2, + "law": 0.2, + "geography": 0.4 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.1842725349096221 + "bleu_score": 0.17678014707244064 }, "prompt_2": { - "bleu_score": 0.16620490569393592 + "bleu_score": 0.17731433484914794 }, "prompt_3": { - "bleu_score": 0.17742394899127048 + "bleu_score": 0.17952240698146069 }, "prompt_4": { - "bleu_score": 0.19649900357191113 + "bleu_score": 0.1839610854412572 }, "prompt_5": { - "bleu_score": 0.1293027691612301 + "bleu_score": 0.16307560288744835 } }, "indommlu": { "prompt_1": { - "accuracy": 0.44816075839508646, + "accuracy": 0.23946858935843515, "category_acc": { - "History": 0.3815261044176707, - "Geography": 0.4122448979591837, - "Lampungic": 0.3469387755102041, - "Social science": 0.659432387312187, - "Balinese": 0.3078556263269639, - "Makassarese": 0.3548387096774194, - "Banjarese": 0.2708333333333333, - "Chemistry": 0.2948905109489051, - "Biology": 0.4059171597633136, - "Science": 0.5727554179566563, - "Christian religion": 0.5422885572139303, - "Art": 0.5307820299500832, - "Islam religion": 0.5078236130867709, - "Hindu religion": 0.35333333333333333, - "Madurese": 0.34576271186440677, - "Sport": 0.5608108108108109, - "Indonesian language": 0.5174346201743462, - "Physics": 0.37777777777777777, - "Minangkabau culture": 0.37185929648241206, + "History": 0.21887550200803213, + "Geography": 0.21428571428571427, + "Lampungic": 0.2925170068027211, + "Social science": 0.2287145242070117, + "Balinese": 0.267515923566879, + "Makassarese": 0.21505376344086022, + "Banjarese": 0.2013888888888889, + "Chemistry": 0.1927007299270073, + "Biology": 0.23550295857988165, + "Science": 0.2579979360165119, + "Christian religion": 0.24875621890547264, + "Art": 0.23627287853577372, + "Islam religion": 0.21621621621621623, + "Hindu religion": 0.2, + "Madurese": 0.23389830508474577, + "Sport": 0.20270270270270271, + "Indonesian language": 0.2534246575342466, + "Physics": 0.21212121212121213, + "Minangkabau culture": 0.22110552763819097, "Dayak language": 0.29357798165137616, - "Sociology": 0.4939516129032258, - "Economy": 0.4569672131147541, - "Sundanese": 0.3500432152117545, - "Javanese": 0.3165322580645161, - "Civic education": 0.5150214592274678 + "Sociology": 0.2217741935483871, + "Economy": 0.21311475409836064, + "Sundanese": 0.2515125324114088, + "Javanese": 0.25806451612903225, + "Civic education": 0.26895565092989987 } }, "prompt_2": { - "accuracy": 0.4552373322651712, + "accuracy": 0.2373990253020896, "category_acc": { - "History": 0.39959839357429716, - "Geography": 0.42857142857142855, - "Lampungic": 0.30612244897959184, - "Social science": 0.669449081803005, - "Balinese": 0.27176220806794055, - "Makassarese": 0.3333333333333333, - "Banjarese": 0.3125, - "Chemistry": 0.310948905109489, - "Biology": 0.4153846153846154, - "Science": 0.5789473684210527, - "Christian religion": 0.5223880597014925, - "Art": 0.5457570715474209, - "Islam religion": 0.5177809388335705, - "Hindu religion": 0.38, - "Madurese": 0.30847457627118646, - "Sport": 0.5472972972972973, - "Indonesian language": 0.538293897882939, - "Physics": 0.39595959595959596, - "Minangkabau culture": 0.36683417085427134, - "Dayak language": 0.3119266055045872, - "Sociology": 0.4798387096774194, - "Economy": 0.4262295081967213, - "Sundanese": 0.3656006914433881, - "Javanese": 0.3175403225806452, - "Civic education": 0.5178826895565093 + "History": 0.20883534136546184, + "Geography": 0.20816326530612245, + "Lampungic": 0.29931972789115646, + "Social science": 0.24707846410684475, + "Balinese": 0.25690021231422505, + "Makassarese": 0.16666666666666666, + "Banjarese": 0.2222222222222222, + "Chemistry": 0.16934306569343066, + "Biology": 0.22011834319526627, + "Science": 0.23426212590299278, + "Christian religion": 0.2935323383084577, + "Art": 0.23294509151414308, + "Islam religion": 0.23044096728307253, + "Hindu religion": 0.21333333333333335, + "Madurese": 0.21016949152542372, + "Sport": 0.21621621621621623, + "Indonesian language": 0.2559153175591532, + "Physics": 0.2202020202020202, + "Minangkabau culture": 0.25125628140703515, + "Dayak language": 0.22935779816513763, + "Sociology": 0.2056451612903226, + "Economy": 0.22950819672131148, + "Sundanese": 0.24632670700086431, + "Javanese": 0.2721774193548387, + "Civic education": 0.26180257510729615 } }, "prompt_3": { - "accuracy": 0.4533012884705254, + "accuracy": 0.23900126844248615, "category_acc": { - "History": 0.39558232931726905, - "Geography": 0.41836734693877553, - "Lampungic": 0.3197278911564626, - "Social science": 0.657762938230384, - "Balinese": 0.27176220806794055, - "Makassarese": 0.3172043010752688, - "Banjarese": 0.3263888888888889, - "Chemistry": 0.3094890510948905, - "Biology": 0.39644970414201186, - "Science": 0.5768833849329206, - "Christian religion": 0.5422885572139303, - "Art": 0.5557404326123128, - "Islam religion": 0.5291607396870555, - "Hindu religion": 0.4, - "Madurese": 0.2983050847457627, - "Sport": 0.5067567567567568, - "Indonesian language": 0.5323785803237858, - "Physics": 0.40606060606060607, - "Minangkabau culture": 0.34673366834170855, - "Dayak language": 0.3302752293577982, - "Sociology": 0.49193548387096775, - "Economy": 0.45491803278688525, - "Sundanese": 0.3474503025064823, - "Javanese": 0.32056451612903225, - "Civic education": 0.525035765379113 + "History": 0.20281124497991967, + "Geography": 0.20612244897959184, + "Lampungic": 0.2585034013605442, + "Social science": 0.2570951585976628, + "Balinese": 0.2653927813163482, + "Makassarese": 0.1881720430107527, + "Banjarese": 0.2222222222222222, + "Chemistry": 0.1635036496350365, + "Biology": 0.2165680473372781, + "Science": 0.23839009287925697, + "Christian religion": 0.26865671641791045, + "Art": 0.24292845257903495, + "Islam religion": 0.21621621621621623, + "Hindu religion": 0.21333333333333335, + "Madurese": 0.21694915254237288, + "Sport": 0.21621621621621623, + "Indonesian language": 0.2559153175591532, + "Physics": 0.22424242424242424, + "Minangkabau culture": 0.23618090452261306, + "Dayak language": 0.3211009174311927, + "Sociology": 0.2056451612903226, + "Economy": 0.25, + "Sundanese": 0.27052722558340536, + "Javanese": 0.2540322580645161, + "Civic education": 0.2632331902718169 } }, "prompt_4": { - "accuracy": 0.4439548701515455, + "accuracy": 0.24113759262968154, "category_acc": { - "History": 0.39558232931726905, - "Geography": 0.4306122448979592, - "Lampungic": 0.29931972789115646, - "Social science": 0.659432387312187, - "Balinese": 0.2781316348195329, - "Makassarese": 0.3333333333333333, - "Banjarese": 0.3263888888888889, - "Chemistry": 0.3051094890510949, - "Biology": 0.40828402366863903, - "Science": 0.5552115583075335, - "Christian religion": 0.5323383084577115, - "Art": 0.5540765391014975, - "Islam religion": 0.519203413940256, - "Hindu religion": 0.3933333333333333, - "Madurese": 0.31186440677966104, - "Sport": 0.5135135135135135, - "Indonesian language": 0.5080946450809465, - "Physics": 0.3696969696969697, - "Minangkabau culture": 0.3417085427135678, - "Dayak language": 0.3119266055045872, - "Sociology": 0.48185483870967744, - "Economy": 0.4385245901639344, - "Sundanese": 0.34053586862575624, - "Javanese": 0.31048387096774194, - "Civic education": 0.525035765379113 + "History": 0.19477911646586346, + "Geography": 0.20816326530612245, + "Lampungic": 0.2857142857142857, + "Social science": 0.2654424040066778, + "Balinese": 0.25690021231422505, + "Makassarese": 0.20967741935483872, + "Banjarese": 0.2222222222222222, + "Chemistry": 0.1678832116788321, + "Biology": 0.22248520710059172, + "Science": 0.24251805985552116, + "Christian religion": 0.29850746268656714, + "Art": 0.23627287853577372, + "Islam religion": 0.21763869132290184, + "Hindu religion": 0.21333333333333335, + "Madurese": 0.2135593220338983, + "Sport": 0.1891891891891892, + "Indonesian language": 0.2602739726027397, + "Physics": 0.2383838383838384, + "Minangkabau culture": 0.23618090452261306, + "Dayak language": 0.29357798165137616, + "Sociology": 0.20161290322580644, + "Economy": 0.23975409836065573, + "Sundanese": 0.26879861711322384, + "Javanese": 0.25806451612903225, + "Civic education": 0.2675250357653791 } }, "prompt_5": { - "accuracy": 0.4452900727685426, + "accuracy": 0.24240603511582884, "category_acc": { - "History": 0.3815261044176707, - "Geography": 0.4, - "Lampungic": 0.3469387755102041, - "Social science": 0.6527545909849749, - "Balinese": 0.25690021231422505, - "Makassarese": 0.34946236559139787, - "Banjarese": 0.3194444444444444, - "Chemistry": 0.2875912408759124, - "Biology": 0.40828402366863903, - "Science": 0.5510835913312694, - "Christian religion": 0.5422885572139303, - "Art": 0.5440931780366056, - "Islam religion": 0.519203413940256, - "Hindu religion": 0.38666666666666666, - "Madurese": 0.31864406779661014, - "Sport": 0.527027027027027, - "Indonesian language": 0.522104607721046, - "Physics": 0.37575757575757573, - "Minangkabau culture": 0.3316582914572864, - "Dayak language": 0.3394495412844037, - "Sociology": 0.4778225806451613, - "Economy": 0.4487704918032787, - "Sundanese": 0.3526361279170268, - "Javanese": 0.31149193548387094, - "Civic education": 0.5207439198855508 + "History": 0.1927710843373494, + "Geography": 0.21224489795918366, + "Lampungic": 0.272108843537415, + "Social science": 0.2637729549248748, + "Balinese": 0.25477707006369427, + "Makassarese": 0.1827956989247312, + "Banjarese": 0.2152777777777778, + "Chemistry": 0.17956204379562044, + "Biology": 0.23076923076923078, + "Science": 0.22910216718266255, + "Christian religion": 0.27860696517412936, + "Art": 0.24625623960066556, + "Islam religion": 0.23897581792318634, + "Hindu religion": 0.25333333333333335, + "Madurese": 0.2033898305084746, + "Sport": 0.18243243243243243, + "Indonesian language": 0.2636986301369863, + "Physics": 0.21414141414141413, + "Minangkabau culture": 0.24623115577889448, + "Dayak language": 0.27522935779816515, + "Sociology": 0.20766129032258066, + "Economy": 0.23770491803278687, + "Sundanese": 0.2662057044079516, + "Javanese": 0.2651209677419355, + "Civic education": 0.2703862660944206 } } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.30344980583150427 + "bleu_score": 0.06407218520167664 }, "prompt_2": { - "bleu_score": 0.29695336030439845 + "bleu_score": 0.06586220325650263 }, "prompt_3": { - "bleu_score": 0.2920872514343988 + "bleu_score": 0.06381655662484863 }, "prompt_4": { - "bleu_score": 0.3091835428432346 + "bleu_score": 0.06512354313124509 }, "prompt_5": { - "bleu_score": 0.2795832375791961 + "bleu_score": 0.06038537914699977 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.2499459409745167 + "bleu_score": 0.03229032150259142 }, "prompt_2": { - "bleu_score": 0.24058192308709833 + "bleu_score": 0.031199003066047656 }, "prompt_3": { - "bleu_score": 0.24181235133752335 + "bleu_score": 0.030584311151675803 }, "prompt_4": { - "bleu_score": 0.2558222306766415 + "bleu_score": 0.03385343853968681 }, "prompt_5": { - "bleu_score": 0.23745271450169794 + "bleu_score": 0.031062141329664804 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.15743792024217895 + "bleu_score": 0.008319239259946272 }, "prompt_2": { - "bleu_score": 0.14484962403361767 + "bleu_score": 0.012894931676973479 }, "prompt_3": { - "bleu_score": 0.15203997884911213 + "bleu_score": 0.013587427733671357 }, "prompt_4": { - "bleu_score": 0.1754927654293425 + "bleu_score": 0.012603283241102662 }, "prompt_5": { - "bleu_score": 0.20176567092810382 + "bleu_score": 0.01183829943175568 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.2820206565776572 + "bleu_score": 0.057558239225713494 }, "prompt_2": { - "bleu_score": 0.2762290245087949 + "bleu_score": 0.05813270509840964 }, "prompt_3": { - "bleu_score": 0.26900636666468153 + "bleu_score": 0.05639021390265389 }, "prompt_4": { - "bleu_score": 0.28471593893642144 + "bleu_score": 0.06088213042386755 }, "prompt_5": { - "bleu_score": 0.25182316612963535 + "bleu_score": 0.056831945580458273 } }, "mmlu": { "prompt_1": { - "accuracy": 0.5764294049008168 + "accuracy": 0.34072345390898484 }, "prompt_2": { - "accuracy": 0.5694282380396732 + "accuracy": 0.3208868144690782 }, "prompt_3": { - "accuracy": 0.5694282380396732 + "accuracy": 0.3278879813302217 }, "prompt_4": { - "accuracy": 0.5484247374562428 + "accuracy": 0.3348891481913652 }, "prompt_5": { - "accuracy": 0.5682613768961493 + "accuracy": 0.3418903150525088 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.5611726850196639, + "accuracy": 0.3511619592420451, "category_acc": { - "high_school_european_history": 0.7560975609756098, - "business_ethics": 0.6161616161616161, - "clinical_knowledge": 0.625, - "medical_genetics": 0.6161616161616161, - "high_school_us_history": 0.7389162561576355, - "high_school_physics": 0.41333333333333333, - "high_school_world_history": 0.7457627118644068, - "virology": 0.4666666666666667, - "high_school_microeconomics": 0.5569620253164557, - "econometrics": 0.40707964601769914, - "college_computer_science": 0.43434343434343436, - "high_school_biology": 0.6601941747572816, - "abstract_algebra": 0.31313131313131315, - "professional_accounting": 0.40213523131672596, - "philosophy": 0.6161290322580645, - "professional_medicine": 0.5830258302583026, - "nutrition": 0.6, - "global_facts": 0.3434343434343434, - "machine_learning": 0.44144144144144143, - "security_studies": 0.5368852459016393, - "public_relations": 0.5779816513761468, - "professional_psychology": 0.5188216039279869, - "prehistory": 0.5975232198142415, - "anatomy": 0.5, - "human_sexuality": 0.6461538461538462, - "college_medicine": 0.5988372093023255, - "high_school_government_and_politics": 0.75, - "college_chemistry": 0.3939393939393939, - "logical_fallacies": 0.6111111111111112, - "high_school_geography": 0.7360406091370558, - "elementary_mathematics": 0.6127320954907162, - "human_aging": 0.5855855855855856, - "college_mathematics": 0.2727272727272727, - "high_school_psychology": 0.7573529411764706, - "formal_logic": 0.376, - "high_school_statistics": 0.44651162790697674, - "international_law": 0.65, - "high_school_mathematics": 0.3048327137546468, - "high_school_computer_science": 0.5858585858585859, - "conceptual_physics": 0.5128205128205128, - "miscellaneous": 0.7774936061381074, - "high_school_chemistry": 0.43564356435643564, - "marketing": 0.8111587982832618, - "professional_law": 0.4135681669928245, - "management": 0.7058823529411765, - "college_physics": 0.32673267326732675, - "jurisprudence": 0.6822429906542056, - "world_religions": 0.7529411764705882, - "sociology": 0.725, - "us_foreign_policy": 0.8282828282828283, - "high_school_macroeconomics": 0.4987146529562982, - "computer_security": 0.6666666666666666, - "moral_scenarios": 0.4015659955257271, - "moral_disputes": 0.6144927536231884, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.5827814569536424, - "college_biology": 0.5804195804195804 + "high_school_european_history": 0.4878048780487805, + "business_ethics": 0.3838383838383838, + "clinical_knowledge": 0.3787878787878788, + "medical_genetics": 0.45454545454545453, + "high_school_us_history": 0.4236453201970443, + "high_school_physics": 0.25333333333333335, + "high_school_world_history": 0.4576271186440678, + "virology": 0.36363636363636365, + "high_school_microeconomics": 0.379746835443038, + "econometrics": 0.24778761061946902, + "college_computer_science": 0.2828282828282828, + "high_school_biology": 0.32362459546925565, + "abstract_algebra": 0.30303030303030304, + "professional_accounting": 0.33807829181494664, + "philosophy": 0.34838709677419355, + "professional_medicine": 0.3025830258302583, + "nutrition": 0.34098360655737703, + "global_facts": 0.30303030303030304, + "machine_learning": 0.23423423423423423, + "security_studies": 0.36065573770491804, + "public_relations": 0.41284403669724773, + "professional_psychology": 0.353518821603928, + "prehistory": 0.35294117647058826, + "anatomy": 0.31343283582089554, + "human_sexuality": 0.3769230769230769, + "college_medicine": 0.3313953488372093, + "high_school_government_and_politics": 0.4270833333333333, + "college_chemistry": 0.20202020202020202, + "logical_fallacies": 0.41975308641975306, + "high_school_geography": 0.41116751269035534, + "elementary_mathematics": 0.26525198938992045, + "human_aging": 0.3783783783783784, + "college_mathematics": 0.25252525252525254, + "high_school_psychology": 0.4485294117647059, + "formal_logic": 0.344, + "high_school_statistics": 0.26976744186046514, + "international_law": 0.525, + "high_school_mathematics": 0.2379182156133829, + "high_school_computer_science": 0.31313131313131315, + "conceptual_physics": 0.2863247863247863, + "miscellaneous": 0.40664961636828645, + "high_school_chemistry": 0.2623762376237624, + "marketing": 0.6008583690987125, + "professional_law": 0.3333333333333333, + "management": 0.4411764705882353, + "college_physics": 0.25742574257425743, + "jurisprudence": 0.3925233644859813, + "world_religions": 0.29411764705882354, + "sociology": 0.535, + "us_foreign_policy": 0.5151515151515151, + "high_school_macroeconomics": 0.33161953727506427, + "computer_security": 0.42424242424242425, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.39420289855072466, + "electrical_engineering": 0.3194444444444444, + "astronomy": 0.304635761589404, + "college_biology": 0.2727272727272727 } }, "prompt_2": { - "accuracy": 0.5437969252770826, + "accuracy": 0.3492313192706471, "category_acc": { - "high_school_european_history": 0.7134146341463414, - "business_ethics": 0.6464646464646465, - "clinical_knowledge": 0.6212121212121212, - "medical_genetics": 0.6060606060606061, - "high_school_us_history": 0.6896551724137931, - "high_school_physics": 0.3333333333333333, - "high_school_world_history": 0.7203389830508474, - "virology": 0.44242424242424244, - "high_school_microeconomics": 0.5316455696202531, - "econometrics": 0.40707964601769914, - "college_computer_science": 0.36363636363636365, - "high_school_biology": 0.6245954692556634, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.3736654804270463, - "philosophy": 0.5580645161290323, - "professional_medicine": 0.5387453874538746, - "nutrition": 0.5737704918032787, - "global_facts": 0.35353535353535354, - "machine_learning": 0.45045045045045046, - "security_studies": 0.5737704918032787, - "public_relations": 0.5137614678899083, - "professional_psychology": 0.5155482815057283, - "prehistory": 0.5356037151702786, - "anatomy": 0.4925373134328358, - "human_sexuality": 0.6615384615384615, - "college_medicine": 0.5406976744186046, - "high_school_government_and_politics": 0.6979166666666666, - "college_chemistry": 0.42424242424242425, - "logical_fallacies": 0.5864197530864198, - "high_school_geography": 0.6903553299492385, - "elementary_mathematics": 0.6074270557029178, - "human_aging": 0.5675675675675675, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.7481617647058824, - "formal_logic": 0.432, - "high_school_statistics": 0.413953488372093, - "international_law": 0.625, - "high_school_mathematics": 0.26765799256505574, - "high_school_computer_science": 0.6565656565656566, - "conceptual_physics": 0.5470085470085471, - "miscellaneous": 0.7659846547314578, - "high_school_chemistry": 0.4504950495049505, - "marketing": 0.7939914163090128, - "professional_law": 0.41226353555120676, - "management": 0.6764705882352942, - "college_physics": 0.3465346534653465, - "jurisprudence": 0.6355140186915887, - "world_religions": 0.7176470588235294, - "sociology": 0.675, - "us_foreign_policy": 0.7373737373737373, - "high_school_macroeconomics": 0.5089974293059126, - "computer_security": 0.7070707070707071, - "moral_scenarios": 0.3970917225950783, - "moral_disputes": 0.5159420289855072, - "electrical_engineering": 0.4236111111111111, - "astronomy": 0.6291390728476821, - "college_biology": 0.5524475524475524 + "high_school_european_history": 0.5, + "business_ethics": 0.3434343434343434, + "clinical_knowledge": 0.4166666666666667, + "medical_genetics": 0.3939393939393939, + "high_school_us_history": 0.4187192118226601, + "high_school_physics": 0.3, + "high_school_world_history": 0.4491525423728814, + "virology": 0.3515151515151515, + "high_school_microeconomics": 0.379746835443038, + "econometrics": 0.30973451327433627, + "college_computer_science": 0.2828282828282828, + "high_school_biology": 0.2977346278317152, + "abstract_algebra": 0.20202020202020202, + "professional_accounting": 0.33807829181494664, + "philosophy": 0.35161290322580646, + "professional_medicine": 0.31365313653136534, + "nutrition": 0.34098360655737703, + "global_facts": 0.2222222222222222, + "machine_learning": 0.3063063063063063, + "security_studies": 0.38114754098360654, + "public_relations": 0.3761467889908257, + "professional_psychology": 0.36661211129296234, + "prehistory": 0.34055727554179566, + "anatomy": 0.30597014925373134, + "human_sexuality": 0.36153846153846153, + "college_medicine": 0.38372093023255816, + "high_school_government_and_politics": 0.4322916666666667, + "college_chemistry": 0.23232323232323232, + "logical_fallacies": 0.41975308641975306, + "high_school_geography": 0.39593908629441626, + "elementary_mathematics": 0.27055702917771884, + "human_aging": 0.3783783783783784, + "college_mathematics": 0.24242424242424243, + "high_school_psychology": 0.44485294117647056, + "formal_logic": 0.272, + "high_school_statistics": 0.2744186046511628, + "international_law": 0.5166666666666667, + "high_school_mathematics": 0.241635687732342, + "high_school_computer_science": 0.3838383838383838, + "conceptual_physics": 0.2777777777777778, + "miscellaneous": 0.391304347826087, + "high_school_chemistry": 0.2623762376237624, + "marketing": 0.5708154506437768, + "professional_law": 0.33659491193737767, + "management": 0.46078431372549017, + "college_physics": 0.2871287128712871, + "jurisprudence": 0.37383177570093457, + "world_religions": 0.29411764705882354, + "sociology": 0.5, + "us_foreign_policy": 0.46464646464646464, + "high_school_macroeconomics": 0.3393316195372751, + "computer_security": 0.3939393939393939, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.37681159420289856, + "electrical_engineering": 0.2777777777777778, + "astronomy": 0.32450331125827814, + "college_biology": 0.2727272727272727 } }, "prompt_3": { - "accuracy": 0.5681801930639971, + "accuracy": 0.34000715051841257, "category_acc": { - "high_school_european_history": 0.7621951219512195, - "business_ethics": 0.6666666666666666, - "clinical_knowledge": 0.625, - "medical_genetics": 0.6363636363636364, - "high_school_us_history": 0.7093596059113301, - "high_school_physics": 0.36, - "high_school_world_history": 0.7627118644067796, - "virology": 0.44242424242424244, - "high_school_microeconomics": 0.5949367088607594, - "econometrics": 0.3805309734513274, - "college_computer_science": 0.42424242424242425, - "high_school_biology": 0.6472491909385113, - "abstract_algebra": 0.36363636363636365, - "professional_accounting": 0.41637010676156583, - "philosophy": 0.6387096774193548, - "professional_medicine": 0.5867158671586716, - "nutrition": 0.6229508196721312, - "global_facts": 0.29292929292929293, - "machine_learning": 0.4594594594594595, - "security_studies": 0.5573770491803278, - "public_relations": 0.6055045871559633, - "professional_psychology": 0.5188216039279869, - "prehistory": 0.6037151702786377, - "anatomy": 0.5149253731343284, - "human_sexuality": 0.6692307692307692, - "college_medicine": 0.5930232558139535, - "high_school_government_and_politics": 0.7552083333333334, - "college_chemistry": 0.3838383838383838, - "logical_fallacies": 0.6419753086419753, - "high_school_geography": 0.751269035532995, - "elementary_mathematics": 0.623342175066313, - "human_aging": 0.5630630630630631, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.7702205882352942, - "formal_logic": 0.416, - "high_school_statistics": 0.4558139534883721, - "international_law": 0.7, - "high_school_mathematics": 0.1970260223048327, - "high_school_computer_science": 0.6666666666666666, - "conceptual_physics": 0.5085470085470085, - "miscellaneous": 0.7774936061381074, - "high_school_chemistry": 0.45544554455445546, - "marketing": 0.7896995708154506, - "professional_law": 0.4233529028049576, - "management": 0.7254901960784313, - "college_physics": 0.36633663366336633, - "jurisprudence": 0.6728971962616822, - "world_religions": 0.7647058823529411, - "sociology": 0.74, - "us_foreign_policy": 0.7575757575757576, - "high_school_macroeconomics": 0.5475578406169666, - "computer_security": 0.6363636363636364, - "moral_scenarios": 0.4161073825503356, - "moral_disputes": 0.6260869565217392, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.6225165562913907, - "college_biology": 0.6503496503496503 + "high_school_european_history": 0.4817073170731707, + "business_ethics": 0.35353535353535354, + "clinical_knowledge": 0.38257575757575757, + "medical_genetics": 0.37373737373737376, + "high_school_us_history": 0.4039408866995074, + "high_school_physics": 0.26, + "high_school_world_history": 0.4533898305084746, + "virology": 0.36363636363636365, + "high_school_microeconomics": 0.350210970464135, + "econometrics": 0.3185840707964602, + "college_computer_science": 0.3434343434343434, + "high_school_biology": 0.29449838187702265, + "abstract_algebra": 0.25252525252525254, + "professional_accounting": 0.3238434163701068, + "philosophy": 0.34838709677419355, + "professional_medicine": 0.2988929889298893, + "nutrition": 0.36721311475409835, + "global_facts": 0.26262626262626265, + "machine_learning": 0.2882882882882883, + "security_studies": 0.3524590163934426, + "public_relations": 0.4036697247706422, + "professional_psychology": 0.3502454991816694, + "prehistory": 0.3281733746130031, + "anatomy": 0.30597014925373134, + "human_sexuality": 0.3153846153846154, + "college_medicine": 0.3430232558139535, + "high_school_government_and_politics": 0.4166666666666667, + "college_chemistry": 0.21212121212121213, + "logical_fallacies": 0.41358024691358025, + "high_school_geography": 0.38071065989847713, + "elementary_mathematics": 0.23076923076923078, + "human_aging": 0.36036036036036034, + "college_mathematics": 0.2828282828282828, + "high_school_psychology": 0.4227941176470588, + "formal_logic": 0.288, + "high_school_statistics": 0.2558139534883721, + "international_law": 0.5, + "high_school_mathematics": 0.21933085501858737, + "high_school_computer_science": 0.36363636363636365, + "conceptual_physics": 0.25213675213675213, + "miscellaneous": 0.38235294117647056, + "high_school_chemistry": 0.2722772277227723, + "marketing": 0.5407725321888412, + "professional_law": 0.33529028049575993, + "management": 0.4215686274509804, + "college_physics": 0.27722772277227725, + "jurisprudence": 0.3644859813084112, + "world_religions": 0.2823529411764706, + "sociology": 0.505, + "us_foreign_policy": 0.4444444444444444, + "high_school_macroeconomics": 0.3213367609254499, + "computer_security": 0.35353535353535354, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.3710144927536232, + "electrical_engineering": 0.2847222222222222, + "astronomy": 0.33112582781456956, + "college_biology": 0.3006993006993007 } }, "prompt_4": { - "accuracy": 0.5457990704326063, + "accuracy": 0.3471576689309975, "category_acc": { - "high_school_european_history": 0.6951219512195121, - "business_ethics": 0.6565656565656566, - "clinical_knowledge": 0.6287878787878788, - "medical_genetics": 0.6161616161616161, - "high_school_us_history": 0.6945812807881774, - "high_school_physics": 0.30666666666666664, - "high_school_world_history": 0.6991525423728814, - "virology": 0.4666666666666667, - "high_school_microeconomics": 0.5907172995780591, - "econometrics": 0.3893805309734513, - "college_computer_science": 0.3838383838383838, - "high_school_biology": 0.627831715210356, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.35587188612099646, - "philosophy": 0.5838709677419355, - "professional_medicine": 0.5276752767527675, - "nutrition": 0.5672131147540984, - "global_facts": 0.36363636363636365, - "machine_learning": 0.44144144144144143, - "security_studies": 0.48360655737704916, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.5155482815057283, - "prehistory": 0.5727554179566563, - "anatomy": 0.47761194029850745, - "human_sexuality": 0.6076923076923076, - "college_medicine": 0.6046511627906976, - "high_school_government_and_politics": 0.7239583333333334, - "college_chemistry": 0.3434343434343434, - "logical_fallacies": 0.6172839506172839, - "high_school_geography": 0.7258883248730964, - "elementary_mathematics": 0.6047745358090185, - "human_aging": 0.5495495495495496, - "college_mathematics": 0.2727272727272727, - "high_school_psychology": 0.7518382352941176, - "formal_logic": 0.376, - "high_school_statistics": 0.40930232558139534, - "international_law": 0.6666666666666666, - "high_school_mathematics": 0.26394052044609667, - "high_school_computer_science": 0.5757575757575758, - "conceptual_physics": 0.5128205128205128, - "miscellaneous": 0.7570332480818415, - "high_school_chemistry": 0.42574257425742573, - "marketing": 0.776824034334764, - "professional_law": 0.42139595564253096, - "management": 0.7156862745098039, - "college_physics": 0.36633663366336633, - "jurisprudence": 0.6822429906542056, - "world_religions": 0.7647058823529411, - "sociology": 0.725, - "us_foreign_policy": 0.7676767676767676, - "high_school_macroeconomics": 0.4884318766066838, - "computer_security": 0.6262626262626263, - "moral_scenarios": 0.39149888143176736, - "moral_disputes": 0.5710144927536231, - "electrical_engineering": 0.5069444444444444, - "astronomy": 0.6158940397350994, - "college_biology": 0.5314685314685315 + "high_school_european_history": 0.49390243902439024, + "business_ethics": 0.3434343434343434, + "clinical_knowledge": 0.38257575757575757, + "medical_genetics": 0.40404040404040403, + "high_school_us_history": 0.4236453201970443, + "high_school_physics": 0.2733333333333333, + "high_school_world_history": 0.4703389830508475, + "virology": 0.3515151515151515, + "high_school_microeconomics": 0.37130801687763715, + "econometrics": 0.25663716814159293, + "college_computer_science": 0.32323232323232326, + "high_school_biology": 0.3042071197411003, + "abstract_algebra": 0.2222222222222222, + "professional_accounting": 0.33807829181494664, + "philosophy": 0.34838709677419355, + "professional_medicine": 0.3173431734317343, + "nutrition": 0.3540983606557377, + "global_facts": 0.21212121212121213, + "machine_learning": 0.2702702702702703, + "security_studies": 0.36475409836065575, + "public_relations": 0.42201834862385323, + "professional_psychology": 0.353518821603928, + "prehistory": 0.33126934984520123, + "anatomy": 0.27611940298507465, + "human_sexuality": 0.35384615384615387, + "college_medicine": 0.3430232558139535, + "high_school_government_and_politics": 0.4114583333333333, + "college_chemistry": 0.21212121212121213, + "logical_fallacies": 0.4074074074074074, + "high_school_geography": 0.4010152284263959, + "elementary_mathematics": 0.26525198938992045, + "human_aging": 0.3918918918918919, + "college_mathematics": 0.25252525252525254, + "high_school_psychology": 0.43566176470588236, + "formal_logic": 0.328, + "high_school_statistics": 0.26976744186046514, + "international_law": 0.5, + "high_school_mathematics": 0.2527881040892193, + "high_school_computer_science": 0.32323232323232326, + "conceptual_physics": 0.3034188034188034, + "miscellaneous": 0.3938618925831202, + "high_school_chemistry": 0.27722772277227725, + "marketing": 0.5793991416309013, + "professional_law": 0.33268101761252444, + "management": 0.45098039215686275, + "college_physics": 0.26732673267326734, + "jurisprudence": 0.3644859813084112, + "world_religions": 0.27058823529411763, + "sociology": 0.54, + "us_foreign_policy": 0.47474747474747475, + "high_school_macroeconomics": 0.3393316195372751, + "computer_security": 0.41414141414141414, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.3565217391304348, + "electrical_engineering": 0.3194444444444444, + "astronomy": 0.32450331125827814, + "college_biology": 0.27972027972027974 } }, "prompt_5": { - "accuracy": 0.5470146585627458, + "accuracy": 0.34522702895959956, "category_acc": { - "high_school_european_history": 0.7012195121951219, - "business_ethics": 0.6161616161616161, - "clinical_knowledge": 0.5568181818181818, - "medical_genetics": 0.5959595959595959, - "high_school_us_history": 0.6896551724137931, - "high_school_physics": 0.32, - "high_school_world_history": 0.7372881355932204, - "virology": 0.41818181818181815, - "high_school_microeconomics": 0.5780590717299579, - "econometrics": 0.35398230088495575, - "college_computer_science": 0.3939393939393939, - "high_school_biology": 0.6504854368932039, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.4199288256227758, - "philosophy": 0.6129032258064516, - "professional_medicine": 0.5166051660516605, - "nutrition": 0.5639344262295082, - "global_facts": 0.36363636363636365, - "machine_learning": 0.38738738738738737, - "security_studies": 0.5245901639344263, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.5335515548281505, - "prehistory": 0.5851393188854489, - "anatomy": 0.44776119402985076, - "human_sexuality": 0.6076923076923076, - "college_medicine": 0.5697674418604651, - "high_school_government_and_politics": 0.734375, - "college_chemistry": 0.45454545454545453, - "logical_fallacies": 0.6419753086419753, - "high_school_geography": 0.6954314720812182, - "elementary_mathematics": 0.6074270557029178, - "human_aging": 0.5765765765765766, - "college_mathematics": 0.29292929292929293, - "high_school_psychology": 0.7408088235294118, - "formal_logic": 0.392, - "high_school_statistics": 0.4186046511627907, - "international_law": 0.6333333333333333, - "high_school_mathematics": 0.27137546468401486, - "high_school_computer_science": 0.6565656565656566, - "conceptual_physics": 0.49572649572649574, - "miscellaneous": 0.7442455242966752, - "high_school_chemistry": 0.44554455445544555, - "marketing": 0.7939914163090128, - "professional_law": 0.42139595564253096, - "management": 0.7745098039215687, - "college_physics": 0.40594059405940597, - "jurisprudence": 0.6261682242990654, - "world_religions": 0.7647058823529411, - "sociology": 0.69, - "us_foreign_policy": 0.7676767676767676, - "high_school_macroeconomics": 0.4884318766066838, - "computer_security": 0.6363636363636364, - "moral_scenarios": 0.3724832214765101, - "moral_disputes": 0.5826086956521739, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.6026490066225165, - "college_biology": 0.5874125874125874 + "high_school_european_history": 0.4817073170731707, + "business_ethics": 0.37373737373737376, + "clinical_knowledge": 0.35984848484848486, + "medical_genetics": 0.3838383838383838, + "high_school_us_history": 0.43349753694581283, + "high_school_physics": 0.24, + "high_school_world_history": 0.4703389830508475, + "virology": 0.3333333333333333, + "high_school_microeconomics": 0.33755274261603374, + "econometrics": 0.22123893805309736, + "college_computer_science": 0.31313131313131315, + "high_school_biology": 0.3106796116504854, + "abstract_algebra": 0.24242424242424243, + "professional_accounting": 0.35231316725978645, + "philosophy": 0.3419354838709677, + "professional_medicine": 0.31365313653136534, + "nutrition": 0.3442622950819672, + "global_facts": 0.31313131313131315, + "machine_learning": 0.25225225225225223, + "security_studies": 0.36065573770491804, + "public_relations": 0.3761467889908257, + "professional_psychology": 0.35515548281505727, + "prehistory": 0.34674922600619196, + "anatomy": 0.29850746268656714, + "human_sexuality": 0.38461538461538464, + "college_medicine": 0.313953488372093, + "high_school_government_and_politics": 0.4166666666666667, + "college_chemistry": 0.21212121212121213, + "logical_fallacies": 0.41975308641975306, + "high_school_geography": 0.4010152284263959, + "elementary_mathematics": 0.2625994694960212, + "human_aging": 0.4009009009009009, + "college_mathematics": 0.26262626262626265, + "high_school_psychology": 0.42463235294117646, + "formal_logic": 0.32, + "high_school_statistics": 0.26976744186046514, + "international_law": 0.5083333333333333, + "high_school_mathematics": 0.2379182156133829, + "high_school_computer_science": 0.3434343434343434, + "conceptual_physics": 0.2863247863247863, + "miscellaneous": 0.39514066496163686, + "high_school_chemistry": 0.2623762376237624, + "marketing": 0.592274678111588, + "professional_law": 0.32811480756686234, + "management": 0.4411764705882353, + "college_physics": 0.25742574257425743, + "jurisprudence": 0.3925233644859813, + "world_religions": 0.27058823529411763, + "sociology": 0.53, + "us_foreign_policy": 0.46464646464646464, + "high_school_macroeconomics": 0.34190231362467866, + "computer_security": 0.40404040404040403, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.3710144927536232, + "electrical_engineering": 0.2986111111111111, + "astronomy": 0.31125827814569534, + "college_biology": 0.2937062937062937 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.6419019316493314 + "accuracy": 0.2436849925705795 }, "prompt_2": { - "accuracy": 0.6419019316493314 + "accuracy": 0.2436849925705795 }, "prompt_3": { - "accuracy": 0.6433878157503715 + "accuracy": 0.24442793462109955 }, "prompt_4": { - "accuracy": 0.5817236255572066 + "accuracy": 0.24145616641901932 }, "prompt_5": { - "accuracy": 0.513372956909361 + "accuracy": 0.24294205052005943 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.6612702366127023, + "accuracy": 0.26214196762141967, "category_acc": { - "computer_network": 0.5, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.6153846153846154, - "college_programming": 0.6666666666666666, - "college_physics": 0.375, - "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.7586206896551724, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.6666666666666666, - "high_school_chemistry": 0.7083333333333334, - "high_school_biology": 0.8333333333333334, - "middle_school_mathematics": 0.5833333333333334, - "middle_school_biology": 0.9230769230769231, - "middle_school_physics": 0.9166666666666666, - "middle_school_chemistry": 0.92, - "veterinary_medicine": 0.6071428571428571, - "college_economics": 0.6333333333333333, - "business_administration": 0.6578947368421053, - "marxism": 0.875, - "mao_zedong_thought": 0.7931034482758621, - "education_science": 0.7058823529411765, - "teacher_qualification": 0.8367346938775511, - "high_school_politics": 0.9166666666666666, - "high_school_geography": 0.875, - "middle_school_politics": 1.0, - "middle_school_geography": 0.9411764705882353, - "modern_chinese_history": 0.8571428571428571, - "ideological_and_moral_cultivation": 0.9583333333333334, - "logic": 0.4444444444444444, - "law": 0.7241379310344828, - "chinese_language_and_literature": 0.6428571428571429, - "art_studies": 0.6578947368421053, - "professional_tour_guide": 0.8235294117647058, - "legal_professional": 0.5357142857142857, - "high_school_chinese": 0.625, - "high_school_history": 0.88, - "middle_school_history": 0.9629629629629629, - "civil_servant": 0.5576923076923077, - "sports_science": 0.7083333333333334, - "plant_protection": 0.7407407407407407, - "basic_medicine": 0.8333333333333334, - "clinical_medicine": 0.6296296296296297, - "urban_and_rural_planner": 0.5686274509803921, - "accountant": 0.6296296296296297, - "fire_engineer": 0.5555555555555556, - "environmental_impact_assessment_engineer": 0.5833333333333334, - "tax_accountant": 0.5185185185185185, - "physician": 0.6296296296296297 + "computer_network": 0.20833333333333334, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.14285714285714285, + "college_physics": 0.16666666666666666, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.21739130434782608, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.2413793103448276, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.20833333333333334, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.17857142857142858, + "college_economics": 0.21666666666666667, + "business_administration": 0.13157894736842105, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.22448979591836735, + "high_school_politics": 0.0, + "high_school_geography": 0.2916666666666667, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.2962962962962963, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.25, + "high_school_history": 0.08, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.3076923076923077, + "sports_science": 0.2916666666666667, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.4166666666666667, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.2549019607843137, + "accountant": 0.3148148148148148, + "fire_engineer": 0.19444444444444445, + "environmental_impact_assessment_engineer": 0.2777777777777778, + "tax_accountant": 0.3148148148148148, + "physician": 0.2962962962962963 } }, "prompt_2": { - "accuracy": 0.6481942714819428, + "accuracy": 0.261519302615193, "category_acc": { - "computer_network": 0.5, - "operating_system": 0.5833333333333334, - "computer_architecture": 0.6153846153846154, - "college_programming": 0.6904761904761905, - "college_physics": 0.375, - "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.13043478260869565, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.4523809523809524, - "metrology_engineer": 0.7931034482758621, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.6666666666666666, - "high_school_chemistry": 0.6666666666666666, - "high_school_biology": 0.8333333333333334, - "middle_school_mathematics": 0.5416666666666666, - "middle_school_biology": 0.9230769230769231, - "middle_school_physics": 0.875, - "middle_school_chemistry": 0.8, - "veterinary_medicine": 0.6071428571428571, - "college_economics": 0.48333333333333334, - "business_administration": 0.631578947368421, - "marxism": 0.875, - "mao_zedong_thought": 0.8275862068965517, - "education_science": 0.6470588235294118, - "teacher_qualification": 0.8571428571428571, - "high_school_politics": 0.9166666666666666, - "high_school_geography": 0.8333333333333334, - "middle_school_politics": 0.9230769230769231, - "middle_school_geography": 0.8823529411764706, - "modern_chinese_history": 0.8928571428571429, - "ideological_and_moral_cultivation": 0.9166666666666666, - "logic": 0.5185185185185185, - "law": 0.7586206896551724, - "chinese_language_and_literature": 0.6071428571428571, - "art_studies": 0.6578947368421053, - "professional_tour_guide": 0.7647058823529411, - "legal_professional": 0.5, - "high_school_chinese": 0.5833333333333334, - "high_school_history": 0.88, - "middle_school_history": 0.9629629629629629, - "civil_servant": 0.5576923076923077, - "sports_science": 0.7083333333333334, - "plant_protection": 0.7037037037037037, - "basic_medicine": 0.8333333333333334, - "clinical_medicine": 0.5555555555555556, - "urban_and_rural_planner": 0.5882352941176471, - "accountant": 0.5740740740740741, - "fire_engineer": 0.5277777777777778, - "environmental_impact_assessment_engineer": 0.6111111111111112, - "tax_accountant": 0.5555555555555556, - "physician": 0.6851851851851852 + "computer_network": 0.20833333333333334, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.14285714285714285, + "college_physics": 0.16666666666666666, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.17391304347826086, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.2413793103448276, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.20833333333333334, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.17857142857142858, + "college_economics": 0.21666666666666667, + "business_administration": 0.13157894736842105, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.22448979591836735, + "high_school_politics": 0.0, + "high_school_geography": 0.2916666666666667, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.2962962962962963, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.25, + "high_school_history": 0.08, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.3076923076923077, + "sports_science": 0.2916666666666667, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.4166666666666667, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.2549019607843137, + "accountant": 0.3148148148148148, + "fire_engineer": 0.19444444444444445, + "environmental_impact_assessment_engineer": 0.2777777777777778, + "tax_accountant": 0.3148148148148148, + "physician": 0.2962962962962963 } }, "prompt_3": { - "accuracy": 0.6625155666251556, + "accuracy": 0.261519302615193, "category_acc": { - "computer_network": 0.5416666666666666, - "operating_system": 0.625, - "computer_architecture": 0.6153846153846154, - "college_programming": 0.6904761904761905, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.47619047619047616, - "metrology_engineer": 0.6896551724137931, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.5833333333333334, - "high_school_chemistry": 0.75, - "high_school_biology": 0.75, - "middle_school_mathematics": 0.7083333333333334, - "middle_school_biology": 0.8461538461538461, - "middle_school_physics": 0.9166666666666666, - "middle_school_chemistry": 0.96, - "veterinary_medicine": 0.6071428571428571, - "college_economics": 0.6, - "business_administration": 0.631578947368421, - "marxism": 0.8333333333333334, - "mao_zedong_thought": 0.8275862068965517, - "education_science": 0.7352941176470589, - "teacher_qualification": 0.8571428571428571, - "high_school_politics": 0.875, - "high_school_geography": 0.875, - "middle_school_politics": 0.9615384615384616, - "middle_school_geography": 1.0, - "modern_chinese_history": 0.7857142857142857, - "ideological_and_moral_cultivation": 0.9583333333333334, - "logic": 0.4444444444444444, - "law": 0.6896551724137931, - "chinese_language_and_literature": 0.6428571428571429, - "art_studies": 0.6578947368421053, - "professional_tour_guide": 0.8235294117647058, - "legal_professional": 0.5357142857142857, - "high_school_chinese": 0.5416666666666666, - "high_school_history": 0.84, - "middle_school_history": 0.9629629629629629, - "civil_servant": 0.5384615384615384, - "sports_science": 0.6666666666666666, - "plant_protection": 0.7037037037037037, - "basic_medicine": 0.8333333333333334, - "clinical_medicine": 0.5925925925925926, - "urban_and_rural_planner": 0.6862745098039216, - "accountant": 0.6111111111111112, - "fire_engineer": 0.5277777777777778, - "environmental_impact_assessment_engineer": 0.6666666666666666, - "tax_accountant": 0.5555555555555556, - "physician": 0.6851851851851852 + "computer_network": 0.20833333333333334, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.14285714285714285, + "college_physics": 0.16666666666666666, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.17391304347826086, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.2413793103448276, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.20833333333333334, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.17857142857142858, + "college_economics": 0.21666666666666667, + "business_administration": 0.13157894736842105, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.22448979591836735, + "high_school_politics": 0.0, + "high_school_geography": 0.2916666666666667, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.2962962962962963, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.25, + "high_school_history": 0.08, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.3076923076923077, + "sports_science": 0.2916666666666667, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.4166666666666667, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.2549019607843137, + "accountant": 0.3148148148148148, + "fire_engineer": 0.19444444444444445, + "environmental_impact_assessment_engineer": 0.2777777777777778, + "tax_accountant": 0.3148148148148148, + "physician": 0.2962962962962963 } }, "prompt_4": { - "accuracy": 0.5840597758405978, + "accuracy": 0.26525529265255293, "category_acc": { - "computer_network": 0.5, - "operating_system": 0.625, - "computer_architecture": 0.46153846153846156, - "college_programming": 0.5476190476190477, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.4523809523809524, - "metrology_engineer": 0.5862068965517241, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.625, - "high_school_chemistry": 0.5416666666666666, - "high_school_biology": 0.5833333333333334, - "middle_school_mathematics": 0.4166666666666667, - "middle_school_biology": 0.7692307692307693, - "middle_school_physics": 0.625, - "middle_school_chemistry": 0.6, - "veterinary_medicine": 0.5714285714285714, - "college_economics": 0.55, - "business_administration": 0.631578947368421, - "marxism": 0.75, - "mao_zedong_thought": 0.7586206896551724, - "education_science": 0.7058823529411765, - "teacher_qualification": 0.8163265306122449, - "high_school_politics": 0.8333333333333334, - "high_school_geography": 0.75, - "middle_school_politics": 0.8846153846153846, - "middle_school_geography": 0.8823529411764706, - "modern_chinese_history": 0.7857142857142857, - "ideological_and_moral_cultivation": 0.875, + "computer_network": 0.20833333333333334, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.14285714285714285, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.30434782608695654, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.2413793103448276, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.375, + "high_school_biology": 0.25, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.17857142857142858, + "college_economics": 0.21666666666666667, + "business_administration": 0.13157894736842105, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.22448979591836735, + "high_school_politics": 0.0, + "high_school_geography": 0.2916666666666667, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.47058823529411764, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.16666666666666666, "logic": 0.3333333333333333, - "law": 0.5862068965517241, - "chinese_language_and_literature": 0.5357142857142857, - "art_studies": 0.6842105263157895, - "professional_tour_guide": 0.7647058823529411, - "legal_professional": 0.4642857142857143, - "high_school_chinese": 0.5, - "high_school_history": 0.76, - "middle_school_history": 0.8518518518518519, - "civil_servant": 0.46153846153846156, - "sports_science": 0.5833333333333334, - "plant_protection": 0.7037037037037037, - "basic_medicine": 0.75, - "clinical_medicine": 0.5925925925925926, - "urban_and_rural_planner": 0.47058823529411764, - "accountant": 0.5, - "fire_engineer": 0.5, - "environmental_impact_assessment_engineer": 0.5277777777777778, - "tax_accountant": 0.46296296296296297, - "physician": 0.5740740740740741 + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.25, + "high_school_history": 0.04, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.3076923076923077, + "sports_science": 0.2916666666666667, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.4166666666666667, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.2549019607843137, + "accountant": 0.3148148148148148, + "fire_engineer": 0.19444444444444445, + "environmental_impact_assessment_engineer": 0.25, + "tax_accountant": 0.3148148148148148, + "physician": 0.2962962962962963 } }, "prompt_5": { - "accuracy": 0.5305105853051059, + "accuracy": 0.2627646326276463, "category_acc": { - "computer_network": 0.5, - "operating_system": 0.5, - "computer_architecture": 0.5384615384615384, - "college_programming": 0.5476190476190477, - "college_physics": 0.25, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.25, + "computer_network": 0.20833333333333334, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.14285714285714285, + "college_physics": 0.16666666666666666, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.5172413793103449, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.5, - "high_school_chemistry": 0.5416666666666666, - "high_school_biology": 0.625, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.6923076923076923, - "middle_school_physics": 0.4166666666666667, - "middle_school_chemistry": 0.92, - "veterinary_medicine": 0.5357142857142857, - "college_economics": 0.43333333333333335, - "business_administration": 0.39473684210526316, - "marxism": 0.7083333333333334, - "mao_zedong_thought": 0.6551724137931034, - "education_science": 0.7352941176470589, - "teacher_qualification": 0.673469387755102, - "high_school_politics": 0.6666666666666666, - "high_school_geography": 0.6666666666666666, - "middle_school_politics": 0.8076923076923077, - "middle_school_geography": 0.8235294117647058, - "modern_chinese_history": 0.7142857142857143, - "ideological_and_moral_cultivation": 0.75, - "logic": 0.25925925925925924, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.6785714285714286, - "art_studies": 0.6578947368421053, - "professional_tour_guide": 0.7647058823529411, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.4583333333333333, - "high_school_history": 0.64, - "middle_school_history": 0.8148148148148148, - "civil_servant": 0.5, - "sports_science": 0.5, - "plant_protection": 0.5925925925925926, - "basic_medicine": 0.7083333333333334, - "clinical_medicine": 0.5185185185185185, - "urban_and_rural_planner": 0.49019607843137253, - "accountant": 0.42592592592592593, - "fire_engineer": 0.4444444444444444, - "environmental_impact_assessment_engineer": 0.5277777777777778, - "tax_accountant": 0.46296296296296297, - "physician": 0.5740740740740741 + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.2413793103448276, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.20833333333333334, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.17857142857142858, + "college_economics": 0.21666666666666667, + "business_administration": 0.13157894736842105, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.22448979591836735, + "high_school_politics": 0.0, + "high_school_geography": 0.2916666666666667, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.2962962962962963, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.25, + "high_school_history": 0.08, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.3076923076923077, + "sports_science": 0.2916666666666667, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.4166666666666667, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.2549019607843137, + "accountant": 0.3148148148148148, + "fire_engineer": 0.19444444444444445, + "environmental_impact_assessment_engineer": 0.2777777777777778, + "tax_accountant": 0.3148148148148148, + "physician": 0.2962962962962963 } } }, "cmmlu": { "prompt_1": { - "accuracy": 0.6881720430107527 + "accuracy": 0.21863799283154123 }, "prompt_2": { - "accuracy": 0.7025089605734767 + "accuracy": 0.2222222222222222 }, "prompt_3": { - "accuracy": 0.6774193548387096 + "accuracy": 0.2222222222222222 }, "prompt_4": { - "accuracy": 0.6630824372759857 + "accuracy": 0.25448028673835127 }, "prompt_5": { - "accuracy": 0.5770609318996416 + "accuracy": 0.21863799283154123 } }, "cmmlu_full": { "prompt_1": { - "accuracy": 0.6457433949231566, + "accuracy": 0.2522880331548955, "category_acc": { - "agronomy": 0.5384615384615384, - "anatomy": 0.6148648648648649, - "ancient_chinese": 0.3902439024390244, - "arts": 0.85625, - "astronomy": 0.41818181818181815, - "business_ethics": 0.6124401913875598, - "chinese_civil_service_exam": 0.6, - "chinese_driving_rule": 0.9007633587786259, - "chinese_food_culture": 0.6176470588235294, - "chinese_foreign_policy": 0.7383177570093458, - "chinese_history": 0.7956656346749226, - "chinese_literature": 0.46078431372549017, - "chinese_teacher_qualification": 0.8044692737430168, - "clinical_knowledge": 0.569620253164557, - "college_actuarial_science": 0.22641509433962265, - "college_education": 0.7850467289719626, - "college_engineering_hydrology": 0.5094339622641509, - "college_law": 0.5925925925925926, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.5188679245283019, - "college_medicine": 0.6593406593406593, - "computer_science": 0.6568627450980392, - "computer_security": 0.7543859649122807, - "conceptual_physics": 0.8435374149659864, - "construction_project_management": 0.5467625899280576, - "economics": 0.6289308176100629, - "education": 0.6871165644171779, - "electrical_engineering": 0.6162790697674418, - "elementary_chinese": 0.6587301587301587, - "elementary_commonsense": 0.6767676767676768, - "elementary_information_and_technology": 0.8151260504201681, - "elementary_mathematics": 0.4956521739130435, - "ethnology": 0.6592592592592592, - "food_science": 0.5664335664335665, - "genetics": 0.5397727272727273, - "global_facts": 0.6510067114093959, - "high_school_biology": 0.7633136094674556, - "high_school_chemistry": 0.6818181818181818, - "high_school_geography": 0.7966101694915254, - "high_school_mathematics": 0.2865853658536585, - "high_school_physics": 0.5636363636363636, - "high_school_politics": 0.7272727272727273, - "human_sexuality": 0.6111111111111112, - "international_law": 0.5405405405405406, - "journalism": 0.5988372093023255, - "jurisprudence": 0.6909975669099757, - "legal_and_moral_basis": 0.9532710280373832, - "logical": 0.5284552845528455, - "machine_learning": 0.5573770491803278, - "management": 0.7619047619047619, - "marketing": 0.7055555555555556, - "marxist_theory": 0.8306878306878307, - "modern_chinese": 0.5431034482758621, - "nutrition": 0.6, - "philosophy": 0.6476190476190476, - "professional_accounting": 0.7257142857142858, - "professional_law": 0.5639810426540285, - "professional_medicine": 0.5292553191489362, - "professional_psychology": 0.7844827586206896, - "public_relations": 0.6379310344827587, - "security_study": 0.725925925925926, - "sociology": 0.6460176991150443, - "sports_science": 0.6666666666666666, - "traditional_chinese_medicine": 0.6648648648648648, - "virology": 0.650887573964497, - "world_history": 0.7577639751552795, - "world_religions": 0.7 + "agronomy": 0.2485207100591716, + "anatomy": 0.25, + "ancient_chinese": 0.2682926829268293, + "arts": 0.28125, + "astronomy": 0.19393939393939394, + "business_ethics": 0.24880382775119617, + "chinese_civil_service_exam": 0.23125, + "chinese_driving_rule": 0.2824427480916031, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.308411214953271, + "chinese_history": 0.23529411764705882, + "chinese_literature": 0.2549019607843137, + "chinese_teacher_qualification": 0.30726256983240224, + "clinical_knowledge": 0.27848101265822783, + "college_actuarial_science": 0.24528301886792453, + "college_education": 0.16822429906542055, + "college_engineering_hydrology": 0.2358490566037736, + "college_law": 0.25, + "college_mathematics": 0.2857142857142857, + "college_medical_statistics": 0.2641509433962264, + "college_medicine": 0.2454212454212454, + "computer_science": 0.2107843137254902, + "computer_security": 0.21637426900584794, + "conceptual_physics": 0.25170068027210885, + "construction_project_management": 0.22302158273381295, + "economics": 0.27044025157232704, + "education": 0.25766871165644173, + "electrical_engineering": 0.23255813953488372, + "elementary_chinese": 0.23015873015873015, + "elementary_commonsense": 0.2828282828282828, + "elementary_information_and_technology": 0.25630252100840334, + "elementary_mathematics": 0.23478260869565218, + "ethnology": 0.25925925925925924, + "food_science": 0.24475524475524477, + "genetics": 0.2784090909090909, + "global_facts": 0.2751677852348993, + "high_school_biology": 0.21301775147928995, + "high_school_chemistry": 0.26515151515151514, + "high_school_geography": 0.2711864406779661, + "high_school_mathematics": 0.3170731707317073, + "high_school_physics": 0.20909090909090908, + "high_school_politics": 0.3146853146853147, + "human_sexuality": 0.2619047619047619, + "international_law": 0.2702702702702703, + "journalism": 0.22674418604651161, + "jurisprudence": 0.23357664233576642, + "legal_and_moral_basis": 0.2803738317757009, + "logical": 0.2601626016260163, + "machine_learning": 0.22950819672131148, + "management": 0.2619047619047619, + "marketing": 0.2611111111111111, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.25, + "nutrition": 0.25517241379310346, + "philosophy": 0.22857142857142856, + "professional_accounting": 0.28, + "professional_law": 0.22748815165876776, + "professional_medicine": 0.2473404255319149, + "professional_psychology": 0.23275862068965517, + "public_relations": 0.26436781609195403, + "security_study": 0.26666666666666666, + "sociology": 0.2831858407079646, + "sports_science": 0.24848484848484848, + "traditional_chinese_medicine": 0.23243243243243245, + "virology": 0.25443786982248523, + "world_history": 0.2422360248447205, + "world_religions": 0.275 } }, "prompt_2": { - "accuracy": 0.6415126921084441, + "accuracy": 0.25125194266965983, "category_acc": { - "agronomy": 0.5384615384615384, - "anatomy": 0.6013513513513513, - "ancient_chinese": 0.3719512195121951, - "arts": 0.86875, - "astronomy": 0.4121212121212121, - "business_ethics": 0.6076555023923444, - "chinese_civil_service_exam": 0.56875, - "chinese_driving_rule": 0.8702290076335878, - "chinese_food_culture": 0.6397058823529411, - "chinese_foreign_policy": 0.6915887850467289, - "chinese_history": 0.7708978328173375, - "chinese_literature": 0.5147058823529411, - "chinese_teacher_qualification": 0.8212290502793296, - "clinical_knowledge": 0.6075949367088608, - "college_actuarial_science": 0.3113207547169811, - "college_education": 0.8037383177570093, - "college_engineering_hydrology": 0.5943396226415094, - "college_law": 0.5740740740740741, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.5, - "college_medicine": 0.6410256410256411, - "computer_science": 0.6715686274509803, - "computer_security": 0.7660818713450293, - "conceptual_physics": 0.8163265306122449, - "construction_project_management": 0.5467625899280576, - "economics": 0.6352201257861635, - "education": 0.6687116564417178, - "electrical_engineering": 0.6511627906976745, - "elementary_chinese": 0.6547619047619048, - "elementary_commonsense": 0.6717171717171717, - "elementary_information_and_technology": 0.8025210084033614, - "elementary_mathematics": 0.5173913043478261, - "ethnology": 0.6518518518518519, - "food_science": 0.6083916083916084, - "genetics": 0.5227272727272727, - "global_facts": 0.6711409395973155, - "high_school_biology": 0.7337278106508875, - "high_school_chemistry": 0.6439393939393939, - "high_school_geography": 0.7627118644067796, - "high_school_mathematics": 0.3719512195121951, - "high_school_physics": 0.5545454545454546, - "high_school_politics": 0.6783216783216783, - "human_sexuality": 0.626984126984127, - "international_law": 0.5405405405405406, - "journalism": 0.5988372093023255, - "jurisprudence": 0.6642335766423357, - "legal_and_moral_basis": 0.9392523364485982, - "logical": 0.5447154471544715, - "machine_learning": 0.5409836065573771, - "management": 0.7333333333333333, - "marketing": 0.7111111111111111, - "marxist_theory": 0.798941798941799, - "modern_chinese": 0.5172413793103449, - "nutrition": 0.6206896551724138, - "philosophy": 0.6857142857142857, - "professional_accounting": 0.6971428571428572, - "professional_law": 0.5450236966824644, - "professional_medicine": 0.5345744680851063, - "professional_psychology": 0.7672413793103449, - "public_relations": 0.6379310344827587, - "security_study": 0.7111111111111111, - "sociology": 0.6194690265486725, - "sports_science": 0.6242424242424243, - "traditional_chinese_medicine": 0.6648648648648648, - "virology": 0.650887573964497, - "world_history": 0.6956521739130435, - "world_religions": 0.6875 + "agronomy": 0.2485207100591716, + "anatomy": 0.24324324324324326, + "ancient_chinese": 0.2804878048780488, + "arts": 0.28125, + "astronomy": 0.19393939393939394, + "business_ethics": 0.24880382775119617, + "chinese_civil_service_exam": 0.24375, + "chinese_driving_rule": 0.2824427480916031, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.308411214953271, + "chinese_history": 0.23839009287925697, + "chinese_literature": 0.24509803921568626, + "chinese_teacher_qualification": 0.30726256983240224, + "clinical_knowledge": 0.26582278481012656, + "college_actuarial_science": 0.2169811320754717, + "college_education": 0.1588785046728972, + "college_engineering_hydrology": 0.2358490566037736, + "college_law": 0.24074074074074073, + "college_mathematics": 0.2571428571428571, + "college_medical_statistics": 0.27358490566037735, + "college_medicine": 0.23076923076923078, + "computer_science": 0.22058823529411764, + "computer_security": 0.21637426900584794, + "conceptual_physics": 0.2653061224489796, + "construction_project_management": 0.2302158273381295, + "economics": 0.27044025157232704, + "education": 0.25766871165644173, + "electrical_engineering": 0.23837209302325582, + "elementary_chinese": 0.23412698412698413, + "elementary_commonsense": 0.2828282828282828, + "elementary_information_and_technology": 0.25210084033613445, + "elementary_mathematics": 0.23043478260869565, + "ethnology": 0.25925925925925924, + "food_science": 0.23776223776223776, + "genetics": 0.26704545454545453, + "global_facts": 0.2751677852348993, + "high_school_biology": 0.21301775147928995, + "high_school_chemistry": 0.2727272727272727, + "high_school_geography": 0.2711864406779661, + "high_school_mathematics": 0.31097560975609756, + "high_school_physics": 0.2, + "high_school_politics": 0.3146853146853147, + "human_sexuality": 0.2619047619047619, + "international_law": 0.2648648648648649, + "journalism": 0.22674418604651161, + "jurisprudence": 0.23357664233576642, + "legal_and_moral_basis": 0.2803738317757009, + "logical": 0.2601626016260163, + "machine_learning": 0.21311475409836064, + "management": 0.2619047619047619, + "marketing": 0.2611111111111111, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.23275862068965517, + "nutrition": 0.2482758620689655, + "philosophy": 0.22857142857142856, + "professional_accounting": 0.2857142857142857, + "professional_law": 0.23696682464454977, + "professional_medicine": 0.2473404255319149, + "professional_psychology": 0.23275862068965517, + "public_relations": 0.26436781609195403, + "security_study": 0.26666666666666666, + "sociology": 0.2831858407079646, + "sports_science": 0.24848484848484848, + "traditional_chinese_medicine": 0.22702702702702704, + "virology": 0.26627218934911245, + "world_history": 0.2484472049689441, + "world_religions": 0.275 } }, "prompt_3": { - "accuracy": 0.6466068036608531, + "accuracy": 0.25116560179589015, "category_acc": { - "agronomy": 0.5443786982248521, - "anatomy": 0.6148648648648649, - "ancient_chinese": 0.3902439024390244, - "arts": 0.8625, - "astronomy": 0.4, - "business_ethics": 0.5980861244019139, - "chinese_civil_service_exam": 0.5875, - "chinese_driving_rule": 0.8931297709923665, - "chinese_food_culture": 0.6102941176470589, - "chinese_foreign_policy": 0.719626168224299, - "chinese_history": 0.7832817337461301, - "chinese_literature": 0.4852941176470588, - "chinese_teacher_qualification": 0.8100558659217877, - "clinical_knowledge": 0.620253164556962, - "college_actuarial_science": 0.3113207547169811, - "college_education": 0.7757009345794392, - "college_engineering_hydrology": 0.5943396226415094, - "college_law": 0.5833333333333334, - "college_mathematics": 0.3238095238095238, - "college_medical_statistics": 0.5188679245283019, - "college_medicine": 0.6593406593406593, - "computer_science": 0.6666666666666666, - "computer_security": 0.7777777777777778, - "conceptual_physics": 0.8367346938775511, - "construction_project_management": 0.5683453237410072, - "economics": 0.6540880503144654, - "education": 0.6748466257668712, - "electrical_engineering": 0.6686046511627907, - "elementary_chinese": 0.6547619047619048, - "elementary_commonsense": 0.696969696969697, - "elementary_information_and_technology": 0.8025210084033614, - "elementary_mathematics": 0.5, - "ethnology": 0.6592592592592592, - "food_science": 0.5874125874125874, - "genetics": 0.5056818181818182, - "global_facts": 0.6510067114093959, - "high_school_biology": 0.7396449704142012, - "high_school_chemistry": 0.6287878787878788, - "high_school_geography": 0.7372881355932204, - "high_school_mathematics": 0.35365853658536583, - "high_school_physics": 0.6363636363636364, - "high_school_politics": 0.7412587412587412, - "human_sexuality": 0.5714285714285714, - "international_law": 0.5297297297297298, - "journalism": 0.6046511627906976, - "jurisprudence": 0.6788321167883211, - "legal_and_moral_basis": 0.9532710280373832, - "logical": 0.5121951219512195, - "machine_learning": 0.47540983606557374, - "management": 0.7761904761904762, - "marketing": 0.7111111111111111, - "marxist_theory": 0.7883597883597884, - "modern_chinese": 0.5258620689655172, - "nutrition": 0.6620689655172414, - "philosophy": 0.638095238095238, - "professional_accounting": 0.7371428571428571, - "professional_law": 0.5829383886255924, - "professional_medicine": 0.5292553191489362, - "professional_psychology": 0.7758620689655172, - "public_relations": 0.6494252873563219, - "security_study": 0.7111111111111111, - "sociology": 0.6283185840707964, - "sports_science": 0.6484848484848484, - "traditional_chinese_medicine": 0.6702702702702703, - "virology": 0.6331360946745562, - "world_history": 0.7391304347826086, - "world_religions": 0.68125 + "agronomy": 0.24260355029585798, + "anatomy": 0.24324324324324326, + "ancient_chinese": 0.2804878048780488, + "arts": 0.28125, + "astronomy": 0.19393939393939394, + "business_ethics": 0.24401913875598086, + "chinese_civil_service_exam": 0.2375, + "chinese_driving_rule": 0.2824427480916031, + "chinese_food_culture": 0.25, + "chinese_foreign_policy": 0.308411214953271, + "chinese_history": 0.23529411764705882, + "chinese_literature": 0.24509803921568626, + "chinese_teacher_qualification": 0.30726256983240224, + "clinical_knowledge": 0.270042194092827, + "college_actuarial_science": 0.2641509433962264, + "college_education": 0.1588785046728972, + "college_engineering_hydrology": 0.2641509433962264, + "college_law": 0.25, + "college_mathematics": 0.2571428571428571, + "college_medical_statistics": 0.2358490566037736, + "college_medicine": 0.23076923076923078, + "computer_science": 0.20588235294117646, + "computer_security": 0.23391812865497075, + "conceptual_physics": 0.2653061224489796, + "construction_project_management": 0.2302158273381295, + "economics": 0.27044025157232704, + "education": 0.25766871165644173, + "electrical_engineering": 0.23255813953488372, + "elementary_chinese": 0.2261904761904762, + "elementary_commonsense": 0.2828282828282828, + "elementary_information_and_technology": 0.25630252100840334, + "elementary_mathematics": 0.2391304347826087, + "ethnology": 0.25925925925925924, + "food_science": 0.24475524475524477, + "genetics": 0.2727272727272727, + "global_facts": 0.2751677852348993, + "high_school_biology": 0.20710059171597633, + "high_school_chemistry": 0.2196969696969697, + "high_school_geography": 0.2796610169491525, + "high_school_mathematics": 0.3048780487804878, + "high_school_physics": 0.19090909090909092, + "high_school_politics": 0.32167832167832167, + "human_sexuality": 0.2619047619047619, + "international_law": 0.2702702702702703, + "journalism": 0.22674418604651161, + "jurisprudence": 0.23357664233576642, + "legal_and_moral_basis": 0.2803738317757009, + "logical": 0.2601626016260163, + "machine_learning": 0.22131147540983606, + "management": 0.2619047619047619, + "marketing": 0.2611111111111111, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.23275862068965517, + "nutrition": 0.25517241379310346, + "philosophy": 0.23809523809523808, + "professional_accounting": 0.2857142857142857, + "professional_law": 0.23696682464454977, + "professional_medicine": 0.2473404255319149, + "professional_psychology": 0.23275862068965517, + "public_relations": 0.26436781609195403, + "security_study": 0.25925925925925924, + "sociology": 0.2831858407079646, + "sports_science": 0.24848484848484848, + "traditional_chinese_medicine": 0.22702702702702704, + "virology": 0.25443786982248523, + "world_history": 0.2422360248447205, + "world_religions": 0.2875 } }, "prompt_4": { - "accuracy": 0.6004144361940943, + "accuracy": 0.25211535140735625, "category_acc": { - "agronomy": 0.47337278106508873, - "anatomy": 0.581081081081081, - "ancient_chinese": 0.3475609756097561, - "arts": 0.83125, - "astronomy": 0.45454545454545453, - "business_ethics": 0.5885167464114832, - "chinese_civil_service_exam": 0.4875, - "chinese_driving_rule": 0.8091603053435115, - "chinese_food_culture": 0.6397058823529411, - "chinese_foreign_policy": 0.6635514018691588, - "chinese_history": 0.6780185758513931, - "chinese_literature": 0.4852941176470588, - "chinese_teacher_qualification": 0.8100558659217877, - "clinical_knowledge": 0.5822784810126582, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.719626168224299, - "college_engineering_hydrology": 0.5188679245283019, - "college_law": 0.46296296296296297, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.46226415094339623, - "college_medicine": 0.6483516483516484, - "computer_science": 0.6372549019607843, - "computer_security": 0.695906432748538, - "conceptual_physics": 0.6462585034013606, - "construction_project_management": 0.4892086330935252, - "economics": 0.610062893081761, - "education": 0.6809815950920245, - "electrical_engineering": 0.5813953488372093, - "elementary_chinese": 0.6190476190476191, - "elementary_commonsense": 0.6515151515151515, - "elementary_information_and_technology": 0.7899159663865546, - "elementary_mathematics": 0.43043478260869567, - "ethnology": 0.6666666666666666, - "food_science": 0.5524475524475524, - "genetics": 0.4659090909090909, - "global_facts": 0.6442953020134228, - "high_school_biology": 0.6153846153846154, - "high_school_chemistry": 0.44696969696969696, - "high_school_geography": 0.7033898305084746, - "high_school_mathematics": 0.34146341463414637, - "high_school_physics": 0.5454545454545454, - "high_school_politics": 0.6923076923076923, - "human_sexuality": 0.5238095238095238, - "international_law": 0.5405405405405406, - "journalism": 0.5872093023255814, - "jurisprudence": 0.5912408759124088, - "legal_and_moral_basis": 0.9158878504672897, - "logical": 0.5203252032520326, - "machine_learning": 0.5409836065573771, - "management": 0.7238095238095238, - "marketing": 0.6277777777777778, - "marxist_theory": 0.7513227513227513, - "modern_chinese": 0.43103448275862066, - "nutrition": 0.6, - "philosophy": 0.6476190476190476, - "professional_accounting": 0.6285714285714286, - "professional_law": 0.4976303317535545, - "professional_medicine": 0.5026595744680851, - "professional_psychology": 0.7413793103448276, - "public_relations": 0.6091954022988506, - "security_study": 0.7037037037037037, - "sociology": 0.6150442477876106, - "sports_science": 0.5878787878787879, - "traditional_chinese_medicine": 0.6324324324324324, - "virology": 0.5976331360946746, - "world_history": 0.6521739130434783, - "world_religions": 0.70625 + "agronomy": 0.24260355029585798, + "anatomy": 0.23648648648648649, + "ancient_chinese": 0.2682926829268293, + "arts": 0.275, + "astronomy": 0.22424242424242424, + "business_ethics": 0.24880382775119617, + "chinese_civil_service_exam": 0.24375, + "chinese_driving_rule": 0.33587786259541985, + "chinese_food_culture": 0.27941176470588236, + "chinese_foreign_policy": 0.2897196261682243, + "chinese_history": 0.23219814241486067, + "chinese_literature": 0.24509803921568626, + "chinese_teacher_qualification": 0.2569832402234637, + "clinical_knowledge": 0.2869198312236287, + "college_actuarial_science": 0.2169811320754717, + "college_education": 0.16822429906542055, + "college_engineering_hydrology": 0.2641509433962264, + "college_law": 0.25, + "college_mathematics": 0.3047619047619048, + "college_medical_statistics": 0.29245283018867924, + "college_medicine": 0.23809523809523808, + "computer_science": 0.2107843137254902, + "computer_security": 0.23391812865497075, + "conceptual_physics": 0.2857142857142857, + "construction_project_management": 0.2158273381294964, + "economics": 0.27044025157232704, + "education": 0.2085889570552147, + "electrical_engineering": 0.23255813953488372, + "elementary_chinese": 0.21428571428571427, + "elementary_commonsense": 0.26262626262626265, + "elementary_information_and_technology": 0.2689075630252101, + "elementary_mathematics": 0.24347826086956523, + "ethnology": 0.3037037037037037, + "food_science": 0.2867132867132867, + "genetics": 0.26136363636363635, + "global_facts": 0.28187919463087246, + "high_school_biology": 0.23076923076923078, + "high_school_chemistry": 0.2196969696969697, + "high_school_geography": 0.2457627118644068, + "high_school_mathematics": 0.32926829268292684, + "high_school_physics": 0.17272727272727273, + "high_school_politics": 0.2867132867132867, + "human_sexuality": 0.23015873015873015, + "international_law": 0.2864864864864865, + "journalism": 0.2441860465116279, + "jurisprudence": 0.25060827250608275, + "legal_and_moral_basis": 0.2523364485981308, + "logical": 0.2764227642276423, + "machine_learning": 0.20491803278688525, + "management": 0.21904761904761905, + "marketing": 0.25555555555555554, + "marxist_theory": 0.25396825396825395, + "modern_chinese": 0.28448275862068967, + "nutrition": 0.2896551724137931, + "philosophy": 0.24761904761904763, + "professional_accounting": 0.22285714285714286, + "professional_law": 0.21800947867298578, + "professional_medicine": 0.23138297872340424, + "professional_psychology": 0.2629310344827586, + "public_relations": 0.25862068965517243, + "security_study": 0.26666666666666666, + "sociology": 0.252212389380531, + "sports_science": 0.24242424242424243, + "traditional_chinese_medicine": 0.25405405405405407, + "virology": 0.26627218934911245, + "world_history": 0.2670807453416149, + "world_religions": 0.2875 } }, "prompt_5": { - "accuracy": 0.542738732515973, + "accuracy": 0.2518563287860473, "category_acc": { - "agronomy": 0.4911242603550296, - "anatomy": 0.5337837837837838, - "ancient_chinese": 0.3475609756097561, - "arts": 0.775, - "astronomy": 0.4484848484848485, - "business_ethics": 0.5215311004784688, - "chinese_civil_service_exam": 0.4, - "chinese_driving_rule": 0.7175572519083969, - "chinese_food_culture": 0.5882352941176471, - "chinese_foreign_policy": 0.5794392523364486, - "chinese_history": 0.6191950464396285, - "chinese_literature": 0.4852941176470588, - "chinese_teacher_qualification": 0.7374301675977654, - "clinical_knowledge": 0.459915611814346, + "agronomy": 0.2485207100591716, + "anatomy": 0.24324324324324326, + "ancient_chinese": 0.27439024390243905, + "arts": 0.275, + "astronomy": 0.2, + "business_ethics": 0.24401913875598086, + "chinese_civil_service_exam": 0.2375, + "chinese_driving_rule": 0.2824427480916031, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.3177570093457944, + "chinese_history": 0.23529411764705882, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.30726256983240224, + "clinical_knowledge": 0.270042194092827, "college_actuarial_science": 0.2358490566037736, - "college_education": 0.6728971962616822, - "college_engineering_hydrology": 0.5566037735849056, - "college_law": 0.5277777777777778, - "college_mathematics": 0.21904761904761905, - "college_medical_statistics": 0.44339622641509435, - "college_medicine": 0.5238095238095238, - "computer_science": 0.5931372549019608, - "computer_security": 0.5789473684210527, - "conceptual_physics": 0.5850340136054422, - "construction_project_management": 0.460431654676259, - "economics": 0.5157232704402516, - "education": 0.5828220858895705, - "electrical_engineering": 0.5232558139534884, - "elementary_chinese": 0.5952380952380952, - "elementary_commonsense": 0.6212121212121212, - "elementary_information_and_technology": 0.7521008403361344, - "elementary_mathematics": 0.4043478260869565, - "ethnology": 0.5777777777777777, - "food_science": 0.5524475524475524, - "genetics": 0.4034090909090909, - "global_facts": 0.610738255033557, - "high_school_biology": 0.5443786982248521, - "high_school_chemistry": 0.3939393939393939, - "high_school_geography": 0.5847457627118644, - "high_school_mathematics": 0.29878048780487804, - "high_school_physics": 0.509090909090909, - "high_school_politics": 0.5734265734265734, - "human_sexuality": 0.5873015873015873, - "international_law": 0.42702702702702705, - "journalism": 0.4941860465116279, - "jurisprudence": 0.5060827250608273, - "legal_and_moral_basis": 0.8130841121495327, - "logical": 0.43902439024390244, - "machine_learning": 0.47540983606557374, - "management": 0.6619047619047619, - "marketing": 0.5777777777777777, - "marxist_theory": 0.656084656084656, - "modern_chinese": 0.46551724137931033, - "nutrition": 0.5862068965517241, - "philosophy": 0.5714285714285714, - "professional_accounting": 0.5828571428571429, - "professional_law": 0.4265402843601896, - "professional_medicine": 0.42021276595744683, - "professional_psychology": 0.7068965517241379, - "public_relations": 0.5517241379310345, - "security_study": 0.6814814814814815, - "sociology": 0.584070796460177, - "sports_science": 0.5272727272727272, - "traditional_chinese_medicine": 0.4594594594594595, - "virology": 0.591715976331361, - "world_history": 0.5527950310559007, - "world_religions": 0.625 + "college_education": 0.1588785046728972, + "college_engineering_hydrology": 0.2830188679245283, + "college_law": 0.25, + "college_mathematics": 0.26666666666666666, + "college_medical_statistics": 0.29245283018867924, + "college_medicine": 0.22344322344322345, + "computer_science": 0.21568627450980393, + "computer_security": 0.22807017543859648, + "conceptual_physics": 0.2585034013605442, + "construction_project_management": 0.2302158273381295, + "economics": 0.27044025157232704, + "education": 0.25766871165644173, + "electrical_engineering": 0.23837209302325582, + "elementary_chinese": 0.23015873015873015, + "elementary_commonsense": 0.2828282828282828, + "elementary_information_and_technology": 0.2605042016806723, + "elementary_mathematics": 0.22608695652173913, + "ethnology": 0.25925925925925924, + "food_science": 0.23776223776223776, + "genetics": 0.2727272727272727, + "global_facts": 0.2751677852348993, + "high_school_biology": 0.22485207100591717, + "high_school_chemistry": 0.22727272727272727, + "high_school_geography": 0.2711864406779661, + "high_school_mathematics": 0.3170731707317073, + "high_school_physics": 0.19090909090909092, + "high_school_politics": 0.32167832167832167, + "human_sexuality": 0.2619047619047619, + "international_law": 0.2756756756756757, + "journalism": 0.22674418604651161, + "jurisprudence": 0.2384428223844282, + "legal_and_moral_basis": 0.2803738317757009, + "logical": 0.2601626016260163, + "machine_learning": 0.21311475409836064, + "management": 0.2619047619047619, + "marketing": 0.2611111111111111, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.23275862068965517, + "nutrition": 0.25517241379310346, + "philosophy": 0.22857142857142856, + "professional_accounting": 0.28, + "professional_law": 0.23222748815165878, + "professional_medicine": 0.2473404255319149, + "professional_psychology": 0.23275862068965517, + "public_relations": 0.26436781609195403, + "security_study": 0.26666666666666666, + "sociology": 0.2831858407079646, + "sports_science": 0.24848484848484848, + "traditional_chinese_medicine": 0.22702702702702704, + "virology": 0.25443786982248523, + "world_history": 0.2484472049689441, + "world_religions": 0.275 } } }, "zbench": { "prompt_1": { - "accuracy": 0.5151515151515151 + "accuracy": 0.21212121212121213 }, "prompt_2": { - "accuracy": 0.5151515151515151 + "accuracy": 0.21212121212121213 }, "prompt_3": { - "accuracy": 0.45454545454545453 + "accuracy": 0.21212121212121213 }, "prompt_4": { - "accuracy": 0.42424242424242425 + "accuracy": 0.24242424242424243 }, "prompt_5": { - "accuracy": 0.42424242424242425 + "accuracy": 0.18181818181818182 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.6 + "accuracy": 0.15454545454545454 }, "prompt_2": { - "accuracy": 0.4818181818181818 + "accuracy": 0.15454545454545454 }, "prompt_3": { - "accuracy": 0.5159090909090909 + "accuracy": 0.13636363636363635 }, "prompt_4": { - "accuracy": 0.5227272727272727 + "accuracy": 0.22954545454545455 }, "prompt_5": { - "accuracy": 0.5295454545454545 + "accuracy": 0.29545454545454547 } }, "ocnli": { "prompt_1": { - "accuracy": 0.6067796610169491 + "accuracy": 0.3474576271186441 }, "prompt_2": { - "accuracy": 0.5972881355932204 + "accuracy": 0.33864406779661016 }, "prompt_3": { - "accuracy": 0.5932203389830508 + "accuracy": 0.33661016949152545 }, "prompt_4": { - "accuracy": 0.5708474576271186 + "accuracy": 0.348135593220339 }, "prompt_5": { - "accuracy": 0.5610169491525424 + "accuracy": 0.348135593220339 } }, "c3": { "prompt_1": { - "accuracy": 0.8919222139117428 + "accuracy": 0.27673896783844426 }, "prompt_2": { - "accuracy": 0.887060583395662 + "accuracy": 0.28085265519820496 }, "prompt_3": { - "accuracy": 0.8855646970830217 + "accuracy": 0.27898279730740466 }, "prompt_4": { - "accuracy": 0.8904263275991025 + "accuracy": 0.27486910994764396 }, "prompt_5": { - "accuracy": 0.8421839940164547 + "accuracy": 0.27898279730740466 } }, "dream": { "prompt_1": { - "accuracy": 0.8912297893189612 + "accuracy": 0.7339539441450269 }, "prompt_2": { - "accuracy": 0.8936795688388045 + "accuracy": 0.732484076433121 }, "prompt_3": { - "accuracy": 0.89171974522293 + "accuracy": 0.7354238118569328 }, "prompt_4": { - "accuracy": 0.876531112199902 + "accuracy": 0.7275845173934345 }, "prompt_5": { - "accuracy": 0.8951494365507104 + "accuracy": 0.732484076433121 } }, "samsum": { "prompt_1": { - "rouge1": 0.3207431038649055, - "rouge2": 0.11284597597167502, - "rougeL": 0.23654812044352486, - "avg_rouge": 0.22337906676003513 + "rouge1": 0.488484040273741, + "rouge2": 0.23998086324118986, + "rougeL": 0.4064468657172112, + "avg_rouge": 0.37830392307738075 }, "prompt_2": { - "rouge1": 0.3500857628840386, - "rouge2": 0.11920612948221446, - "rougeL": 0.2580799945426485, - "avg_rouge": 0.2424572956363005 + "rouge1": 0.4849018980243311, + "rouge2": 0.237551200146336, + "rougeL": 0.40312373253579026, + "avg_rouge": 0.37519227690215246 }, "prompt_3": { - "rouge1": 0.3584255030755962, - "rouge2": 0.12202249099086714, - "rougeL": 0.26665273861509947, - "avg_rouge": 0.24903357756052094 + "rouge1": 0.4895295318667229, + "rouge2": 0.24117191448518707, + "rougeL": 0.40644791748868025, + "avg_rouge": 0.37904978794686334 }, "prompt_4": { - "rouge1": 0.32964990975229463, - "rouge2": 0.11180045514342556, - "rougeL": 0.2443147362455602, - "avg_rouge": 0.22858836704709348 + "rouge1": 0.4866886581108752, + "rouge2": 0.2394549990808094, + "rougeL": 0.4058628019738131, + "avg_rouge": 0.37733548638849923 }, "prompt_5": { - "rouge1": 0.3597346344760821, - "rouge2": 0.12246859600679422, - "rougeL": 0.2650354199698709, - "avg_rouge": 0.24907955015091576 + "rouge1": 0.4883999304094335, + "rouge2": 0.2396140799702983, + "rougeL": 0.40482381144754065, + "avg_rouge": 0.37761260727575746 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.21557574906056598, - "rouge2": 0.05119950399859938, - "rougeL": 0.1575169279706156, - "avg_rouge": 0.14143072700992698 + "rouge1": 0.2655584776294273, + "rouge2": 0.08400121911084955, + "rougeL": 0.2043071525241852, + "avg_rouge": 0.184622283088154 }, "prompt_2": { - "rouge1": 0.212404956960776, - "rouge2": 0.04852666249234976, - "rougeL": 0.15413554058713955, - "avg_rouge": 0.13835572001342178 + "rouge1": 0.2739249487548124, + "rouge2": 0.08831062539365067, + "rougeL": 0.2093411832754166, + "avg_rouge": 0.1905255858079599 }, "prompt_3": { - "rouge1": 0.21355176988178426, - "rouge2": 0.04948466294953414, - "rougeL": 0.15503255149100825, - "avg_rouge": 0.13935632810744222 + "rouge1": 0.23397379438148797, + "rouge2": 0.06834623109477767, + "rougeL": 0.17495323413594485, + "avg_rouge": 0.15909108653740348 }, "prompt_4": { - "rouge1": 0.2152779603935761, - "rouge2": 0.05235103303530447, - "rougeL": 0.15740898285672367, - "avg_rouge": 0.14167932542853476 + "rouge1": 0.23679257285465277, + "rouge2": 0.06983511641069447, + "rougeL": 0.17733703138085358, + "avg_rouge": 0.16132157354873358 }, "prompt_5": { - "rouge1": 0.2188515520344668, - "rouge2": 0.05236781969139806, - "rougeL": 0.16045569638914942, - "avg_rouge": 0.14389168937167143 + "rouge1": 0.2708748715755265, + "rouge2": 0.0875519573364796, + "rougeL": 0.20647193742575837, + "avg_rouge": 0.18829958877925482 } }, "sst2": { "prompt_1": { - "accuracy": 0.9380733944954128 + "accuracy": 0.9094036697247706 }, "prompt_2": { - "accuracy": 0.911697247706422 + "accuracy": 0.908256880733945 }, "prompt_3": { - "accuracy": 0.9197247706422018 + "accuracy": 0.9036697247706422 }, "prompt_4": { - "accuracy": 0.9323394495412844 + "accuracy": 0.908256880733945 }, "prompt_5": { - "accuracy": 0.8658256880733946 + "accuracy": 0.9139908256880734 } }, "cola": { "prompt_1": { - "accuracy": 0.8245445829338447 + "accuracy": 0.5081495685522531 }, "prompt_2": { - "accuracy": 0.8149568552253116 + "accuracy": 0.6864813039309684 }, "prompt_3": { - "accuracy": 0.8120805369127517 + "accuracy": 0.660594439117929 }, "prompt_4": { - "accuracy": 0.825503355704698 + "accuracy": 0.6078619367209971 }, "prompt_5": { - "accuracy": 0.8178331735378715 + "accuracy": 0.5225311601150527 } }, "qqp": { "prompt_1": { - "accuracy": 0.793 + "accuracy": 0.81 }, "prompt_2": { - "accuracy": 0.7845 + "accuracy": 0.8175 }, "prompt_3": { - "accuracy": 0.7805 + "accuracy": 0.8055 }, "prompt_4": { - "accuracy": 0.785 + "accuracy": 0.8175 }, "prompt_5": { - "accuracy": 0.77 + "accuracy": 0.812 } }, "mnli": { "prompt_1": { - "accuracy": 0.6905 + "accuracy": 0.7 }, "prompt_2": { - "accuracy": 0.7075 + "accuracy": 0.7 }, "prompt_3": { - "accuracy": 0.7005 + "accuracy": 0.7 }, "prompt_4": { - "accuracy": 0.6925 + "accuracy": 0.7 }, "prompt_5": { - "accuracy": 0.6655 + "accuracy": 0.5 } }, "qnli": { "prompt_1": { - "accuracy": 0.8315 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.8355 + "accuracy": 0.7 }, "prompt_3": { - "accuracy": 0.839 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.831 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.7995 + "accuracy": 0.9 } }, "wnli": { "prompt_1": { - "accuracy": 0.6056338028169014 + "accuracy": 0.3 }, "prompt_2": { - "accuracy": 0.6619718309859155 + "accuracy": 0.3 }, "prompt_3": { - "accuracy": 0.5352112676056338 + "accuracy": 0.3 }, "prompt_4": { - "accuracy": 0.5070422535211268 + "accuracy": 0.3 }, "prompt_5": { - "accuracy": 0.6338028169014085 + "accuracy": 0.3 } }, "rte": { "prompt_1": { - "accuracy": 0.8592057761732852 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.8736462093862816 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.8447653429602888 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.7725631768953068 + "accuracy": 1.0 }, "prompt_5": { - "accuracy": 0.8375451263537906 + "accuracy": 1.0 } }, "mrpc": { "prompt_1": { - "accuracy": 0.7524509803921569 + "accuracy": 0.4 }, "prompt_2": { - "accuracy": 0.7598039215686274 + "accuracy": 0.5 }, "prompt_3": { - "accuracy": 0.7401960784313726 + "accuracy": 0.4 }, "prompt_4": { - "accuracy": 0.7450980392156863 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.7671568627450981 + "accuracy": 0.6 } } }, "five_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.9239495798319327, - "language_acc": { - "English": 0.9394957983193277, - "Chinese": 0.9226890756302522, - "Spanish": 0.9277310924369748, - "Vietnamese": 0.9058823529411765 - }, - "consistency_score_2": 0.9156862745098039, - "consistency_score_3": 0.8771008403361344, - "consistency_score_4": 0.8529411764705882, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.9235294117647059, - "English,Spanish": 0.934453781512605, - "English,Vietnamese": 0.9100840336134454, - "Chinese,Spanish": 0.9142857142857143, - "Chinese,Vietnamese": 0.9033613445378151, - "Spanish,Vietnamese": 0.9084033613445378 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8890756302521008, - "English,Chinese,Vietnamese": 0.8722689075630252, - "English,Spanish,Vietnamese": 0.8798319327731092, - "Chinese,Spanish,Vietnamese": 0.8672268907563025 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8529411764705882 - } - }, - "AC3_2": 0.9197993684983339, - "AC3_3": 0.8999158977217169, - "AC3_4": 0.8870265533001973 - } + "prompt_1": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.5409523809523809, - "language_acc": { - "Spanish": 0.5933333333333334, - "English": 0.6733333333333333, - "Chinese": 0.6, - "Vietnamese": 0.5333333333333333, - "Filipino": 0.4, - "Indonesian": 0.5266666666666666, - "Malay": 0.46 - }, - "consistency_score_2": 0.5603174603174603, - "consistency_score_3": 0.39123809523809516, - "consistency_score_4": 0.2975238095238095, - "consistency_score_5": 0.23682539682539683, - "consistency_score_6": 0.19333333333333333, - "consistency_score_7": 0.16, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6733333333333333, - "Spanish,Chinese": 0.6666666666666666, - "Spanish,Vietnamese": 0.6533333333333333, - "Spanish,Filipino": 0.4666666666666667, - "Spanish,Indonesian": 0.5933333333333334, - "Spanish,Malay": 0.54, - "English,Chinese": 0.6533333333333333, - "English,Vietnamese": 0.6133333333333333, - "English,Filipino": 0.4866666666666667, - "English,Indonesian": 0.5266666666666666, - "English,Malay": 0.5066666666666667, - "Chinese,Vietnamese": 0.58, - "Chinese,Filipino": 0.46, - "Chinese,Indonesian": 0.5533333333333333, - "Chinese,Malay": 0.5333333333333333, - "Vietnamese,Filipino": 0.5133333333333333, - "Vietnamese,Indonesian": 0.58, - "Vietnamese,Malay": 0.5466666666666666, - "Filipino,Indonesian": 0.5, - "Filipino,Malay": 0.49333333333333335, - "Indonesian,Malay": 0.6266666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.52, - "Spanish,English,Vietnamese": 0.5, - "Spanish,English,Filipino": 0.36666666666666664, - "Spanish,English,Indonesian": 0.43333333333333335, - "Spanish,English,Malay": 0.4066666666666667, - "Spanish,Chinese,Vietnamese": 0.47333333333333333, - "Spanish,Chinese,Filipino": 0.3333333333333333, - "Spanish,Chinese,Indonesian": 0.44, - "Spanish,Chinese,Malay": 0.4066666666666667, - "Spanish,Vietnamese,Filipino": 0.37333333333333335, - "Spanish,Vietnamese,Indonesian": 0.4533333333333333, - "Spanish,Vietnamese,Malay": 0.41333333333333333, - "Spanish,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Filipino,Malay": 0.3333333333333333, - "Spanish,Indonesian,Malay": 0.43333333333333335, - "English,Chinese,Vietnamese": 0.4666666666666667, - "English,Chinese,Filipino": 0.35333333333333333, - "English,Chinese,Indonesian": 0.4, - "English,Chinese,Malay": 0.38666666666666666, - "English,Vietnamese,Filipino": 0.37333333333333335, - "English,Vietnamese,Indonesian": 0.42, - "English,Vietnamese,Malay": 0.38666666666666666, - "English,Filipino,Indonesian": 0.32666666666666666, - "English,Filipino,Malay": 0.31333333333333335, - "English,Indonesian,Malay": 0.38666666666666666, - "Chinese,Vietnamese,Filipino": 0.3466666666666667, - "Chinese,Vietnamese,Indonesian": 0.4, - "Chinese,Vietnamese,Malay": 0.36666666666666664, - "Chinese,Filipino,Indonesian": 0.31333333333333335, - "Chinese,Filipino,Malay": 0.31333333333333335, - "Chinese,Indonesian,Malay": 0.4, - "Vietnamese,Filipino,Indonesian": 0.34, - "Vietnamese,Filipino,Malay": 0.34, - "Vietnamese,Indonesian,Malay": 0.44, - "Filipino,Indonesian,Malay": 0.38 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4, - "Spanish,English,Chinese,Filipino": 0.28, - "Spanish,English,Chinese,Indonesian": 0.3466666666666667, - "Spanish,English,Chinese,Malay": 0.3333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.3, - "Spanish,English,Vietnamese,Indonesian": 0.36666666666666664, - "Spanish,English,Vietnamese,Malay": 0.32666666666666666, - "Spanish,English,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Indonesian,Malay": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.2733333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.3466666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.31333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.26, - "Spanish,Chinese,Filipino,Malay": 0.24, - "Spanish,Chinese,Indonesian,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Spanish,Filipino,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.29333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.32666666666666666, - "English,Chinese,Vietnamese,Malay": 0.30666666666666664, - "English,Chinese,Filipino,Indonesian": 0.25333333333333335, - "English,Chinese,Filipino,Malay": 0.24666666666666667, - "English,Chinese,Indonesian,Malay": 0.30666666666666664, - "English,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "English,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "English,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Chinese,Filipino,Indonesian,Malay": 0.26, - "Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.29333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.26666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian": 0.22, - "Spanish,English,Chinese,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.28, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.22, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.26, - "English,Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.24, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16 - } - }, - "AC3_2": 0.5504646597257158, - "AC3_3": 0.454072819925213, - "AC3_4": 0.38390168966235566, - "AC3_5": 0.32942896011313555, - "AC3_6": 0.28485948980124043, - "AC3_7": 0.24695652170389884 - } + "prompt_1": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.47483766233766234, - "language_acc": { - "English": 0.6022727272727273, - "Filipino": 0.3181818181818182, - "Spanish": 0.48863636363636365, - "Malay": 0.3806818181818182, - "Chinese": 0.6193181818181818, - "Indonesian": 0.4602272727272727, - "Vietnamese": 0.45454545454545453 - }, - "consistency_score_2": 0.5830627705627706, - "consistency_score_3": 0.42110389610389615, - "consistency_score_4": 0.3313311688311688, - "consistency_score_5": 0.27326839826839827, - "consistency_score_6": 0.23133116883116883, - "consistency_score_7": 0.19886363636363635, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.44886363636363635, - "English,Spanish": 0.6931818181818182, - "English,Malay": 0.5625, - "English,Chinese": 0.6818181818181818, - "English,Indonesian": 0.5965909090909091, - "English,Vietnamese": 0.6420454545454546, - "Filipino,Spanish": 0.5170454545454546, - "Filipino,Malay": 0.5, - "Filipino,Chinese": 0.4772727272727273, - "Filipino,Indonesian": 0.5454545454545454, - "Filipino,Vietnamese": 0.4943181818181818, - "Spanish,Malay": 0.5795454545454546, - "Spanish,Chinese": 0.6477272727272727, - "Spanish,Indonesian": 0.6818181818181818, - "Spanish,Vietnamese": 0.5965909090909091, - "Malay,Chinese": 0.5340909090909091, - "Malay,Indonesian": 0.6306818181818182, - "Malay,Vietnamese": 0.5625, - "Chinese,Indonesian": 0.625, - "Chinese,Vietnamese": 0.5965909090909091, - "Indonesian,Vietnamese": 0.6306818181818182 - }, - "3_combine": { - "English,Filipino,Spanish": 0.35795454545454547, - "English,Filipino,Malay": 0.32386363636363635, - "English,Filipino,Chinese": 0.3465909090909091, - "English,Filipino,Indonesian": 0.3409090909090909, - "English,Filipino,Vietnamese": 0.3409090909090909, - "English,Spanish,Malay": 0.45454545454545453, - "English,Spanish,Chinese": 0.5227272727272727, - "English,Spanish,Indonesian": 0.5170454545454546, - "English,Spanish,Vietnamese": 0.4943181818181818, - "English,Malay,Chinese": 0.4318181818181818, - "English,Malay,Indonesian": 0.4431818181818182, - "English,Malay,Vietnamese": 0.4318181818181818, - "English,Chinese,Indonesian": 0.48295454545454547, - "English,Chinese,Vietnamese": 0.5056818181818182, - "English,Indonesian,Vietnamese": 0.48863636363636365, - "Filipino,Spanish,Malay": 0.36363636363636365, - "Filipino,Spanish,Chinese": 0.375, - "Filipino,Spanish,Indonesian": 0.4090909090909091, - "Filipino,Spanish,Vietnamese": 0.3522727272727273, - "Filipino,Malay,Chinese": 0.3409090909090909, - "Filipino,Malay,Indonesian": 0.3977272727272727, - "Filipino,Malay,Vietnamese": 0.3352272727272727, - "Filipino,Chinese,Indonesian": 0.375, - "Filipino,Chinese,Vietnamese": 0.3522727272727273, - "Filipino,Indonesian,Vietnamese": 0.3806818181818182, - "Spanish,Malay,Chinese": 0.4375, - "Spanish,Malay,Indonesian": 0.48295454545454547, - "Spanish,Malay,Vietnamese": 0.42613636363636365, - "Spanish,Chinese,Indonesian": 0.5227272727272727, - "Spanish,Chinese,Vietnamese": 0.4715909090909091, - "Spanish,Indonesian,Vietnamese": 0.4943181818181818, - "Malay,Chinese,Indonesian": 0.4375, - "Malay,Chinese,Vietnamese": 0.39204545454545453, - "Malay,Indonesian,Vietnamese": 0.44886363636363635, - "Chinese,Indonesian,Vietnamese": 0.4602272727272727 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.2727272727272727, - "English,Filipino,Spanish,Chinese": 0.2784090909090909, - "English,Filipino,Spanish,Indonesian": 0.3068181818181818, - "English,Filipino,Spanish,Vietnamese": 0.2840909090909091, - "English,Filipino,Malay,Chinese": 0.26704545454545453, - "English,Filipino,Malay,Indonesian": 0.2784090909090909, - "English,Filipino,Malay,Vietnamese": 0.26704545454545453, - "English,Filipino,Chinese,Indonesian": 0.2840909090909091, - "English,Filipino,Chinese,Vietnamese": 0.2897727272727273, - "English,Filipino,Indonesian,Vietnamese": 0.29545454545454547, - "English,Spanish,Malay,Chinese": 0.3806818181818182, - "English,Spanish,Malay,Indonesian": 0.39204545454545453, - "English,Spanish,Malay,Vietnamese": 0.375, - "English,Spanish,Chinese,Indonesian": 0.4318181818181818, - "English,Spanish,Chinese,Vietnamese": 0.4147727272727273, - "English,Spanish,Indonesian,Vietnamese": 0.4318181818181818, - "English,Malay,Chinese,Indonesian": 0.35795454545454547, - "English,Malay,Chinese,Vietnamese": 0.3522727272727273, - "English,Malay,Indonesian,Vietnamese": 0.375, - "English,Chinese,Indonesian,Vietnamese": 0.4090909090909091, - "Filipino,Spanish,Malay,Chinese": 0.2897727272727273, - "Filipino,Spanish,Malay,Indonesian": 0.32954545454545453, - "Filipino,Spanish,Malay,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.32386363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.29545454545454547, - "Filipino,Spanish,Indonesian,Vietnamese": 0.3181818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.3068181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.26136363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.29545454545454547, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.38636363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.35795454545454547, - "Spanish,Malay,Indonesian,Vietnamese": 0.3806818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.3977272727272727, - "Malay,Chinese,Indonesian,Vietnamese": 0.3465909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.23295454545454544, - "English,Filipino,Spanish,Malay,Indonesian": 0.2556818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, - "English,Filipino,Spanish,Chinese,Indonesian": 0.2556818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.24431818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.26704545454545453, - "English,Filipino,Malay,Chinese,Indonesian": 0.23863636363636365, - "English,Filipino,Malay,Chinese,Vietnamese": 0.22727272727272727, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.3352272727272727, - "English,Spanish,Malay,Chinese,Vietnamese": 0.32386363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.3409090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.36363636363636365, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.3181818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.32386363636363635 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.2215909090909091, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.20454545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.29545454545454547, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635 - } - }, - "AC3_2": 0.5234144052383302, - "AC3_3": 0.4463594477138676, - "AC3_4": 0.39031158627102414, - "AC3_5": 0.34689767732147003, - "AC3_6": 0.3111005373495861, - "AC3_7": 0.28032584880833544 - } + "prompt_1": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.6310679611650486 - } + "prompt_1": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.6095238095238096 - } + "prompt_1": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.6074766355140186 - } + "prompt_1": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.41, - "category_acc": { - "brand": 0.5, - "demographics": 0.4, - "biology": 0.4, - "history": 0.4, - "literature": 0.3, - "politics": 0.4, - "culture": 0.5, - "film": 0.3, - "law": 0.3, - "geography": 0.6 - } - } + "prompt_1": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.15915666736266532 - } + "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.2870207072364554 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.23416047729758283 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.20233811814211664 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.2697836656622985 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.5834305717619603 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.5715409367179121, - "category_acc": { - "high_school_european_history": 0.7439024390243902, - "business_ethics": 0.6666666666666666, - "clinical_knowledge": 0.6401515151515151, - "medical_genetics": 0.6464646464646465, - "high_school_us_history": 0.6995073891625616, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.7627118644067796, - "virology": 0.4484848484848485, - "high_school_microeconomics": 0.6118143459915611, - "econometrics": 0.36283185840707965, - "college_computer_science": 0.48484848484848486, - "high_school_biology": 0.7281553398058253, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.39501779359430605, - "philosophy": 0.6838709677419355, - "professional_medicine": 0.5830258302583026, - "nutrition": 0.6098360655737705, - "global_facts": 0.35353535353535354, - "machine_learning": 0.3153153153153153, - "security_studies": 0.5942622950819673, - "public_relations": 0.5963302752293578, - "professional_psychology": 0.5220949263502455, - "prehistory": 0.6160990712074303, - "anatomy": 0.5074626865671642, - "human_sexuality": 0.6692307692307692, - "college_medicine": 0.5813953488372093, - "high_school_government_and_politics": 0.7708333333333334, - "college_chemistry": 0.48484848484848486, - "logical_fallacies": 0.654320987654321, - "high_school_geography": 0.766497461928934, - "elementary_mathematics": 0.5517241379310345, - "human_aging": 0.6441441441441441, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.8051470588235294, - "formal_logic": 0.44, - "high_school_statistics": 0.4697674418604651, - "international_law": 0.7166666666666667, - "high_school_mathematics": 0.27137546468401486, - "high_school_computer_science": 0.6565656565656566, - "conceptual_physics": 0.5213675213675214, - "miscellaneous": 0.7468030690537084, - "high_school_chemistry": 0.504950495049505, - "marketing": 0.7725321888412017, - "professional_law": 0.4292237442922374, - "management": 0.7647058823529411, - "college_physics": 0.3564356435643564, - "jurisprudence": 0.7102803738317757, - "world_religions": 0.7529411764705882, - "sociology": 0.74, - "us_foreign_policy": 0.797979797979798, - "high_school_macroeconomics": 0.5604113110539846, - "computer_security": 0.6666666666666666, - "moral_scenarios": 0.36577181208053694, - "moral_disputes": 0.6260869565217392, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.6357615894039735, - "college_biology": 0.6013986013986014 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.6352154531946508 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.6519302615193027, - "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.6923076923076923, - "college_programming": 0.6428571428571429, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.42857142857142855, - "electrical_engineer": 0.4523809523809524, - "metrology_engineer": 0.6551724137931034, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.75, - "high_school_chemistry": 0.6666666666666666, - "high_school_biology": 0.8333333333333334, - "middle_school_mathematics": 0.625, - "middle_school_biology": 0.9230769230769231, - "middle_school_physics": 0.9166666666666666, - "middle_school_chemistry": 0.92, - "veterinary_medicine": 0.7142857142857143, - "college_economics": 0.5666666666666667, - "business_administration": 0.7105263157894737, - "marxism": 0.9166666666666666, - "mao_zedong_thought": 0.8275862068965517, - "education_science": 0.7058823529411765, - "teacher_qualification": 0.8163265306122449, - "high_school_politics": 0.8333333333333334, - "high_school_geography": 0.875, - "middle_school_politics": 0.8846153846153846, - "middle_school_geography": 0.9411764705882353, - "modern_chinese_history": 0.8214285714285714, - "ideological_and_moral_cultivation": 0.9583333333333334, - "logic": 0.4444444444444444, - "law": 0.5862068965517241, - "chinese_language_and_literature": 0.5714285714285714, - "art_studies": 0.6578947368421053, - "professional_tour_guide": 0.7647058823529411, - "legal_professional": 0.4642857142857143, - "high_school_chinese": 0.6666666666666666, - "high_school_history": 0.88, - "middle_school_history": 0.9629629629629629, - "civil_servant": 0.5576923076923077, - "sports_science": 0.7083333333333334, - "plant_protection": 0.7037037037037037, - "basic_medicine": 0.75, - "clinical_medicine": 0.5925925925925926, - "urban_and_rural_planner": 0.6274509803921569, - "accountant": 0.6296296296296297, - "fire_engineer": 0.5, - "environmental_impact_assessment_engineer": 0.5, - "tax_accountant": 0.5925925925925926, - "physician": 0.6111111111111112 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.5985663082437276 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.6376273527888102, - "category_acc": { - "agronomy": 0.5562130177514792, - "anatomy": 0.5135135135135135, - "ancient_chinese": 0.40853658536585363, - "arts": 0.875, - "astronomy": 0.38181818181818183, - "business_ethics": 0.5933014354066986, - "chinese_civil_service_exam": 0.60625, - "chinese_driving_rule": 0.8244274809160306, - "chinese_food_culture": 0.6029411764705882, - "chinese_foreign_policy": 0.6822429906542056, - "chinese_history": 0.7832817337461301, - "chinese_literature": 0.5294117647058824, - "chinese_teacher_qualification": 0.8268156424581006, - "clinical_knowledge": 0.6033755274261603, - "college_actuarial_science": 0.3113207547169811, - "college_education": 0.7476635514018691, - "college_engineering_hydrology": 0.5660377358490566, - "college_law": 0.5277777777777778, - "college_mathematics": 0.3238095238095238, - "college_medical_statistics": 0.5471698113207547, - "college_medicine": 0.608058608058608, - "computer_science": 0.6519607843137255, - "computer_security": 0.8011695906432749, - "conceptual_physics": 0.7891156462585034, - "construction_project_management": 0.4892086330935252, - "economics": 0.6289308176100629, - "education": 0.6319018404907976, - "electrical_engineering": 0.5872093023255814, - "elementary_chinese": 0.6150793650793651, - "elementary_commonsense": 0.7070707070707071, - "elementary_information_and_technology": 0.8025210084033614, - "elementary_mathematics": 0.47391304347826085, - "ethnology": 0.6518518518518519, - "food_science": 0.5734265734265734, - "genetics": 0.5454545454545454, - "global_facts": 0.6845637583892618, - "high_school_biology": 0.7692307692307693, - "high_school_chemistry": 0.6363636363636364, - "high_school_geography": 0.711864406779661, - "high_school_mathematics": 0.4634146341463415, - "high_school_physics": 0.509090909090909, - "high_school_politics": 0.6923076923076923, - "human_sexuality": 0.5634920634920635, - "international_law": 0.5081081081081081, - "journalism": 0.5755813953488372, - "jurisprudence": 0.6326034063260341, - "legal_and_moral_basis": 0.9205607476635514, - "logical": 0.5691056910569106, - "machine_learning": 0.5491803278688525, - "management": 0.7666666666666667, - "marketing": 0.6722222222222223, - "marxist_theory": 0.8095238095238095, - "modern_chinese": 0.5603448275862069, - "nutrition": 0.6413793103448275, - "philosophy": 0.7238095238095238, - "professional_accounting": 0.7085714285714285, - "professional_law": 0.5781990521327014, - "professional_medicine": 0.523936170212766, - "professional_psychology": 0.7801724137931034, - "public_relations": 0.6551724137931034, - "security_study": 0.7185185185185186, - "sociology": 0.5973451327433629, - "sports_science": 0.6121212121212121, - "traditional_chinese_medicine": 0.6756756756756757, - "virology": 0.6449704142011834, - "world_history": 0.7701863354037267, - "world_religions": 0.71875 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.45454545454545453 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.5840909090909091 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.5742372881355933 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.8855646970830217 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.89171974522293 - } + "prompt_1": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.39505299509995395, - "rouge2": 0.15381448536484357, - "rougeL": 0.2973732163679409, - "avg_rouge": 0.28208023227757945 - } + "prompt_1": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.3449029389896441, - "rouge2": 0.11090245646203131, - "rougeL": 0.2606177248720784, - "avg_rouge": 0.23880770677458463 - } + "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.9346330275229358 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.8082454458293384 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.7395 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.76 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.717 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.6056338028169014 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.8122743682310469 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.7352941176470589 - } + "prompt_1": -1 } } }, - "mt0_xxl": { - "model_size": "13B", - "model_link": "https://huggingface.co/bigscience/mt0-xxl", + "flan_t5_large": { + "model_size": "0.78B", + "model_link": "https://huggingface.co/google/flan-t5-large", "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.9680672268907563, + "overall_acc": 0.7676470588235293, "language_acc": { - "English": 0.9714285714285714, - "Chinese": 0.957983193277311, - "Spanish": 0.9722689075630252, - "Vietnamese": 0.9705882352941176 + "Spanish": 0.9319327731092437, + "English": 0.9705882352941176, + "Chinese": 0.37563025210084033, + "Vietnamese": 0.7924369747899159 }, - "consistency_score_2": 0.9642857142857143, - "consistency_score_3": 0.9476890756302521, - "consistency_score_4": 0.9352941176470588, + "consistency_score_2": 0.6124649859943977, + "consistency_score_3": 0.44621848739495795, + "consistency_score_4": 0.32184873949579834, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9588235294117647, - "English,Spanish": 0.9781512605042016, - "English,Vietnamese": 0.9663865546218487, - "Chinese,Spanish": 0.957983193277311, - "Chinese,Vietnamese": 0.9546218487394958, - "Spanish,Vietnamese": 0.9697478991596639 + "Spanish,English": 0.9361344537815126, + "Spanish,Chinese": 0.38487394957983195, + "Spanish,Vietnamese": 0.7798319327731092, + "English,Chinese": 0.3764705882352941, + "English,Vietnamese": 0.788235294117647, + "Chinese,Vietnamese": 0.4092436974789916 }, "3_combine": { - "English,Chinese,Spanish": 0.9487394957983193, - "English,Chinese,Vietnamese": 0.9411764705882353, - "English,Spanish,Vietnamese": 0.9588235294117647, - "Chinese,Spanish,Vietnamese": 0.9420168067226891 + "Spanish,English,Chinese": 0.3638655462184874, + "Spanish,English,Vietnamese": 0.761344537815126, + "Spanish,Chinese,Vietnamese": 0.3302521008403361, + "English,Chinese,Vietnamese": 0.32941176470588235 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.9352941176470588 + "Spanish,English,Chinese,Vietnamese": 0.32184873949579834 } }, - "AC3_2": 0.9661727704280544, - "AC3_3": 0.9577697686755681, - "AC3_4": 0.9513985196227847 + "AC3_2": 0.6813315583503264, + "AC3_3": 0.564376030739652, + "AC3_4": 0.45354234246421293 }, "prompt_2": { - "overall_acc": 0.9697478991596639, + "overall_acc": 0.7485294117647059, "language_acc": { - "English": 0.9747899159663865, - "Chinese": 0.9588235294117647, - "Spanish": 0.973109243697479, - "Vietnamese": 0.9722689075630252 + "Spanish": 0.9168067226890756, + "English": 0.9638655462184874, + "Chinese": 0.35798319327731093, + "Vietnamese": 0.7554621848739496 }, - "consistency_score_2": 0.9665266106442577, - "consistency_score_3": 0.9506302521008403, - "consistency_score_4": 0.938655462184874, + "consistency_score_2": 0.59593837535014, + "consistency_score_3": 0.42626050420168066, + "consistency_score_4": 0.3042016806722689, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9596638655462185, - "English,Spanish": 0.9798319327731092, - "English,Vietnamese": 0.9714285714285714, - "Chinese,Spanish": 0.957983193277311, - "Chinese,Vietnamese": 0.9571428571428572, - "Spanish,Vietnamese": 0.973109243697479 + "Spanish,English": 0.9201680672268907, + "Spanish,Chinese": 0.3739495798319328, + "Spanish,Vietnamese": 0.7546218487394958, + "English,Chinese": 0.3621848739495798, + "English,Vietnamese": 0.7563025210084033, + "Chinese,Vietnamese": 0.4084033613445378 }, "3_combine": { - "English,Chinese,Spanish": 0.9495798319327731, - "English,Chinese,Vietnamese": 0.9453781512605042, - "English,Spanish,Vietnamese": 0.9630252100840336, - "Chinese,Spanish,Vietnamese": 0.9445378151260504 + "Spanish,English,Chinese": 0.346218487394958, + "Spanish,English,Vietnamese": 0.7277310924369748, + "Spanish,Chinese,Vietnamese": 0.3176470588235294, + "English,Chinese,Vietnamese": 0.3134453781512605 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.938655462184874 + "Spanish,English,Chinese,Vietnamese": 0.3042016806722689 } }, - "AC3_2": 0.968134575299127, - "AC3_3": 0.9600939160855435, - "AC3_4": 0.9539483956887579 + "AC3_2": 0.6635746959366482, + "AC3_3": 0.5431924808087655, + "AC3_4": 0.4325965229025488 }, "prompt_3": { - "overall_acc": 0.9684873949579832, + "overall_acc": 0.7565126050420168, "language_acc": { - "English": 0.9756302521008403, - "Chinese": 0.9546218487394958, - "Spanish": 0.9714285714285714, - "Vietnamese": 0.9722689075630252 + "Spanish": 0.9218487394957983, + "English": 0.965546218487395, + "Chinese": 0.36722689075630255, + "Vietnamese": 0.7714285714285715 }, - "consistency_score_2": 0.9645658263305322, - "consistency_score_3": 0.948109243697479, - "consistency_score_4": 0.9352941176470588, + "consistency_score_2": 0.6046218487394958, + "consistency_score_3": 0.4373949579831933, + "consistency_score_4": 0.31512605042016806, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9546218487394958, - "English,Spanish": 0.9798319327731092, - "English,Vietnamese": 0.9739495798319328, - "Chinese,Spanish": 0.9521008403361344, - "Chinese,Vietnamese": 0.9546218487394958, - "Spanish,Vietnamese": 0.9722689075630252 + "Spanish,English": 0.9277310924369748, + "Spanish,Chinese": 0.3840336134453782, + "Spanish,Vietnamese": 0.7638655462184873, + "English,Chinese": 0.37058823529411766, + "English,Vietnamese": 0.7714285714285715, + "Chinese,Vietnamese": 0.41008403361344536 }, "3_combine": { - "English,Chinese,Spanish": 0.9445378151260504, - "English,Chinese,Vietnamese": 0.9428571428571428, - "English,Spanish,Vietnamese": 0.9647058823529412, - "Chinese,Spanish,Vietnamese": 0.9403361344537815 + "Spanish,English,Chinese": 0.3596638655462185, + "Spanish,English,Vietnamese": 0.7453781512605042, + "Spanish,Chinese,Vietnamese": 0.3235294117647059, + "English,Chinese,Vietnamese": 0.32100840336134456 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.9352941176470588 + "Spanish,English,Chinese,Vietnamese": 0.31512605042016806 } }, - "AC3_2": 0.9665226327677992, - "AC3_3": 0.9581899842633627, - "AC3_4": 0.9516013864768176 + "AC3_2": 0.6720923837477901, + "AC3_3": 0.5543055581789739, + "AC3_4": 0.44492017543898615 }, "prompt_4": { - "overall_acc": 0.9686974789915966, + "overall_acc": 0.7596638655462185, "language_acc": { - "English": 0.9747899159663865, - "Chinese": 0.9563025210084034, - "Spanish": 0.9722689075630252, - "Vietnamese": 0.9714285714285714 + "Spanish": 0.926890756302521, + "English": 0.9680672268907563, + "Chinese": 0.36050420168067226, + "Vietnamese": 0.7831932773109244 }, - "consistency_score_2": 0.9656862745098039, - "consistency_score_3": 0.9495798319327731, - "consistency_score_4": 0.9369747899159664, + "consistency_score_2": 0.6058823529411764, + "consistency_score_3": 0.4373949579831933, + "consistency_score_4": 0.3126050420168067, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9588235294117647, - "English,Spanish": 0.980672268907563, - "English,Vietnamese": 0.9705882352941176, - "Chinese,Spanish": 0.9563025210084034, - "Chinese,Vietnamese": 0.9554621848739496, - "Spanish,Vietnamese": 0.9722689075630252 + "Spanish,English": 0.9277310924369748, + "Spanish,Chinese": 0.37478991596638656, + "Spanish,Vietnamese": 0.7789915966386555, + "English,Chinese": 0.36554621848739494, + "English,Vietnamese": 0.7823529411764706, + "Chinese,Vietnamese": 0.40588235294117647 }, "3_combine": { - "English,Chinese,Spanish": 0.9487394957983193, - "English,Chinese,Vietnamese": 0.9436974789915966, - "English,Spanish,Vietnamese": 0.9630252100840336, - "Chinese,Spanish,Vietnamese": 0.9428571428571428 + "Spanish,English,Chinese": 0.3521008403361345, + "Spanish,English,Vietnamese": 0.7554621848739496, + "Spanish,Chinese,Vietnamese": 0.3226890756302521, + "English,Chinese,Vietnamese": 0.31932773109243695 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.9369747899159664 + "Spanish,English,Chinese,Vietnamese": 0.3126050420168067 } }, - "AC3_2": 0.9671895329691594, - "AC3_3": 0.959043391701624, - "AC3_4": 0.9525720992340004 + "AC3_2": 0.6741140270999553, + "AC3_3": 0.5551492340944547, + "AC3_4": 0.44293880557498183 }, "prompt_5": { - "overall_acc": 0.9697478991596639, + "overall_acc": 0.7495798319327731, "language_acc": { - "English": 0.9747899159663865, - "Chinese": 0.9588235294117647, - "Spanish": 0.9714285714285714, - "Vietnamese": 0.9739495798319328 + "Spanish": 0.9159663865546218, + "English": 0.9647058823529412, + "Chinese": 0.3621848739495798, + "Vietnamese": 0.7554621848739496 }, - "consistency_score_2": 0.9668067226890756, - "consistency_score_3": 0.9512605042016806, - "consistency_score_4": 0.9394957983193277, + "consistency_score_2": 0.6012605042016806, + "consistency_score_3": 0.4315126050420168, + "consistency_score_4": 0.3100840336134454, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9588235294117647, - "English,Spanish": 0.9798319327731092, - "English,Vietnamese": 0.9739495798319328, - "Chinese,Spanish": 0.957983193277311, - "Chinese,Vietnamese": 0.9596638655462185, - "Spanish,Vietnamese": 0.9705882352941176 + "Spanish,English": 0.9176470588235294, + "Spanish,Chinese": 0.38235294117647056, + "Spanish,Vietnamese": 0.7554621848739496, + "English,Chinese": 0.3680672268907563, + "English,Vietnamese": 0.7554621848739496, + "Chinese,Vietnamese": 0.42857142857142855 }, "3_combine": { - "English,Chinese,Spanish": 0.9495798319327731, - "English,Chinese,Vietnamese": 0.9470588235294117, - "English,Spanish,Vietnamese": 0.9630252100840336, - "Chinese,Spanish,Vietnamese": 0.9453781512605042 + "Spanish,English,Chinese": 0.35294117647058826, + "Spanish,English,Vietnamese": 0.7285714285714285, + "Spanish,Chinese,Vietnamese": 0.32521008403361346, + "English,Chinese,Vietnamese": 0.31932773109243695 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.9394957983193277 + "Spanish,English,Chinese,Vietnamese": 0.3100840336134454 } }, - "AC3_2": 0.9682750773925631, - "AC3_3": 0.9604152421623752, - "AC3_4": 0.954382175355386 + "AC3_2": 0.667277598392779, + "AC3_3": 0.5477185964637349, + "AC3_4": 0.43869144798904225 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.4504761904761904, + "overall_acc": 0.31904761904761897, "language_acc": { - "Spanish": 0.42, - "English": 0.46, - "Chinese": 0.4533333333333333, - "Vietnamese": 0.43333333333333335, - "Filipino": 0.4533333333333333, - "Indonesian": 0.46, - "Malay": 0.47333333333333333 + "Filipino": 0.26666666666666666, + "Vietnamese": 0.2866666666666667, + "Chinese": 0.28, + "Spanish": 0.34, + "Malay": 0.24666666666666667, + "Indonesian": 0.36, + "English": 0.4533333333333333 }, - "consistency_score_2": 0.6968253968253969, - "consistency_score_3": 0.5702857142857144, - "consistency_score_4": 0.4952380952380953, - "consistency_score_5": 0.44317460317460317, - "consistency_score_6": 0.40380952380952373, - "consistency_score_7": 0.37333333333333335, + "consistency_score_2": 0.5473015873015873, + "consistency_score_3": 0.381904761904762, + "consistency_score_4": 0.29599999999999993, + "consistency_score_5": 0.24285714285714283, + "consistency_score_6": 0.2066666666666667, + "consistency_score_7": 0.18, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.7733333333333333, - "Spanish,Chinese": 0.6333333333333333, - "Spanish,Vietnamese": 0.6733333333333333, - "Spanish,Filipino": 0.7266666666666667, - "Spanish,Indonesian": 0.7333333333333333, - "Spanish,Malay": 0.7266666666666667, - "English,Chinese": 0.6466666666666666, - "English,Vietnamese": 0.7133333333333334, - "English,Filipino": 0.7733333333333333, - "English,Indonesian": 0.7333333333333333, - "English,Malay": 0.7533333333333333, - "Chinese,Vietnamese": 0.6266666666666667, - "Chinese,Filipino": 0.5933333333333334, - "Chinese,Indonesian": 0.6533333333333333, - "Chinese,Malay": 0.5733333333333334, - "Vietnamese,Filipino": 0.7066666666666667, - "Vietnamese,Indonesian": 0.7, - "Vietnamese,Malay": 0.7, - "Filipino,Indonesian": 0.7, - "Filipino,Malay": 0.74, - "Indonesian,Malay": 0.7533333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5466666666666666, - "Spanish,English,Vietnamese": 0.6066666666666667, - "Spanish,English,Filipino": 0.6666666666666666, - "Spanish,English,Indonesian": 0.64, - "Spanish,English,Malay": 0.6533333333333333, - "Spanish,Chinese,Vietnamese": 0.5066666666666667, - "Spanish,Chinese,Filipino": 0.5133333333333333, - "Spanish,Chinese,Indonesian": 0.5466666666666666, - "Spanish,Chinese,Malay": 0.5, - "Spanish,Vietnamese,Filipino": 0.5733333333333334, - "Spanish,Vietnamese,Indonesian": 0.5866666666666667, - "Spanish,Vietnamese,Malay": 0.5666666666666667, - "Spanish,Filipino,Indonesian": 0.6, - "Spanish,Filipino,Malay": 0.6133333333333333, - "Spanish,Indonesian,Malay": 0.6333333333333333, - "English,Chinese,Vietnamese": 0.5133333333333333, - "English,Chinese,Filipino": 0.5333333333333333, - "English,Chinese,Indonesian": 0.5333333333333333, - "English,Chinese,Malay": 0.5066666666666667, - "English,Vietnamese,Filipino": 0.6133333333333333, - "English,Vietnamese,Indonesian": 0.5933333333333334, - "English,Vietnamese,Malay": 0.6, - "English,Filipino,Indonesian": 0.62, - "English,Filipino,Malay": 0.6466666666666666, - "English,Indonesian,Malay": 0.64, - "Chinese,Vietnamese,Filipino": 0.49333333333333335, - "Chinese,Vietnamese,Indonesian": 0.52, - "Chinese,Vietnamese,Malay": 0.4866666666666667, - "Chinese,Filipino,Indonesian": 0.49333333333333335, - "Chinese,Filipino,Malay": 0.4866666666666667, - "Chinese,Indonesian,Malay": 0.52, - "Vietnamese,Filipino,Indonesian": 0.5866666666666667, - "Vietnamese,Filipino,Malay": 0.6, - "Vietnamese,Indonesian,Malay": 0.6, - "Filipino,Indonesian,Malay": 0.62 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.46, - "Spanish,English,Chinese,Filipino": 0.48, - "Spanish,English,Chinese,Indonesian": 0.4866666666666667, - "Spanish,English,Chinese,Malay": 0.46, - "Spanish,English,Vietnamese,Filipino": 0.5466666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.5466666666666666, - "Spanish,English,Vietnamese,Malay": 0.54, - "Spanish,English,Filipino,Indonesian": 0.5666666666666667, - "Spanish,English,Filipino,Malay": 0.58, - "Spanish,English,Indonesian,Malay": 0.5866666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.44, - "Spanish,Chinese,Vietnamese,Indonesian": 0.4533333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.44, - "Spanish,Chinese,Filipino,Indonesian": 0.4533333333333333, - "Spanish,Chinese,Filipino,Malay": 0.43333333333333335, - "Spanish,Chinese,Indonesian,Malay": 0.48, - "Spanish,Vietnamese,Filipino,Indonesian": 0.52, - "Spanish,Vietnamese,Filipino,Malay": 0.5133333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.54, - "Spanish,Filipino,Indonesian,Malay": 0.5466666666666666, - "English,Chinese,Vietnamese,Filipino": 0.46, - "English,Chinese,Vietnamese,Indonesian": 0.44666666666666666, - "English,Chinese,Vietnamese,Malay": 0.44666666666666666, - "English,Chinese,Filipino,Indonesian": 0.46, - "English,Chinese,Filipino,Malay": 0.4533333333333333, - "English,Chinese,Indonesian,Malay": 0.47333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.5333333333333333, - "English,Vietnamese,Filipino,Malay": 0.5533333333333333, - "English,Vietnamese,Indonesian,Malay": 0.5466666666666666, - "English,Filipino,Indonesian,Malay": 0.5666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.43333333333333335, - "Chinese,Vietnamese,Filipino,Malay": 0.44, - "Chinese,Vietnamese,Indonesian,Malay": 0.4533333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.4533333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.54 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.42, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.42, - "Spanish,English,Chinese,Vietnamese,Malay": 0.41333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.43333333333333335, - "Spanish,English,Chinese,Filipino,Malay": 0.41333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.44666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.5, - "Spanish,English,Vietnamese,Filipino,Malay": 0.5, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.5133333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.52, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.4, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.4, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.42, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.4866666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.4066666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.42, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.42, - "English,Chinese,Filipino,Indonesian,Malay": 0.4266666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.5066666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.41333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.38666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.4, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.4, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.47333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.38666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3933333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.37333333333333335 - } - }, - "AC3_2": 0.54720267737081, - "AC3_3": 0.5033497720297014, - "AC3_4": 0.47179782280533666, - "AC3_5": 0.44679556791075775, - "AC3_6": 0.4258682379923002, - "AC3_7": 0.40829287085602606 + "Filipino,Vietnamese": 0.5333333333333333, + "Filipino,Chinese": 0.56, + "Filipino,Spanish": 0.54, + "Filipino,Malay": 0.6266666666666667, + "Filipino,Indonesian": 0.58, + "Filipino,English": 0.5066666666666667, + "Vietnamese,Chinese": 0.7, + "Vietnamese,Spanish": 0.5466666666666666, + "Vietnamese,Malay": 0.62, + "Vietnamese,Indonesian": 0.58, + "Vietnamese,English": 0.47333333333333333, + "Chinese,Spanish": 0.5066666666666667, + "Chinese,Malay": 0.6066666666666667, + "Chinese,Indonesian": 0.5866666666666667, + "Chinese,English": 0.44666666666666666, + "Spanish,Malay": 0.4533333333333333, + "Spanish,Indonesian": 0.5, + "Spanish,English": 0.5533333333333333, + "Malay,Indonesian": 0.6, + "Malay,English": 0.48, + "Indonesian,English": 0.49333333333333335 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.44, + "Filipino,Vietnamese,Spanish": 0.36666666666666664, + "Filipino,Vietnamese,Malay": 0.4533333333333333, + "Filipino,Vietnamese,Indonesian": 0.3933333333333333, + "Filipino,Vietnamese,English": 0.31333333333333335, + "Filipino,Chinese,Spanish": 0.36666666666666664, + "Filipino,Chinese,Malay": 0.46, + "Filipino,Chinese,Indonesian": 0.4266666666666667, + "Filipino,Chinese,English": 0.32666666666666666, + "Filipino,Spanish,Malay": 0.4, + "Filipino,Spanish,Indonesian": 0.36666666666666664, + "Filipino,Spanish,English": 0.38, + "Filipino,Malay,Indonesian": 0.46, + "Filipino,Malay,English": 0.37333333333333335, + "Filipino,Indonesian,English": 0.35333333333333333, + "Vietnamese,Chinese,Spanish": 0.42, + "Vietnamese,Chinese,Malay": 0.5066666666666667, + "Vietnamese,Chinese,Indonesian": 0.47333333333333333, + "Vietnamese,Chinese,English": 0.36, + "Vietnamese,Spanish,Malay": 0.36, + "Vietnamese,Spanish,Indonesian": 0.36666666666666664, + "Vietnamese,Spanish,English": 0.3466666666666667, + "Vietnamese,Malay,Indonesian": 0.43333333333333335, + "Vietnamese,Malay,English": 0.3466666666666667, + "Vietnamese,Indonesian,English": 0.3333333333333333, + "Chinese,Spanish,Malay": 0.36, + "Chinese,Spanish,Indonesian": 0.38, + "Chinese,Spanish,English": 0.3333333333333333, + "Chinese,Malay,Indonesian": 0.44666666666666666, + "Chinese,Malay,English": 0.3333333333333333, + "Chinese,Indonesian,English": 0.3466666666666667, + "Spanish,Malay,Indonesian": 0.3333333333333333, + "Spanish,Malay,English": 0.30666666666666664, + "Spanish,Indonesian,English": 0.34, + "Malay,Indonesian,English": 0.36 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Malay": 0.38666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian": 0.3466666666666667, + "Filipino,Vietnamese,Chinese,English": 0.26, + "Filipino,Vietnamese,Spanish,Malay": 0.32666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.28, + "Filipino,Vietnamese,Spanish,English": 0.24666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.36, + "Filipino,Vietnamese,Malay,English": 0.2866666666666667, + "Filipino,Vietnamese,Indonesian,English": 0.26, + "Filipino,Chinese,Spanish,Malay": 0.32666666666666666, + "Filipino,Chinese,Spanish,Indonesian": 0.29333333333333333, + "Filipino,Chinese,Spanish,English": 0.26, + "Filipino,Chinese,Malay,Indonesian": 0.38, + "Filipino,Chinese,Malay,English": 0.3, + "Filipino,Chinese,Indonesian,English": 0.28, + "Filipino,Spanish,Malay,Indonesian": 0.3, + "Filipino,Spanish,Malay,English": 0.29333333333333333, + "Filipino,Spanish,Indonesian,English": 0.28, + "Filipino,Malay,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish,Malay": 0.32, + "Vietnamese,Chinese,Spanish,Indonesian": 0.3333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.2733333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.37333333333333335, + "Vietnamese,Chinese,Malay,English": 0.28, + "Vietnamese,Chinese,Indonesian,English": 0.29333333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.2866666666666667, + "Vietnamese,Spanish,Malay,English": 0.24, + "Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, + "Vietnamese,Malay,Indonesian,English": 0.28, + "Chinese,Spanish,Malay,Indonesian": 0.3, + "Chinese,Spanish,Malay,English": 0.25333333333333335, + "Chinese,Spanish,Indonesian,English": 0.28, + "Chinese,Malay,Indonesian,English": 0.28, + "Spanish,Malay,Indonesian,English": 0.24 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.32, + "Filipino,Vietnamese,Chinese,Malay,English": 0.24666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.26, + "Filipino,Vietnamese,Spanish,Malay,English": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.25333333333333335, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, + "Filipino,Chinese,Spanish,Malay,English": 0.24, + "Filipino,Chinese,Spanish,Indonesian,English": 0.22666666666666666, + "Filipino,Chinese,Malay,Indonesian,English": 0.26666666666666666, + "Filipino,Spanish,Malay,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,Malay,English": 0.21333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.24, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.2, + "Chinese,Spanish,Malay,Indonesian,English": 0.22 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.2, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.21333333333333335, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18 + } + }, + "AC3_2": 0.4031059693474576, + "AC3_3": 0.3476578674452261, + "AC3_4": 0.3070919788909187, + "AC3_5": 0.27578692489038653, + "AC3_6": 0.25084541058030413, + "AC3_7": 0.2301526717096068 }, "prompt_2": { - "overall_acc": 0.4447619047619048, + "overall_acc": 0.31428571428571433, "language_acc": { - "Spanish": 0.4, - "English": 0.4666666666666667, - "Chinese": 0.46, - "Vietnamese": 0.44, - "Filipino": 0.4266666666666667, - "Indonesian": 0.43333333333333335, - "Malay": 0.4866666666666667 + "Filipino": 0.30666666666666664, + "Vietnamese": 0.28, + "Chinese": 0.26, + "Spanish": 0.3466666666666667, + "Malay": 0.23333333333333334, + "Indonesian": 0.30666666666666664, + "English": 0.4666666666666667 }, - "consistency_score_2": 0.7069841269841268, - "consistency_score_3": 0.5841904761904765, - "consistency_score_4": 0.5085714285714286, - "consistency_score_5": 0.45396825396825397, - "consistency_score_6": 0.4123809523809524, - "consistency_score_7": 0.38, + "consistency_score_2": 0.5374603174603175, + "consistency_score_3": 0.3678095238095238, + "consistency_score_4": 0.2811428571428571, + "consistency_score_5": 0.22825396825396824, + "consistency_score_6": 0.1904761904761905, + "consistency_score_7": 0.16, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.74, - "Spanish,Chinese": 0.64, - "Spanish,Vietnamese": 0.66, - "Spanish,Filipino": 0.7466666666666667, - "Spanish,Indonesian": 0.7133333333333334, - "Spanish,Malay": 0.7333333333333333, - "English,Chinese": 0.6466666666666666, - "English,Vietnamese": 0.7, - "English,Filipino": 0.7666666666666667, - "English,Indonesian": 0.7466666666666667, - "English,Malay": 0.7733333333333333, - "Chinese,Vietnamese": 0.68, - "Chinese,Filipino": 0.62, - "Chinese,Indonesian": 0.6666666666666666, - "Chinese,Malay": 0.6133333333333333, - "Vietnamese,Filipino": 0.7066666666666667, - "Vietnamese,Indonesian": 0.7266666666666667, - "Vietnamese,Malay": 0.7133333333333334, - "Filipino,Indonesian": 0.7333333333333333, - "Filipino,Malay": 0.74, - "Indonesian,Malay": 0.78 + "Filipino,Vietnamese": 0.47333333333333333, + "Filipino,Chinese": 0.5333333333333333, + "Filipino,Spanish": 0.5333333333333333, + "Filipino,Malay": 0.6133333333333333, + "Filipino,Indonesian": 0.6066666666666667, + "Filipino,English": 0.49333333333333335, + "Vietnamese,Chinese": 0.6266666666666667, + "Vietnamese,Spanish": 0.5133333333333333, + "Vietnamese,Malay": 0.5933333333333334, + "Vietnamese,Indonesian": 0.54, + "Vietnamese,English": 0.47333333333333333, + "Chinese,Spanish": 0.49333333333333335, + "Chinese,Malay": 0.6266666666666667, + "Chinese,Indonesian": 0.5666666666666667, + "Chinese,English": 0.46, + "Spanish,Malay": 0.4533333333333333, + "Spanish,Indonesian": 0.48, + "Spanish,English": 0.6, + "Malay,Indonesian": 0.6466666666666666, + "Malay,English": 0.46, + "Indonesian,English": 0.5 }, "3_combine": { - "Spanish,English,Chinese": 0.54, - "Spanish,English,Vietnamese": 0.5866666666666667, - "Spanish,English,Filipino": 0.64, - "Spanish,English,Indonesian": 0.62, - "Spanish,English,Malay": 0.6333333333333333, - "Spanish,Chinese,Vietnamese": 0.5266666666666666, - "Spanish,Chinese,Filipino": 0.5333333333333333, - "Spanish,Chinese,Indonesian": 0.54, - "Spanish,Chinese,Malay": 0.5066666666666667, - "Spanish,Vietnamese,Filipino": 0.58, - "Spanish,Vietnamese,Indonesian": 0.5866666666666667, - "Spanish,Vietnamese,Malay": 0.5733333333333334, - "Spanish,Filipino,Indonesian": 0.6266666666666667, - "Spanish,Filipino,Malay": 0.6266666666666667, - "Spanish,Indonesian,Malay": 0.6333333333333333, - "English,Chinese,Vietnamese": 0.5466666666666666, - "English,Chinese,Filipino": 0.5466666666666666, - "English,Chinese,Indonesian": 0.56, - "English,Chinese,Malay": 0.54, - "English,Vietnamese,Filipino": 0.6066666666666667, - "English,Vietnamese,Indonesian": 0.6133333333333333, - "English,Vietnamese,Malay": 0.6066666666666667, - "English,Filipino,Indonesian": 0.6466666666666666, - "English,Filipino,Malay": 0.6533333333333333, - "English,Indonesian,Malay": 0.66, - "Chinese,Vietnamese,Filipino": 0.5266666666666666, - "Chinese,Vietnamese,Indonesian": 0.5666666666666667, - "Chinese,Vietnamese,Malay": 0.5333333333333333, - "Chinese,Filipino,Indonesian": 0.5333333333333333, - "Chinese,Filipino,Malay": 0.5, - "Chinese,Indonesian,Malay": 0.5466666666666666, - "Vietnamese,Filipino,Indonesian": 0.62, - "Vietnamese,Filipino,Malay": 0.6066666666666667, - "Vietnamese,Indonesian,Malay": 0.6333333333333333, - "Filipino,Indonesian,Malay": 0.6466666666666666 + "Filipino,Vietnamese,Chinese": 0.38, + "Filipino,Vietnamese,Spanish": 0.3333333333333333, + "Filipino,Vietnamese,Malay": 0.4, + "Filipino,Vietnamese,Indonesian": 0.36666666666666664, + "Filipino,Vietnamese,English": 0.30666666666666664, + "Filipino,Chinese,Spanish": 0.35333333333333333, + "Filipino,Chinese,Malay": 0.44666666666666666, + "Filipino,Chinese,Indonesian": 0.4266666666666667, + "Filipino,Chinese,English": 0.32666666666666666, + "Filipino,Spanish,Malay": 0.38, + "Filipino,Spanish,Indonesian": 0.38666666666666666, + "Filipino,Spanish,English": 0.38, + "Filipino,Malay,Indonesian": 0.49333333333333335, + "Filipino,Malay,English": 0.35333333333333333, + "Filipino,Indonesian,English": 0.38, + "Vietnamese,Chinese,Spanish": 0.36666666666666664, + "Vietnamese,Chinese,Malay": 0.4666666666666667, + "Vietnamese,Chinese,Indonesian": 0.4, + "Vietnamese,Chinese,English": 0.3333333333333333, + "Vietnamese,Spanish,Malay": 0.34, + "Vietnamese,Spanish,Indonesian": 0.32, + "Vietnamese,Spanish,English": 0.3333333333333333, + "Vietnamese,Malay,Indonesian": 0.44, + "Vietnamese,Malay,English": 0.32666666666666666, + "Vietnamese,Indonesian,English": 0.32, + "Chinese,Spanish,Malay": 0.35333333333333333, + "Chinese,Spanish,Indonesian": 0.34, + "Chinese,Spanish,English": 0.34, + "Chinese,Malay,Indonesian": 0.4666666666666667, + "Chinese,Malay,English": 0.3333333333333333, + "Chinese,Indonesian,English": 0.3333333333333333, + "Spanish,Malay,Indonesian": 0.34, + "Spanish,Malay,English": 0.31333333333333335, + "Spanish,Indonesian,English": 0.3333333333333333, + "Malay,Indonesian,English": 0.36 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.47333333333333333, - "Spanish,English,Chinese,Filipino": 0.4866666666666667, - "Spanish,English,Chinese,Indonesian": 0.49333333333333335, - "Spanish,English,Chinese,Malay": 0.46, - "Spanish,English,Vietnamese,Filipino": 0.5333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.5333333333333333, - "Spanish,English,Vietnamese,Malay": 0.52, - "Spanish,English,Filipino,Indonesian": 0.5666666666666667, - "Spanish,English,Filipino,Malay": 0.5733333333333334, - "Spanish,English,Indonesian,Malay": 0.56, - "Spanish,Chinese,Vietnamese,Filipino": 0.4666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.47333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.4533333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.47333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.44, - "Spanish,Chinese,Indonesian,Malay": 0.4666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.5333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.5266666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.54, - "Spanish,Filipino,Indonesian,Malay": 0.56, - "English,Chinese,Vietnamese,Filipino": 0.48, - "English,Chinese,Vietnamese,Indonesian": 0.5, - "English,Chinese,Vietnamese,Malay": 0.48, - "English,Chinese,Filipino,Indonesian": 0.49333333333333335, - "English,Chinese,Filipino,Malay": 0.47333333333333333, - "English,Chinese,Indonesian,Malay": 0.5, - "English,Vietnamese,Filipino,Indonesian": 0.56, - "English,Vietnamese,Filipino,Malay": 0.5533333333333333, - "English,Vietnamese,Indonesian,Malay": 0.5666666666666667, - "English,Filipino,Indonesian,Malay": 0.5866666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.48, - "Chinese,Vietnamese,Filipino,Malay": 0.46, - "Chinese,Vietnamese,Indonesian,Malay": 0.49333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.4666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.5733333333333334 + "Filipino,Vietnamese,Chinese,Spanish": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Malay": 0.32666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian": 0.31333333333333335, + "Filipino,Vietnamese,Chinese,English": 0.26, + "Filipino,Vietnamese,Spanish,Malay": 0.28, + "Filipino,Vietnamese,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Spanish,English": 0.24666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.32666666666666666, + "Filipino,Vietnamese,Malay,English": 0.2733333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.26, + "Filipino,Chinese,Spanish,Malay": 0.30666666666666664, + "Filipino,Chinese,Spanish,Indonesian": 0.29333333333333333, + "Filipino,Chinese,Spanish,English": 0.26, + "Filipino,Chinese,Malay,Indonesian": 0.38666666666666666, + "Filipino,Chinese,Malay,English": 0.2866666666666667, + "Filipino,Chinese,Indonesian,English": 0.29333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.30666666666666664, + "Filipino,Spanish,Malay,English": 0.28, + "Filipino,Spanish,Indonesian,English": 0.29333333333333333, + "Filipino,Malay,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish,Malay": 0.28, + "Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,English": 0.25333333333333335, + "Vietnamese,Chinese,Malay,Indonesian": 0.36, + "Vietnamese,Chinese,Malay,English": 0.26666666666666666, + "Vietnamese,Chinese,Indonesian,English": 0.26, + "Vietnamese,Spanish,Malay,Indonesian": 0.26, + "Vietnamese,Spanish,Malay,English": 0.24, + "Vietnamese,Spanish,Indonesian,English": 0.22, + "Vietnamese,Malay,Indonesian,English": 0.2733333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, + "Chinese,Spanish,Malay,English": 0.24666666666666667, + "Chinese,Spanish,Indonesian,English": 0.24666666666666667, + "Chinese,Malay,Indonesian,English": 0.29333333333333333, + "Spanish,Malay,Indonesian,English": 0.24666666666666667 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.43333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.44, - "Spanish,English,Chinese,Vietnamese,Malay": 0.42, - "Spanish,English,Chinese,Filipino,Indonesian": 0.44666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.42, - "Spanish,English,Chinese,Indonesian,Malay": 0.43333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.5, - "Spanish,English,Vietnamese,Filipino,Malay": 0.49333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.49333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.52, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.4266666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.41333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.41333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.5, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.44666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.43333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.46, - "English,Chinese,Filipino,Indonesian,Malay": 0.44666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.5266666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.44 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Malay,English": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.23333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Filipino,Chinese,Spanish,Malay,English": 0.22666666666666666, + "Filipino,Chinese,Spanish,Indonesian,English": 0.22666666666666666, + "Filipino,Chinese,Malay,Indonesian,English": 0.26666666666666666, + "Filipino,Spanish,Malay,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Malay,English": 0.2, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.19333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.24, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.18666666666666668, + "Chinese,Spanish,Malay,Indonesian,English": 0.22 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.4066666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.3933333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.4, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.4, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.47333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3933333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.42 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.18, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.38 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16 } }, - "AC3_2": 0.5460224706828904, - "AC3_3": 0.5050295304599008, - "AC3_4": 0.47453118305283565, - "AC3_5": 0.4493179256367052, - "AC3_6": 0.4279597883098597, - "AC3_7": 0.40983833713275636 + "AC3_2": 0.3966348969237751, + "AC3_3": 0.33894762032769654, + "AC3_4": 0.29679188369021525, + "AC3_5": 0.2644487168282948, + "AC3_6": 0.2371967654516605, + "AC3_7": 0.2120481927263754 }, "prompt_3": { - "overall_acc": 0.43714285714285717, + "overall_acc": 0.30666666666666664, "language_acc": { - "Spanish": 0.37333333333333335, - "English": 0.4533333333333333, - "Chinese": 0.47333333333333333, - "Vietnamese": 0.4533333333333333, - "Filipino": 0.4066666666666667, - "Indonesian": 0.4266666666666667, - "Malay": 0.47333333333333333 + "Filipino": 0.29333333333333333, + "Vietnamese": 0.24666666666666667, + "Chinese": 0.2733333333333333, + "Spanish": 0.34, + "Malay": 0.24666666666666667, + "Indonesian": 0.30666666666666664, + "English": 0.44 }, - "consistency_score_2": 0.6971428571428572, - "consistency_score_3": 0.5697142857142857, - "consistency_score_4": 0.49561904761904757, - "consistency_score_5": 0.4453968253968254, - "consistency_score_6": 0.40857142857142864, - "consistency_score_7": 0.38, + "consistency_score_2": 0.567936507936508, + "consistency_score_3": 0.4022857142857142, + "consistency_score_4": 0.3154285714285714, + "consistency_score_5": 0.26412698412698415, + "consistency_score_6": 0.23047619047619047, + "consistency_score_7": 0.20666666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.74, - "Spanish,Chinese": 0.6333333333333333, - "Spanish,Vietnamese": 0.64, - "Spanish,Filipino": 0.72, - "Spanish,Indonesian": 0.7266666666666667, - "Spanish,Malay": 0.7, - "English,Chinese": 0.6933333333333334, - "English,Vietnamese": 0.7133333333333334, - "English,Filipino": 0.7666666666666667, - "English,Indonesian": 0.6933333333333334, - "English,Malay": 0.76, - "Chinese,Vietnamese": 0.68, - "Chinese,Filipino": 0.6266666666666667, - "Chinese,Indonesian": 0.6866666666666666, - "Chinese,Malay": 0.58, - "Vietnamese,Filipino": 0.6666666666666666, - "Vietnamese,Indonesian": 0.7333333333333333, - "Vietnamese,Malay": 0.7, - "Filipino,Indonesian": 0.6933333333333334, - "Filipino,Malay": 0.72, - "Indonesian,Malay": 0.7666666666666667 + "Filipino,Vietnamese": 0.5533333333333333, + "Filipino,Chinese": 0.5933333333333334, + "Filipino,Spanish": 0.5533333333333333, + "Filipino,Malay": 0.6733333333333333, + "Filipino,Indonesian": 0.62, + "Filipino,English": 0.49333333333333335, + "Vietnamese,Chinese": 0.6666666666666666, + "Vietnamese,Spanish": 0.5533333333333333, + "Vietnamese,Malay": 0.6066666666666667, + "Vietnamese,Indonesian": 0.5866666666666667, + "Vietnamese,English": 0.48, + "Chinese,Spanish": 0.5333333333333333, + "Chinese,Malay": 0.6466666666666666, + "Chinese,Indonesian": 0.6066666666666667, + "Chinese,English": 0.48, + "Spanish,Malay": 0.5, + "Spanish,Indonesian": 0.5066666666666667, + "Spanish,English": 0.58, + "Malay,Indonesian": 0.7066666666666667, + "Malay,English": 0.5066666666666667, + "Indonesian,English": 0.48 }, "3_combine": { - "Spanish,English,Chinese": 0.5533333333333333, - "Spanish,English,Vietnamese": 0.58, - "Spanish,English,Filipino": 0.6266666666666667, - "Spanish,English,Indonesian": 0.6066666666666667, - "Spanish,English,Malay": 0.6133333333333333, - "Spanish,Chinese,Vietnamese": 0.5066666666666667, - "Spanish,Chinese,Filipino": 0.5133333333333333, - "Spanish,Chinese,Indonesian": 0.56, - "Spanish,Chinese,Malay": 0.48, - "Spanish,Vietnamese,Filipino": 0.54, - "Spanish,Vietnamese,Indonesian": 0.58, - "Spanish,Vietnamese,Malay": 0.54, - "Spanish,Filipino,Indonesian": 0.5933333333333334, - "Spanish,Filipino,Malay": 0.5933333333333334, - "Spanish,Indonesian,Malay": 0.6133333333333333, - "English,Chinese,Vietnamese": 0.5666666666666667, - "English,Chinese,Filipino": 0.5666666666666667, - "English,Chinese,Indonesian": 0.5666666666666667, - "English,Chinese,Malay": 0.54, - "English,Vietnamese,Filipino": 0.5933333333333334, - "English,Vietnamese,Indonesian": 0.6, - "English,Vietnamese,Malay": 0.6, - "English,Filipino,Indonesian": 0.6066666666666667, - "English,Filipino,Malay": 0.6333333333333333, - "English,Indonesian,Malay": 0.6266666666666667, - "Chinese,Vietnamese,Filipino": 0.5066666666666667, - "Chinese,Vietnamese,Indonesian": 0.5733333333333334, - "Chinese,Vietnamese,Malay": 0.5066666666666667, - "Chinese,Filipino,Indonesian": 0.5266666666666666, - "Chinese,Filipino,Malay": 0.4866666666666667, - "Chinese,Indonesian,Malay": 0.54, - "Vietnamese,Filipino,Indonesian": 0.5866666666666667, - "Vietnamese,Filipino,Malay": 0.5733333333333334, - "Vietnamese,Indonesian,Malay": 0.6266666666666667, - "Filipino,Indonesian,Malay": 0.6133333333333333 + "Filipino,Vietnamese,Chinese": 0.46, + "Filipino,Vietnamese,Spanish": 0.38666666666666666, + "Filipino,Vietnamese,Malay": 0.4666666666666667, + "Filipino,Vietnamese,Indonesian": 0.4266666666666667, + "Filipino,Vietnamese,English": 0.32666666666666666, + "Filipino,Chinese,Spanish": 0.4, + "Filipino,Chinese,Malay": 0.49333333333333335, + "Filipino,Chinese,Indonesian": 0.4666666666666667, + "Filipino,Chinese,English": 0.34, + "Filipino,Spanish,Malay": 0.41333333333333333, + "Filipino,Spanish,Indonesian": 0.38666666666666666, + "Filipino,Spanish,English": 0.37333333333333335, + "Filipino,Malay,Indonesian": 0.54, + "Filipino,Malay,English": 0.38, + "Filipino,Indonesian,English": 0.36666666666666664, + "Vietnamese,Chinese,Spanish": 0.4266666666666667, + "Vietnamese,Chinese,Malay": 0.5, + "Vietnamese,Chinese,Indonesian": 0.4666666666666667, + "Vietnamese,Chinese,English": 0.36, + "Vietnamese,Spanish,Malay": 0.38, + "Vietnamese,Spanish,Indonesian": 0.36, + "Vietnamese,Spanish,English": 0.3466666666666667, + "Vietnamese,Malay,Indonesian": 0.48, + "Vietnamese,Malay,English": 0.36, + "Vietnamese,Indonesian,English": 0.32666666666666666, + "Chinese,Spanish,Malay": 0.4, + "Chinese,Spanish,Indonesian": 0.38666666666666666, + "Chinese,Spanish,English": 0.36, + "Chinese,Malay,Indonesian": 0.5066666666666667, + "Chinese,Malay,English": 0.36666666666666664, + "Chinese,Indonesian,English": 0.3466666666666667, + "Spanish,Malay,Indonesian": 0.38666666666666666, + "Spanish,Malay,English": 0.35333333333333333, + "Spanish,Indonesian,English": 0.3466666666666667, + "Malay,Indonesian,English": 0.3933333333333333 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4666666666666667, - "Spanish,English,Chinese,Filipino": 0.4866666666666667, - "Spanish,English,Chinese,Indonesian": 0.5066666666666667, - "Spanish,English,Chinese,Malay": 0.46, - "Spanish,English,Vietnamese,Filipino": 0.5066666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.5266666666666666, - "Spanish,English,Vietnamese,Malay": 0.5133333333333333, - "Spanish,English,Filipino,Indonesian": 0.5466666666666666, - "Spanish,English,Filipino,Malay": 0.5533333333333333, - "Spanish,English,Indonesian,Malay": 0.5466666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.43333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian": 0.47333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.42, - "Spanish,Chinese,Filipino,Indonesian": 0.4666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.4266666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.4666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.5066666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.49333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.5133333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.54, - "English,Chinese,Vietnamese,Filipino": 0.47333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.49333333333333335, - "English,Chinese,Vietnamese,Malay": 0.47333333333333333, - "English,Chinese,Filipino,Indonesian": 0.49333333333333335, - "English,Chinese,Filipino,Malay": 0.48, - "English,Chinese,Indonesian,Malay": 0.5066666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.5333333333333333, - "English,Vietnamese,Filipino,Malay": 0.54, - "English,Vietnamese,Indonesian,Malay": 0.5533333333333333, - "English,Filipino,Indonesian,Malay": 0.5666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.4533333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.43333333333333335, - "Chinese,Vietnamese,Indonesian,Malay": 0.48, - "Chinese,Filipino,Indonesian,Malay": 0.4666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.5466666666666666 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.41333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.44, - "Spanish,English,Chinese,Vietnamese,Malay": 0.4066666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.4533333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.4266666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.44666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.48, - "Spanish,English,Vietnamese,Filipino,Malay": 0.48, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.4866666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.5133333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.4066666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.38666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.41333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.42, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.48, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.4266666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.4266666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.4533333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.46, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.5133333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.42 + "Filipino,Vietnamese,Chinese,Spanish": 0.3333333333333333, + "Filipino,Vietnamese,Chinese,Malay": 0.4, + "Filipino,Vietnamese,Chinese,Indonesian": 0.38, + "Filipino,Vietnamese,Chinese,English": 0.2733333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.3333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Spanish,English": 0.26, + "Filipino,Vietnamese,Malay,Indonesian": 0.3933333333333333, + "Filipino,Vietnamese,Malay,English": 0.30666666666666664, + "Filipino,Vietnamese,Indonesian,English": 0.28, + "Filipino,Chinese,Spanish,Malay": 0.3466666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.32666666666666666, + "Filipino,Chinese,Spanish,English": 0.2733333333333333, + "Filipino,Chinese,Malay,Indonesian": 0.4266666666666667, + "Filipino,Chinese,Malay,English": 0.30666666666666664, + "Filipino,Chinese,Indonesian,English": 0.3, + "Filipino,Spanish,Malay,Indonesian": 0.3333333333333333, + "Filipino,Spanish,Malay,English": 0.29333333333333333, + "Filipino,Spanish,Indonesian,English": 0.28, + "Filipino,Malay,Indonesian,English": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Malay": 0.34, + "Vietnamese,Chinese,Spanish,Indonesian": 0.3333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.28, + "Vietnamese,Chinese,Malay,Indonesian": 0.4066666666666667, + "Vietnamese,Chinese,Malay,English": 0.30666666666666664, + "Vietnamese,Chinese,Indonesian,English": 0.28, + "Vietnamese,Spanish,Malay,Indonesian": 0.3, + "Vietnamese,Spanish,Malay,English": 0.26666666666666666, + "Vietnamese,Spanish,Indonesian,English": 0.24, + "Vietnamese,Malay,Indonesian,English": 0.3, + "Chinese,Spanish,Malay,Indonesian": 0.3333333333333333, + "Chinese,Spanish,Malay,English": 0.28, + "Chinese,Spanish,Indonesian,English": 0.28, + "Chinese,Malay,Indonesian,English": 0.32, + "Spanish,Malay,Indonesian,English": 0.29333333333333333 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.3, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Spanish,Malay,English": 0.24, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.22, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.26666666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.3, + "Filipino,Chinese,Spanish,Malay,English": 0.24666666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.24, + "Filipino,Chinese,Malay,Indonesian,English": 0.2866666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.25333333333333335, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, + "Vietnamese,Chinese,Spanish,Malay,English": 0.24, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.2733333333333333, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.22666666666666666, + "Chinese,Spanish,Malay,Indonesian,English": 0.26 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.3933333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.38666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.4, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.42, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.4666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.38, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.41333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.22, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.24666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.38 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667 } }, - "AC3_2": 0.5373443684304968, - "AC3_3": 0.4947008269331921, - "AC3_4": 0.46454797392843394, - "AC3_5": 0.4412312435265718, - "AC3_6": 0.42237451732457443, - "AC3_7": 0.40657342652367107 + "AC3_2": 0.39827707194478257, + "AC3_3": 0.34802794191759406, + "AC3_4": 0.3109859154429676, + "AC3_5": 0.2838116425160916, + "AC3_6": 0.2631678486507696, + "AC3_7": 0.24692640687830436 }, "prompt_4": { - "overall_acc": 0.4390476190476191, + "overall_acc": 0.3152380952380952, "language_acc": { - "Spanish": 0.4, - "English": 0.46, - "Chinese": 0.43333333333333335, - "Vietnamese": 0.46, - "Filipino": 0.4266666666666667, - "Indonesian": 0.41333333333333333, - "Malay": 0.48 + "Filipino": 0.29333333333333333, + "Vietnamese": 0.28, + "Chinese": 0.2733333333333333, + "Spanish": 0.34, + "Malay": 0.24, + "Indonesian": 0.32666666666666666, + "English": 0.4533333333333333 }, - "consistency_score_2": 0.6946031746031747, - "consistency_score_3": 0.5664761904761905, - "consistency_score_4": 0.49085714285714277, - "consistency_score_5": 0.43936507936507935, - "consistency_score_6": 0.40190476190476193, - "consistency_score_7": 0.37333333333333335, + "consistency_score_2": 0.5666666666666667, + "consistency_score_3": 0.4081904761904762, + "consistency_score_4": 0.3259047619047619, + "consistency_score_5": 0.27650793650793654, + "consistency_score_6": 0.24380952380952384, + "consistency_score_7": 0.22, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.76, - "Spanish,Chinese": 0.6266666666666667, - "Spanish,Vietnamese": 0.66, - "Spanish,Filipino": 0.7333333333333333, - "Spanish,Indonesian": 0.74, - "Spanish,Malay": 0.6933333333333334, - "English,Chinese": 0.66, - "English,Vietnamese": 0.72, - "English,Filipino": 0.7666666666666667, - "English,Indonesian": 0.7466666666666667, - "English,Malay": 0.76, - "Chinese,Vietnamese": 0.68, - "Chinese,Filipino": 0.6, - "Chinese,Indonesian": 0.6666666666666666, - "Chinese,Malay": 0.5733333333333334, - "Vietnamese,Filipino": 0.6733333333333333, - "Vietnamese,Indonesian": 0.72, - "Vietnamese,Malay": 0.7066666666666667, - "Filipino,Indonesian": 0.68, - "Filipino,Malay": 0.6933333333333334, - "Indonesian,Malay": 0.7266666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.54, - "Spanish,English,Vietnamese": 0.5933333333333334, - "Spanish,English,Filipino": 0.6466666666666666, - "Spanish,English,Indonesian": 0.6533333333333333, - "Spanish,English,Malay": 0.6266666666666667, - "Spanish,Chinese,Vietnamese": 0.52, - "Spanish,Chinese,Filipino": 0.5133333333333333, - "Spanish,Chinese,Indonesian": 0.5466666666666666, - "Spanish,Chinese,Malay": 0.4666666666666667, - "Spanish,Vietnamese,Filipino": 0.56, - "Spanish,Vietnamese,Indonesian": 0.5933333333333334, - "Spanish,Vietnamese,Malay": 0.5466666666666666, - "Spanish,Filipino,Indonesian": 0.6, - "Spanish,Filipino,Malay": 0.5866666666666667, - "Spanish,Indonesian,Malay": 0.6066666666666667, - "English,Chinese,Vietnamese": 0.5533333333333333, - "English,Chinese,Filipino": 0.5333333333333333, - "English,Chinese,Indonesian": 0.56, - "English,Chinese,Malay": 0.5133333333333333, - "English,Vietnamese,Filipino": 0.5933333333333334, - "English,Vietnamese,Indonesian": 0.62, - "English,Vietnamese,Malay": 0.6066666666666667, - "English,Filipino,Indonesian": 0.62, - "English,Filipino,Malay": 0.6266666666666667, - "English,Indonesian,Malay": 0.6266666666666667, - "Chinese,Vietnamese,Filipino": 0.5066666666666667, - "Chinese,Vietnamese,Indonesian": 0.5533333333333333, - "Chinese,Vietnamese,Malay": 0.5133333333333333, - "Chinese,Filipino,Indonesian": 0.5066666666666667, - "Chinese,Filipino,Malay": 0.4533333333333333, - "Chinese,Indonesian,Malay": 0.5066666666666667, - "Vietnamese,Filipino,Indonesian": 0.5733333333333334, - "Vietnamese,Filipino,Malay": 0.5733333333333334, - "Vietnamese,Indonesian,Malay": 0.6066666666666667, - "Filipino,Indonesian,Malay": 0.58 + "Filipino,Vietnamese": 0.5266666666666666, + "Filipino,Chinese": 0.6, + "Filipino,Spanish": 0.5866666666666667, + "Filipino,Malay": 0.68, + "Filipino,Indonesian": 0.62, + "Filipino,English": 0.5466666666666666, + "Vietnamese,Chinese": 0.6466666666666666, + "Vietnamese,Spanish": 0.5533333333333333, + "Vietnamese,Malay": 0.5666666666666667, + "Vietnamese,Indonesian": 0.5333333333333333, + "Vietnamese,English": 0.47333333333333333, + "Chinese,Spanish": 0.52, + "Chinese,Malay": 0.64, + "Chinese,Indonesian": 0.6, + "Chinese,English": 0.48, + "Spanish,Malay": 0.5066666666666667, + "Spanish,Indonesian": 0.5733333333333334, + "Spanish,English": 0.5866666666666667, + "Malay,Indonesian": 0.66, + "Malay,English": 0.4866666666666667, + "Indonesian,English": 0.5133333333333333 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.44, + "Filipino,Vietnamese,Spanish": 0.4066666666666667, + "Filipino,Vietnamese,Malay": 0.44, + "Filipino,Vietnamese,Indonesian": 0.4, + "Filipino,Vietnamese,English": 0.34, + "Filipino,Chinese,Spanish": 0.42, + "Filipino,Chinese,Malay": 0.5, + "Filipino,Chinese,Indonesian": 0.46, + "Filipino,Chinese,English": 0.38, + "Filipino,Spanish,Malay": 0.44, + "Filipino,Spanish,Indonesian": 0.44, + "Filipino,Spanish,English": 0.44, + "Filipino,Malay,Indonesian": 0.52, + "Filipino,Malay,English": 0.41333333333333333, + "Filipino,Indonesian,English": 0.4066666666666667, + "Vietnamese,Chinese,Spanish": 0.4, + "Vietnamese,Chinese,Malay": 0.48, + "Vietnamese,Chinese,Indonesian": 0.4266666666666667, + "Vietnamese,Chinese,English": 0.3466666666666667, + "Vietnamese,Spanish,Malay": 0.36666666666666664, + "Vietnamese,Spanish,Indonesian": 0.36666666666666664, + "Vietnamese,Spanish,English": 0.37333333333333335, + "Vietnamese,Malay,Indonesian": 0.44, + "Vietnamese,Malay,English": 0.3333333333333333, + "Vietnamese,Indonesian,English": 0.32, + "Chinese,Spanish,Malay": 0.38666666666666666, + "Chinese,Spanish,Indonesian": 0.4066666666666667, + "Chinese,Spanish,English": 0.3933333333333333, + "Chinese,Malay,Indonesian": 0.49333333333333335, + "Chinese,Malay,English": 0.36666666666666664, + "Chinese,Indonesian,English": 0.38, + "Spanish,Malay,Indonesian": 0.4066666666666667, + "Spanish,Malay,English": 0.36, + "Spanish,Indonesian,English": 0.38666666666666666, + "Malay,Indonesian,English": 0.4066666666666667 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.48, - "Spanish,English,Chinese,Filipino": 0.47333333333333333, - "Spanish,English,Chinese,Indonesian": 0.5066666666666667, - "Spanish,English,Chinese,Malay": 0.44666666666666666, - "Spanish,English,Vietnamese,Filipino": 0.5266666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.5533333333333333, - "Spanish,English,Vietnamese,Malay": 0.5133333333333333, - "Spanish,English,Filipino,Indonesian": 0.5666666666666667, - "Spanish,English,Filipino,Malay": 0.5533333333333333, - "Spanish,English,Indonesian,Malay": 0.56, - "Spanish,Chinese,Vietnamese,Filipino": 0.4533333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.47333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.43333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.4533333333333333, - "Spanish,Chinese,Filipino,Malay": 0.4066666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.44666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.5133333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.49333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.52, - "Spanish,Filipino,Indonesian,Malay": 0.52, - "English,Chinese,Vietnamese,Filipino": 0.47333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.49333333333333335, - "English,Chinese,Vietnamese,Malay": 0.47333333333333333, - "English,Chinese,Filipino,Indonesian": 0.48, - "English,Chinese,Filipino,Malay": 0.44, - "English,Chinese,Indonesian,Malay": 0.47333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.5333333333333333, - "English,Vietnamese,Filipino,Malay": 0.5333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.5533333333333333, - "English,Filipino,Indonesian,Malay": 0.54, - "Chinese,Vietnamese,Filipino,Indonesian": 0.4533333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.43333333333333335, - "Chinese,Vietnamese,Indonesian,Malay": 0.4666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.42, - "Vietnamese,Filipino,Indonesian,Malay": 0.52 + "Filipino,Vietnamese,Chinese,Spanish": 0.34, + "Filipino,Vietnamese,Chinese,Malay": 0.38666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,Chinese,English": 0.29333333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.3466666666666667, + "Filipino,Vietnamese,Spanish,Indonesian": 0.32666666666666666, + "Filipino,Vietnamese,Spanish,English": 0.3, + "Filipino,Vietnamese,Malay,Indonesian": 0.36666666666666664, + "Filipino,Vietnamese,Malay,English": 0.3, + "Filipino,Vietnamese,Indonesian,English": 0.2866666666666667, + "Filipino,Chinese,Spanish,Malay": 0.35333333333333333, + "Filipino,Chinese,Spanish,Indonesian": 0.36, + "Filipino,Chinese,Spanish,English": 0.32666666666666666, + "Filipino,Chinese,Malay,Indonesian": 0.42, + "Filipino,Chinese,Malay,English": 0.32666666666666666, + "Filipino,Chinese,Indonesian,English": 0.3333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.36666666666666664, + "Filipino,Spanish,Malay,English": 0.3333333333333333, + "Filipino,Spanish,Indonesian,English": 0.3466666666666667, + "Filipino,Malay,Indonesian,English": 0.35333333333333333, + "Vietnamese,Chinese,Spanish,Malay": 0.32, + "Vietnamese,Chinese,Spanish,Indonesian": 0.30666666666666664, + "Vietnamese,Chinese,Spanish,English": 0.29333333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.37333333333333335, + "Vietnamese,Chinese,Malay,English": 0.2866666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.28, + "Vietnamese,Spanish,Malay,Indonesian": 0.30666666666666664, + "Vietnamese,Spanish,Malay,English": 0.26666666666666666, + "Vietnamese,Spanish,Indonesian,English": 0.25333333333333335, + "Vietnamese,Malay,Indonesian,English": 0.3, + "Chinese,Spanish,Malay,Indonesian": 0.3466666666666667, + "Chinese,Spanish,Malay,English": 0.29333333333333333, + "Chinese,Spanish,Indonesian,English": 0.32, + "Chinese,Malay,Indonesian,English": 0.3333333333333333, + "Spanish,Malay,Indonesian,English": 0.30666666666666664 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.43333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.4533333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.42, - "Spanish,English,Chinese,Filipino,Indonesian": 0.44, - "Spanish,English,Chinese,Filipino,Malay": 0.4, - "Spanish,English,Chinese,Indonesian,Malay": 0.43333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.49333333333333335, - "Spanish,English,Vietnamese,Filipino,Malay": 0.47333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.49333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.5, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.41333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.3933333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.41333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.38666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.4666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.43333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.42, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.44666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.41333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.49333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.4066666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.26, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.3333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.26666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.29333333333333333, + "Filipino,Vietnamese,Spanish,Malay,English": 0.26, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.2733333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.32, + "Filipino,Chinese,Spanish,Malay,English": 0.2733333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.29333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.3, + "Filipino,Spanish,Malay,Indonesian,English": 0.29333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.28, + "Vietnamese,Chinese,Spanish,Malay,English": 0.24, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.26666666666666666, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.23333333333333334, + "Chinese,Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.4066666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.38666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.38666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.4533333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.37333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.4 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.23333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.37333333333333335 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22 } }, - "AC3_2": 0.5380208291187075, - "AC3_3": 0.49468748585632266, - "AC3_4": 0.4635091004359046, - "AC3_5": 0.4392062917909142, - "AC3_6": 0.4196559348042204, - "AC3_7": 0.40353262988389493 + "AC3_2": 0.4051115910267782, + "AC3_3": 0.3557426213806507, + "AC3_4": 0.32048269807977076, + "AC3_5": 0.29460555891199636, + "AC3_6": 0.274960655422545, + "AC3_7": 0.2591459074248927 }, "prompt_5": { - "overall_acc": 0.4361904761904762, + "overall_acc": 0.31523809523809526, "language_acc": { - "Spanish": 0.38666666666666666, - "English": 0.4666666666666667, - "Chinese": 0.44666666666666666, - "Vietnamese": 0.44, - "Filipino": 0.44, - "Indonesian": 0.4066666666666667, - "Malay": 0.4666666666666667 + "Filipino": 0.30666666666666664, + "Vietnamese": 0.25333333333333335, + "Chinese": 0.26, + "Spanish": 0.3466666666666667, + "Malay": 0.26, + "Indonesian": 0.28, + "English": 0.5 }, - "consistency_score_2": 0.7000000000000002, - "consistency_score_3": 0.5779047619047619, - "consistency_score_4": 0.5095238095238096, - "consistency_score_5": 0.4647619047619048, - "consistency_score_6": 0.4323809523809525, - "consistency_score_7": 0.4066666666666667, + "consistency_score_2": 0.5269841269841271, + "consistency_score_3": 0.3487619047619048, + "consistency_score_4": 0.25904761904761914, + "consistency_score_5": 0.206984126984127, + "consistency_score_6": 0.1742857142857143, + "consistency_score_7": 0.15333333333333332, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.7666666666666667, - "Spanish,Chinese": 0.6533333333333333, - "Spanish,Vietnamese": 0.6533333333333333, - "Spanish,Filipino": 0.7066666666666667, - "Spanish,Indonesian": 0.7533333333333333, - "Spanish,Malay": 0.7266666666666667, - "English,Chinese": 0.66, - "English,Vietnamese": 0.72, - "English,Filipino": 0.7533333333333333, - "English,Indonesian": 0.7466666666666667, - "English,Malay": 0.78, - "Chinese,Vietnamese": 0.6533333333333333, - "Chinese,Filipino": 0.6066666666666667, - "Chinese,Indonesian": 0.6466666666666666, - "Chinese,Malay": 0.5733333333333334, - "Vietnamese,Filipino": 0.6733333333333333, - "Vietnamese,Indonesian": 0.7266666666666667, - "Vietnamese,Malay": 0.7066666666666667, - "Filipino,Indonesian": 0.6933333333333334, - "Filipino,Malay": 0.7333333333333333, - "Indonesian,Malay": 0.7666666666666667 + "Filipino,Vietnamese": 0.5, + "Filipino,Chinese": 0.54, + "Filipino,Spanish": 0.48, + "Filipino,Malay": 0.6066666666666667, + "Filipino,Indonesian": 0.5866666666666667, + "Filipino,English": 0.47333333333333333, + "Vietnamese,Chinese": 0.6266666666666667, + "Vietnamese,Spanish": 0.48, + "Vietnamese,Malay": 0.6066666666666667, + "Vietnamese,Indonesian": 0.5066666666666667, + "Vietnamese,English": 0.4533333333333333, + "Chinese,Spanish": 0.4666666666666667, + "Chinese,Malay": 0.6333333333333333, + "Chinese,Indonesian": 0.5466666666666666, + "Chinese,English": 0.46, + "Spanish,Malay": 0.42, + "Spanish,Indonesian": 0.48, + "Spanish,English": 0.56, + "Malay,Indonesian": 0.6733333333333333, + "Malay,English": 0.4866666666666667, + "Indonesian,English": 0.48 }, "3_combine": { - "Spanish,English,Chinese": 0.56, - "Spanish,English,Vietnamese": 0.6, - "Spanish,English,Filipino": 0.6266666666666667, - "Spanish,English,Indonesian": 0.66, - "Spanish,English,Malay": 0.6533333333333333, - "Spanish,Chinese,Vietnamese": 0.52, - "Spanish,Chinese,Filipino": 0.52, - "Spanish,Chinese,Indonesian": 0.5533333333333333, - "Spanish,Chinese,Malay": 0.5066666666666667, - "Spanish,Vietnamese,Filipino": 0.5533333333333333, - "Spanish,Vietnamese,Indonesian": 0.6, - "Spanish,Vietnamese,Malay": 0.5666666666666667, - "Spanish,Filipino,Indonesian": 0.6, - "Spanish,Filipino,Malay": 0.6066666666666667, - "Spanish,Indonesian,Malay": 0.6333333333333333, - "English,Chinese,Vietnamese": 0.54, - "English,Chinese,Filipino": 0.54, - "English,Chinese,Indonesian": 0.56, - "English,Chinese,Malay": 0.5333333333333333, - "English,Vietnamese,Filipino": 0.6, - "English,Vietnamese,Indonesian": 0.6333333333333333, - "English,Vietnamese,Malay": 0.62, - "English,Filipino,Indonesian": 0.6133333333333333, - "English,Filipino,Malay": 0.64, - "English,Indonesian,Malay": 0.66, - "Chinese,Vietnamese,Filipino": 0.5066666666666667, - "Chinese,Vietnamese,Indonesian": 0.5466666666666666, - "Chinese,Vietnamese,Malay": 0.5066666666666667, - "Chinese,Filipino,Indonesian": 0.52, - "Chinese,Filipino,Malay": 0.48, - "Chinese,Indonesian,Malay": 0.5333333333333333, - "Vietnamese,Filipino,Indonesian": 0.5933333333333334, - "Vietnamese,Filipino,Malay": 0.5866666666666667, - "Vietnamese,Indonesian,Malay": 0.6333333333333333, - "Filipino,Indonesian,Malay": 0.62 + "Filipino,Vietnamese,Chinese": 0.3933333333333333, + "Filipino,Vietnamese,Spanish": 0.30666666666666664, + "Filipino,Vietnamese,Malay": 0.4, + "Filipino,Vietnamese,Indonesian": 0.3466666666666667, + "Filipino,Vietnamese,English": 0.3, + "Filipino,Chinese,Spanish": 0.32, + "Filipino,Chinese,Malay": 0.43333333333333335, + "Filipino,Chinese,Indonesian": 0.3933333333333333, + "Filipino,Chinese,English": 0.30666666666666664, + "Filipino,Spanish,Malay": 0.3333333333333333, + "Filipino,Spanish,Indonesian": 0.34, + "Filipino,Spanish,English": 0.3333333333333333, + "Filipino,Malay,Indonesian": 0.4666666666666667, + "Filipino,Malay,English": 0.34, + "Filipino,Indonesian,English": 0.3333333333333333, + "Vietnamese,Chinese,Spanish": 0.3466666666666667, + "Vietnamese,Chinese,Malay": 0.48, + "Vietnamese,Chinese,Indonesian": 0.38, + "Vietnamese,Chinese,English": 0.32666666666666666, + "Vietnamese,Spanish,Malay": 0.31333333333333335, + "Vietnamese,Spanish,Indonesian": 0.2866666666666667, + "Vietnamese,Spanish,English": 0.30666666666666664, + "Vietnamese,Malay,Indonesian": 0.4266666666666667, + "Vietnamese,Malay,English": 0.34, + "Vietnamese,Indonesian,English": 0.29333333333333333, + "Chinese,Spanish,Malay": 0.3333333333333333, + "Chinese,Spanish,Indonesian": 0.30666666666666664, + "Chinese,Spanish,English": 0.32666666666666666, + "Chinese,Malay,Indonesian": 0.46, + "Chinese,Malay,English": 0.3466666666666667, + "Chinese,Indonesian,English": 0.31333333333333335, + "Spanish,Malay,Indonesian": 0.31333333333333335, + "Spanish,Malay,English": 0.3, + "Spanish,Indonesian,English": 0.29333333333333333, + "Malay,Indonesian,English": 0.36666666666666664 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.48, - "Spanish,English,Chinese,Filipino": 0.4866666666666667, - "Spanish,English,Chinese,Indonesian": 0.5133333333333333, - "Spanish,English,Chinese,Malay": 0.49333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.5333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.56, - "Spanish,English,Vietnamese,Malay": 0.54, - "Spanish,English,Filipino,Indonesian": 0.5666666666666667, - "Spanish,English,Filipino,Malay": 0.5666666666666667, - "Spanish,English,Indonesian,Malay": 0.5866666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.46, - "Spanish,Chinese,Vietnamese,Indonesian": 0.48, - "Spanish,Chinese,Vietnamese,Malay": 0.4533333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.48, - "Spanish,Chinese,Filipino,Malay": 0.44666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.48, - "Spanish,Vietnamese,Filipino,Indonesian": 0.5133333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.5133333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.54, - "Spanish,Filipino,Indonesian,Malay": 0.5533333333333333, - "English,Chinese,Vietnamese,Filipino": 0.48, - "English,Chinese,Vietnamese,Indonesian": 0.49333333333333335, - "English,Chinese,Vietnamese,Malay": 0.47333333333333333, - "English,Chinese,Filipino,Indonesian": 0.49333333333333335, - "English,Chinese,Filipino,Malay": 0.4666666666666667, - "English,Chinese,Indonesian,Malay": 0.5066666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.5466666666666666, - "English,Vietnamese,Filipino,Malay": 0.5466666666666666, - "English,Vietnamese,Indonesian,Malay": 0.5733333333333334, - "English,Filipino,Indonesian,Malay": 0.5666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.47333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.4533333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.48, - "Chinese,Filipino,Indonesian,Malay": 0.47333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.56 + "Filipino,Vietnamese,Chinese,Spanish": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Malay": 0.3333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian": 0.24, + "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, + "Filipino,Vietnamese,Malay,Indonesian": 0.31333333333333335, + "Filipino,Vietnamese,Malay,English": 0.26, + "Filipino,Vietnamese,Indonesian,English": 0.23333333333333334, + "Filipino,Chinese,Spanish,Malay": 0.2733333333333333, + "Filipino,Chinese,Spanish,Indonesian": 0.26, + "Filipino,Chinese,Spanish,English": 0.24, + "Filipino,Chinese,Malay,Indonesian": 0.3466666666666667, + "Filipino,Chinese,Malay,English": 0.26666666666666666, + "Filipino,Chinese,Indonesian,English": 0.24666666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.26666666666666666, + "Filipino,Spanish,Malay,English": 0.24666666666666667, + "Filipino,Spanish,Indonesian,English": 0.24, + "Filipino,Malay,Indonesian,English": 0.26, + "Vietnamese,Chinese,Spanish,Malay": 0.28, + "Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,English": 0.24, + "Vietnamese,Chinese,Malay,Indonesian": 0.34, + "Vietnamese,Chinese,Malay,English": 0.28, + "Vietnamese,Chinese,Indonesian,English": 0.24, + "Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, + "Vietnamese,Spanish,Malay,English": 0.22, + "Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, + "Vietnamese,Malay,Indonesian,English": 0.26, + "Chinese,Spanish,Malay,Indonesian": 0.26, + "Chinese,Spanish,Malay,English": 0.24, + "Chinese,Spanish,Indonesian,English": 0.22, + "Chinese,Malay,Indonesian,English": 0.2733333333333333, + "Spanish,Malay,Indonesian,English": 0.21333333333333335 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.44666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.4533333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.44, - "Spanish,English,Chinese,Filipino,Indonesian": 0.4666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.44, - "Spanish,English,Chinese,Indonesian,Malay": 0.4666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.5, - "Spanish,English,Vietnamese,Filipino,Malay": 0.5, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.5133333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.5266666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.43333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.42, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.44, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.44, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.49333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.4533333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.44, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.46, - "English,Chinese,Filipino,Indonesian,Malay": 0.46, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.52, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.44666666666666666 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Malay,English": 0.22666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.19333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.20666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, + "Filipino,Chinese,Spanish,Malay,English": 0.20666666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.19333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.21333333333333335, + "Filipino,Spanish,Malay,Indonesian,English": 0.18666666666666668, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22, + "Vietnamese,Chinese,Spanish,Malay,English": 0.2, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.22, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.16, + "Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.4266666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.41333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.43333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.48, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.41333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.43333333333333335 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.2, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.18, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.4066666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 } }, - "AC3_2": 0.5374685665914215, - "AC3_3": 0.497145719245456, - "AC3_4": 0.4700139068222136, - "AC3_5": 0.45002315508948454, - "AC3_6": 0.43427736001683476, - "AC3_7": 0.4209114877090067 + "AC3_2": 0.3944932062066381, + "AC3_3": 0.3311537305186601, + "AC3_4": 0.28439390344885984, + "AC3_5": 0.24989086693724047, + "AC3_6": 0.2244691494816607, + "AC3_7": 0.2063143630996009 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.4862012987012987, + "overall_acc": 0.3003246753246754, "language_acc": { - "English": 0.5056818181818182, - "Filipino": 0.5056818181818182, - "Spanish": 0.4772727272727273, - "Malay": 0.4772727272727273, - "Chinese": 0.48863636363636365, - "Indonesian": 0.4772727272727273, - "Vietnamese": 0.4715909090909091 + "Indonesian": 0.26136363636363635, + "English": 0.5113636363636364, + "Filipino": 0.2556818181818182, + "Spanish": 0.32954545454545453, + "Chinese": 0.24431818181818182, + "Malay": 0.24431818181818182, + "Vietnamese": 0.2556818181818182 }, - "consistency_score_2": 0.7194264069264069, - "consistency_score_3": 0.5957792207792205, - "consistency_score_4": 0.5186688311688311, - "consistency_score_5": 0.4637445887445888, - "consistency_score_6": 0.42126623376623373, - "consistency_score_7": 0.38636363636363635, + "consistency_score_2": 0.449404761904762, + "consistency_score_3": 0.2490259740259741, + "consistency_score_4": 0.14853896103896105, + "consistency_score_5": 0.09063852813852814, + "consistency_score_6": 0.05519480519480519, + "consistency_score_7": 0.03409090909090909, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.7443181818181818, - "English,Spanish": 0.7897727272727273, - "English,Malay": 0.7102272727272727, - "English,Chinese": 0.6761363636363636, - "English,Indonesian": 0.7386363636363636, - "English,Vietnamese": 0.7954545454545454, - "Filipino,Spanish": 0.75, - "Filipino,Malay": 0.6761363636363636, - "Filipino,Chinese": 0.6647727272727273, - "Filipino,Indonesian": 0.7386363636363636, - "Filipino,Vietnamese": 0.7215909090909091, - "Spanish,Malay": 0.7386363636363636, - "Spanish,Chinese": 0.6306818181818182, - "Spanish,Indonesian": 0.7215909090909091, - "Spanish,Vietnamese": 0.7727272727272727, - "Malay,Chinese": 0.6363636363636364, - "Malay,Indonesian": 0.7215909090909091, - "Malay,Vietnamese": 0.7386363636363636, - "Chinese,Indonesian": 0.6818181818181818, - "Chinese,Vietnamese": 0.6931818181818182, - "Indonesian,Vietnamese": 0.7670454545454546 + "Indonesian,English": 0.3352272727272727, + "Indonesian,Filipino": 0.5909090909090909, + "Indonesian,Spanish": 0.4659090909090909, + "Indonesian,Chinese": 0.48295454545454547, + "Indonesian,Malay": 0.6193181818181818, + "Indonesian,Vietnamese": 0.5056818181818182, + "English,Filipino": 0.3181818181818182, + "English,Spanish": 0.4602272727272727, + "English,Chinese": 0.2840909090909091, + "English,Malay": 0.32386363636363635, + "English,Vietnamese": 0.3409090909090909, + "Filipino,Spanish": 0.4147727272727273, + "Filipino,Chinese": 0.5056818181818182, + "Filipino,Malay": 0.5511363636363636, + "Filipino,Vietnamese": 0.5170454545454546, + "Spanish,Chinese": 0.3522727272727273, + "Spanish,Malay": 0.42613636363636365, + "Spanish,Vietnamese": 0.4090909090909091, + "Chinese,Malay": 0.44886363636363635, + "Chinese,Vietnamese": 0.5738636363636364, + "Malay,Vietnamese": 0.5113636363636364 }, "3_combine": { - "English,Filipino,Spanish": 0.6477272727272727, - "English,Filipino,Malay": 0.5795454545454546, - "English,Filipino,Chinese": 0.5568181818181818, - "English,Filipino,Indonesian": 0.6193181818181818, - "English,Filipino,Vietnamese": 0.6420454545454546, - "English,Spanish,Malay": 0.6306818181818182, - "English,Spanish,Chinese": 0.5681818181818182, - "English,Spanish,Indonesian": 0.6363636363636364, - "English,Spanish,Vietnamese": 0.6931818181818182, - "English,Malay,Chinese": 0.5284090909090909, - "English,Malay,Indonesian": 0.6022727272727273, - "English,Malay,Vietnamese": 0.6477272727272727, - "English,Chinese,Indonesian": 0.5625, - "English,Chinese,Vietnamese": 0.6022727272727273, - "English,Indonesian,Vietnamese": 0.6647727272727273, - "Filipino,Spanish,Malay": 0.6022727272727273, - "Filipino,Spanish,Chinese": 0.5397727272727273, - "Filipino,Spanish,Indonesian": 0.6193181818181818, - "Filipino,Spanish,Vietnamese": 0.6363636363636364, - "Filipino,Malay,Chinese": 0.5113636363636364, - "Filipino,Malay,Indonesian": 0.5852272727272727, - "Filipino,Malay,Vietnamese": 0.5965909090909091, - "Filipino,Chinese,Indonesian": 0.5568181818181818, - "Filipino,Chinese,Vietnamese": 0.5568181818181818, - "Filipino,Indonesian,Vietnamese": 0.6306818181818182, - "Spanish,Malay,Chinese": 0.5170454545454546, - "Spanish,Malay,Indonesian": 0.6079545454545454, - "Spanish,Malay,Vietnamese": 0.6420454545454546, - "Spanish,Chinese,Indonesian": 0.5284090909090909, - "Spanish,Chinese,Vietnamese": 0.5625, - "Spanish,Indonesian,Vietnamese": 0.6420454545454546, - "Malay,Chinese,Indonesian": 0.5397727272727273, - "Malay,Chinese,Vietnamese": 0.5625, - "Malay,Indonesian,Vietnamese": 0.6420454545454546, - "Chinese,Indonesian,Vietnamese": 0.5909090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.5340909090909091, - "English,Filipino,Spanish,Chinese": 0.4943181818181818, - "English,Filipino,Spanish,Indonesian": 0.5625, - "English,Filipino,Spanish,Vietnamese": 0.5852272727272727, - "English,Filipino,Malay,Chinese": 0.45454545454545453, - "English,Filipino,Malay,Indonesian": 0.5113636363636364, - "English,Filipino,Malay,Vietnamese": 0.5397727272727273, - "English,Filipino,Chinese,Indonesian": 0.4943181818181818, - "English,Filipino,Chinese,Vietnamese": 0.5170454545454546, - "English,Filipino,Indonesian,Vietnamese": 0.5738636363636364, - "English,Spanish,Malay,Chinese": 0.4715909090909091, - "English,Spanish,Malay,Indonesian": 0.5511363636363636, - "English,Spanish,Malay,Vietnamese": 0.5909090909090909, - "English,Spanish,Chinese,Indonesian": 0.4943181818181818, - "English,Spanish,Chinese,Vietnamese": 0.5340909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.5965909090909091, - "English,Malay,Chinese,Indonesian": 0.4715909090909091, - "English,Malay,Chinese,Vietnamese": 0.5170454545454546, - "English,Malay,Indonesian,Vietnamese": 0.5625, - "English,Chinese,Indonesian,Vietnamese": 0.5397727272727273, - "Filipino,Spanish,Malay,Chinese": 0.4602272727272727, - "Filipino,Spanish,Malay,Indonesian": 0.5284090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.5454545454545454, - "Filipino,Spanish,Chinese,Indonesian": 0.4659090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.4943181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.5738636363636364, - "Filipino,Malay,Chinese,Indonesian": 0.45454545454545453, - "Filipino,Malay,Chinese,Vietnamese": 0.4772727272727273, - "Filipino,Malay,Indonesian,Vietnamese": 0.5397727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.4943181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.4602272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.4943181818181818, - "Spanish,Malay,Indonesian,Vietnamese": 0.5625, - "Spanish,Chinese,Indonesian,Vietnamese": 0.5, - "Malay,Chinese,Indonesian,Vietnamese": 0.5056818181818182 + "Indonesian,English,Filipino": 0.19318181818181818, + "Indonesian,English,Spanish": 0.22727272727272727, + "Indonesian,English,Chinese": 0.1590909090909091, + "Indonesian,English,Malay": 0.23863636363636365, + "Indonesian,English,Vietnamese": 0.18181818181818182, + "Indonesian,Filipino,Spanish": 0.30113636363636365, + "Indonesian,Filipino,Chinese": 0.3522727272727273, + "Indonesian,Filipino,Malay": 0.4431818181818182, + "Indonesian,Filipino,Vietnamese": 0.35795454545454547, + "Indonesian,Spanish,Chinese": 0.2215909090909091, + "Indonesian,Spanish,Malay": 0.3125, + "Indonesian,Spanish,Vietnamese": 0.25, + "Indonesian,Chinese,Malay": 0.3409090909090909, + "Indonesian,Chinese,Vietnamese": 0.3465909090909091, + "Indonesian,Malay,Vietnamese": 0.36363636363636365, + "English,Filipino,Spanish": 0.1875, + "English,Filipino,Chinese": 0.14204545454545456, + "English,Filipino,Malay": 0.17045454545454544, + "English,Filipino,Vietnamese": 0.1590909090909091, + "English,Spanish,Chinese": 0.14772727272727273, + "English,Spanish,Malay": 0.19886363636363635, + "English,Spanish,Vietnamese": 0.19886363636363635, + "English,Chinese,Malay": 0.14772727272727273, + "English,Chinese,Vietnamese": 0.17045454545454544, + "English,Malay,Vietnamese": 0.19318181818181818, + "Filipino,Spanish,Chinese": 0.2215909090909091, + "Filipino,Spanish,Malay": 0.2784090909090909, + "Filipino,Spanish,Vietnamese": 0.22727272727272727, + "Filipino,Chinese,Malay": 0.3181818181818182, + "Filipino,Chinese,Vietnamese": 0.3522727272727273, + "Filipino,Malay,Vietnamese": 0.32386363636363635, + "Spanish,Chinese,Malay": 0.19886363636363635, + "Spanish,Chinese,Vietnamese": 0.25, + "Spanish,Malay,Vietnamese": 0.2215909090909091, + "Chinese,Malay,Vietnamese": 0.3181818181818182 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.13068181818181818, + "Indonesian,English,Filipino,Chinese": 0.10795454545454546, + "Indonesian,English,Filipino,Malay": 0.1534090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.10795454545454546, + "Indonesian,English,Spanish,Chinese": 0.09090909090909091, + "Indonesian,English,Spanish,Malay": 0.1590909090909091, + "Indonesian,English,Spanish,Vietnamese": 0.10795454545454546, + "Indonesian,English,Chinese,Malay": 0.125, + "Indonesian,English,Chinese,Vietnamese": 0.11931818181818182, + "Indonesian,English,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Spanish,Chinese": 0.16477272727272727, + "Indonesian,Filipino,Spanish,Malay": 0.23863636363636365, + "Indonesian,Filipino,Spanish,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Chinese,Malay": 0.26136363636363635, + "Indonesian,Filipino,Chinese,Vietnamese": 0.26704545454545453, + "Indonesian,Filipino,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,Spanish,Chinese,Malay": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Vietnamese": 0.17613636363636365, + "Indonesian,Spanish,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,Chinese,Malay,Vietnamese": 0.25, + "English,Filipino,Spanish,Chinese": 0.07954545454545454, + "English,Filipino,Spanish,Malay": 0.11363636363636363, + "English,Filipino,Spanish,Vietnamese": 0.09090909090909091, + "English,Filipino,Chinese,Malay": 0.09090909090909091, + "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, + "English,Filipino,Malay,Vietnamese": 0.09659090909090909, + "English,Spanish,Chinese,Malay": 0.07954545454545454, + "English,Spanish,Chinese,Vietnamese": 0.10795454545454546, + "English,Spanish,Malay,Vietnamese": 0.11363636363636363, + "English,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Filipino,Spanish,Chinese,Malay": 0.1534090909090909, + "Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, + "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, + "Filipino,Chinese,Malay,Vietnamese": 0.23863636363636365, + "Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.42045454545454547, - "English,Filipino,Spanish,Malay,Indonesian": 0.48295454545454547, - "English,Filipino,Spanish,Malay,Vietnamese": 0.5056818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.4431818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.4715909090909091, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.5340909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.4147727272727273, - "English,Filipino,Malay,Chinese,Vietnamese": 0.44886363636363635, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.48863636363636365, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.4715909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.4318181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.4715909090909091, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.5227272727272727, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.48295454545454547, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.4659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.4090909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.4431818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.5, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.44886363636363635, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.4318181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.44886363636363635 + "Indonesian,English,Filipino,Spanish,Chinese": 0.056818181818181816, + "Indonesian,English,Filipino,Spanish,Malay": 0.10795454545454546, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Chinese,Malay": 0.08522727272727272, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,English,Spanish,Chinese,Malay": 0.06818181818181818, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182, + "English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, + "English,Filipino,Chinese,Malay,Vietnamese": 0.0625, + "English,Spanish,Chinese,Malay,Vietnamese": 0.0625, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.38636363636363635, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.42045454545454547, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.4659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.4318181818181818, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.4090909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.4318181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.4034090909090909 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.05113636363636364, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.38636363636363635 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 } }, - "AC3_2": 0.5802554996137891, - "AC3_3": 0.5354414901932032, - "AC3_4": 0.5019105490132669, - "AC3_5": 0.4747075054892054, - "AC3_6": 0.4514105080468106, - "AC3_7": 0.4305708244749675 + "AC3_2": 0.3600427900741609, + "AC3_3": 0.27228017249656605, + "AC3_4": 0.19876822991743748, + "AC3_5": 0.13925088748516634, + "AC3_6": 0.09325149733485677, + "AC3_7": 0.06123124446536174 }, "prompt_2": { - "overall_acc": 0.49269480519480524, + "overall_acc": 0.30113636363636365, "language_acc": { - "English": 0.5227272727272727, - "Filipino": 0.4943181818181818, - "Spanish": 0.48863636363636365, - "Malay": 0.48863636363636365, - "Chinese": 0.4943181818181818, - "Indonesian": 0.48295454545454547, - "Vietnamese": 0.4772727272727273 + "Indonesian": 0.24431818181818182, + "English": 0.5113636363636364, + "Filipino": 0.26704545454545453, + "Spanish": 0.3181818181818182, + "Chinese": 0.24431818181818182, + "Malay": 0.23295454545454544, + "Vietnamese": 0.2897727272727273 }, - "consistency_score_2": 0.7083333333333334, - "consistency_score_3": 0.5780844155844156, - "consistency_score_4": 0.4996753246753248, - "consistency_score_5": 0.4456168831168832, - "consistency_score_6": 0.40422077922077915, - "consistency_score_7": 0.3693181818181818, + "consistency_score_2": 0.4396645021645022, + "consistency_score_3": 0.2446428571428572, + "consistency_score_4": 0.15194805194805192, + "consistency_score_5": 0.0992965367965368, + "consistency_score_6": 0.06655844155844155, + "consistency_score_7": 0.045454545454545456, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.75, - "English,Spanish": 0.7727272727272727, - "English,Malay": 0.6875, - "English,Chinese": 0.6818181818181818, - "English,Indonesian": 0.7159090909090909, - "English,Vietnamese": 0.7556818181818182, - "Filipino,Spanish": 0.7159090909090909, - "Filipino,Malay": 0.6988636363636364, - "Filipino,Chinese": 0.6704545454545454, - "Filipino,Indonesian": 0.7159090909090909, - "Filipino,Vietnamese": 0.6988636363636364, - "Spanish,Malay": 0.7272727272727273, - "Spanish,Chinese": 0.6193181818181818, - "Spanish,Indonesian": 0.7159090909090909, - "Spanish,Vietnamese": 0.7556818181818182, - "Malay,Chinese": 0.625, - "Malay,Indonesian": 0.7613636363636364, - "Malay,Vietnamese": 0.7272727272727273, - "Chinese,Indonesian": 0.6647727272727273, - "Chinese,Vietnamese": 0.6590909090909091, - "Indonesian,Vietnamese": 0.7556818181818182 + "Indonesian,English": 0.3181818181818182, + "Indonesian,Filipino": 0.5227272727272727, + "Indonesian,Spanish": 0.4375, + "Indonesian,Chinese": 0.48295454545454547, + "Indonesian,Malay": 0.625, + "Indonesian,Vietnamese": 0.48863636363636365, + "English,Filipino": 0.3181818181818182, + "English,Spanish": 0.4943181818181818, + "English,Chinese": 0.2556818181818182, + "English,Malay": 0.2727272727272727, + "English,Vietnamese": 0.35795454545454547, + "Filipino,Spanish": 0.4090909090909091, + "Filipino,Chinese": 0.4431818181818182, + "Filipino,Malay": 0.5625, + "Filipino,Vietnamese": 0.4772727272727273, + "Spanish,Chinese": 0.3693181818181818, + "Spanish,Malay": 0.45454545454545453, + "Spanish,Vietnamese": 0.4431818181818182, + "Chinese,Malay": 0.4318181818181818, + "Chinese,Vietnamese": 0.5568181818181818, + "Malay,Vietnamese": 0.5113636363636364 }, "3_combine": { - "English,Filipino,Spanish": 0.625, - "English,Filipino,Malay": 0.5795454545454546, - "English,Filipino,Chinese": 0.5681818181818182, - "English,Filipino,Indonesian": 0.6022727272727273, - "English,Filipino,Vietnamese": 0.6193181818181818, - "English,Spanish,Malay": 0.6022727272727273, - "English,Spanish,Chinese": 0.5511363636363636, - "English,Spanish,Indonesian": 0.6079545454545454, - "English,Spanish,Vietnamese": 0.6590909090909091, - "English,Malay,Chinese": 0.5113636363636364, - "English,Malay,Indonesian": 0.5965909090909091, - "English,Malay,Vietnamese": 0.6079545454545454, - "English,Chinese,Indonesian": 0.5397727272727273, - "English,Chinese,Vietnamese": 0.5681818181818182, - "English,Indonesian,Vietnamese": 0.625, - "Filipino,Spanish,Malay": 0.5852272727272727, - "Filipino,Spanish,Chinese": 0.5284090909090909, - "Filipino,Spanish,Indonesian": 0.5852272727272727, - "Filipino,Spanish,Vietnamese": 0.6079545454545454, - "Filipino,Malay,Chinese": 0.5227272727272727, - "Filipino,Malay,Indonesian": 0.6079545454545454, - "Filipino,Malay,Vietnamese": 0.5909090909090909, - "Filipino,Chinese,Indonesian": 0.5397727272727273, - "Filipino,Chinese,Vietnamese": 0.5284090909090909, - "Filipino,Indonesian,Vietnamese": 0.6022727272727273, - "Spanish,Malay,Chinese": 0.4943181818181818, - "Spanish,Malay,Indonesian": 0.6136363636363636, - "Spanish,Malay,Vietnamese": 0.6193181818181818, - "Spanish,Chinese,Indonesian": 0.5113636363636364, - "Spanish,Chinese,Vietnamese": 0.5284090909090909, - "Spanish,Indonesian,Vietnamese": 0.625, - "Malay,Chinese,Indonesian": 0.5454545454545454, - "Malay,Chinese,Vietnamese": 0.5284090909090909, - "Malay,Indonesian,Vietnamese": 0.6420454545454546, - "Chinese,Indonesian,Vietnamese": 0.5625 + "Indonesian,English,Filipino": 0.17613636363636365, + "Indonesian,English,Spanish": 0.2215909090909091, + "Indonesian,English,Chinese": 0.1534090909090909, + "Indonesian,English,Malay": 0.19886363636363635, + "Indonesian,English,Vietnamese": 0.19318181818181818, + "Indonesian,Filipino,Spanish": 0.2784090909090909, + "Indonesian,Filipino,Chinese": 0.2897727272727273, + "Indonesian,Filipino,Malay": 0.4147727272727273, + "Indonesian,Filipino,Vietnamese": 0.3068181818181818, + "Indonesian,Spanish,Chinese": 0.2215909090909091, + "Indonesian,Spanish,Malay": 0.3352272727272727, + "Indonesian,Spanish,Vietnamese": 0.2784090909090909, + "Indonesian,Chinese,Malay": 0.32386363636363635, + "Indonesian,Chinese,Vietnamese": 0.3181818181818182, + "Indonesian,Malay,Vietnamese": 0.36363636363636365, + "English,Filipino,Spanish": 0.19318181818181818, + "English,Filipino,Chinese": 0.13068181818181818, + "English,Filipino,Malay": 0.1590909090909091, + "English,Filipino,Vietnamese": 0.1875, + "English,Spanish,Chinese": 0.14772727272727273, + "English,Spanish,Malay": 0.19886363636363635, + "English,Spanish,Vietnamese": 0.24431818181818182, + "English,Chinese,Malay": 0.125, + "English,Chinese,Vietnamese": 0.17045454545454544, + "English,Malay,Vietnamese": 0.19886363636363635, + "Filipino,Spanish,Chinese": 0.2159090909090909, + "Filipino,Spanish,Malay": 0.2784090909090909, + "Filipino,Spanish,Vietnamese": 0.25, + "Filipino,Chinese,Malay": 0.2840909090909091, + "Filipino,Chinese,Vietnamese": 0.29545454545454547, + "Filipino,Malay,Vietnamese": 0.32954545454545453, + "Spanish,Chinese,Malay": 0.22727272727272727, + "Spanish,Chinese,Vietnamese": 0.26136363636363635, + "Spanish,Malay,Vietnamese": 0.2840909090909091, + "Chinese,Malay,Vietnamese": 0.3068181818181818 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.5113636363636364, - "English,Filipino,Spanish,Chinese": 0.4772727272727273, - "English,Filipino,Spanish,Indonesian": 0.5284090909090909, - "English,Filipino,Spanish,Vietnamese": 0.5568181818181818, - "English,Filipino,Malay,Chinese": 0.4659090909090909, - "English,Filipino,Malay,Indonesian": 0.5227272727272727, - "English,Filipino,Malay,Vietnamese": 0.5284090909090909, - "English,Filipino,Chinese,Indonesian": 0.4772727272727273, - "English,Filipino,Chinese,Vietnamese": 0.4943181818181818, - "English,Filipino,Indonesian,Vietnamese": 0.5511363636363636, - "English,Spanish,Malay,Chinese": 0.44886363636363635, - "English,Spanish,Malay,Indonesian": 0.5340909090909091, - "English,Spanish,Malay,Vietnamese": 0.5568181818181818, - "English,Spanish,Chinese,Indonesian": 0.4659090909090909, - "English,Spanish,Chinese,Vietnamese": 0.5, - "English,Spanish,Indonesian,Vietnamese": 0.5681818181818182, - "English,Malay,Chinese,Indonesian": 0.4659090909090909, - "English,Malay,Chinese,Vietnamese": 0.48295454545454547, - "English,Malay,Indonesian,Vietnamese": 0.5397727272727273, - "English,Chinese,Indonesian,Vietnamese": 0.5056818181818182, - "Filipino,Spanish,Malay,Chinese": 0.44886363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.5284090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.5227272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.44886363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.4659090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.5284090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.4715909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.4602272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.5397727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.4715909090909091, - "Spanish,Malay,Chinese,Indonesian": 0.4602272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.45454545454545453, - "Spanish,Malay,Indonesian,Vietnamese": 0.5454545454545454, - "Spanish,Chinese,Indonesian,Vietnamese": 0.4659090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.4943181818181818 + "Indonesian,English,Filipino,Spanish": 0.13636363636363635, + "Indonesian,English,Filipino,Chinese": 0.10227272727272728, + "Indonesian,English,Filipino,Malay": 0.13068181818181818, + "Indonesian,English,Filipino,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese": 0.10795454545454546, + "Indonesian,English,Spanish,Malay": 0.1590909090909091, + "Indonesian,English,Spanish,Vietnamese": 0.1534090909090909, + "Indonesian,English,Chinese,Malay": 0.10795454545454546, + "Indonesian,English,Chinese,Vietnamese": 0.11363636363636363, + "Indonesian,English,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,Filipino,Spanish,Chinese": 0.1534090909090909, + "Indonesian,Filipino,Spanish,Malay": 0.24431818181818182, + "Indonesian,Filipino,Spanish,Vietnamese": 0.1875, + "Indonesian,Filipino,Chinese,Malay": 0.20454545454545456, + "Indonesian,Filipino,Chinese,Vietnamese": 0.19318181818181818, + "Indonesian,Filipino,Malay,Vietnamese": 0.24431818181818182, + "Indonesian,Spanish,Chinese,Malay": 0.1875, + "Indonesian,Spanish,Chinese,Vietnamese": 0.17613636363636365, + "Indonesian,Spanish,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,Chinese,Malay,Vietnamese": 0.23295454545454544, + "English,Filipino,Spanish,Chinese": 0.09090909090909091, + "English,Filipino,Spanish,Malay": 0.11931818181818182, + "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, + "English,Filipino,Chinese,Malay": 0.07954545454545454, + "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, + "English,Filipino,Malay,Vietnamese": 0.125, + "English,Spanish,Chinese,Malay": 0.09090909090909091, + "English,Spanish,Chinese,Vietnamese": 0.11931818181818182, + "English,Spanish,Malay,Vietnamese": 0.14204545454545456, + "English,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Filipino,Spanish,Chinese,Malay": 0.14772727272727273, + "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, + "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, + "Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, + "Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.4090909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.4715909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.48295454545454547, - "English,Filipino,Spanish,Chinese,Indonesian": 0.4147727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.4431818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.5, - "English,Filipino,Malay,Chinese,Indonesian": 0.42613636363636365, - "English,Filipino,Malay,Chinese,Vietnamese": 0.4375, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.48863636363636365, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.44886363636363635, - "English,Spanish,Malay,Chinese,Indonesian": 0.42045454545454547, - "English,Spanish,Malay,Chinese,Vietnamese": 0.4375, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.5, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.44886363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.44886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.4147727272727273, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.4147727272727273, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.4772727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.4147727272727273, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.4318181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.42613636363636365 + "Indonesian,English,Filipino,Spanish,Chinese": 0.07386363636363637, + "Indonesian,English,Filipino,Spanish,Malay": 0.11363636363636363, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, + "Indonesian,English,Filipino,Chinese,Malay": 0.06818181818181818, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,English,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.125, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.125, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, + "English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, + "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, + "English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, + "English,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.3806818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.3977272727272727, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.44886363636363635, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.3977272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.4090909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.4090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.38636363636363635 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.3693181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 } }, - "AC3_2": 0.5811556656420827, - "AC3_3": 0.5319849002834597, - "AC3_4": 0.4961605137368864, - "AC3_5": 0.46797482356941317, - "AC3_6": 0.44409414110345113, - "AC3_7": 0.42217728123847503 + "AC3_2": 0.357448203683736, + "AC3_3": 0.26996579412067256, + "AC3_4": 0.20198039147361257, + "AC3_5": 0.14934735868506224, + "AC3_6": 0.10902067024929864, + "AC3_7": 0.07898658716051893 }, "prompt_3": { - "overall_acc": 0.4983766233766234, + "overall_acc": 0.288961038961039, "language_acc": { - "English": 0.5284090909090909, - "Filipino": 0.4943181818181818, - "Spanish": 0.48295454545454547, - "Malay": 0.5284090909090909, - "Chinese": 0.5056818181818182, - "Indonesian": 0.48295454545454547, - "Vietnamese": 0.4659090909090909 + "Indonesian": 0.25, + "English": 0.44886363636363635, + "Filipino": 0.26136363636363635, + "Spanish": 0.32386363636363635, + "Chinese": 0.25, + "Malay": 0.23295454545454544, + "Vietnamese": 0.2556818181818182 }, - "consistency_score_2": 0.7126623376623377, - "consistency_score_3": 0.5842532467532466, - "consistency_score_4": 0.50762987012987, - "consistency_score_5": 0.45508658008658, - "consistency_score_6": 0.41477272727272724, - "consistency_score_7": 0.3806818181818182, + "consistency_score_2": 0.4507575757575758, + "consistency_score_3": 0.2568181818181819, + "consistency_score_4": 0.16103896103896104, + "consistency_score_5": 0.10470779220779221, + "consistency_score_6": 0.0689935064935065, + "consistency_score_7": 0.045454545454545456, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.7670454545454546, - "English,Spanish": 0.8011363636363636, - "English,Malay": 0.7102272727272727, - "English,Chinese": 0.6818181818181818, - "English,Indonesian": 0.7272727272727273, - "English,Vietnamese": 0.7670454545454546, - "Filipino,Spanish": 0.7272727272727273, - "Filipino,Malay": 0.7272727272727273, - "Filipino,Chinese": 0.6704545454545454, - "Filipino,Indonesian": 0.7272727272727273, - "Filipino,Vietnamese": 0.6931818181818182, - "Spanish,Malay": 0.75, - "Spanish,Chinese": 0.6193181818181818, - "Spanish,Indonesian": 0.7329545454545454, - "Spanish,Vietnamese": 0.75, - "Malay,Chinese": 0.6306818181818182, - "Malay,Indonesian": 0.7556818181818182, - "Malay,Vietnamese": 0.7159090909090909, - "Chinese,Indonesian": 0.6306818181818182, - "Chinese,Vietnamese": 0.6363636363636364, - "Indonesian,Vietnamese": 0.7443181818181818 + "Indonesian,English": 0.3465909090909091, + "Indonesian,Filipino": 0.5625, + "Indonesian,Spanish": 0.44886363636363635, + "Indonesian,Chinese": 0.48295454545454547, + "Indonesian,Malay": 0.6647727272727273, + "Indonesian,Vietnamese": 0.4772727272727273, + "English,Filipino": 0.29545454545454547, + "English,Spanish": 0.48863636363636365, + "English,Chinese": 0.30113636363636365, + "English,Malay": 0.2840909090909091, + "English,Vietnamese": 0.3352272727272727, + "Filipino,Spanish": 0.45454545454545453, + "Filipino,Chinese": 0.4772727272727273, + "Filipino,Malay": 0.5681818181818182, + "Filipino,Vietnamese": 0.4659090909090909, + "Spanish,Chinese": 0.35795454545454547, + "Spanish,Malay": 0.48295454545454547, + "Spanish,Vietnamese": 0.4090909090909091, + "Chinese,Malay": 0.4715909090909091, + "Chinese,Vietnamese": 0.5681818181818182, + "Malay,Vietnamese": 0.5227272727272727 }, "3_combine": { - "English,Filipino,Spanish": 0.6534090909090909, - "English,Filipino,Malay": 0.6193181818181818, - "English,Filipino,Chinese": 0.5795454545454546, - "English,Filipino,Indonesian": 0.6193181818181818, - "English,Filipino,Vietnamese": 0.6306818181818182, - "English,Spanish,Malay": 0.6363636363636364, - "English,Spanish,Chinese": 0.5625, - "English,Spanish,Indonesian": 0.6363636363636364, - "English,Spanish,Vietnamese": 0.6761363636363636, - "English,Malay,Chinese": 0.5227272727272727, - "English,Malay,Indonesian": 0.6079545454545454, - "English,Malay,Vietnamese": 0.6193181818181818, - "English,Chinese,Indonesian": 0.5284090909090909, - "English,Chinese,Vietnamese": 0.5568181818181818, - "English,Indonesian,Vietnamese": 0.6363636363636364, - "Filipino,Spanish,Malay": 0.6193181818181818, - "Filipino,Spanish,Chinese": 0.5284090909090909, - "Filipino,Spanish,Indonesian": 0.6022727272727273, - "Filipino,Spanish,Vietnamese": 0.6079545454545454, - "Filipino,Malay,Chinese": 0.5340909090909091, - "Filipino,Malay,Indonesian": 0.625, - "Filipino,Malay,Vietnamese": 0.6022727272727273, - "Filipino,Chinese,Indonesian": 0.5340909090909091, - "Filipino,Chinese,Vietnamese": 0.5170454545454546, - "Filipino,Indonesian,Vietnamese": 0.6022727272727273, - "Spanish,Malay,Chinese": 0.5056818181818182, - "Spanish,Malay,Indonesian": 0.6306818181818182, - "Spanish,Malay,Vietnamese": 0.6193181818181818, - "Spanish,Chinese,Indonesian": 0.5, - "Spanish,Chinese,Vietnamese": 0.5170454545454546, - "Spanish,Indonesian,Vietnamese": 0.625, - "Malay,Chinese,Indonesian": 0.5227272727272727, - "Malay,Chinese,Vietnamese": 0.5170454545454546, - "Malay,Indonesian,Vietnamese": 0.6306818181818182, - "Chinese,Indonesian,Vietnamese": 0.5227272727272727 + "Indonesian,English,Filipino": 0.19318181818181818, + "Indonesian,English,Spanish": 0.22727272727272727, + "Indonesian,English,Chinese": 0.18181818181818182, + "Indonesian,English,Malay": 0.22727272727272727, + "Indonesian,English,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Spanish": 0.3068181818181818, + "Indonesian,Filipino,Chinese": 0.32386363636363635, + "Indonesian,Filipino,Malay": 0.44886363636363635, + "Indonesian,Filipino,Vietnamese": 0.3125, + "Indonesian,Spanish,Chinese": 0.21022727272727273, + "Indonesian,Spanish,Malay": 0.35795454545454547, + "Indonesian,Spanish,Vietnamese": 0.25, + "Indonesian,Chinese,Malay": 0.35795454545454547, + "Indonesian,Chinese,Vietnamese": 0.3181818181818182, + "Indonesian,Malay,Vietnamese": 0.375, + "English,Filipino,Spanish": 0.2159090909090909, + "English,Filipino,Chinese": 0.1534090909090909, + "English,Filipino,Malay": 0.17613636363636365, + "English,Filipino,Vietnamese": 0.16477272727272727, + "English,Spanish,Chinese": 0.17045454545454544, + "English,Spanish,Malay": 0.2215909090909091, + "English,Spanish,Vietnamese": 0.2215909090909091, + "English,Chinese,Malay": 0.14772727272727273, + "English,Chinese,Vietnamese": 0.19318181818181818, + "English,Malay,Vietnamese": 0.17045454545454544, + "Filipino,Spanish,Chinese": 0.23863636363636365, + "Filipino,Spanish,Malay": 0.32386363636363635, + "Filipino,Spanish,Vietnamese": 0.24431818181818182, + "Filipino,Chinese,Malay": 0.3181818181818182, + "Filipino,Chinese,Vietnamese": 0.3068181818181818, + "Filipino,Malay,Vietnamese": 0.32386363636363635, + "Spanish,Chinese,Malay": 0.23863636363636365, + "Spanish,Chinese,Vietnamese": 0.2556818181818182, + "Spanish,Malay,Vietnamese": 0.2727272727272727, + "Chinese,Malay,Vietnamese": 0.3409090909090909 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.5625, - "English,Filipino,Spanish,Chinese": 0.4943181818181818, - "English,Filipino,Spanish,Indonesian": 0.5568181818181818, - "English,Filipino,Spanish,Vietnamese": 0.5795454545454546, - "English,Filipino,Malay,Chinese": 0.48863636363636365, - "English,Filipino,Malay,Indonesian": 0.5454545454545454, - "English,Filipino,Malay,Vietnamese": 0.5568181818181818, - "English,Filipino,Chinese,Indonesian": 0.48295454545454547, - "English,Filipino,Chinese,Vietnamese": 0.48863636363636365, - "English,Filipino,Indonesian,Vietnamese": 0.5568181818181818, - "English,Spanish,Malay,Chinese": 0.4659090909090909, - "English,Spanish,Malay,Indonesian": 0.5568181818181818, - "English,Spanish,Malay,Vietnamese": 0.5738636363636364, - "English,Spanish,Chinese,Indonesian": 0.4659090909090909, - "English,Spanish,Chinese,Vietnamese": 0.5, - "English,Spanish,Indonesian,Vietnamese": 0.5795454545454546, - "English,Malay,Chinese,Indonesian": 0.4659090909090909, - "English,Malay,Chinese,Vietnamese": 0.48295454545454547, - "English,Malay,Indonesian,Vietnamese": 0.5454545454545454, - "English,Chinese,Indonesian,Vietnamese": 0.48295454545454547, - "Filipino,Spanish,Malay,Chinese": 0.4715909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.5511363636363636, - "Filipino,Spanish,Malay,Vietnamese": 0.5454545454545454, - "Filipino,Spanish,Chinese,Indonesian": 0.44886363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.4602272727272727, - "Filipino,Spanish,Indonesian,Vietnamese": 0.5397727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.4715909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.4659090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.5511363636363636, - "Filipino,Chinese,Indonesian,Vietnamese": 0.45454545454545453, - "Spanish,Malay,Chinese,Indonesian": 0.45454545454545453, - "Spanish,Malay,Chinese,Vietnamese": 0.45454545454545453, - "Spanish,Malay,Indonesian,Vietnamese": 0.5511363636363636, - "Spanish,Chinese,Indonesian,Vietnamese": 0.44886363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.4659090909090909 + "Indonesian,English,Filipino,Spanish": 0.13636363636363635, + "Indonesian,English,Filipino,Chinese": 0.11931818181818182, + "Indonesian,English,Filipino,Malay": 0.1534090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese": 0.10227272727272728, + "Indonesian,English,Spanish,Malay": 0.17613636363636365, + "Indonesian,English,Spanish,Vietnamese": 0.13636363636363635, + "Indonesian,English,Chinese,Malay": 0.13068181818181818, + "Indonesian,English,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,English,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Spanish,Chinese": 0.16477272727272727, + "Indonesian,Filipino,Spanish,Malay": 0.2840909090909091, + "Indonesian,Filipino,Spanish,Vietnamese": 0.18181818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.26136363636363635, + "Indonesian,Filipino,Chinese,Vietnamese": 0.2159090909090909, + "Indonesian,Filipino,Malay,Vietnamese": 0.26136363636363635, + "Indonesian,Spanish,Chinese,Malay": 0.1875, + "Indonesian,Spanish,Chinese,Vietnamese": 0.1590909090909091, + "Indonesian,Spanish,Malay,Vietnamese": 0.2159090909090909, + "Indonesian,Chinese,Malay,Vietnamese": 0.2556818181818182, + "English,Filipino,Spanish,Chinese": 0.10227272727272728, + "English,Filipino,Spanish,Malay": 0.14204545454545456, + "English,Filipino,Spanish,Vietnamese": 0.11931818181818182, + "English,Filipino,Chinese,Malay": 0.09659090909090909, + "English,Filipino,Chinese,Vietnamese": 0.10227272727272728, + "English,Filipino,Malay,Vietnamese": 0.10795454545454546, + "English,Spanish,Chinese,Malay": 0.10227272727272728, + "English,Spanish,Chinese,Vietnamese": 0.13068181818181818, + "English,Spanish,Malay,Vietnamese": 0.13068181818181818, + "English,Chinese,Malay,Vietnamese": 0.11363636363636363, + "Filipino,Spanish,Chinese,Malay": 0.18181818181818182, + "Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, + "Filipino,Spanish,Malay,Vietnamese": 0.1875, + "Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, + "Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.4431818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.5056818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.5227272727272727, - "English,Filipino,Spanish,Chinese,Indonesian": 0.42613636363636365, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.44886363636363635, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.5170454545454546, - "English,Filipino,Malay,Chinese,Indonesian": 0.4375, - "English,Filipino,Malay,Chinese,Vietnamese": 0.44886363636363635, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.5056818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.4375, - "English,Spanish,Malay,Chinese,Indonesian": 0.42613636363636365, - "English,Spanish,Malay,Chinese,Vietnamese": 0.4431818181818182, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.5113636363636364, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.4375, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.4375, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.42613636363636365, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.4318181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.5, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.4090909090909091, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.42613636363636365, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.4147727272727273 + "Indonesian,English,Filipino,Spanish,Chinese": 0.06818181818181818, + "Indonesian,English,Filipino,Spanish,Malay": 0.125, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.09090909090909091, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,English,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.1534090909090909, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, + "English,Filipino,Spanish,Chinese,Malay": 0.06818181818181818, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, + "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, + "English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, + "English,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.4034090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.42045454545454547, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.4772727272727273, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.3977272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.4090909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.4034090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.39204545454545453 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.0625, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.3806818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 } }, - "AC3_2": 0.586561226928342, - "AC3_3": 0.537908972069813, - "AC3_4": 0.5029606910259123, - "AC3_5": 0.4757488538204098, - "AC3_6": 0.45274747469789384, - "AC3_7": 0.4316502979446846 + "AC3_2": 0.352164660456482, + "AC3_3": 0.27194310740785554, + "AC3_4": 0.20681771326526033, + "AC3_5": 0.15371535676909345, + "AC3_6": 0.11139087666588042, + "AC3_7": 0.07855251542222984 }, "prompt_4": { - "overall_acc": 0.4935064935064935, + "overall_acc": 0.2938311688311689, "language_acc": { - "English": 0.5113636363636364, - "Filipino": 0.5, - "Spanish": 0.48295454545454547, - "Malay": 0.5056818181818182, - "Chinese": 0.5056818181818182, - "Indonesian": 0.48295454545454547, - "Vietnamese": 0.4659090909090909 + "Indonesian": 0.22727272727272727, + "English": 0.4943181818181818, + "Filipino": 0.2556818181818182, + "Spanish": 0.32954545454545453, + "Chinese": 0.26136363636363635, + "Malay": 0.23295454545454544, + "Vietnamese": 0.2556818181818182 }, - "consistency_score_2": 0.7048160173160174, - "consistency_score_3": 0.5735389610389612, - "consistency_score_4": 0.4939935064935066, - "consistency_score_5": 0.43858225108225113, - "consistency_score_6": 0.3969155844155844, - "consistency_score_7": 0.36363636363636365, + "consistency_score_2": 0.43290043290043295, + "consistency_score_3": 0.23538961038961043, + "consistency_score_4": 0.14090909090909093, + "consistency_score_5": 0.08793290043290043, + "consistency_score_6": 0.056818181818181816, + "consistency_score_7": 0.03977272727272727, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.7556818181818182, - "English,Spanish": 0.7784090909090909, - "English,Malay": 0.6647727272727273, - "English,Chinese": 0.6818181818181818, - "English,Indonesian": 0.7272727272727273, - "English,Vietnamese": 0.75, - "Filipino,Spanish": 0.7329545454545454, - "Filipino,Malay": 0.6988636363636364, - "Filipino,Chinese": 0.6704545454545454, - "Filipino,Indonesian": 0.7215909090909091, - "Filipino,Vietnamese": 0.6875, - "Spanish,Malay": 0.7159090909090909, - "Spanish,Chinese": 0.6306818181818182, - "Spanish,Indonesian": 0.7443181818181818, - "Spanish,Vietnamese": 0.7386363636363636, - "Malay,Chinese": 0.625, - "Malay,Indonesian": 0.7386363636363636, - "Malay,Vietnamese": 0.6988636363636364, - "Chinese,Indonesian": 0.6534090909090909, - "Chinese,Vietnamese": 0.6477272727272727, - "Indonesian,Vietnamese": 0.7386363636363636 + "Indonesian,English": 0.30113636363636365, + "Indonesian,Filipino": 0.5681818181818182, + "Indonesian,Spanish": 0.3977272727272727, + "Indonesian,Chinese": 0.45454545454545453, + "Indonesian,Malay": 0.6193181818181818, + "Indonesian,Vietnamese": 0.48863636363636365, + "English,Filipino": 0.3352272727272727, + "English,Spanish": 0.4659090909090909, + "English,Chinese": 0.30113636363636365, + "English,Malay": 0.3068181818181818, + "English,Vietnamese": 0.36363636363636365, + "Filipino,Spanish": 0.4147727272727273, + "Filipino,Chinese": 0.4715909090909091, + "Filipino,Malay": 0.5625, + "Filipino,Vietnamese": 0.48295454545454547, + "Spanish,Chinese": 0.32386363636363635, + "Spanish,Malay": 0.4318181818181818, + "Spanish,Vietnamese": 0.3806818181818182, + "Chinese,Malay": 0.42613636363636365, + "Chinese,Vietnamese": 0.5170454545454546, + "Malay,Vietnamese": 0.4772727272727273 }, "3_combine": { - "English,Filipino,Spanish": 0.6363636363636364, - "English,Filipino,Malay": 0.5738636363636364, - "English,Filipino,Chinese": 0.5738636363636364, - "English,Filipino,Indonesian": 0.6079545454545454, - "English,Filipino,Vietnamese": 0.6193181818181818, - "English,Spanish,Malay": 0.5909090909090909, - "English,Spanish,Chinese": 0.5625, - "English,Spanish,Indonesian": 0.6306818181818182, - "English,Spanish,Vietnamese": 0.6534090909090909, - "English,Malay,Chinese": 0.5, - "English,Malay,Indonesian": 0.5795454545454546, - "English,Malay,Vietnamese": 0.5795454545454546, - "English,Chinese,Indonesian": 0.5397727272727273, - "English,Chinese,Vietnamese": 0.5568181818181818, - "English,Indonesian,Vietnamese": 0.625, - "Filipino,Spanish,Malay": 0.5909090909090909, - "Filipino,Spanish,Chinese": 0.5397727272727273, - "Filipino,Spanish,Indonesian": 0.6079545454545454, - "Filipino,Spanish,Vietnamese": 0.6022727272727273, - "Filipino,Malay,Chinese": 0.5170454545454546, - "Filipino,Malay,Indonesian": 0.5965909090909091, - "Filipino,Malay,Vietnamese": 0.5738636363636364, - "Filipino,Chinese,Indonesian": 0.5454545454545454, - "Filipino,Chinese,Vietnamese": 0.5170454545454546, - "Filipino,Indonesian,Vietnamese": 0.6022727272727273, - "Spanish,Malay,Chinese": 0.4943181818181818, - "Spanish,Malay,Indonesian": 0.6136363636363636, - "Spanish,Malay,Vietnamese": 0.5965909090909091, - "Spanish,Chinese,Indonesian": 0.5227272727272727, - "Spanish,Chinese,Vietnamese": 0.5227272727272727, - "Spanish,Indonesian,Vietnamese": 0.6193181818181818, - "Malay,Chinese,Indonesian": 0.5227272727272727, - "Malay,Chinese,Vietnamese": 0.5113636363636364, - "Malay,Indonesian,Vietnamese": 0.6079545454545454, - "Chinese,Indonesian,Vietnamese": 0.5397727272727273 + "Indonesian,English,Filipino": 0.19318181818181818, + "Indonesian,English,Spanish": 0.18181818181818182, + "Indonesian,English,Chinese": 0.14772727272727273, + "Indonesian,English,Malay": 0.20454545454545456, + "Indonesian,English,Vietnamese": 0.19318181818181818, + "Indonesian,Filipino,Spanish": 0.2727272727272727, + "Indonesian,Filipino,Chinese": 0.3125, + "Indonesian,Filipino,Malay": 0.4318181818181818, + "Indonesian,Filipino,Vietnamese": 0.3352272727272727, + "Indonesian,Spanish,Chinese": 0.18181818181818182, + "Indonesian,Spanish,Malay": 0.2784090909090909, + "Indonesian,Spanish,Vietnamese": 0.22727272727272727, + "Indonesian,Chinese,Malay": 0.30113636363636365, + "Indonesian,Chinese,Vietnamese": 0.3125, + "Indonesian,Malay,Vietnamese": 0.3522727272727273, + "English,Filipino,Spanish": 0.19886363636363635, + "English,Filipino,Chinese": 0.1590909090909091, + "English,Filipino,Malay": 0.17613636363636365, + "English,Filipino,Vietnamese": 0.1875, + "English,Spanish,Chinese": 0.13636363636363635, + "English,Spanish,Malay": 0.19318181818181818, + "English,Spanish,Vietnamese": 0.19886363636363635, + "English,Chinese,Malay": 0.14772727272727273, + "English,Chinese,Vietnamese": 0.1875, + "English,Malay,Vietnamese": 0.19886363636363635, + "Filipino,Spanish,Chinese": 0.20454545454545456, + "Filipino,Spanish,Malay": 0.2840909090909091, + "Filipino,Spanish,Vietnamese": 0.2215909090909091, + "Filipino,Chinese,Malay": 0.2897727272727273, + "Filipino,Chinese,Vietnamese": 0.29545454545454547, + "Filipino,Malay,Vietnamese": 0.3181818181818182, + "Spanish,Chinese,Malay": 0.19318181818181818, + "Spanish,Chinese,Vietnamese": 0.20454545454545456, + "Spanish,Malay,Vietnamese": 0.23295454545454544, + "Chinese,Malay,Vietnamese": 0.2840909090909091 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.5170454545454546, - "English,Filipino,Spanish,Chinese": 0.4943181818181818, - "English,Filipino,Spanish,Indonesian": 0.5511363636363636, - "English,Filipino,Spanish,Vietnamese": 0.5625, - "English,Filipino,Malay,Chinese": 0.4659090909090909, - "English,Filipino,Malay,Indonesian": 0.5113636363636364, - "English,Filipino,Malay,Vietnamese": 0.5170454545454546, - "English,Filipino,Chinese,Indonesian": 0.48863636363636365, - "English,Filipino,Chinese,Vietnamese": 0.48863636363636365, - "English,Filipino,Indonesian,Vietnamese": 0.5511363636363636, - "English,Spanish,Malay,Chinese": 0.44886363636363635, - "English,Spanish,Malay,Indonesian": 0.5227272727272727, - "English,Spanish,Malay,Vietnamese": 0.5397727272727273, - "English,Spanish,Chinese,Indonesian": 0.4772727272727273, - "English,Spanish,Chinese,Vietnamese": 0.4943181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.5625, - "English,Malay,Chinese,Indonesian": 0.44886363636363635, - "English,Malay,Chinese,Vietnamese": 0.4602272727272727, - "English,Malay,Indonesian,Vietnamese": 0.5113636363636364, - "English,Chinese,Indonesian,Vietnamese": 0.4943181818181818, - "Filipino,Spanish,Malay,Chinese": 0.45454545454545453, - "Filipino,Spanish,Malay,Indonesian": 0.5284090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.5170454545454546, - "Filipino,Spanish,Chinese,Indonesian": 0.4659090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.4602272727272727, - "Filipino,Spanish,Indonesian,Vietnamese": 0.5340909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.4602272727272727, - "Filipino,Malay,Chinese,Vietnamese": 0.4431818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.5227272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.4602272727272727, - "Spanish,Malay,Chinese,Indonesian": 0.44886363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.4431818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.5284090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.45454545454545453, - "Malay,Chinese,Indonesian,Vietnamese": 0.4602272727272727 + "Indonesian,English,Filipino,Spanish": 0.11931818181818182, + "Indonesian,English,Filipino,Chinese": 0.10795454545454546, + "Indonesian,English,Filipino,Malay": 0.13636363636363635, + "Indonesian,English,Filipino,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese": 0.07386363636363637, + "Indonesian,English,Spanish,Malay": 0.14204545454545456, + "Indonesian,English,Spanish,Vietnamese": 0.10227272727272728, + "Indonesian,English,Chinese,Malay": 0.10795454545454546, + "Indonesian,English,Chinese,Vietnamese": 0.11931818181818182, + "Indonesian,English,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,Filipino,Spanish,Chinese": 0.14772727272727273, + "Indonesian,Filipino,Spanish,Malay": 0.2159090909090909, + "Indonesian,Filipino,Spanish,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Chinese,Malay": 0.23295454545454544, + "Indonesian,Filipino,Chinese,Vietnamese": 0.21022727272727273, + "Indonesian,Filipino,Malay,Vietnamese": 0.26136363636363635, + "Indonesian,Spanish,Chinese,Malay": 0.13636363636363635, + "Indonesian,Spanish,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,Spanish,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,Chinese,Malay,Vietnamese": 0.2215909090909091, + "English,Filipino,Spanish,Chinese": 0.08522727272727272, + "English,Filipino,Spanish,Malay": 0.11931818181818182, + "English,Filipino,Spanish,Vietnamese": 0.11363636363636363, + "English,Filipino,Chinese,Malay": 0.09090909090909091, + "English,Filipino,Chinese,Vietnamese": 0.10795454545454546, + "English,Filipino,Malay,Vietnamese": 0.11363636363636363, + "English,Spanish,Chinese,Malay": 0.07954545454545454, + "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, + "English,Spanish,Malay,Vietnamese": 0.11931818181818182, + "English,Chinese,Malay,Vietnamese": 0.11931818181818182, + "Filipino,Spanish,Chinese,Malay": 0.14204545454545456, + "Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, + "Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, + "Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, + "Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.42045454545454547, - "English,Filipino,Spanish,Malay,Indonesian": 0.4715909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.48295454545454547, - "English,Filipino,Spanish,Chinese,Indonesian": 0.4375, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.4431818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.5056818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.42045454545454547, - "English,Filipino,Malay,Chinese,Vietnamese": 0.42613636363636365, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.4715909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.4431818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.4090909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.42613636363636365, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.4772727272727273, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.4375, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.42045454545454547, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.4147727272727273, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.4090909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.4715909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.4090909090909091, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.4090909090909091, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.4034090909090909 + "Indonesian,English,Filipino,Spanish,Chinese": 0.056818181818181816, + "Indonesian,English,Filipino,Spanish,Malay": 0.09659090909090909, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07386363636363637, + "Indonesian,English,Filipino,Chinese,Malay": 0.07386363636363637, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,English,Spanish,Chinese,Malay": 0.0625, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0625, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, + "English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, + "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, + "English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, + "English,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.38636363636363635, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.3977272727272727, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.4431818181818182, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.3977272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.39204545454545453, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.38636363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.375 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.36363636363636365 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 } }, - "AC3_2": 0.5805303298609771, - "AC3_3": 0.5305213574912646, - "AC3_4": 0.4937498798580256, - "AC3_5": 0.4644261398480116, - "AC3_6": 0.43997206070660533, - "AC3_7": 0.4187327823202939 + "AC3_2": 0.3500594713813695, + "AC3_3": 0.26138355504582134, + "AC3_4": 0.1904745739259062, + "AC3_5": 0.13535808624396273, + "AC3_6": 0.09522306394590777, + "AC3_7": 0.0700619331796178 }, "prompt_5": { - "overall_acc": 0.4886363636363637, + "overall_acc": 0.3060064935064935, "language_acc": { - "English": 0.5284090909090909, - "Filipino": 0.48863636363636365, - "Spanish": 0.48295454545454547, - "Malay": 0.5056818181818182, - "Chinese": 0.5056818181818182, - "Indonesian": 0.45454545454545453, - "Vietnamese": 0.45454545454545453 + "Indonesian": 0.2215909090909091, + "English": 0.5340909090909091, + "Filipino": 0.2556818181818182, + "Spanish": 0.3352272727272727, + "Chinese": 0.26136363636363635, + "Malay": 0.24431818181818182, + "Vietnamese": 0.2897727272727273 }, - "consistency_score_2": 0.6999458874458876, - "consistency_score_3": 0.5681818181818181, - "consistency_score_4": 0.48814935064935056, - "consistency_score_5": 0.4320887445887447, - "consistency_score_6": 0.38879870129870125, - "consistency_score_7": 0.3522727272727273, + "consistency_score_2": 0.4293831168831169, + "consistency_score_3": 0.23311688311688308, + "consistency_score_4": 0.14139610389610388, + "consistency_score_5": 0.08982683982683981, + "consistency_score_6": 0.05844155844155845, + "consistency_score_7": 0.03977272727272727, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.75, - "English,Spanish": 0.7613636363636364, - "English,Malay": 0.7045454545454546, - "English,Chinese": 0.6647727272727273, - "English,Indonesian": 0.6761363636363636, - "English,Vietnamese": 0.7443181818181818, - "Filipino,Spanish": 0.7386363636363636, - "Filipino,Malay": 0.7159090909090909, - "Filipino,Chinese": 0.6420454545454546, - "Filipino,Indonesian": 0.7272727272727273, - "Filipino,Vietnamese": 0.7045454545454546, - "Spanish,Malay": 0.7329545454545454, - "Spanish,Chinese": 0.6193181818181818, - "Spanish,Indonesian": 0.7159090909090909, - "Spanish,Vietnamese": 0.7443181818181818, - "Malay,Chinese": 0.6136363636363636, - "Malay,Indonesian": 0.7159090909090909, - "Malay,Vietnamese": 0.7159090909090909, - "Chinese,Indonesian": 0.6136363636363636, - "Chinese,Vietnamese": 0.6534090909090909, - "Indonesian,Vietnamese": 0.7443181818181818 + "Indonesian,English": 0.3409090909090909, + "Indonesian,Filipino": 0.5, + "Indonesian,Spanish": 0.4090909090909091, + "Indonesian,Chinese": 0.4431818181818182, + "Indonesian,Malay": 0.6647727272727273, + "Indonesian,Vietnamese": 0.4715909090909091, + "English,Filipino": 0.32954545454545453, + "English,Spanish": 0.4772727272727273, + "English,Chinese": 0.2840909090909091, + "English,Malay": 0.2840909090909091, + "English,Vietnamese": 0.3806818181818182, + "Filipino,Spanish": 0.38636363636363635, + "Filipino,Chinese": 0.4375, + "Filipino,Malay": 0.5454545454545454, + "Filipino,Vietnamese": 0.45454545454545453, + "Spanish,Chinese": 0.3125, + "Spanish,Malay": 0.4318181818181818, + "Spanish,Vietnamese": 0.4147727272727273, + "Chinese,Malay": 0.4147727272727273, + "Chinese,Vietnamese": 0.5625, + "Malay,Vietnamese": 0.4715909090909091 }, "3_combine": { - "English,Filipino,Spanish": 0.6306818181818182, - "English,Filipino,Malay": 0.6022727272727273, - "English,Filipino,Chinese": 0.5511363636363636, - "English,Filipino,Indonesian": 0.5909090909090909, - "English,Filipino,Vietnamese": 0.6193181818181818, - "English,Spanish,Malay": 0.6136363636363636, - "English,Spanish,Chinese": 0.5397727272727273, - "English,Spanish,Indonesian": 0.5909090909090909, - "English,Spanish,Vietnamese": 0.6420454545454546, - "English,Malay,Chinese": 0.5056818181818182, - "English,Malay,Indonesian": 0.5681818181818182, - "English,Malay,Vietnamese": 0.6079545454545454, - "English,Chinese,Indonesian": 0.5, - "English,Chinese,Vietnamese": 0.5568181818181818, - "English,Indonesian,Vietnamese": 0.6022727272727273, - "Filipino,Spanish,Malay": 0.6079545454545454, - "Filipino,Spanish,Chinese": 0.5227272727272727, - "Filipino,Spanish,Indonesian": 0.5965909090909091, - "Filipino,Spanish,Vietnamese": 0.6136363636363636, - "Filipino,Malay,Chinese": 0.5113636363636364, - "Filipino,Malay,Indonesian": 0.5909090909090909, - "Filipino,Malay,Vietnamese": 0.5965909090909091, - "Filipino,Chinese,Indonesian": 0.5056818181818182, - "Filipino,Chinese,Vietnamese": 0.5227272727272727, - "Filipino,Indonesian,Vietnamese": 0.6079545454545454, - "Spanish,Malay,Chinese": 0.4943181818181818, - "Spanish,Malay,Indonesian": 0.5965909090909091, - "Spanish,Malay,Vietnamese": 0.6136363636363636, - "Spanish,Chinese,Indonesian": 0.48863636363636365, - "Spanish,Chinese,Vietnamese": 0.5284090909090909, - "Spanish,Indonesian,Vietnamese": 0.6136363636363636, - "Malay,Chinese,Indonesian": 0.4943181818181818, - "Malay,Chinese,Vietnamese": 0.5170454545454546, - "Malay,Indonesian,Vietnamese": 0.6136363636363636, - "Chinese,Indonesian,Vietnamese": 0.5284090909090909 + "Indonesian,English,Filipino": 0.19318181818181818, + "Indonesian,English,Spanish": 0.21022727272727273, + "Indonesian,English,Chinese": 0.1590909090909091, + "Indonesian,English,Malay": 0.23295454545454544, + "Indonesian,English,Vietnamese": 0.2159090909090909, + "Indonesian,Filipino,Spanish": 0.22727272727272727, + "Indonesian,Filipino,Chinese": 0.26704545454545453, + "Indonesian,Filipino,Malay": 0.4090909090909091, + "Indonesian,Filipino,Vietnamese": 0.29545454545454547, + "Indonesian,Spanish,Chinese": 0.16477272727272727, + "Indonesian,Spanish,Malay": 0.3125, + "Indonesian,Spanish,Vietnamese": 0.23863636363636365, + "Indonesian,Chinese,Malay": 0.30113636363636365, + "Indonesian,Chinese,Vietnamese": 0.3068181818181818, + "Indonesian,Malay,Vietnamese": 0.35795454545454547, + "English,Filipino,Spanish": 0.19318181818181818, + "English,Filipino,Chinese": 0.14772727272727273, + "English,Filipino,Malay": 0.17045454545454544, + "English,Filipino,Vietnamese": 0.19318181818181818, + "English,Spanish,Chinese": 0.14204545454545456, + "English,Spanish,Malay": 0.20454545454545456, + "English,Spanish,Vietnamese": 0.23295454545454544, + "English,Chinese,Malay": 0.13636363636363635, + "English,Chinese,Vietnamese": 0.19318181818181818, + "English,Malay,Vietnamese": 0.18181818181818182, + "Filipino,Spanish,Chinese": 0.1875, + "Filipino,Spanish,Malay": 0.26136363636363635, + "Filipino,Spanish,Vietnamese": 0.21022727272727273, + "Filipino,Chinese,Malay": 0.2784090909090909, + "Filipino,Chinese,Vietnamese": 0.30113636363636365, + "Filipino,Malay,Vietnamese": 0.2897727272727273, + "Spanish,Chinese,Malay": 0.17613636363636365, + "Spanish,Chinese,Vietnamese": 0.23295454545454544, + "Spanish,Malay,Vietnamese": 0.23863636363636365, + "Chinese,Malay,Vietnamese": 0.29545454545454547 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.5397727272727273, - "English,Filipino,Spanish,Chinese": 0.4715909090909091, - "English,Filipino,Spanish,Indonesian": 0.5284090909090909, - "English,Filipino,Spanish,Vietnamese": 0.5568181818181818, - "English,Filipino,Malay,Chinese": 0.45454545454545453, - "English,Filipino,Malay,Indonesian": 0.5, - "English,Filipino,Malay,Vietnamese": 0.5397727272727273, - "English,Filipino,Chinese,Indonesian": 0.45454545454545453, - "English,Filipino,Chinese,Vietnamese": 0.48295454545454547, - "English,Filipino,Indonesian,Vietnamese": 0.5397727272727273, - "English,Spanish,Malay,Chinese": 0.4431818181818182, - "English,Spanish,Malay,Indonesian": 0.5113636363636364, - "English,Spanish,Malay,Vietnamese": 0.5511363636363636, - "English,Spanish,Chinese,Indonesian": 0.4431818181818182, - "English,Spanish,Chinese,Vietnamese": 0.48863636363636365, - "English,Spanish,Indonesian,Vietnamese": 0.5397727272727273, - "English,Malay,Chinese,Indonesian": 0.4318181818181818, - "English,Malay,Chinese,Vietnamese": 0.4772727272727273, - "English,Malay,Indonesian,Vietnamese": 0.5284090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.4772727272727273, - "Filipino,Spanish,Malay,Chinese": 0.44886363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.5170454545454546, - "Filipino,Spanish,Malay,Vietnamese": 0.5397727272727273, - "Filipino,Spanish,Chinese,Indonesian": 0.4431818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.4715909090909091, - "Filipino,Spanish,Indonesian,Vietnamese": 0.5340909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.4318181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.44886363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.5284090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.45454545454545453, - "Spanish,Malay,Chinese,Indonesian": 0.42613636363636365, - "Spanish,Malay,Chinese,Vietnamese": 0.45454545454545453, - "Spanish,Malay,Indonesian,Vietnamese": 0.5340909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.4431818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.44886363636363635 + "Indonesian,English,Filipino,Spanish": 0.125, + "Indonesian,English,Filipino,Chinese": 0.10795454545454546, + "Indonesian,English,Filipino,Malay": 0.14772727272727273, + "Indonesian,English,Filipino,Vietnamese": 0.14772727272727273, + "Indonesian,English,Spanish,Chinese": 0.08522727272727272, + "Indonesian,English,Spanish,Malay": 0.16477272727272727, + "Indonesian,English,Spanish,Vietnamese": 0.14772727272727273, + "Indonesian,English,Chinese,Malay": 0.11931818181818182, + "Indonesian,English,Chinese,Vietnamese": 0.13636363636363635, + "Indonesian,English,Malay,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Chinese": 0.11363636363636363, + "Indonesian,Filipino,Spanish,Malay": 0.19886363636363635, + "Indonesian,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "Indonesian,Filipino,Chinese,Malay": 0.21022727272727273, + "Indonesian,Filipino,Chinese,Vietnamese": 0.20454545454545456, + "Indonesian,Filipino,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,Spanish,Chinese,Malay": 0.125, + "Indonesian,Spanish,Chinese,Vietnamese": 0.125, + "Indonesian,Spanish,Malay,Vietnamese": 0.1875, + "Indonesian,Chinese,Malay,Vietnamese": 0.2215909090909091, + "English,Filipino,Spanish,Chinese": 0.09090909090909091, + "English,Filipino,Spanish,Malay": 0.11931818181818182, + "English,Filipino,Spanish,Vietnamese": 0.11931818181818182, + "English,Filipino,Chinese,Malay": 0.09090909090909091, + "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, + "English,Filipino,Malay,Vietnamese": 0.11363636363636363, + "English,Spanish,Chinese,Malay": 0.07954545454545454, + "English,Spanish,Chinese,Vietnamese": 0.11931818181818182, + "English,Spanish,Malay,Vietnamese": 0.13068181818181818, + "English,Chinese,Malay,Vietnamese": 0.11363636363636363, + "Filipino,Spanish,Chinese,Malay": 0.125, + "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, + "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, + "Filipino,Chinese,Malay,Vietnamese": 0.19886363636363635, + "Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.4090909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.4602272727272727, - "English,Filipino,Spanish,Malay,Vietnamese": 0.5, - "English,Filipino,Spanish,Chinese,Indonesian": 0.4090909090909091, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.4375, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.48863636363636365, - "English,Filipino,Malay,Chinese,Indonesian": 0.39204545454545453, - "English,Filipino,Malay,Chinese,Vietnamese": 0.42613636363636365, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.4772727272727273, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.4318181818181818, - "English,Spanish,Malay,Chinese,Indonesian": 0.39204545454545453, - "English,Spanish,Malay,Chinese,Vietnamese": 0.4318181818181818, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.48295454545454547, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.42045454545454547, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.42045454545454547, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.39204545454545453, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.42045454545454547, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.4772727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.4090909090909091, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.3977272727272727, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.3977272727272727 + "Indonesian,English,Filipino,Spanish,Chinese": 0.0625, + "Indonesian,English,Filipino,Spanish,Malay": 0.10227272727272728, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, + "Indonesian,English,Filipino,Chinese,Malay": 0.08522727272727272, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09659090909090909, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,English,Spanish,Chinese,Malay": 0.0625, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.07954545454545454, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, + "English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, + "English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, + "English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, + "English,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.36363636363636365, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.3977272727272727, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.4431818181818182, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.38636363636363635, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.3806818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.3806818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.3693181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.3522727272727273 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 } }, - "AC3_2": 0.575507522249451, - "AC3_3": 0.5254154447205667, - "AC3_4": 0.48839273568360325, - "AC3_5": 0.45862607854394283, - "AC3_6": 0.4330375914063701, - "AC3_7": 0.4093980343493492 + "AC3_2": 0.3573453312205796, + "AC3_3": 0.26463434182102813, + "AC3_4": 0.19341919868257093, + "AC3_5": 0.13888469694940525, + "AC3_6": 0.098140167155555, + "AC3_7": 0.07039585998817745 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.5048543689320388 + "accuracy": 0.4368932038834951 }, "prompt_2": { - "accuracy": 0.49514563106796117 + "accuracy": 0.4368932038834951 }, "prompt_3": { - "accuracy": 0.5242718446601942 + "accuracy": 0.44660194174757284 }, "prompt_4": { - "accuracy": 0.5145631067961165 + "accuracy": 0.44660194174757284 }, "prompt_5": { - "accuracy": 0.5339805825242718 + "accuracy": 0.42718446601941745 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.3047619047619048 + "accuracy": 0.2857142857142857 }, "prompt_2": { - "accuracy": 0.3142857142857143 + "accuracy": 0.21904761904761905 }, "prompt_3": { - "accuracy": 0.29523809523809524 + "accuracy": 0.22857142857142856 }, "prompt_4": { - "accuracy": 0.3142857142857143 + "accuracy": 0.21904761904761905 }, "prompt_5": { - "accuracy": 0.3238095238095238 + "accuracy": 0.24761904761904763 } }, "us_eval": { "prompt_1": { - "accuracy": 0.2523364485981308 + "accuracy": 0.38317757009345793 }, "prompt_2": { - "accuracy": 0.2336448598130841 + "accuracy": 0.3644859813084112 }, "prompt_3": { - "accuracy": 0.24299065420560748 + "accuracy": 0.38317757009345793 }, "prompt_4": { - "accuracy": 0.2523364485981308 + "accuracy": 0.40186915887850466 }, "prompt_5": { - "accuracy": 0.24299065420560748 + "accuracy": 0.40186915887850466 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.3, + "accuracy": 0.24, "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.6, - "history": 0.2, - "literature": 0.0, - "politics": 0.5, - "culture": 0.3, + "brand": 0.1, + "demographics": 0.0, + "biology": 0.3, + "history": 0.13333333333333333, + "literature": 0.5, + "politics": 0.4, + "culture": 0.1, "film": 0.1, "law": 0.3, - "geography": 0.5 + "geography": 0.4 } }, "prompt_2": { - "accuracy": 0.33, + "accuracy": 0.23, "category_acc": { - "brand": 0.3, + "brand": 0.1, "demographics": 0.2, - "biology": 0.6, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.5, - "culture": 0.2, + "biology": 0.3, + "history": 0.13333333333333333, + "literature": 0.4, + "politics": 0.3, + "culture": 0.1, "film": 0.1, "law": 0.3, - "geography": 0.5 + "geography": 0.4 } }, "prompt_3": { - "accuracy": 0.31, + "accuracy": 0.23, "category_acc": { - "brand": 0.4, - "demographics": 0.0, - "biology": 0.5, + "brand": 0.1, + "demographics": 0.2, + "biology": 0.3, "history": 0.2, - "literature": 0.2, - "politics": 0.4, - "culture": 0.4, + "literature": 0.3, + "politics": 0.2, + "culture": 0.1, "film": 0.1, "law": 0.3, "geography": 0.5 } }, "prompt_4": { - "accuracy": 0.31, + "accuracy": 0.26, "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.5, + "brand": 0.1, + "demographics": 0.4, + "biology": 0.3, "history": 0.2, - "literature": 0.2, - "politics": 0.5, - "culture": 0.3, + "literature": 0.4, + "politics": 0.4, + "culture": 0.1, "film": 0.1, "law": 0.3, - "geography": 0.5 + "geography": 0.4 } }, "prompt_5": { - "accuracy": 0.32, + "accuracy": 0.25, "category_acc": { - "brand": 0.4, - "demographics": 0.0, - "biology": 0.5, + "brand": 0.1, + "demographics": 0.2, + "biology": 0.3, "history": 0.2, "literature": 0.3, - "politics": 0.4, - "culture": 0.3, + "politics": 0.2, + "culture": 0.1, "film": 0.1, "law": 0.3, - "geography": 0.6 + "geography": 0.7 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.1415726644994035 + "bleu_score": 0.17858137998030377 }, "prompt_2": { - "bleu_score": 0.13732215511867826 + "bleu_score": 0.17980302230654807 }, "prompt_3": { - "bleu_score": 0.1372456478046343 + "bleu_score": 0.18216557256738 }, "prompt_4": { - "bleu_score": 0.13881717658413173 + "bleu_score": 0.18530270379921296 }, "prompt_5": { - "bleu_score": 0.1359329437121942 + "bleu_score": 0.16536410084591763 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "prompt_1": { + "accuracy": 0.2544896187996529, + "category_acc": { + "History": 0.25502008032128515, + "Geography": 0.22448979591836735, + "Lampungic": 0.2857142857142857, + "Social science": 0.28046744574290483, + "Balinese": 0.29936305732484075, + "Makassarese": 0.2849462365591398, + "Banjarese": 0.2916666666666667, + "Chemistry": 0.20583941605839415, + "Biology": 0.22248520710059172, + "Science": 0.24664602683178535, + "Christian religion": 0.30845771144278605, + "Art": 0.259567387687188, + "Islam religion": 0.2233285917496444, + "Hindu religion": 0.2733333333333333, + "Madurese": 0.2847457627118644, + "Sport": 0.32432432432432434, + "Indonesian language": 0.2693026151930262, + "Physics": 0.21616161616161617, + "Minangkabau culture": 0.25125628140703515, + "Dayak language": 0.25688073394495414, + "Sociology": 0.20766129032258066, + "Economy": 0.22745901639344263, + "Sundanese": 0.28608470181503887, + "Javanese": 0.2540322580645161, + "Civic education": 0.2374821173104435 + } + }, + "prompt_2": { + "accuracy": 0.25769410508044593, + "category_acc": { + "History": 0.25301204819277107, + "Geography": 0.2163265306122449, + "Lampungic": 0.29931972789115646, + "Social science": 0.28213689482470783, + "Balinese": 0.29936305732484075, + "Makassarese": 0.2956989247311828, + "Banjarese": 0.2916666666666667, + "Chemistry": 0.21751824817518248, + "Biology": 0.24142011834319527, + "Science": 0.239422084623323, + "Christian religion": 0.30845771144278605, + "Art": 0.26622296173044924, + "Islam religion": 0.24893314366998578, + "Hindu religion": 0.24666666666666667, + "Madurese": 0.27796610169491526, + "Sport": 0.31756756756756754, + "Indonesian language": 0.27179327521793273, + "Physics": 0.20404040404040405, + "Minangkabau culture": 0.271356783919598, + "Dayak language": 0.26605504587155965, + "Sociology": 0.19959677419354838, + "Economy": 0.2069672131147541, + "Sundanese": 0.29559204840103714, + "Javanese": 0.2550403225806452, + "Civic education": 0.2532188841201717 + } + }, + "prompt_3": { + "accuracy": 0.25936310835169235, + "category_acc": { + "History": 0.25903614457831325, + "Geography": 0.22448979591836735, + "Lampungic": 0.30612244897959184, + "Social science": 0.2921535893155259, + "Balinese": 0.3248407643312102, + "Makassarese": 0.27956989247311825, + "Banjarese": 0.2847222222222222, + "Chemistry": 0.20437956204379562, + "Biology": 0.23668639053254437, + "Science": 0.239422084623323, + "Christian religion": 0.3482587064676617, + "Art": 0.26455906821963393, + "Islam religion": 0.24324324324324326, + "Hindu religion": 0.26666666666666666, + "Madurese": 0.26101694915254237, + "Sport": 0.30405405405405406, + "Indonesian language": 0.26805728518057287, + "Physics": 0.21414141414141413, + "Minangkabau culture": 0.2663316582914573, + "Dayak language": 0.23853211009174313, + "Sociology": 0.20362903225806453, + "Economy": 0.23155737704918034, + "Sundanese": 0.3016421780466724, + "Javanese": 0.2600806451612903, + "Civic education": 0.25608011444921314 + } + }, + "prompt_4": { + "accuracy": 0.25475665932305225, + "category_acc": { + "History": 0.26104417670682734, + "Geography": 0.21224489795918366, + "Lampungic": 0.2925170068027211, + "Social science": 0.2854757929883139, + "Balinese": 0.3184713375796178, + "Makassarese": 0.2849462365591398, + "Banjarese": 0.2569444444444444, + "Chemistry": 0.21751824817518248, + "Biology": 0.20946745562130178, + "Science": 0.24251805985552116, + "Christian religion": 0.2885572139303483, + "Art": 0.26788685524126454, + "Islam religion": 0.23470839260312945, + "Hindu religion": 0.26666666666666666, + "Madurese": 0.2745762711864407, + "Sport": 0.32432432432432434, + "Indonesian language": 0.26525529265255293, + "Physics": 0.2202020202020202, + "Minangkabau culture": 0.2914572864321608, + "Dayak language": 0.25688073394495414, + "Sociology": 0.2056451612903226, + "Economy": 0.1987704918032787, + "Sundanese": 0.2990492653414002, + "Javanese": 0.25806451612903225, + "Civic education": 0.2374821173104435 + } + }, + "prompt_5": { + "accuracy": 0.2565591828559984, + "category_acc": { + "History": 0.2389558232931727, + "Geography": 0.22040816326530613, + "Lampungic": 0.3129251700680272, + "Social science": 0.2570951585976628, + "Balinese": 0.3184713375796178, + "Makassarese": 0.2849462365591398, + "Banjarese": 0.2847222222222222, + "Chemistry": 0.19708029197080293, + "Biology": 0.21301775147928995, + "Science": 0.2435500515995872, + "Christian religion": 0.2736318407960199, + "Art": 0.2562396006655574, + "Islam religion": 0.25177809388335703, + "Hindu religion": 0.26, + "Madurese": 0.2677966101694915, + "Sport": 0.34459459459459457, + "Indonesian language": 0.27677459526774595, + "Physics": 0.21414141414141413, + "Minangkabau culture": 0.271356783919598, + "Dayak language": 0.24770642201834864, + "Sociology": 0.20766129032258066, + "Economy": 0.19672131147540983, + "Sundanese": 0.3025064822817632, + "Javanese": 0.2590725806451613, + "Civic education": 0.2632331902718169 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.3702499286280674 + "bleu_score": 0.12760103596967726 }, "prompt_2": { - "bleu_score": 0.3725328734892377 + "bleu_score": 0.12728182784238143 }, "prompt_3": { - "bleu_score": 0.3682673142036171 + "bleu_score": 0.12596753154582943 }, "prompt_4": { - "bleu_score": 0.3729836664115557 + "bleu_score": 0.12849571351474165 }, "prompt_5": { - "bleu_score": 0.3738645804241501 + "bleu_score": 0.11764504442840339 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.3331210760216344 + "bleu_score": 0.03668322829140508 }, "prompt_2": { - "bleu_score": 0.3385870277660897 + "bleu_score": 0.03663401506892185 }, "prompt_3": { - "bleu_score": 0.3340723812360414 + "bleu_score": 0.037271563791032004 }, "prompt_4": { - "bleu_score": 0.34067568908971596 + "bleu_score": 0.03656009783069245 }, "prompt_5": { - "bleu_score": 0.3404965901668535 + "bleu_score": 0.03556398570352625 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.2777190863401374 + "bleu_score": 0.014429676623784958 }, "prompt_2": { - "bleu_score": 0.28465043692335784 + "bleu_score": 0.016031539247195952 }, "prompt_3": { - "bleu_score": 0.27745720654550554 + "bleu_score": 0.015074674723810412 }, "prompt_4": { - "bleu_score": 0.28738979676124116 + "bleu_score": 0.01475034003355935 }, "prompt_5": { - "bleu_score": 0.28484775494547243 + "bleu_score": 0.012008477296306986 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.37879242173955846 + "bleu_score": 0.1184246883322347 }, "prompt_2": { - "bleu_score": 0.3834367453020953 + "bleu_score": 0.11819809679873207 }, "prompt_3": { - "bleu_score": 0.37836560188389307 + "bleu_score": 0.11660955144396412 }, "prompt_4": { - "bleu_score": 0.38465772917800367 + "bleu_score": 0.11840549537817129 }, "prompt_5": { - "bleu_score": 0.382863345203959 + "bleu_score": 0.11277536636039184 } }, "mmlu": { "prompt_1": { - "accuracy": 0.45857642940490084 + "accuracy": 0.456242707117853 }, "prompt_2": { - "accuracy": 0.44807467911318555 + "accuracy": 0.43990665110851807 }, "prompt_3": { - "accuracy": 0.44807467911318555 + "accuracy": 0.4364060676779463 }, "prompt_4": { - "accuracy": 0.43990665110851807 + "accuracy": 0.4364060676779463 }, "prompt_5": { - "accuracy": 0.45274212368728123 + "accuracy": 0.43990665110851807 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.4276725062567036, + "accuracy": 0.41344297461565965, "category_acc": { - "high_school_european_history": 0.6646341463414634, + "high_school_european_history": 0.5426829268292683, "business_ethics": 0.5555555555555556, - "clinical_knowledge": 0.4393939393939394, - "medical_genetics": 0.3939393939393939, - "high_school_us_history": 0.47783251231527096, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.6483050847457628, - "virology": 0.3878787878787879, - "high_school_microeconomics": 0.4345991561181435, + "clinical_knowledge": 0.44696969696969696, + "medical_genetics": 0.43434343434343436, + "high_school_us_history": 0.5369458128078818, + "high_school_physics": 0.31333333333333335, + "high_school_world_history": 0.576271186440678, + "virology": 0.34545454545454546, + "high_school_microeconomics": 0.459915611814346, "econometrics": 0.25663716814159293, - "college_computer_science": 0.40404040404040403, - "high_school_biology": 0.45307443365695793, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.3202846975088968, - "philosophy": 0.4483870967741935, + "college_computer_science": 0.30303030303030304, + "high_school_biology": 0.42394822006472493, + "abstract_algebra": 0.2222222222222222, + "professional_accounting": 0.3416370106761566, + "philosophy": 0.4161290322580645, "professional_medicine": 0.4095940959409594, - "nutrition": 0.4524590163934426, - "global_facts": 0.29292929292929293, - "machine_learning": 0.23423423423423423, - "security_studies": 0.5245901639344263, - "public_relations": 0.4954128440366973, - "professional_psychology": 0.40098199672667756, - "prehistory": 0.42724458204334365, - "anatomy": 0.48507462686567165, - "human_sexuality": 0.5, - "college_medicine": 0.38372093023255816, - "high_school_government_and_politics": 0.53125, - "college_chemistry": 0.32323232323232326, - "logical_fallacies": 0.5864197530864198, - "high_school_geography": 0.5888324873096447, - "elementary_mathematics": 0.3050397877984085, - "human_aging": 0.4369369369369369, - "college_mathematics": 0.35353535353535354, - "high_school_psychology": 0.5735294117647058, - "formal_logic": 0.32, + "nutrition": 0.46557377049180326, + "global_facts": 0.30303030303030304, + "machine_learning": 0.24324324324324326, + "security_studies": 0.5573770491803278, + "public_relations": 0.47706422018348627, + "professional_psychology": 0.3698854337152209, + "prehistory": 0.4055727554179567, + "anatomy": 0.3880597014925373, + "human_sexuality": 0.4, + "college_medicine": 0.4186046511627907, + "high_school_government_and_politics": 0.546875, + "college_chemistry": 0.36363636363636365, + "logical_fallacies": 0.5246913580246914, + "high_school_geography": 0.5482233502538071, + "elementary_mathematics": 0.27851458885941643, + "human_aging": 0.45495495495495497, + "college_mathematics": 0.26262626262626265, + "high_school_psychology": 0.5661764705882353, + "formal_logic": 0.36, "high_school_statistics": 0.33488372093023255, - "international_law": 0.65, - "high_school_mathematics": 0.26394052044609667, - "high_school_computer_science": 0.42424242424242425, - "conceptual_physics": 0.32905982905982906, - "miscellaneous": 0.5332480818414322, - "high_school_chemistry": 0.297029702970297, - "marketing": 0.759656652360515, - "professional_law": 0.3268101761252446, + "international_law": 0.5583333333333333, + "high_school_mathematics": 0.2825278810408922, + "high_school_computer_science": 0.36363636363636365, + "conceptual_physics": 0.358974358974359, + "miscellaneous": 0.5127877237851662, + "high_school_chemistry": 0.29207920792079206, + "marketing": 0.7167381974248928, + "professional_law": 0.3378995433789954, "management": 0.6372549019607843, - "college_physics": 0.27722772277227725, - "jurisprudence": 0.5887850467289719, - "world_religions": 0.48823529411764705, - "sociology": 0.605, - "us_foreign_policy": 0.5858585858585859, - "high_school_macroeconomics": 0.4473007712082262, - "computer_security": 0.5454545454545454, - "moral_scenarios": 0.2785234899328859, - "moral_disputes": 0.4927536231884058, - "electrical_engineering": 0.4583333333333333, - "astronomy": 0.3841059602649007, - "college_biology": 0.2937062937062937 + "college_physics": 0.31683168316831684, + "jurisprudence": 0.4766355140186916, + "world_religions": 0.38235294117647056, + "sociology": 0.565, + "us_foreign_policy": 0.5757575757575758, + "high_school_macroeconomics": 0.43958868894601544, + "computer_security": 0.47474747474747475, + "moral_scenarios": 0.23937360178970918, + "moral_disputes": 0.48985507246376814, + "electrical_engineering": 0.4027777777777778, + "astronomy": 0.37748344370860926, + "college_biology": 0.3706293706293706 } }, "prompt_2": { - "accuracy": 0.424168752234537, + "accuracy": 0.41329996424740795, "category_acc": { - "high_school_european_history": 0.6646341463414634, - "business_ethics": 0.5757575757575758, + "high_school_european_history": 0.573170731707317, + "business_ethics": 0.5050505050505051, "clinical_knowledge": 0.4431818181818182, - "medical_genetics": 0.43434343434343436, - "high_school_us_history": 0.46798029556650245, - "high_school_physics": 0.28, - "high_school_world_history": 0.6313559322033898, - "virology": 0.3939393939393939, + "medical_genetics": 0.4444444444444444, + "high_school_us_history": 0.5467980295566502, + "high_school_physics": 0.30666666666666664, + "high_school_world_history": 0.5889830508474576, + "virology": 0.36363636363636365, "high_school_microeconomics": 0.4345991561181435, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.43434343434343436, - "high_school_biology": 0.44660194174757284, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.3309608540925267, - "philosophy": 0.43870967741935485, - "professional_medicine": 0.3837638376383764, - "nutrition": 0.4426229508196721, - "global_facts": 0.3333333333333333, - "machine_learning": 0.23423423423423423, - "security_studies": 0.5122950819672131, - "public_relations": 0.45871559633027525, - "professional_psychology": 0.3911620294599018, - "prehistory": 0.4179566563467492, - "anatomy": 0.4925373134328358, - "human_sexuality": 0.5307692307692308, - "college_medicine": 0.36627906976744184, + "econometrics": 0.24778761061946902, + "college_computer_science": 0.31313131313131315, + "high_school_biology": 0.43042071197411, + "abstract_algebra": 0.29292929292929293, + "professional_accounting": 0.37722419928825623, + "philosophy": 0.4161290322580645, + "professional_medicine": 0.39114391143911437, + "nutrition": 0.4557377049180328, + "global_facts": 0.29292929292929293, + "machine_learning": 0.3153153153153153, + "security_studies": 0.5573770491803278, + "public_relations": 0.46788990825688076, + "professional_psychology": 0.37479541734860883, + "prehistory": 0.3931888544891641, + "anatomy": 0.417910447761194, + "human_sexuality": 0.4230769230769231, + "college_medicine": 0.43023255813953487, "high_school_government_and_politics": 0.5260416666666666, "college_chemistry": 0.3333333333333333, - "logical_fallacies": 0.5925925925925926, - "high_school_geography": 0.5786802030456852, - "elementary_mathematics": 0.3050397877984085, - "human_aging": 0.42792792792792794, - "college_mathematics": 0.3434343434343434, - "high_school_psychology": 0.5625, - "formal_logic": 0.312, - "high_school_statistics": 0.3116279069767442, - "international_law": 0.65, - "high_school_mathematics": 0.275092936802974, - "high_school_computer_science": 0.45454545454545453, - "conceptual_physics": 0.3418803418803419, - "miscellaneous": 0.5281329923273658, - "high_school_chemistry": 0.26732673267326734, - "marketing": 0.7553648068669528, - "professional_law": 0.32093933463796476, - "management": 0.6176470588235294, - "college_physics": 0.2871287128712871, - "jurisprudence": 0.5420560747663551, - "world_religions": 0.4823529411764706, - "sociology": 0.59, - "us_foreign_policy": 0.6060606060606061, - "high_school_macroeconomics": 0.42159383033419023, - "computer_security": 0.5353535353535354, - "moral_scenarios": 0.28187919463087246, + "logical_fallacies": 0.5308641975308642, + "high_school_geography": 0.5076142131979695, + "elementary_mathematics": 0.2838196286472148, + "human_aging": 0.40540540540540543, + "college_mathematics": 0.2727272727272727, + "high_school_psychology": 0.5477941176470589, + "formal_logic": 0.304, + "high_school_statistics": 0.33488372093023255, + "international_law": 0.525, + "high_school_mathematics": 0.2899628252788104, + "high_school_computer_science": 0.42424242424242425, + "conceptual_physics": 0.32905982905982906, + "miscellaneous": 0.5025575447570333, + "high_school_chemistry": 0.27722772277227725, + "marketing": 0.703862660944206, + "professional_law": 0.33268101761252444, + "management": 0.6078431372549019, + "college_physics": 0.33663366336633666, + "jurisprudence": 0.4672897196261682, + "world_religions": 0.38235294117647056, + "sociology": 0.575, + "us_foreign_policy": 0.6161616161616161, + "high_school_macroeconomics": 0.4241645244215938, + "computer_security": 0.5050505050505051, + "moral_scenarios": 0.27293064876957496, "moral_disputes": 0.5043478260869565, - "electrical_engineering": 0.4583333333333333, - "astronomy": 0.3841059602649007, - "college_biology": 0.3356643356643357 + "electrical_engineering": 0.4027777777777778, + "astronomy": 0.40397350993377484, + "college_biology": 0.34265734265734266 } }, "prompt_3": { - "accuracy": 0.4221666070790132, + "accuracy": 0.41623167679656775, "category_acc": { - "high_school_european_history": 0.6585365853658537, - "business_ethics": 0.5151515151515151, - "clinical_knowledge": 0.4356060606060606, - "medical_genetics": 0.41414141414141414, - "high_school_us_history": 0.4729064039408867, + "high_school_european_history": 0.5792682926829268, + "business_ethics": 0.5757575757575758, + "clinical_knowledge": 0.4583333333333333, + "medical_genetics": 0.40404040404040403, + "high_school_us_history": 0.5467980295566502, "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.635593220338983, - "virology": 0.3878787878787879, - "high_school_microeconomics": 0.4345991561181435, - "econometrics": 0.2920353982300885, - "college_computer_science": 0.42424242424242425, - "high_school_biology": 0.4401294498381877, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.3416370106761566, - "philosophy": 0.44516129032258067, - "professional_medicine": 0.3874538745387454, - "nutrition": 0.45901639344262296, - "global_facts": 0.29292929292929293, - "machine_learning": 0.23423423423423423, - "security_studies": 0.5286885245901639, - "public_relations": 0.48623853211009177, - "professional_psychology": 0.39279869067103107, - "prehistory": 0.4148606811145511, - "anatomy": 0.5, - "human_sexuality": 0.5230769230769231, - "college_medicine": 0.38953488372093026, - "high_school_government_and_politics": 0.53125, - "college_chemistry": 0.31313131313131315, - "logical_fallacies": 0.5802469135802469, - "high_school_geography": 0.5634517766497462, - "elementary_mathematics": 0.2917771883289125, - "human_aging": 0.44144144144144143, - "college_mathematics": 0.3434343434343434, - "high_school_psychology": 0.5569852941176471, - "formal_logic": 0.312, - "high_school_statistics": 0.3023255813953488, - "international_law": 0.6416666666666667, - "high_school_mathematics": 0.2899628252788104, - "high_school_computer_science": 0.41414141414141414, - "conceptual_physics": 0.34615384615384615, - "miscellaneous": 0.5421994884910486, - "high_school_chemistry": 0.26732673267326734, - "marketing": 0.7467811158798283, - "professional_law": 0.31637312459230266, - "management": 0.6078431372549019, - "college_physics": 0.2871287128712871, - "jurisprudence": 0.5607476635514018, - "world_religions": 0.47058823529411764, - "sociology": 0.595, - "us_foreign_policy": 0.5757575757575758, - "high_school_macroeconomics": 0.4370179948586118, - "computer_security": 0.5454545454545454, - "moral_scenarios": 0.2494407158836689, + "high_school_world_history": 0.5932203389830508, + "virology": 0.3696969696969697, + "high_school_microeconomics": 0.45147679324894513, + "econometrics": 0.25663716814159293, + "college_computer_science": 0.3333333333333333, + "high_school_biology": 0.42394822006472493, + "abstract_algebra": 0.2828282828282828, + "professional_accounting": 0.3736654804270463, + "philosophy": 0.432258064516129, + "professional_medicine": 0.4022140221402214, + "nutrition": 0.46885245901639344, + "global_facts": 0.2828282828282828, + "machine_learning": 0.32432432432432434, + "security_studies": 0.5655737704918032, + "public_relations": 0.47706422018348627, + "professional_psychology": 0.3698854337152209, + "prehistory": 0.3993808049535604, + "anatomy": 0.3880597014925373, + "human_sexuality": 0.4153846153846154, + "college_medicine": 0.42441860465116277, + "high_school_government_and_politics": 0.5260416666666666, + "college_chemistry": 0.3333333333333333, + "logical_fallacies": 0.5246913580246914, + "high_school_geography": 0.5126903553299492, + "elementary_mathematics": 0.2838196286472148, + "human_aging": 0.4189189189189189, + "college_mathematics": 0.2828282828282828, + "high_school_psychology": 0.5514705882352942, + "formal_logic": 0.328, + "high_school_statistics": 0.32558139534883723, + "international_law": 0.5416666666666666, + "high_school_mathematics": 0.2862453531598513, + "high_school_computer_science": 0.37373737373737376, + "conceptual_physics": 0.33760683760683763, + "miscellaneous": 0.5, + "high_school_chemistry": 0.31683168316831684, + "marketing": 0.703862660944206, + "professional_law": 0.33920417482061316, + "management": 0.6470588235294118, + "college_physics": 0.3465346534653465, + "jurisprudence": 0.4485981308411215, + "world_religions": 0.4, + "sociology": 0.585, + "us_foreign_policy": 0.6161616161616161, + "high_school_macroeconomics": 0.442159383033419, + "computer_security": 0.5050505050505051, + "moral_scenarios": 0.26174496644295303, "moral_disputes": 0.5072463768115942, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.40397350993377484, - "college_biology": 0.3076923076923077 + "electrical_engineering": 0.4027777777777778, + "astronomy": 0.3973509933774834, + "college_biology": 0.3146853146853147 } }, "prompt_4": { - "accuracy": 0.42366821594565607, + "accuracy": 0.4138720057204147, "category_acc": { - "high_school_european_history": 0.6585365853658537, - "business_ethics": 0.5252525252525253, - "clinical_knowledge": 0.4393939393939394, - "medical_genetics": 0.41414141414141414, - "high_school_us_history": 0.46798029556650245, + "high_school_european_history": 0.5487804878048781, + "business_ethics": 0.5050505050505051, + "clinical_knowledge": 0.45454545454545453, + "medical_genetics": 0.40404040404040403, + "high_school_us_history": 0.5517241379310345, "high_school_physics": 0.3, - "high_school_world_history": 0.635593220338983, - "virology": 0.4, - "high_school_microeconomics": 0.4388185654008439, - "econometrics": 0.2743362831858407, - "college_computer_science": 0.40404040404040403, - "high_school_biology": 0.4336569579288026, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.3238434163701068, - "philosophy": 0.43548387096774194, - "professional_medicine": 0.3948339483394834, - "nutrition": 0.46885245901639344, - "global_facts": 0.31313131313131315, - "machine_learning": 0.24324324324324326, - "security_studies": 0.5163934426229508, - "public_relations": 0.45871559633027525, - "professional_psychology": 0.3993453355155483, - "prehistory": 0.43343653250773995, - "anatomy": 0.4701492537313433, - "human_sexuality": 0.5384615384615384, - "college_medicine": 0.37790697674418605, - "high_school_government_and_politics": 0.5052083333333334, + "high_school_world_history": 0.5889830508474576, + "virology": 0.36363636363636365, + "high_school_microeconomics": 0.45147679324894513, + "econometrics": 0.23893805309734514, + "college_computer_science": 0.3333333333333333, + "high_school_biology": 0.44660194174757284, + "abstract_algebra": 0.2828282828282828, + "professional_accounting": 0.36298932384341637, + "philosophy": 0.41935483870967744, + "professional_medicine": 0.4059040590405904, + "nutrition": 0.46557377049180326, + "global_facts": 0.2828282828282828, + "machine_learning": 0.2702702702702703, + "security_studies": 0.5614754098360656, + "public_relations": 0.44036697247706424, + "professional_psychology": 0.3617021276595745, + "prehistory": 0.3993808049535604, + "anatomy": 0.40298507462686567, + "human_sexuality": 0.4230769230769231, + "college_medicine": 0.4011627906976744, + "high_school_government_and_politics": 0.546875, "college_chemistry": 0.3333333333333333, - "logical_fallacies": 0.5864197530864198, - "high_school_geography": 0.5685279187817259, - "elementary_mathematics": 0.2917771883289125, - "human_aging": 0.42342342342342343, - "college_mathematics": 0.3434343434343434, - "high_school_psychology": 0.5606617647058824, - "formal_logic": 0.344, - "high_school_statistics": 0.3209302325581395, - "international_law": 0.6333333333333333, - "high_school_mathematics": 0.2936802973977695, - "high_school_computer_science": 0.46464646464646464, - "conceptual_physics": 0.3547008547008547, - "miscellaneous": 0.5319693094629157, - "high_school_chemistry": 0.25742574257425743, - "marketing": 0.759656652360515, - "professional_law": 0.324853228962818, - "management": 0.6176470588235294, - "college_physics": 0.297029702970297, - "jurisprudence": 0.5794392523364486, - "world_religions": 0.47058823529411764, - "sociology": 0.61, - "us_foreign_policy": 0.5858585858585859, - "high_school_macroeconomics": 0.43444730077120824, - "computer_security": 0.5353535353535354, - "moral_scenarios": 0.24720357941834453, - "moral_disputes": 0.4927536231884058, - "electrical_engineering": 0.4583333333333333, - "astronomy": 0.40397350993377484, - "college_biology": 0.32867132867132864 + "logical_fallacies": 0.5308641975308642, + "high_school_geography": 0.5279187817258884, + "elementary_mathematics": 0.2864721485411141, + "human_aging": 0.4144144144144144, + "college_mathematics": 0.25252525252525254, + "high_school_psychology": 0.5551470588235294, + "formal_logic": 0.304, + "high_school_statistics": 0.34418604651162793, + "international_law": 0.5333333333333333, + "high_school_mathematics": 0.2899628252788104, + "high_school_computer_science": 0.37373737373737376, + "conceptual_physics": 0.33760683760683763, + "miscellaneous": 0.5127877237851662, + "high_school_chemistry": 0.2871287128712871, + "marketing": 0.6995708154506438, + "professional_law": 0.33463796477495106, + "management": 0.6176470588235294, + "college_physics": 0.3465346534653465, + "jurisprudence": 0.4672897196261682, + "world_religions": 0.38235294117647056, + "sociology": 0.59, + "us_foreign_policy": 0.6262626262626263, + "high_school_macroeconomics": 0.42930591259640105, + "computer_security": 0.48484848484848486, + "moral_scenarios": 0.25727069351230425, + "moral_disputes": 0.5101449275362319, + "electrical_engineering": 0.4097222222222222, + "astronomy": 0.4105960264900662, + "college_biology": 0.34265734265734266 } }, "prompt_5": { - "accuracy": 0.42323918484090095, + "accuracy": 0.4199499463711119, "category_acc": { - "high_school_european_history": 0.6585365853658537, - "business_ethics": 0.5555555555555556, + "high_school_european_history": 0.573170731707317, + "business_ethics": 0.5252525252525253, "clinical_knowledge": 0.45075757575757575, - "medical_genetics": 0.45454545454545453, - "high_school_us_history": 0.45320197044334976, - "high_school_physics": 0.2733333333333333, - "high_school_world_history": 0.6271186440677966, - "virology": 0.3878787878787879, - "high_school_microeconomics": 0.4388185654008439, - "econometrics": 0.2743362831858407, - "college_computer_science": 0.4444444444444444, - "high_school_biology": 0.44983818770226536, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.33807829181494664, - "philosophy": 0.43870967741935485, - "professional_medicine": 0.3948339483394834, - "nutrition": 0.4459016393442623, - "global_facts": 0.30303030303030304, - "machine_learning": 0.25225225225225223, - "security_studies": 0.5122950819672131, - "public_relations": 0.45871559633027525, - "professional_psychology": 0.3895253682487725, - "prehistory": 0.4086687306501548, - "anatomy": 0.5149253731343284, - "human_sexuality": 0.5230769230769231, - "college_medicine": 0.38953488372093026, - "high_school_government_and_politics": 0.5208333333333334, + "medical_genetics": 0.42424242424242425, + "high_school_us_history": 0.5467980295566502, + "high_school_physics": 0.31333333333333335, + "high_school_world_history": 0.597457627118644, + "virology": 0.38181818181818183, + "high_school_microeconomics": 0.4345991561181435, + "econometrics": 0.23893805309734514, + "college_computer_science": 0.29292929292929293, + "high_school_biology": 0.44660194174757284, + "abstract_algebra": 0.30303030303030304, + "professional_accounting": 0.3736654804270463, + "philosophy": 0.43548387096774194, + "professional_medicine": 0.3800738007380074, + "nutrition": 0.4721311475409836, + "global_facts": 0.29292929292929293, + "machine_learning": 0.3153153153153153, + "security_studies": 0.5614754098360656, + "public_relations": 0.46788990825688076, + "professional_psychology": 0.3862520458265139, + "prehistory": 0.4179566563467492, + "anatomy": 0.417910447761194, + "human_sexuality": 0.4153846153846154, + "college_medicine": 0.436046511627907, + "high_school_government_and_politics": 0.53125, "college_chemistry": 0.32323232323232326, - "logical_fallacies": 0.5802469135802469, - "high_school_geography": 0.5685279187817259, - "elementary_mathematics": 0.30238726790450926, - "human_aging": 0.43243243243243246, - "college_mathematics": 0.3434343434343434, - "high_school_psychology": 0.5625, - "formal_logic": 0.336, - "high_school_statistics": 0.3209302325581395, - "international_law": 0.6166666666666667, - "high_school_mathematics": 0.26765799256505574, - "high_school_computer_science": 0.4444444444444444, - "conceptual_physics": 0.36752136752136755, - "miscellaneous": 0.5358056265984654, - "high_school_chemistry": 0.28217821782178215, - "marketing": 0.759656652360515, - "professional_law": 0.31833007175472927, - "management": 0.6176470588235294, - "college_physics": 0.2871287128712871, - "jurisprudence": 0.5420560747663551, - "world_religions": 0.4764705882352941, - "sociology": 0.605, - "us_foreign_policy": 0.5656565656565656, - "high_school_macroeconomics": 0.4267352185089974, - "computer_security": 0.5454545454545454, - "moral_scenarios": 0.2550335570469799, - "moral_disputes": 0.5043478260869565, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.37748344370860926, - "college_biology": 0.3356643356643357 + "logical_fallacies": 0.5555555555555556, + "high_school_geography": 0.5482233502538071, + "elementary_mathematics": 0.2864721485411141, + "human_aging": 0.42342342342342343, + "college_mathematics": 0.2828282828282828, + "high_school_psychology": 0.5588235294117647, + "formal_logic": 0.304, + "high_school_statistics": 0.32558139534883723, + "international_law": 0.5666666666666667, + "high_school_mathematics": 0.275092936802974, + "high_school_computer_science": 0.3939393939393939, + "conceptual_physics": 0.3333333333333333, + "miscellaneous": 0.5089514066496164, + "high_school_chemistry": 0.30198019801980197, + "marketing": 0.7339055793991416, + "professional_law": 0.33659491193737767, + "management": 0.6078431372549019, + "college_physics": 0.3564356435643564, + "jurisprudence": 0.5046728971962616, + "world_religions": 0.3764705882352941, + "sociology": 0.595, + "us_foreign_policy": 0.6161616161616161, + "high_school_macroeconomics": 0.42930591259640105, + "computer_security": 0.5151515151515151, + "moral_scenarios": 0.27628635346756153, + "moral_disputes": 0.5014492753623189, + "electrical_engineering": 0.4166666666666667, + "astronomy": 0.39072847682119205, + "college_biology": 0.34965034965034963 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.40713224368499257 + "accuracy": 0.24962852897473997 }, "prompt_2": { - "accuracy": 0.40193164933135217 + "accuracy": 0.25482912332838037 }, "prompt_3": { - "accuracy": 0.4049034175334324 + "accuracy": 0.2473997028231798 }, "prompt_4": { - "accuracy": 0.41084695393759285 + "accuracy": 0.2451708766716196 }, "prompt_5": { - "accuracy": 0.4078751857355126 + "accuracy": 0.2473997028231798 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.3929016189290162, + "accuracy": 0.24719800747198006, "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.375, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.2857142857142857, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.4782608695652174, + "computer_network": 0.25, + "operating_system": 0.16666666666666666, + "computer_architecture": 0.3076923076923077, + "college_programming": 0.19047619047619047, + "college_physics": 0.041666666666666664, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.4166666666666667, + "probability_and_statistics": 0.21739130434782608, "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.5172413793103449, - "high_school_mathematics": 0.30434782608695654, + "electrical_engineer": 0.23809523809523808, + "metrology_engineer": 0.1724137931034483, + "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.25, + "high_school_chemistry": 0.375, + "high_school_biology": 0.4583333333333333, "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.5, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.4166666666666667, - "business_administration": 0.39473684210526316, - "marxism": 0.4583333333333333, - "mao_zedong_thought": 0.4827586206896552, - "education_science": 0.5588235294117647, - "teacher_qualification": 0.5918367346938775, - "high_school_politics": 0.25, - "high_school_geography": 0.5, - "middle_school_politics": 0.6923076923076923, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.7083333333333334, - "logic": 0.4444444444444444, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.5, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.25, - "high_school_history": 0.48, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.40384615384615385, - "sports_science": 0.4166666666666667, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.375, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.47058823529411764, - "accountant": 0.3148148148148148, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.3611111111111111, + "middle_school_biology": 0.15384615384615385, + "middle_school_physics": 0.16666666666666666, + "middle_school_chemistry": 0.24, + "veterinary_medicine": 0.21428571428571427, + "college_economics": 0.31666666666666665, + "business_administration": 0.15789473684210525, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.3103448275862069, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.30612244897959184, + "high_school_politics": 0.20833333333333334, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.19230769230769232, + "middle_school_geography": 0.23529411764705882, + "modern_chinese_history": 0.07142857142857142, + "ideological_and_moral_cultivation": 0.2916666666666667, + "logic": 0.25925925925925924, + "law": 0.2413793103448276, + "chinese_language_and_literature": 0.25, + "art_studies": 0.42105263157894735, + "professional_tour_guide": 0.20588235294117646, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.3333333333333333, + "high_school_history": 0.12, + "middle_school_history": 0.2962962962962963, + "civil_servant": 0.23076923076923078, + "sports_science": 0.25, + "plant_protection": 0.2222222222222222, + "basic_medicine": 0.125, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.2549019607843137, + "accountant": 0.18518518518518517, + "fire_engineer": 0.25, + "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.2962962962962963, - "physician": 0.3888888888888889 + "physician": 0.2777777777777778 } }, "prompt_2": { - "accuracy": 0.3891656288916563, + "accuracy": 0.2559153175591532, "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.46153846153846156, - "college_programming": 0.3333333333333333, - "college_physics": 0.25, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.41379310344827586, + "computer_network": 0.25, + "operating_system": 0.125, + "computer_architecture": 0.34615384615384615, + "college_programming": 0.21428571428571427, + "college_physics": 0.125, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.4166666666666667, + "probability_and_statistics": 0.43478260869565216, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.19047619047619047, + "metrology_engineer": 0.1724137931034483, "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.20833333333333334, + "high_school_physics": 0.3333333333333333, "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.46153846153846156, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.4166666666666667, - "business_administration": 0.39473684210526316, - "marxism": 0.5, - "mao_zedong_thought": 0.5517241379310345, - "education_science": 0.5588235294117647, - "teacher_qualification": 0.5714285714285714, - "high_school_politics": 0.375, - "high_school_geography": 0.5, - "middle_school_politics": 0.6538461538461539, + "high_school_biology": 0.375, + "middle_school_mathematics": 0.16666666666666666, + "middle_school_biology": 0.15384615384615385, + "middle_school_physics": 0.16666666666666666, + "middle_school_chemistry": 0.28, + "veterinary_medicine": 0.25, + "college_economics": 0.3333333333333333, + "business_administration": 0.18421052631578946, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.3448275862068966, + "education_science": 0.2647058823529412, + "teacher_qualification": 0.30612244897959184, + "high_school_politics": 0.125, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.3076923076923077, "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.6666666666666666, - "logic": 0.48148148148148145, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.5, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.48, - "middle_school_history": 0.4074074074074074, - "civil_servant": 0.40384615384615385, - "sports_science": 0.4583333333333333, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.375, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.49019607843137253, - "accountant": 0.35185185185185186, + "modern_chinese_history": 0.17857142857142858, + "ideological_and_moral_cultivation": 0.3333333333333333, + "logic": 0.25925925925925924, + "law": 0.2413793103448276, + "chinese_language_and_literature": 0.25, + "art_studies": 0.42105263157894735, + "professional_tour_guide": 0.23529411764705882, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.25, + "high_school_history": 0.2, + "middle_school_history": 0.25925925925925924, + "civil_servant": 0.19230769230769232, + "sports_science": 0.25, + "plant_protection": 0.25925925925925924, + "basic_medicine": 0.20833333333333334, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.27450980392156865, + "accountant": 0.2037037037037037, "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.3611111111111111, + "environmental_impact_assessment_engineer": 0.2777777777777778, "tax_accountant": 0.25925925925925924, - "physician": 0.3888888888888889 + "physician": 0.2962962962962963 } }, "prompt_3": { - "accuracy": 0.39476961394769616, + "accuracy": 0.24782067247820672, "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.375, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.30952380952380953, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.16666666666666666, - "probability_and_statistics": 0.4782608695652174, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.41379310344827586, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.20833333333333334, + "computer_network": 0.25, + "operating_system": 0.16666666666666666, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.2619047619047619, + "college_physics": 0.08333333333333333, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.4166666666666667, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.2857142857142857, + "metrology_engineer": 0.1724137931034483, + "high_school_mathematics": 0.17391304347826086, + "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.25, + "high_school_biology": 0.4166666666666667, "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.5, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.4166666666666667, - "business_administration": 0.42105263157894735, - "marxism": 0.5, - "mao_zedong_thought": 0.4827586206896552, - "education_science": 0.5588235294117647, - "teacher_qualification": 0.5918367346938775, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.4583333333333333, - "middle_school_politics": 0.6538461538461539, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.75, - "logic": 0.48148148148148145, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.5294117647058824, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.56, - "middle_school_history": 0.4074074074074074, - "civil_servant": 0.4230769230769231, - "sports_science": 0.4166666666666667, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.375, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.5098039215686274, - "accountant": 0.3148148148148148, + "middle_school_biology": 0.15384615384615385, + "middle_school_physics": 0.16666666666666666, + "middle_school_chemistry": 0.24, + "veterinary_medicine": 0.25, + "college_economics": 0.2833333333333333, + "business_administration": 0.15789473684210525, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.3448275862068966, + "education_science": 0.2647058823529412, + "teacher_qualification": 0.32653061224489793, + "high_school_politics": 0.125, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.19230769230769232, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.14285714285714285, + "ideological_and_moral_cultivation": 0.2916666666666667, + "logic": 0.2962962962962963, + "law": 0.2413793103448276, + "chinese_language_and_literature": 0.25, + "art_studies": 0.4473684210526316, + "professional_tour_guide": 0.20588235294117646, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.3333333333333333, + "high_school_history": 0.16, + "middle_school_history": 0.25925925925925924, + "civil_servant": 0.23076923076923078, + "sports_science": 0.16666666666666666, + "plant_protection": 0.2222222222222222, + "basic_medicine": 0.20833333333333334, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.27450980392156865, + "accountant": 0.14814814814814814, "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.3888888888888889, - "tax_accountant": 0.2962962962962963, - "physician": 0.4074074074074074 + "environmental_impact_assessment_engineer": 0.2777777777777778, + "tax_accountant": 0.24074074074074073, + "physician": 0.2777777777777778 } }, "prompt_4": { - "accuracy": 0.387920298879203, + "accuracy": 0.24097135740971357, "category_acc": { - "computer_network": 0.375, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.3333333333333333, - "college_physics": 0.25, - "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.16666666666666666, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.4482758620689655, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.16666666666666666, + "computer_network": 0.20833333333333334, + "operating_system": 0.125, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.2857142857142857, + "college_physics": 0.16666666666666666, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.3333333333333333, + "probability_and_statistics": 0.30434782608695654, + "discrete_mathematics": 0.42857142857142855, + "electrical_engineer": 0.23809523809523808, + "metrology_engineer": 0.13793103448275862, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.3333333333333333, + "high_school_chemistry": 0.25, "high_school_biology": 0.25, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.5, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.43333333333333335, - "business_administration": 0.3684210526315789, - "marxism": 0.5416666666666666, - "mao_zedong_thought": 0.4827586206896552, - "education_science": 0.5294117647058824, - "teacher_qualification": 0.5714285714285714, - "high_school_politics": 0.375, - "high_school_geography": 0.4583333333333333, - "middle_school_politics": 0.6923076923076923, + "middle_school_mathematics": 0.125, + "middle_school_biology": 0.19230769230769232, + "middle_school_physics": 0.16666666666666666, + "middle_school_chemistry": 0.2, + "veterinary_medicine": 0.35714285714285715, + "college_economics": 0.31666666666666665, + "business_administration": 0.23684210526315788, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.27586206896551724, + "education_science": 0.2647058823529412, + "teacher_qualification": 0.24489795918367346, + "high_school_politics": 0.16666666666666666, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.15384615384615385, "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.6666666666666666, - "logic": 0.48148148148148145, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.5, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.44, - "middle_school_history": 0.4074074074074074, - "civil_servant": 0.40384615384615385, - "sports_science": 0.375, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.5098039215686274, - "accountant": 0.3148148148148148, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.3611111111111111, - "tax_accountant": 0.24074074074074073, - "physician": 0.37037037037037035 + "modern_chinese_history": 0.07142857142857142, + "ideological_and_moral_cultivation": 0.25, + "logic": 0.25925925925925924, + "law": 0.27586206896551724, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.47368421052631576, + "professional_tour_guide": 0.23529411764705882, + "legal_professional": 0.25, + "high_school_chinese": 0.25, + "high_school_history": 0.12, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.19230769230769232, + "sports_science": 0.08333333333333333, + "plant_protection": 0.1111111111111111, + "basic_medicine": 0.16666666666666666, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.29411764705882354, + "accountant": 0.1111111111111111, + "fire_engineer": 0.3055555555555556, + "environmental_impact_assessment_engineer": 0.2222222222222222, + "tax_accountant": 0.25925925925925924, + "physician": 0.2962962962962963 } }, "prompt_5": { - "accuracy": 0.39352428393524286, + "accuracy": 0.2521793275217933, "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.2857142857142857, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.25, + "computer_network": 0.20833333333333334, + "operating_system": 0.16666666666666666, + "computer_architecture": 0.2692307692307692, + "college_programming": 0.23809523809523808, + "college_physics": 0.16666666666666666, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.4583333333333333, + "probability_and_statistics": 0.30434782608695654, + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.16666666666666666, + "metrology_engineer": 0.13793103448275862, + "high_school_mathematics": 0.21739130434782608, + "high_school_physics": 0.2916666666666667, "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.5, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.43333333333333335, - "business_administration": 0.42105263157894735, - "marxism": 0.5, - "mao_zedong_thought": 0.5862068965517241, - "education_science": 0.5882352941176471, - "teacher_qualification": 0.5918367346938775, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.5, - "middle_school_politics": 0.8076923076923077, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.48148148148148145, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.4411764705882353, - "legal_professional": 0.39285714285714285, + "high_school_biology": 0.375, + "middle_school_mathematics": 0.16666666666666666, + "middle_school_biology": 0.15384615384615385, + "middle_school_physics": 0.16666666666666666, + "middle_school_chemistry": 0.24, + "veterinary_medicine": 0.21428571428571427, + "college_economics": 0.31666666666666665, + "business_administration": 0.15789473684210525, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.3448275862068966, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.3673469387755102, + "high_school_politics": 0.20833333333333334, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.23076923076923078, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.2857142857142857, + "ideological_and_moral_cultivation": 0.3333333333333333, + "logic": 0.25925925925925924, + "law": 0.2413793103448276, + "chinese_language_and_literature": 0.25, + "art_studies": 0.42105263157894735, + "professional_tour_guide": 0.23529411764705882, + "legal_professional": 0.32142857142857145, "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.48, - "middle_school_history": 0.4074074074074074, - "civil_servant": 0.40384615384615385, - "sports_science": 0.375, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.375, - "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.47058823529411764, - "accountant": 0.3148148148148148, + "high_school_history": 0.16, + "middle_school_history": 0.25925925925925924, + "civil_servant": 0.2692307692307692, + "sports_science": 0.16666666666666666, + "plant_protection": 0.18518518518518517, + "basic_medicine": 0.125, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.29411764705882354, + "accountant": 0.16666666666666666, "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.3611111111111111, - "tax_accountant": 0.2962962962962963, - "physician": 0.4074074074074074 + "environmental_impact_assessment_engineer": 0.2222222222222222, + "tax_accountant": 0.25925925925925924, + "physician": 0.2962962962962963 } } }, "cmmlu": { "prompt_1": { - "accuracy": 0.4121863799283154 + "accuracy": 0.2831541218637993 }, "prompt_2": { - "accuracy": 0.40860215053763443 + "accuracy": 0.2867383512544803 }, "prompt_3": { - "accuracy": 0.3978494623655914 + "accuracy": 0.2724014336917563 }, "prompt_4": { - "accuracy": 0.3978494623655914 + "accuracy": 0.26523297491039427 }, "prompt_5": { - "accuracy": 0.4265232974910394 + "accuracy": 0.2724014336917563 } }, "cmmlu_full": { "prompt_1": { - "accuracy": 0.4135727853565878, + "accuracy": 0.2515109652909687, "category_acc": { - "agronomy": 0.40828402366863903, - "anatomy": 0.32432432432432434, - "ancient_chinese": 0.23170731707317074, - "arts": 0.43125, - "astronomy": 0.2, - "business_ethics": 0.46411483253588515, - "chinese_civil_service_exam": 0.425, - "chinese_driving_rule": 0.6564885496183206, - "chinese_food_culture": 0.36764705882352944, - "chinese_foreign_policy": 0.45794392523364486, - "chinese_history": 0.47678018575851394, - "chinese_literature": 0.30392156862745096, - "chinese_teacher_qualification": 0.5083798882681564, - "clinical_knowledge": 0.3206751054852321, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.5700934579439252, - "college_engineering_hydrology": 0.33962264150943394, - "college_law": 0.32407407407407407, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.4056603773584906, - "college_medicine": 0.31868131868131866, - "computer_science": 0.44607843137254904, - "computer_security": 0.4444444444444444, - "conceptual_physics": 0.3197278911564626, - "construction_project_management": 0.381294964028777, - "economics": 0.4716981132075472, - "education": 0.4785276073619632, - "electrical_engineering": 0.38953488372093026, - "elementary_chinese": 0.3134920634920635, - "elementary_commonsense": 0.398989898989899, - "elementary_information_and_technology": 0.6050420168067226, - "elementary_mathematics": 0.30434782608695654, - "ethnology": 0.4074074074074074, - "food_science": 0.46153846153846156, - "genetics": 0.2840909090909091, - "global_facts": 0.35570469798657717, - "high_school_biology": 0.3076923076923077, - "high_school_chemistry": 0.32575757575757575, - "high_school_geography": 0.3813559322033898, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.33636363636363636, - "high_school_politics": 0.40559440559440557, - "human_sexuality": 0.4444444444444444, - "international_law": 0.43243243243243246, - "journalism": 0.5232558139534884, - "jurisprudence": 0.4793187347931874, - "legal_and_moral_basis": 0.7850467289719626, - "logical": 0.3983739837398374, - "machine_learning": 0.36065573770491804, - "management": 0.5, - "marketing": 0.5444444444444444, - "marxist_theory": 0.5238095238095238, - "modern_chinese": 0.31896551724137934, - "nutrition": 0.45517241379310347, - "philosophy": 0.4095238095238095, - "professional_accounting": 0.45714285714285713, - "professional_law": 0.3459715639810427, - "professional_medicine": 0.2898936170212766, - "professional_psychology": 0.5129310344827587, - "public_relations": 0.5114942528735632, - "security_study": 0.5481481481481482, - "sociology": 0.43805309734513276, - "sports_science": 0.49696969696969695, - "traditional_chinese_medicine": 0.32972972972972975, - "virology": 0.4260355029585799, - "world_history": 0.4409937888198758, - "world_religions": 0.41875 + "agronomy": 0.2485207100591716, + "anatomy": 0.25675675675675674, + "ancient_chinese": 0.23780487804878048, + "arts": 0.25625, + "astronomy": 0.296969696969697, + "business_ethics": 0.2583732057416268, + "chinese_civil_service_exam": 0.225, + "chinese_driving_rule": 0.29770992366412213, + "chinese_food_culture": 0.20588235294117646, + "chinese_foreign_policy": 0.22429906542056074, + "chinese_history": 0.25696594427244585, + "chinese_literature": 0.2549019607843137, + "chinese_teacher_qualification": 0.22346368715083798, + "clinical_knowledge": 0.25738396624472576, + "college_actuarial_science": 0.19811320754716982, + "college_education": 0.2897196261682243, + "college_engineering_hydrology": 0.29245283018867924, + "college_law": 0.24074074074074073, + "college_mathematics": 0.24761904761904763, + "college_medical_statistics": 0.20754716981132076, + "college_medicine": 0.2454212454212454, + "computer_science": 0.27941176470588236, + "computer_security": 0.21637426900584794, + "conceptual_physics": 0.22448979591836735, + "construction_project_management": 0.23741007194244604, + "economics": 0.23270440251572327, + "education": 0.26380368098159507, + "electrical_engineering": 0.29069767441860467, + "elementary_chinese": 0.2857142857142857, + "elementary_commonsense": 0.25757575757575757, + "elementary_information_and_technology": 0.2647058823529412, + "elementary_mathematics": 0.25217391304347825, + "ethnology": 0.26666666666666666, + "food_science": 0.27972027972027974, + "genetics": 0.25, + "global_facts": 0.2483221476510067, + "high_school_biology": 0.2603550295857988, + "high_school_chemistry": 0.25757575757575757, + "high_school_geography": 0.2457627118644068, + "high_school_mathematics": 0.27439024390243905, + "high_school_physics": 0.24545454545454545, + "high_school_politics": 0.25874125874125875, + "human_sexuality": 0.24603174603174602, + "international_law": 0.2594594594594595, + "journalism": 0.2616279069767442, + "jurisprudence": 0.25790754257907544, + "legal_and_moral_basis": 0.2616822429906542, + "logical": 0.2764227642276423, + "machine_learning": 0.2459016393442623, + "management": 0.22380952380952382, + "marketing": 0.2, + "marxist_theory": 0.2222222222222222, + "modern_chinese": 0.2413793103448276, + "nutrition": 0.2689655172413793, + "philosophy": 0.3238095238095238, + "professional_accounting": 0.24, + "professional_law": 0.25118483412322273, + "professional_medicine": 0.24202127659574468, + "professional_psychology": 0.24568965517241378, + "public_relations": 0.2471264367816092, + "security_study": 0.23703703703703705, + "sociology": 0.22566371681415928, + "sports_science": 0.24848484848484848, + "traditional_chinese_medicine": 0.2810810810810811, + "virology": 0.24260355029585798, + "world_history": 0.21739130434782608, + "world_religions": 0.2625 } }, "prompt_2": { - "accuracy": 0.4108962182697289, + "accuracy": 0.25090657917458126, "category_acc": { - "agronomy": 0.39644970414201186, - "anatomy": 0.31756756756756754, - "ancient_chinese": 0.25, - "arts": 0.45625, - "astronomy": 0.21212121212121213, - "business_ethics": 0.4688995215311005, - "chinese_civil_service_exam": 0.425, - "chinese_driving_rule": 0.648854961832061, - "chinese_food_culture": 0.36764705882352944, - "chinese_foreign_policy": 0.45794392523364486, - "chinese_history": 0.47368421052631576, - "chinese_literature": 0.30392156862745096, - "chinese_teacher_qualification": 0.5195530726256983, - "clinical_knowledge": 0.32489451476793246, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.5607476635514018, - "college_engineering_hydrology": 0.330188679245283, - "college_law": 0.3148148148148148, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.3867924528301887, - "college_medicine": 0.3076923076923077, - "computer_science": 0.4362745098039216, - "computer_security": 0.43859649122807015, - "conceptual_physics": 0.32653061224489793, - "construction_project_management": 0.3381294964028777, - "economics": 0.46540880503144655, - "education": 0.49693251533742333, - "electrical_engineering": 0.38372093023255816, - "elementary_chinese": 0.2976190476190476, - "elementary_commonsense": 0.3838383838383838, - "elementary_information_and_technology": 0.592436974789916, - "elementary_mathematics": 0.29130434782608694, - "ethnology": 0.4074074074074074, - "food_science": 0.48951048951048953, - "genetics": 0.26136363636363635, - "global_facts": 0.33557046979865773, - "high_school_biology": 0.3254437869822485, - "high_school_chemistry": 0.32575757575757575, - "high_school_geography": 0.3813559322033898, - "high_school_mathematics": 0.21341463414634146, - "high_school_physics": 0.34545454545454546, - "high_school_politics": 0.3986013986013986, - "human_sexuality": 0.4603174603174603, - "international_law": 0.41621621621621624, - "journalism": 0.5348837209302325, - "jurisprudence": 0.4793187347931874, - "legal_and_moral_basis": 0.7523364485981309, - "logical": 0.3821138211382114, - "machine_learning": 0.3442622950819672, - "management": 0.4857142857142857, - "marketing": 0.5388888888888889, - "marxist_theory": 0.5343915343915344, - "modern_chinese": 0.3017241379310345, - "nutrition": 0.4482758620689655, - "philosophy": 0.4095238095238095, - "professional_accounting": 0.44, - "professional_law": 0.36018957345971564, - "professional_medicine": 0.30319148936170215, - "professional_psychology": 0.5129310344827587, - "public_relations": 0.5172413793103449, - "security_study": 0.5407407407407407, - "sociology": 0.43805309734513276, - "sports_science": 0.49696969696969695, - "traditional_chinese_medicine": 0.32432432432432434, - "virology": 0.4319526627218935, - "world_history": 0.4409937888198758, - "world_religions": 0.40625 + "agronomy": 0.2485207100591716, + "anatomy": 0.27702702702702703, + "ancient_chinese": 0.24390243902439024, + "arts": 0.25, + "astronomy": 0.23030303030303031, + "business_ethics": 0.2631578947368421, + "chinese_civil_service_exam": 0.25625, + "chinese_driving_rule": 0.26717557251908397, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.29906542056074764, + "chinese_history": 0.24148606811145512, + "chinese_literature": 0.2647058823529412, + "chinese_teacher_qualification": 0.2346368715083799, + "clinical_knowledge": 0.25316455696202533, + "college_actuarial_science": 0.20754716981132076, + "college_education": 0.29906542056074764, + "college_engineering_hydrology": 0.24528301886792453, + "college_law": 0.19444444444444445, + "college_mathematics": 0.2761904761904762, + "college_medical_statistics": 0.18867924528301888, + "college_medicine": 0.2454212454212454, + "computer_science": 0.27941176470588236, + "computer_security": 0.21637426900584794, + "conceptual_physics": 0.23809523809523808, + "construction_project_management": 0.2517985611510791, + "economics": 0.2389937106918239, + "education": 0.26380368098159507, + "electrical_engineering": 0.2616279069767442, + "elementary_chinese": 0.28174603174603174, + "elementary_commonsense": 0.25757575757575757, + "elementary_information_and_technology": 0.2647058823529412, + "elementary_mathematics": 0.2826086956521739, + "ethnology": 0.2740740740740741, + "food_science": 0.23776223776223776, + "genetics": 0.23863636363636365, + "global_facts": 0.24161073825503357, + "high_school_biology": 0.2485207100591716, + "high_school_chemistry": 0.3181818181818182, + "high_school_geography": 0.2796610169491525, + "high_school_mathematics": 0.22560975609756098, + "high_school_physics": 0.2545454545454545, + "high_school_politics": 0.24475524475524477, + "human_sexuality": 0.25396825396825395, + "international_law": 0.2648648648648649, + "journalism": 0.26744186046511625, + "jurisprudence": 0.25304136253041365, + "legal_and_moral_basis": 0.24299065420560748, + "logical": 0.2682926829268293, + "machine_learning": 0.2459016393442623, + "management": 0.21428571428571427, + "marketing": 0.22777777777777777, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.25862068965517243, + "nutrition": 0.2413793103448276, + "philosophy": 0.26666666666666666, + "professional_accounting": 0.26285714285714284, + "professional_law": 0.26540284360189575, + "professional_medicine": 0.2393617021276596, + "professional_psychology": 0.25, + "public_relations": 0.23563218390804597, + "security_study": 0.21481481481481482, + "sociology": 0.25663716814159293, + "sports_science": 0.22424242424242424, + "traditional_chinese_medicine": 0.2756756756756757, + "virology": 0.23668639053254437, + "world_history": 0.2484472049689441, + "world_religions": 0.225 } }, "prompt_3": { - "accuracy": 0.4117596270074253, - "category_acc": { - "agronomy": 0.42011834319526625, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.27439024390243905, - "arts": 0.43125, - "astronomy": 0.20606060606060606, - "business_ethics": 0.45933014354066987, - "chinese_civil_service_exam": 0.41875, - "chinese_driving_rule": 0.648854961832061, - "chinese_food_culture": 0.39705882352941174, - "chinese_foreign_policy": 0.4485981308411215, - "chinese_history": 0.47058823529411764, - "chinese_literature": 0.3235294117647059, - "chinese_teacher_qualification": 0.5139664804469274, - "clinical_knowledge": 0.3080168776371308, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.5607476635514018, - "college_engineering_hydrology": 0.32075471698113206, - "college_law": 0.3333333333333333, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.3867924528301887, - "college_medicine": 0.31135531135531136, - "computer_science": 0.43137254901960786, - "computer_security": 0.4327485380116959, - "conceptual_physics": 0.3333333333333333, - "construction_project_management": 0.35251798561151076, - "economics": 0.46540880503144655, - "education": 0.4723926380368098, - "electrical_engineering": 0.37790697674418605, - "elementary_chinese": 0.31746031746031744, - "elementary_commonsense": 0.3686868686868687, - "elementary_information_and_technology": 0.6176470588235294, - "elementary_mathematics": 0.32608695652173914, - "ethnology": 0.4222222222222222, - "food_science": 0.46153846153846156, - "genetics": 0.26704545454545453, - "global_facts": 0.37583892617449666, - "high_school_biology": 0.31952662721893493, - "high_school_chemistry": 0.32575757575757575, - "high_school_geography": 0.3898305084745763, - "high_school_mathematics": 0.23170731707317074, - "high_school_physics": 0.35454545454545455, - "high_school_politics": 0.40559440559440557, - "human_sexuality": 0.4444444444444444, - "international_law": 0.41621621621621624, - "journalism": 0.5523255813953488, - "jurisprudence": 0.48418491484184917, - "legal_and_moral_basis": 0.7523364485981309, - "logical": 0.3821138211382114, - "machine_learning": 0.3360655737704918, - "management": 0.47619047619047616, - "marketing": 0.5333333333333333, - "marxist_theory": 0.5238095238095238, - "modern_chinese": 0.3275862068965517, - "nutrition": 0.4689655172413793, - "philosophy": 0.3904761904761905, - "professional_accounting": 0.44571428571428573, - "professional_law": 0.33175355450236965, - "professional_medicine": 0.30319148936170215, - "professional_psychology": 0.5172413793103449, - "public_relations": 0.5057471264367817, - "security_study": 0.562962962962963, - "sociology": 0.4557522123893805, - "sports_science": 0.49696969696969695, - "traditional_chinese_medicine": 0.3027027027027027, - "virology": 0.4319526627218935, - "world_history": 0.40993788819875776, - "world_religions": 0.39375 - } - }, - "prompt_4": { - "accuracy": 0.4123640131238128, + "accuracy": 0.25125194266965983, "category_acc": { - "agronomy": 0.41420118343195267, - "anatomy": 0.31756756756756754, - "ancient_chinese": 0.23780487804878048, - "arts": 0.425, - "astronomy": 0.2, - "business_ethics": 0.45933014354066987, - "chinese_civil_service_exam": 0.41875, - "chinese_driving_rule": 0.6259541984732825, - "chinese_food_culture": 0.375, - "chinese_foreign_policy": 0.4766355140186916, - "chinese_history": 0.48606811145510836, - "chinese_literature": 0.29901960784313725, - "chinese_teacher_qualification": 0.5139664804469274, - "clinical_knowledge": 0.31223628691983124, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.5607476635514018, - "college_engineering_hydrology": 0.33962264150943394, - "college_law": 0.3333333333333333, - "college_mathematics": 0.18095238095238095, - "college_medical_statistics": 0.3867924528301887, - "college_medicine": 0.30036630036630035, - "computer_science": 0.45588235294117646, - "computer_security": 0.4327485380116959, - "conceptual_physics": 0.3197278911564626, - "construction_project_management": 0.34532374100719426, - "economics": 0.4968553459119497, - "education": 0.4785276073619632, - "electrical_engineering": 0.38372093023255816, - "elementary_chinese": 0.2976190476190476, - "elementary_commonsense": 0.3838383838383838, - "elementary_information_and_technology": 0.6050420168067226, - "elementary_mathematics": 0.3, - "ethnology": 0.4148148148148148, - "food_science": 0.46153846153846156, - "genetics": 0.30113636363636365, - "global_facts": 0.348993288590604, - "high_school_biology": 0.31952662721893493, - "high_school_chemistry": 0.30303030303030304, - "high_school_geography": 0.3983050847457627, - "high_school_mathematics": 0.24390243902439024, - "high_school_physics": 0.35454545454545455, - "high_school_politics": 0.40559440559440557, - "human_sexuality": 0.4603174603174603, - "international_law": 0.42702702702702705, - "journalism": 0.5116279069767442, - "jurisprudence": 0.48661800486618007, - "legal_and_moral_basis": 0.7757009345794392, - "logical": 0.3902439024390244, - "machine_learning": 0.3524590163934426, - "management": 0.5, - "marketing": 0.5444444444444444, - "marxist_theory": 0.5079365079365079, - "modern_chinese": 0.3275862068965517, - "nutrition": 0.4482758620689655, - "philosophy": 0.3619047619047619, - "professional_accounting": 0.4514285714285714, - "professional_law": 0.3459715639810427, - "professional_medicine": 0.30851063829787234, - "professional_psychology": 0.521551724137931, - "public_relations": 0.5287356321839081, - "security_study": 0.5481481481481482, - "sociology": 0.4469026548672566, - "sports_science": 0.5333333333333333, - "traditional_chinese_medicine": 0.32972972972972975, - "virology": 0.40236686390532544, - "world_history": 0.4161490683229814, - "world_religions": 0.41875 + "agronomy": 0.21893491124260356, + "anatomy": 0.2702702702702703, + "ancient_chinese": 0.25, + "arts": 0.25625, + "astronomy": 0.24848484848484848, + "business_ethics": 0.2679425837320574, + "chinese_civil_service_exam": 0.2125, + "chinese_driving_rule": 0.2748091603053435, + "chinese_food_culture": 0.25735294117647056, + "chinese_foreign_policy": 0.24299065420560748, + "chinese_history": 0.23839009287925697, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.25139664804469275, + "clinical_knowledge": 0.25738396624472576, + "college_actuarial_science": 0.2358490566037736, + "college_education": 0.2897196261682243, + "college_engineering_hydrology": 0.3018867924528302, + "college_law": 0.23148148148148148, + "college_mathematics": 0.3047619047619048, + "college_medical_statistics": 0.22641509433962265, + "college_medicine": 0.2490842490842491, + "computer_science": 0.22058823529411764, + "computer_security": 0.21637426900584794, + "conceptual_physics": 0.24489795918367346, + "construction_project_management": 0.28776978417266186, + "economics": 0.2389937106918239, + "education": 0.26380368098159507, + "electrical_engineering": 0.27325581395348836, + "elementary_chinese": 0.2896825396825397, + "elementary_commonsense": 0.24242424242424243, + "elementary_information_and_technology": 0.2689075630252101, + "elementary_mathematics": 0.22608695652173913, + "ethnology": 0.25925925925925924, + "food_science": 0.2937062937062937, + "genetics": 0.23863636363636365, + "global_facts": 0.2550335570469799, + "high_school_biology": 0.2603550295857988, + "high_school_chemistry": 0.2878787878787879, + "high_school_geography": 0.2627118644067797, + "high_school_mathematics": 0.25609756097560976, + "high_school_physics": 0.24545454545454545, + "high_school_politics": 0.2727272727272727, + "human_sexuality": 0.2698412698412698, + "international_law": 0.2594594594594595, + "journalism": 0.28488372093023256, + "jurisprudence": 0.24330900243309003, + "legal_and_moral_basis": 0.24299065420560748, + "logical": 0.22764227642276422, + "machine_learning": 0.2459016393442623, + "management": 0.24285714285714285, + "marketing": 0.2222222222222222, + "marxist_theory": 0.2222222222222222, + "modern_chinese": 0.27586206896551724, + "nutrition": 0.2689655172413793, + "philosophy": 0.29523809523809524, + "professional_accounting": 0.24, + "professional_law": 0.27014218009478674, + "professional_medicine": 0.2553191489361702, + "professional_psychology": 0.2413793103448276, + "public_relations": 0.22988505747126436, + "security_study": 0.21481481481481482, + "sociology": 0.22566371681415928, + "sports_science": 0.24242424242424243, + "traditional_chinese_medicine": 0.2918918918918919, + "virology": 0.23076923076923078, + "world_history": 0.18012422360248448, + "world_religions": 0.2625 } }, - "prompt_5": { - "accuracy": 0.41080987739595926, + "prompt_4": { + "accuracy": 0.2493524434467277, "category_acc": { - "agronomy": 0.3905325443786982, - "anatomy": 0.30405405405405406, - "ancient_chinese": 0.24390243902439024, - "arts": 0.45, - "astronomy": 0.2, - "business_ethics": 0.47368421052631576, - "chinese_civil_service_exam": 0.41875, - "chinese_driving_rule": 0.6564885496183206, - "chinese_food_culture": 0.3602941176470588, - "chinese_foreign_policy": 0.4672897196261682, - "chinese_history": 0.47987616099071206, - "chinese_literature": 0.3088235294117647, - "chinese_teacher_qualification": 0.4972067039106145, - "clinical_knowledge": 0.31223628691983124, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.5420560747663551, + "agronomy": 0.23076923076923078, + "anatomy": 0.2635135135135135, + "ancient_chinese": 0.23780487804878048, + "arts": 0.26875, + "astronomy": 0.2727272727272727, + "business_ethics": 0.2583732057416268, + "chinese_civil_service_exam": 0.21875, + "chinese_driving_rule": 0.2595419847328244, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.205607476635514, + "chinese_history": 0.21981424148606812, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.24581005586592178, + "clinical_knowledge": 0.2742616033755274, + "college_actuarial_science": 0.2358490566037736, + "college_education": 0.308411214953271, "college_engineering_hydrology": 0.3113207547169811, - "college_law": 0.32407407407407407, + "college_law": 0.21296296296296297, "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.39622641509433965, - "college_medicine": 0.31868131868131866, - "computer_science": 0.4411764705882353, - "computer_security": 0.42105263157894735, - "conceptual_physics": 0.3401360544217687, - "construction_project_management": 0.3381294964028777, - "economics": 0.48427672955974843, - "education": 0.48466257668711654, - "electrical_engineering": 0.38953488372093026, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.3939393939393939, - "elementary_information_and_technology": 0.6008403361344538, - "elementary_mathematics": 0.3130434782608696, - "ethnology": 0.4, - "food_science": 0.4755244755244755, - "genetics": 0.2556818181818182, - "global_facts": 0.33557046979865773, - "high_school_biology": 0.3136094674556213, - "high_school_chemistry": 0.3484848484848485, - "high_school_geography": 0.4067796610169492, + "college_medical_statistics": 0.2169811320754717, + "college_medicine": 0.25274725274725274, + "computer_science": 0.23039215686274508, + "computer_security": 0.24561403508771928, + "conceptual_physics": 0.2585034013605442, + "construction_project_management": 0.2589928057553957, + "economics": 0.2389937106918239, + "education": 0.2392638036809816, + "electrical_engineering": 0.2441860465116279, + "elementary_chinese": 0.24603174603174602, + "elementary_commonsense": 0.25757575757575757, + "elementary_information_and_technology": 0.25630252100840334, + "elementary_mathematics": 0.29130434782608694, + "ethnology": 0.2814814814814815, + "food_science": 0.25874125874125875, + "genetics": 0.23295454545454544, + "global_facts": 0.24161073825503357, + "high_school_biology": 0.25443786982248523, + "high_school_chemistry": 0.25757575757575757, + "high_school_geography": 0.2711864406779661, "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.33636363636363636, - "high_school_politics": 0.3986013986013986, - "human_sexuality": 0.4603174603174603, - "international_law": 0.42162162162162165, - "journalism": 0.5174418604651163, - "jurisprudence": 0.48661800486618007, - "legal_and_moral_basis": 0.7663551401869159, - "logical": 0.3821138211382114, - "machine_learning": 0.36885245901639346, - "management": 0.49047619047619045, - "marketing": 0.5388888888888889, - "marxist_theory": 0.5291005291005291, - "modern_chinese": 0.3017241379310345, - "nutrition": 0.46206896551724136, - "philosophy": 0.4, - "professional_accounting": 0.44571428571428573, - "professional_law": 0.3412322274881517, - "professional_medicine": 0.30319148936170215, - "professional_psychology": 0.5086206896551724, - "public_relations": 0.5, - "security_study": 0.5185185185185185, - "sociology": 0.4336283185840708, - "sports_science": 0.509090909090909, - "traditional_chinese_medicine": 0.31351351351351353, - "virology": 0.41420118343195267, - "world_history": 0.4409937888198758, - "world_religions": 0.4125 + "high_school_physics": 0.22727272727272727, + "high_school_politics": 0.23076923076923078, + "human_sexuality": 0.25396825396825395, + "international_law": 0.24864864864864866, + "journalism": 0.2558139534883721, + "jurisprudence": 0.2384428223844282, + "legal_and_moral_basis": 0.2336448598130841, + "logical": 0.2601626016260163, + "machine_learning": 0.20491803278688525, + "management": 0.24285714285714285, + "marketing": 0.25, + "marxist_theory": 0.24338624338624337, + "modern_chinese": 0.22413793103448276, + "nutrition": 0.2689655172413793, + "philosophy": 0.2857142857142857, + "professional_accounting": 0.24571428571428572, + "professional_law": 0.25118483412322273, + "professional_medicine": 0.2393617021276596, + "professional_psychology": 0.2413793103448276, + "public_relations": 0.2413793103448276, + "security_study": 0.23703703703703705, + "sociology": 0.26548672566371684, + "sports_science": 0.2545454545454545, + "traditional_chinese_medicine": 0.2918918918918919, + "virology": 0.24260355029585798, + "world_history": 0.2546583850931677, + "world_religions": 0.2625 + } + }, + "prompt_5": { + "accuracy": 0.25116560179589015, + "category_acc": { + "agronomy": 0.23668639053254437, + "anatomy": 0.2635135135135135, + "ancient_chinese": 0.2682926829268293, + "arts": 0.275, + "astronomy": 0.23030303030303031, + "business_ethics": 0.2631578947368421, + "chinese_civil_service_exam": 0.25, + "chinese_driving_rule": 0.26717557251908397, + "chinese_food_culture": 0.20588235294117646, + "chinese_foreign_policy": 0.21495327102803738, + "chinese_history": 0.26006191950464397, + "chinese_literature": 0.24509803921568626, + "chinese_teacher_qualification": 0.21787709497206703, + "clinical_knowledge": 0.24050632911392406, + "college_actuarial_science": 0.19811320754716982, + "college_education": 0.2803738317757009, + "college_engineering_hydrology": 0.25471698113207547, + "college_law": 0.17592592592592593, + "college_mathematics": 0.26666666666666666, + "college_medical_statistics": 0.1792452830188679, + "college_medicine": 0.23076923076923078, + "computer_science": 0.2696078431372549, + "computer_security": 0.2046783625730994, + "conceptual_physics": 0.2585034013605442, + "construction_project_management": 0.2589928057553957, + "economics": 0.2578616352201258, + "education": 0.26380368098159507, + "electrical_engineering": 0.28488372093023256, + "elementary_chinese": 0.30158730158730157, + "elementary_commonsense": 0.26262626262626265, + "elementary_information_and_technology": 0.24789915966386555, + "elementary_mathematics": 0.2782608695652174, + "ethnology": 0.3111111111111111, + "food_science": 0.25874125874125875, + "genetics": 0.2727272727272727, + "global_facts": 0.2348993288590604, + "high_school_biology": 0.24260355029585798, + "high_school_chemistry": 0.25757575757575757, + "high_school_geography": 0.2457627118644068, + "high_school_mathematics": 0.2621951219512195, + "high_school_physics": 0.2727272727272727, + "high_school_politics": 0.23776223776223776, + "human_sexuality": 0.23809523809523808, + "international_law": 0.25405405405405407, + "journalism": 0.27325581395348836, + "jurisprudence": 0.25304136253041365, + "legal_and_moral_basis": 0.27102803738317754, + "logical": 0.25203252032520324, + "machine_learning": 0.27049180327868855, + "management": 0.22857142857142856, + "marketing": 0.2611111111111111, + "marxist_theory": 0.2275132275132275, + "modern_chinese": 0.25862068965517243, + "nutrition": 0.2620689655172414, + "philosophy": 0.29523809523809524, + "professional_accounting": 0.24, + "professional_law": 0.27014218009478674, + "professional_medicine": 0.23404255319148937, + "professional_psychology": 0.22844827586206898, + "public_relations": 0.22413793103448276, + "security_study": 0.22962962962962963, + "sociology": 0.2345132743362832, + "sports_science": 0.23030303030303031, + "traditional_chinese_medicine": 0.3027027027027027, + "virology": 0.24260355029585798, + "world_history": 0.2484472049689441, + "world_religions": 0.25 } } }, "zbench": { "prompt_1": { - "accuracy": 0.3333333333333333 + "accuracy": 0.36363636363636365 }, "prompt_2": { - "accuracy": 0.36363636363636365 + "accuracy": 0.2727272727272727 }, "prompt_3": { - "accuracy": 0.36363636363636365 + "accuracy": 0.30303030303030304 }, "prompt_4": { - "accuracy": 0.3333333333333333 + "accuracy": 0.2727272727272727 }, "prompt_5": { - "accuracy": 0.36363636363636365 + "accuracy": 0.30303030303030304 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.6227272727272727 + "accuracy": 0.2727272727272727 }, "prompt_2": { - "accuracy": 0.6136363636363636 + "accuracy": 0.30227272727272725 }, "prompt_3": { - "accuracy": 0.5977272727272728 + "accuracy": 0.23636363636363636 }, "prompt_4": { - "accuracy": 0.6136363636363636 + "accuracy": 0.325 }, "prompt_5": { - "accuracy": 0.6113636363636363 + "accuracy": 0.2840909090909091 } }, "ocnli": { "prompt_1": { - "accuracy": 0.38813559322033897 + "accuracy": 0.3305084745762712 }, "prompt_2": { - "accuracy": 0.40135593220338983 + "accuracy": 0.32 }, "prompt_3": { - "accuracy": 0.37322033898305085 + "accuracy": 0.3342372881355932 }, "prompt_4": { - "accuracy": 0.4152542372881356 + "accuracy": 0.3176271186440678 }, "prompt_5": { - "accuracy": 0.4542372881355932 + "accuracy": 0.3342372881355932 } }, "c3": { "prompt_1": { - "accuracy": 0.8702318623784593 + "accuracy": 0.26028421839940163 }, "prompt_2": { - "accuracy": 0.8709798055347794 + "accuracy": 0.2655198204936425 }, "prompt_3": { - "accuracy": 0.8739715781600599 + "accuracy": 0.27786088257292446 }, "prompt_4": { - "accuracy": 0.868362004487659 + "accuracy": 0.2801047120418848 }, "prompt_5": { - "accuracy": 0.8713537771129394 + "accuracy": 0.27599102468212416 } }, "dream": { "prompt_1": { - "accuracy": 0.8892699657030867 + "accuracy": 0.875061244487996 }, "prompt_2": { - "accuracy": 0.8848603625673689 + "accuracy": 0.8682018618324351 }, "prompt_3": { - "accuracy": 0.8878000979911808 + "accuracy": 0.8740813326800588 }, "prompt_4": { - "accuracy": 0.8863302302792748 + "accuracy": 0.8726114649681529 }, "prompt_5": { - "accuracy": 0.8878000979911808 + "accuracy": 0.8716315531602156 } }, "samsum": { "prompt_1": { - "rouge1": 0.41259329359093655, - "rouge2": 0.2115510244457223, - "rougeL": 0.3530618026494755, - "avg_rouge": 0.3257353735620448 + "rouge1": 0.5185659226290711, + "rouge2": 0.26879953180572064, + "rougeL": 0.43275638022449625, + "avg_rouge": 0.40670727821976266 }, "prompt_2": { - "rouge1": 0.41707374512390494, - "rouge2": 0.2124796660781807, - "rougeL": 0.354080814542538, - "avg_rouge": 0.32787807524820783 + "rouge1": 0.5190628568296515, + "rouge2": 0.26969716664146526, + "rougeL": 0.4317910039013235, + "avg_rouge": 0.4068503424574801 }, "prompt_3": { - "rouge1": 0.41321357066826325, - "rouge2": 0.20913215733645862, - "rougeL": 0.3501808312576957, - "avg_rouge": 0.3241755197541392 + "rouge1": 0.5164177682133352, + "rouge2": 0.2634396180719505, + "rougeL": 0.4281279644017568, + "avg_rouge": 0.40266178356234744 }, "prompt_4": { - "rouge1": 0.41878254040446516, - "rouge2": 0.2122266216884994, - "rougeL": 0.35334307500453, - "avg_rouge": 0.32811741236583153 + "rouge1": 0.518744098149516, + "rouge2": 0.2687583498609372, + "rougeL": 0.4321035265896283, + "avg_rouge": 0.40653532486669386 }, "prompt_5": { - "rouge1": 0.41256561894917226, - "rouge2": 0.2101843136359623, - "rougeL": 0.34918946838957315, - "avg_rouge": 0.3239798003249026 + "rouge1": 0.5175029512092139, + "rouge2": 0.26827791419756786, + "rougeL": 0.4309740774155282, + "avg_rouge": 0.40558498094076995 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.19650427568779455, - "rouge2": 0.05364004839716688, - "rougeL": 0.14780535686319093, - "avg_rouge": 0.13264989364938415 + "rouge1": 0.24423627199761502, + "rouge2": 0.07472364536524811, + "rougeL": 0.18416102098715098, + "avg_rouge": 0.1677069794500047 }, "prompt_2": { - "rouge1": 0.1973451031462774, - "rouge2": 0.05506989252399569, - "rougeL": 0.14860981077086455, - "avg_rouge": 0.1336749354803792 + "rouge1": 0.2438277162660386, + "rouge2": 0.07408111256345526, + "rougeL": 0.18586363370004302, + "avg_rouge": 0.1679241541765123 }, "prompt_3": { - "rouge1": 0.20097510069358493, - "rouge2": 0.06074357546102678, - "rougeL": 0.15732985995613014, - "avg_rouge": 0.13968284537024728 + "rouge1": 0.23272168097538612, + "rouge2": 0.06643126452079036, + "rougeL": 0.17368331730278494, + "avg_rouge": 0.1576120875996538 }, "prompt_4": { - "rouge1": 0.19718866521664802, - "rouge2": 0.055280092355201836, - "rougeL": 0.14855552896452479, - "avg_rouge": 0.13367476217879154 + "rouge1": 0.2331686673676627, + "rouge2": 0.06578694871923757, + "rougeL": 0.1738804407479083, + "avg_rouge": 0.15761201894493618 }, "prompt_5": { - "rouge1": 0.2004555565400454, - "rouge2": 0.05905148166979365, - "rougeL": 0.15259325268376314, - "avg_rouge": 0.13736676363120073 + "rouge1": 0.24261531328574346, + "rouge2": 0.07032891941011479, + "rougeL": 0.18516565082477832, + "avg_rouge": 0.1660366278402122 } }, "sst2": { "prompt_1": { - "accuracy": 0.948394495412844 + "accuracy": 0.9529816513761468 }, "prompt_2": { - "accuracy": 0.9461009174311926 + "accuracy": 0.9506880733944955 }, "prompt_3": { - "accuracy": 0.9461009174311926 + "accuracy": 0.9495412844036697 }, "prompt_4": { - "accuracy": 0.9472477064220184 + "accuracy": 0.9529816513761468 }, "prompt_5": { - "accuracy": 0.9243119266055045 + "accuracy": 0.948394495412844 } }, "cola": { "prompt_1": { - "accuracy": 0.5656759348034516 + "accuracy": 0.7171620325982742 }, "prompt_2": { - "accuracy": 0.573346116970278 + "accuracy": 0.7277085330776606 }, "prompt_3": { - "accuracy": 0.5695110258868649 + "accuracy": 0.7229146692233941 }, "prompt_4": { - "accuracy": 0.573346116970278 + "accuracy": 0.713326941514861 }, "prompt_5": { - "accuracy": 0.5043144774688398 + "accuracy": 0.7248322147651006 } }, "qqp": { "prompt_1": { - "accuracy": 0.69 + "accuracy": 0.824 }, "prompt_2": { - "accuracy": 0.5155 + "accuracy": 0.853 }, "prompt_3": { - "accuracy": 0.7685 + "accuracy": 0.821 }, "prompt_4": { - "accuracy": 0.643 + "accuracy": 0.8275 }, "prompt_5": { - "accuracy": 0.5565 + "accuracy": 0.8465 } }, "mnli": { "prompt_1": { - "accuracy": 0.603 + "accuracy": 0.7 }, "prompt_2": { - "accuracy": 0.591 + "accuracy": 0.7 }, "prompt_3": { - "accuracy": 0.6025 + "accuracy": 0.7 }, "prompt_4": { - "accuracy": 0.586 + "accuracy": 0.7 }, "prompt_5": { - "accuracy": 0.568 + "accuracy": 0.7 } }, "qnli": { "prompt_1": { - "accuracy": 0.609 + "accuracy": 0.8 }, "prompt_2": { - "accuracy": 0.6035 + "accuracy": 0.5 }, "prompt_3": { - "accuracy": 0.576 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.6195 + "accuracy": 0.7 }, "prompt_5": { - "accuracy": 0.631 + "accuracy": 0.9 } }, "wnli": { "prompt_1": { - "accuracy": 0.4788732394366197 + "accuracy": 0.5 }, "prompt_2": { - "accuracy": 0.5352112676056338 + "accuracy": 0.6 }, "prompt_3": { - "accuracy": 0.6056338028169014 + "accuracy": 0.6 }, "prompt_4": { - "accuracy": 0.43661971830985913 + "accuracy": 0.5 }, "prompt_5": { - "accuracy": 0.5492957746478874 + "accuracy": 0.5 } }, "rte": { "prompt_1": { - "accuracy": 0.5667870036101083 + "accuracy": 1.0 }, "prompt_2": { - "accuracy": 0.5126353790613718 + "accuracy": 1.0 }, "prompt_3": { - "accuracy": 0.5451263537906137 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.5812274368231047 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.6425992779783394 + "accuracy": 1.0 } }, "mrpc": { "prompt_1": { - "accuracy": 0.7058823529411765 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.47058823529411764 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.7009803921568627 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.4681372549019608 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.5147058823529411 + "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.9153361344537815, - "language_acc": { - "English": 0.9260504201680673, - "Chinese": 0.8949579831932774, - "Spanish": 0.9352941176470588, - "Vietnamese": 0.9050420168067227 - }, - "consistency_score_2": 0.8780112044817926, - "consistency_score_3": 0.8279411764705882, - "consistency_score_4": 0.7915966386554621, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8756302521008403, - "English,Spanish": 0.9109243697478991, - "English,Vietnamese": 0.8756302521008403, - "Chinese,Spanish": 0.8773109243697479, - "Chinese,Vietnamese": 0.853781512605042, - "Spanish,Vietnamese": 0.8747899159663866 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8394957983193277, - "English,Chinese,Vietnamese": 0.8184873949579832, - "English,Spanish,Vietnamese": 0.8394957983193277, - "Chinese,Spanish,Vietnamese": 0.8142857142857143 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7915966386554621 - } - }, - "AC3_2": 0.8962852476193186, - "AC3_3": 0.8694479888347612, - "AC3_4": 0.8489813057033332 - } + "prompt_1": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.40190476190476193, - "language_acc": { - "Spanish": 0.38666666666666666, - "English": 0.44, - "Chinese": 0.3933333333333333, - "Vietnamese": 0.4066666666666667, - "Filipino": 0.4, - "Indonesian": 0.38666666666666666, - "Malay": 0.4 - }, - "consistency_score_2": 0.5984126984126984, - "consistency_score_3": 0.44476190476190475, - "consistency_score_4": 0.35619047619047617, - "consistency_score_5": 0.2968253968253968, - "consistency_score_6": 0.25333333333333335, - "consistency_score_7": 0.22, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.68, - "Spanish,Chinese": 0.5866666666666667, - "Spanish,Vietnamese": 0.66, - "Spanish,Filipino": 0.6333333333333333, - "Spanish,Indonesian": 0.5933333333333334, - "Spanish,Malay": 0.62, - "English,Chinese": 0.6333333333333333, - "English,Vietnamese": 0.6066666666666667, - "English,Filipino": 0.6333333333333333, - "English,Indonesian": 0.5933333333333334, - "English,Malay": 0.5666666666666667, - "Chinese,Vietnamese": 0.64, - "Chinese,Filipino": 0.5333333333333333, - "Chinese,Indonesian": 0.4866666666666667, - "Chinese,Malay": 0.5333333333333333, - "Vietnamese,Filipino": 0.6066666666666667, - "Vietnamese,Indonesian": 0.6, - "Vietnamese,Malay": 0.6266666666666667, - "Filipino,Indonesian": 0.5666666666666667, - "Filipino,Malay": 0.5333333333333333, - "Indonesian,Malay": 0.6333333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.4866666666666667, - "Spanish,English,Vietnamese": 0.5, - "Spanish,English,Filipino": 0.5133333333333333, - "Spanish,English,Indonesian": 0.47333333333333333, - "Spanish,English,Malay": 0.46, - "Spanish,Chinese,Vietnamese": 0.48, - "Spanish,Chinese,Filipino": 0.43333333333333335, - "Spanish,Chinese,Indonesian": 0.38666666666666666, - "Spanish,Chinese,Malay": 0.42, - "Spanish,Vietnamese,Filipino": 0.5, - "Spanish,Vietnamese,Indonesian": 0.4866666666666667, - "Spanish,Vietnamese,Malay": 0.5133333333333333, - "Spanish,Filipino,Indonesian": 0.46, - "Spanish,Filipino,Malay": 0.4533333333333333, - "Spanish,Indonesian,Malay": 0.46, - "English,Chinese,Vietnamese": 0.48, - "English,Chinese,Filipino": 0.4533333333333333, - "English,Chinese,Indonesian": 0.4, - "English,Chinese,Malay": 0.41333333333333333, - "English,Vietnamese,Filipino": 0.46, - "English,Vietnamese,Indonesian": 0.4533333333333333, - "English,Vietnamese,Malay": 0.44666666666666666, - "English,Filipino,Indonesian": 0.4666666666666667, - "English,Filipino,Malay": 0.4266666666666667, - "English,Indonesian,Malay": 0.4266666666666667, - "Chinese,Vietnamese,Filipino": 0.43333333333333335, - "Chinese,Vietnamese,Indonesian": 0.4066666666666667, - "Chinese,Vietnamese,Malay": 0.44666666666666666, - "Chinese,Filipino,Indonesian": 0.36, - "Chinese,Filipino,Malay": 0.36666666666666664, - "Chinese,Indonesian,Malay": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian": 0.44, - "Vietnamese,Filipino,Malay": 0.42, - "Vietnamese,Indonesian,Malay": 0.4666666666666667, - "Filipino,Indonesian,Malay": 0.4066666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4, - "Spanish,English,Chinese,Filipino": 0.38666666666666666, - "Spanish,English,Chinese,Indonesian": 0.3333333333333333, - "Spanish,English,Chinese,Malay": 0.3333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.4066666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.3933333333333333, - "Spanish,English,Vietnamese,Malay": 0.38666666666666666, - "Spanish,English,Filipino,Indonesian": 0.4066666666666667, - "Spanish,English,Filipino,Malay": 0.38, - "Spanish,English,Indonesian,Malay": 0.36, - "Spanish,Chinese,Vietnamese,Filipino": 0.37333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian": 0.35333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.38, - "Spanish,Chinese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.32666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.3933333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.38666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.4, - "Spanish,Filipino,Indonesian,Malay": 0.36, - "English,Chinese,Vietnamese,Filipino": 0.38, - "English,Chinese,Vietnamese,Indonesian": 0.35333333333333333, - "English,Chinese,Vietnamese,Malay": 0.36, - "English,Chinese,Filipino,Indonesian": 0.32666666666666666, - "English,Chinese,Filipino,Malay": 0.32, - "English,Chinese,Indonesian,Malay": 0.30666666666666664, - "English,Vietnamese,Filipino,Indonesian": 0.36666666666666664, - "English,Vietnamese,Filipino,Malay": 0.35333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.36, - "English,Filipino,Indonesian,Malay": 0.34, - "Chinese,Vietnamese,Filipino,Indonesian": 0.32, - "Chinese,Vietnamese,Filipino,Malay": 0.32, - "Chinese,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.28, - "Vietnamese,Filipino,Indonesian,Malay": 0.3466666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.30666666666666664, - "Spanish,English,Chinese,Vietnamese,Malay": 0.30666666666666664, - "Spanish,English,Chinese,Filipino,Indonesian": 0.3, - "Spanish,English,Chinese,Filipino,Malay": 0.2866666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.26, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.34, - "Spanish,English,Vietnamese,Filipino,Malay": 0.32666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.26, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.26, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22 - } - }, - "AC3_2": 0.4808571730337933, - "AC3_3": 0.4222486474406204, - "AC3_4": 0.3776692988255108, - "AC3_5": 0.3414638367352765, - "AC3_6": 0.31077519375102025, - "AC3_7": 0.2843491576878152 - } + "prompt_1": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.34172077922077915, - "language_acc": { - "English": 0.3693181818181818, - "Filipino": 0.375, - "Spanish": 0.3522727272727273, - "Malay": 0.2897727272727273, - "Chinese": 0.3352272727272727, - "Indonesian": 0.3352272727272727, - "Vietnamese": 0.3352272727272727 - }, - "consistency_score_2": 0.458874458874459, - "consistency_score_3": 0.26249999999999996, - "consistency_score_4": 0.16688311688311686, - "consistency_score_5": 0.11336580086580085, - "consistency_score_6": 0.08035714285714286, - "consistency_score_7": 0.056818181818181816, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.5, - "English,Spanish": 0.4943181818181818, - "English,Malay": 0.4602272727272727, - "English,Chinese": 0.48863636363636365, - "English,Indonesian": 0.5170454545454546, - "English,Vietnamese": 0.45454545454545453, - "Filipino,Spanish": 0.4431818181818182, - "Filipino,Malay": 0.4715909090909091, - "Filipino,Chinese": 0.4375, - "Filipino,Indonesian": 0.4772727272727273, - "Filipino,Vietnamese": 0.4034090909090909, - "Spanish,Malay": 0.44886363636363635, - "Spanish,Chinese": 0.4318181818181818, - "Spanish,Indonesian": 0.5, - "Spanish,Vietnamese": 0.4602272727272727, - "Malay,Chinese": 0.44886363636363635, - "Malay,Indonesian": 0.4147727272727273, - "Malay,Vietnamese": 0.45454545454545453, - "Chinese,Indonesian": 0.45454545454545453, - "Chinese,Vietnamese": 0.44886363636363635, - "Indonesian,Vietnamese": 0.42613636363636365 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Malay": 0.2840909090909091, - "English,Filipino,Chinese": 0.2897727272727273, - "English,Filipino,Indonesian": 0.32386363636363635, - "English,Filipino,Vietnamese": 0.25, - "English,Spanish,Malay": 0.26704545454545453, - "English,Spanish,Chinese": 0.2727272727272727, - "English,Spanish,Indonesian": 0.3181818181818182, - "English,Spanish,Vietnamese": 0.26136363636363635, - "English,Malay,Chinese": 0.2727272727272727, - "English,Malay,Indonesian": 0.2784090909090909, - "English,Malay,Vietnamese": 0.26136363636363635, - "English,Chinese,Indonesian": 0.2784090909090909, - "English,Chinese,Vietnamese": 0.26136363636363635, - "English,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay": 0.25, - "Filipino,Spanish,Chinese": 0.23863636363636365, - "Filipino,Spanish,Indonesian": 0.2897727272727273, - "Filipino,Spanish,Vietnamese": 0.23295454545454544, - "Filipino,Malay,Chinese": 0.26136363636363635, - "Filipino,Malay,Indonesian": 0.2727272727272727, - "Filipino,Malay,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Indonesian": 0.26704545454545453, - "Filipino,Chinese,Vietnamese": 0.22727272727272727, - "Filipino,Indonesian,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Chinese": 0.23863636363636365, - "Spanish,Malay,Indonesian": 0.26136363636363635, - "Spanish,Malay,Vietnamese": 0.25, - "Spanish,Chinese,Indonesian": 0.26136363636363635, - "Spanish,Chinese,Vietnamese": 0.24431818181818182, - "Spanish,Indonesian,Vietnamese": 0.26136363636363635, - "Malay,Chinese,Indonesian": 0.23863636363636365, - "Malay,Chinese,Vietnamese": 0.24431818181818182, - "Malay,Indonesian,Vietnamese": 0.22727272727272727, - "Chinese,Indonesian,Vietnamese": 0.23863636363636365 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17045454545454544, - "English,Filipino,Spanish,Chinese": 0.18181818181818182, - "English,Filipino,Spanish,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese": 0.1875, - "English,Filipino,Malay,Indonesian": 0.19886363636363635, - "English,Filipino,Malay,Vietnamese": 0.17045454545454544, - "English,Filipino,Chinese,Indonesian": 0.1875, - "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Chinese": 0.16477272727272727, - "English,Spanish,Malay,Indonesian": 0.1875, - "English,Spanish,Malay,Vietnamese": 0.1534090909090909, - "English,Spanish,Chinese,Indonesian": 0.18181818181818182, - "English,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Malay,Chinese,Indonesian": 0.17045454545454544, - "English,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.125, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 - } - }, - "AC3_2": 0.3917258813694157, - "AC3_3": 0.29691698006907546, - "AC3_4": 0.22425085283746676, - "AC3_5": 0.17025089947924388, - "AC3_6": 0.13011675821093058, - "AC3_7": 0.09743566003850308 - } + "prompt_1": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.44660194174757284 - } + "prompt_1": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.3047619047619048 - } + "prompt_1": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.35514018691588783 - } + "prompt_1": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.34, - "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.5, - "politics": 0.2, - "culture": 0.4, - "film": 0.2, - "law": 0.2, - "geography": 0.5 - } - } + "prompt_1": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.18316391933943926 - } + "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.4979094272744674 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.4627197410981695 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.37798118287462545 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.5211110850608137 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.43757292882147025 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.39249195566678585, - "category_acc": { - "high_school_european_history": 0.6280487804878049, - "business_ethics": 0.45454545454545453, - "clinical_knowledge": 0.4090909090909091, - "medical_genetics": 0.41414141414141414, - "high_school_us_history": 0.4630541871921182, - "high_school_physics": 0.3, - "high_school_world_history": 0.5847457627118644, - "virology": 0.3333333333333333, - "high_school_microeconomics": 0.379746835443038, - "econometrics": 0.2743362831858407, - "college_computer_science": 0.41414141414141414, - "high_school_biology": 0.42394822006472493, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.3309608540925267, - "philosophy": 0.45161290322580644, - "professional_medicine": 0.34686346863468637, - "nutrition": 0.419672131147541, - "global_facts": 0.31313131313131315, - "machine_learning": 0.18018018018018017, - "security_studies": 0.5040983606557377, - "public_relations": 0.44036697247706424, - "professional_psychology": 0.38461538461538464, - "prehistory": 0.3746130030959752, - "anatomy": 0.4701492537313433, - "human_sexuality": 0.46923076923076923, - "college_medicine": 0.37790697674418605, - "high_school_government_and_politics": 0.4739583333333333, - "college_chemistry": 0.32323232323232326, - "logical_fallacies": 0.5185185185185185, - "high_school_geography": 0.5583756345177665, - "elementary_mathematics": 0.27320954907161804, - "human_aging": 0.36936936936936937, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.5330882352941176, - "formal_logic": 0.312, - "high_school_statistics": 0.32558139534883723, - "international_law": 0.55, - "high_school_mathematics": 0.26022304832713755, - "high_school_computer_science": 0.42424242424242425, - "conceptual_physics": 0.26495726495726496, - "miscellaneous": 0.4680306905370844, - "high_school_chemistry": 0.2871287128712871, - "marketing": 0.6437768240343348, - "professional_law": 0.294194390084801, - "management": 0.6078431372549019, - "college_physics": 0.31683168316831684, - "jurisprudence": 0.5233644859813084, - "world_religions": 0.4117647058823529, - "sociology": 0.52, - "us_foreign_policy": 0.5454545454545454, - "high_school_macroeconomics": 0.39331619537275064, - "computer_security": 0.4444444444444444, - "moral_scenarios": 0.2516778523489933, - "moral_disputes": 0.45217391304347826, - "electrical_engineering": 0.3611111111111111, - "astronomy": 0.3841059602649007, - "college_biology": 0.3356643356643357 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.37592867756315007 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.3605230386052304, - "category_acc": { - "computer_network": 0.5416666666666666, - "operating_system": 0.25, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.23809523809523808, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.4782608695652174, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.21428571428571427, - "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.125, - "high_school_chemistry": 0.375, - "high_school_biology": 0.16666666666666666, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.38461538461538464, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.35, - "business_administration": 0.39473684210526316, - "marxism": 0.5416666666666666, - "mao_zedong_thought": 0.5172413793103449, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.5306122448979592, - "high_school_politics": 0.5, - "high_school_geography": 0.4583333333333333, - "middle_school_politics": 0.6923076923076923, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.4583333333333333, - "logic": 0.4074074074074074, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.47058823529411764, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.36, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.5, - "sports_science": 0.4166666666666667, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.375, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.3333333333333333, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.2962962962962963, - "physician": 0.3148148148148148 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.4014336917562724 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.3860300466240718, - "category_acc": { - "agronomy": 0.3727810650887574, - "anatomy": 0.2905405405405405, - "ancient_chinese": 0.21951219512195122, - "arts": 0.425, - "astronomy": 0.23030303030303031, - "business_ethics": 0.45933014354066987, - "chinese_civil_service_exam": 0.3625, - "chinese_driving_rule": 0.5648854961832062, - "chinese_food_culture": 0.40441176470588236, - "chinese_foreign_policy": 0.4392523364485981, - "chinese_history": 0.43962848297213625, - "chinese_literature": 0.29411764705882354, - "chinese_teacher_qualification": 0.4245810055865922, - "clinical_knowledge": 0.29535864978902954, - "college_actuarial_science": 0.3490566037735849, - "college_education": 0.514018691588785, - "college_engineering_hydrology": 0.3113207547169811, - "college_law": 0.25925925925925924, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.4056603773584906, - "college_medicine": 0.2893772893772894, - "computer_science": 0.37254901960784315, - "computer_security": 0.3684210526315789, - "conceptual_physics": 0.3197278911564626, - "construction_project_management": 0.3597122302158273, - "economics": 0.44654088050314467, - "education": 0.4110429447852761, - "electrical_engineering": 0.313953488372093, - "elementary_chinese": 0.30952380952380953, - "elementary_commonsense": 0.3333333333333333, - "elementary_information_and_technology": 0.5588235294117647, - "elementary_mathematics": 0.2565217391304348, - "ethnology": 0.3925925925925926, - "food_science": 0.3916083916083916, - "genetics": 0.3125, - "global_facts": 0.348993288590604, - "high_school_biology": 0.3609467455621302, - "high_school_chemistry": 0.3409090909090909, - "high_school_geography": 0.3898305084745763, - "high_school_mathematics": 0.24390243902439024, - "high_school_physics": 0.3, - "high_school_politics": 0.44755244755244755, - "human_sexuality": 0.3888888888888889, - "international_law": 0.43243243243243246, - "journalism": 0.4476744186046512, - "jurisprudence": 0.41849148418491483, - "legal_and_moral_basis": 0.7289719626168224, - "logical": 0.37398373983739835, - "machine_learning": 0.29508196721311475, - "management": 0.46190476190476193, - "marketing": 0.49444444444444446, - "marxist_theory": 0.4973544973544973, - "modern_chinese": 0.3448275862068966, - "nutrition": 0.3931034482758621, - "philosophy": 0.3523809523809524, - "professional_accounting": 0.42857142857142855, - "professional_law": 0.3696682464454976, - "professional_medicine": 0.30319148936170215, - "professional_psychology": 0.47413793103448276, - "public_relations": 0.5114942528735632, - "security_study": 0.45925925925925926, - "sociology": 0.4424778761061947, - "sports_science": 0.3878787878787879, - "traditional_chinese_medicine": 0.3081081081081081, - "virology": 0.4260355029585799, - "world_history": 0.422360248447205, - "world_religions": 0.3375 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.36363636363636365 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.24545454545454545 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.34711864406779663 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.7965594614809275 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.8388045075943165 - } + "prompt_1": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.3373788959857585, - "rouge2": 0.1532879767132338, - "rougeL": 0.29066058936402633, - "avg_rouge": 0.26044248735433956 - } + "prompt_1": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.26177030340883045, - "rouge2": 0.09129933955167958, - "rougeL": 0.2258691574012535, - "avg_rouge": 0.19297960012058782 - } + "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.6594036697247706 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.45445829338446786 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.522 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.3825 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.517 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.5070422535211268 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.51985559566787 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.5196078431372549 - } + "prompt_1": -1 } } }, - "flan_t5_xxl": { - "model_size": "11B", - "model_link": "https://huggingface.co/google/flan-t5-xxl", + "flan_t5_xl": { + "model_size": "3B", + "model_link": "https://huggingface.co/google/flan-t5-xl", "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.7771008403361345, + "overall_acc": 0.7733193277310924, "language_acc": { + "Spanish": 0.9563025210084034, "English": 0.9697478991596639, - "Chinese": 0.3445378151260504, - "Spanish": 0.9613445378151261, - "Vietnamese": 0.8327731092436975 + "Chinese": 0.3563025210084034, + "Vietnamese": 0.8109243697478992 }, - "consistency_score_2": 0.6124649859943978, - "consistency_score_3": 0.4403361344537815, - "consistency_score_4": 0.3025210084033613, + "consistency_score_2": 0.6148459383753501, + "consistency_score_3": 0.4455882352941176, + "consistency_score_4": 0.3142857142857143, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.33865546218487397, - "English,Spanish": 0.973109243697479, - "English,Vietnamese": 0.8294117647058824, - "Chinese,Spanish": 0.33949579831932775, - "Chinese,Vietnamese": 0.37058823529411766, - "Spanish,Vietnamese": 0.8235294117647058 + "Spanish,English": 0.9537815126050421, + "Spanish,Chinese": 0.3621848739495798, + "Spanish,Vietnamese": 0.807563025210084, + "English,Chinese": 0.3621848739495798, + "English,Vietnamese": 0.8142857142857143, + "Chinese,Vietnamese": 0.38907563025210085 }, "3_combine": { - "English,Chinese,Spanish": 0.3319327731092437, - "English,Chinese,Vietnamese": 0.3067226890756303, - "English,Spanish,Vietnamese": 0.8176470588235294, - "Chinese,Spanish,Vietnamese": 0.3050420168067227 + "Spanish,English,Chinese": 0.3495798319327731, + "Spanish,English,Vietnamese": 0.7915966386554621, + "Spanish,Chinese,Vietnamese": 0.32016806722689073, + "English,Chinese,Vietnamese": 0.32100840336134456 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3025210084033613 + "Spanish,English,Chinese,Vietnamese": 0.3142857142857143 } }, - "AC3_2": 0.6850298794628555, - "AC3_3": 0.5621409357069018, - "AC3_4": 0.43550309787133057 + "AC3_2": 0.6850369466770796, + "AC3_3": 0.5653947928242272, + "AC3_4": 0.4469328623447341 }, "prompt_2": { - "overall_acc": 0.778781512605042, + "overall_acc": 0.7762605042016806, "language_acc": { - "English": 0.9689075630252101, - "Chinese": 0.34789915966386553, - "Spanish": 0.9621848739495799, - "Vietnamese": 0.8361344537815126 + "Spanish": 0.9588235294117647, + "English": 0.9697478991596639, + "Chinese": 0.36470588235294116, + "Vietnamese": 0.8117647058823529 }, - "consistency_score_2": 0.6145658263305322, - "consistency_score_3": 0.4434873949579832, - "consistency_score_4": 0.3050420168067227, + "consistency_score_2": 0.6179271708683473, + "consistency_score_3": 0.4521008403361344, + "consistency_score_4": 0.3226890756302521, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.34285714285714286, - "English,Spanish": 0.9739495798319328, - "English,Vietnamese": 0.8302521008403362, - "Chinese,Spanish": 0.34285714285714286, - "Chinese,Vietnamese": 0.36722689075630255, - "Spanish,Vietnamese": 0.8302521008403362 + "Spanish,English": 0.9529411764705882, + "Spanish,Chinese": 0.37058823529411766, + "Spanish,Vietnamese": 0.807563025210084, + "English,Chinese": 0.37142857142857144, + "English,Vietnamese": 0.8117647058823529, + "Chinese,Vietnamese": 0.39327731092436974 }, "3_combine": { - "English,Chinese,Spanish": 0.33613445378151263, - "English,Chinese,Vietnamese": 0.30756302521008405, - "English,Spanish,Vietnamese": 0.8218487394957983, - "Chinese,Spanish,Vietnamese": 0.30840336134453783 + "Spanish,English,Chinese": 0.35798319327731093, + "Spanish,English,Vietnamese": 0.7907563025210084, + "Spanish,Chinese,Vietnamese": 0.32941176470588235, + "English,Chinese,Vietnamese": 0.3302521008403361 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3050420168067227 + "Spanish,English,Chinese,Vietnamese": 0.3226890756302521 } }, - "AC3_2": 0.6869966883581781, - "AC3_3": 0.5651453327517151, - "AC3_4": 0.43837594735343577 + "AC3_2": 0.6881031381369555, + "AC3_3": 0.5714084504552835, + "AC3_4": 0.4558731157886766 }, "prompt_3": { - "overall_acc": 0.7775210084033614, + "overall_acc": 0.774579831932773, "language_acc": { - "English": 0.9689075630252101, - "Chinese": 0.3487394957983193, - "Spanish": 0.9588235294117647, - "Vietnamese": 0.8336134453781513 + "Spanish": 0.9554621848739496, + "English": 0.9697478991596639, + "Chinese": 0.3621848739495798, + "Vietnamese": 0.8109243697478992 }, - "consistency_score_2": 0.6162464985994398, - "consistency_score_3": 0.44453781512605045, - "consistency_score_4": 0.30840336134453783, + "consistency_score_2": 0.6170868347338935, + "consistency_score_3": 0.4491596638655462, + "consistency_score_4": 0.31848739495798317, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.3453781512605042, - "English,Spanish": 0.9714285714285714, - "English,Vietnamese": 0.8277310924369747, - "Chinese,Spanish": 0.3453781512605042, - "Chinese,Vietnamese": 0.3815126050420168, - "Spanish,Vietnamese": 0.8260504201680672 + "Spanish,English": 0.9529411764705882, + "Spanish,Chinese": 0.36722689075630255, + "Spanish,Vietnamese": 0.8084033613445378, + "English,Chinese": 0.3689075630252101, + "English,Vietnamese": 0.8117647058823529, + "Chinese,Vietnamese": 0.39327731092436974 }, "3_combine": { - "English,Chinese,Spanish": 0.3369747899159664, - "English,Chinese,Vietnamese": 0.31176470588235294, - "English,Spanish,Vietnamese": 0.8168067226890756, - "Chinese,Spanish,Vietnamese": 0.3126050420168067 + "Spanish,English,Chinese": 0.3546218487394958, + "Spanish,English,Vietnamese": 0.7907563025210084, + "Spanish,Chinese,Vietnamese": 0.32521008403361346, + "English,Chinese,Vietnamese": 0.32605042016806723 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.30840336134453783 + "Spanish,English,Chinese,Vietnamese": 0.31848739495798317 } }, - "AC3_2": 0.6875531199777589, - "AC3_3": 0.5656642439906772, - "AC3_4": 0.4416331360924843 + "AC3_2": 0.686921700649236, + "AC3_3": 0.5686014353774906, + "AC3_4": 0.4513792139545092 }, "prompt_4": { - "overall_acc": 0.7779411764705882, + "overall_acc": 0.7731092436974789, "language_acc": { - "English": 0.9705882352941176, - "Chinese": 0.3487394957983193, - "Spanish": 0.9613445378151261, - "Vietnamese": 0.83109243697479 + "Spanish": 0.9571428571428572, + "English": 0.9680672268907563, + "Chinese": 0.36554621848739494, + "Vietnamese": 0.8016806722689076 }, - "consistency_score_2": 0.6147058823529412, - "consistency_score_3": 0.4432773109243698, - "consistency_score_4": 0.3058823529411765, + "consistency_score_2": 0.6168067226890755, + "consistency_score_3": 0.45042016806722684, + "consistency_score_4": 0.32184873949579834, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.34369747899159664, - "English,Spanish": 0.9747899159663865, - "English,Vietnamese": 0.826890756302521, - "Chinese,Spanish": 0.3453781512605042, - "Chinese,Vietnamese": 0.373109243697479, - "Spanish,Vietnamese": 0.8243697478991596 + "Spanish,English": 0.9529411764705882, + "Spanish,Chinese": 0.37058823529411766, + "Spanish,Vietnamese": 0.8016806722689076, + "English,Chinese": 0.3722689075630252, + "English,Vietnamese": 0.8033613445378152, + "Chinese,Vietnamese": 0.4 }, "3_combine": { - "English,Chinese,Spanish": 0.3378151260504202, - "English,Chinese,Vietnamese": 0.3092436974789916, - "English,Spanish,Vietnamese": 0.8168067226890756, - "Chinese,Spanish,Vietnamese": 0.3092436974789916 + "Spanish,English,Chinese": 0.3588235294117647, + "Spanish,English,Vietnamese": 0.7840336134453781, + "Spanish,Chinese,Vietnamese": 0.32941176470588235, + "English,Chinese,Vietnamese": 0.32941176470588235 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3058823529411765 + "Spanish,English,Chinese,Vietnamese": 0.32184873949579834 } }, - "AC3_2": 0.6867569413756203, - "AC3_3": 0.5647534430478341, - "AC3_4": 0.4391092664612012 + "AC3_2": 0.686169510076327, + "AC3_3": 0.5692123002582151, + "AC3_4": 0.45449092910067884 }, "prompt_5": { - "overall_acc": 0.7655462184873949, + "overall_acc": 0.7752100840336134, "language_acc": { - "English": 0.9605042016806723, - "Chinese": 0.3420168067226891, - "Spanish": 0.9495798319327731, - "Vietnamese": 0.8100840336134454 + "Spanish": 0.957983193277311, + "English": 0.9722689075630252, + "Chinese": 0.3563025210084034, + "Vietnamese": 0.8142857142857143 }, - "consistency_score_2": 0.5990196078431373, - "consistency_score_3": 0.4273109243697479, - "consistency_score_4": 0.28823529411764703, + "consistency_score_2": 0.6134453781512604, + "consistency_score_3": 0.4426470588235294, + "consistency_score_4": 0.30756302521008405, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.34369747899159664, - "English,Spanish": 0.9546218487394958, - "English,Vietnamese": 0.807563025210084, - "Chinese,Spanish": 0.3411764705882353, - "Chinese,Vietnamese": 0.33949579831932775, - "Spanish,Vietnamese": 0.807563025210084 + "Spanish,English": 0.9554621848739496, + "Spanish,Chinese": 0.35798319327731093, + "Spanish,Vietnamese": 0.8142857142857143, + "English,Chinese": 0.3596638655462185, + "English,Vietnamese": 0.8168067226890756, + "Chinese,Vietnamese": 0.3764705882352941 }, "3_combine": { - "English,Chinese,Spanish": 0.3310924369747899, - "English,Chinese,Vietnamese": 0.2949579831932773, - "English,Spanish,Vietnamese": 0.7907563025210084, - "Chinese,Spanish,Vietnamese": 0.292436974789916 + "Spanish,English,Chinese": 0.3445378151260504, + "Spanish,English,Vietnamese": 0.7983193277310925, + "Spanish,Chinese,Vietnamese": 0.3142857142857143, + "English,Chinese,Vietnamese": 0.3134453781512605 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.28823529411764703 + "Spanish,English,Chinese,Vietnamese": 0.30756302521008405 } }, - "AC3_2": 0.6721217645962518, - "AC3_3": 0.5484751702264262, - "AC3_4": 0.41879163144538817 + "AC3_2": 0.6849057322889995, + "AC3_3": 0.5635216998187702, + "AC3_4": 0.44039874385735167 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.37714285714285717, + "overall_acc": 0.36, "language_acc": { - "Spanish": 0.44, - "English": 0.5933333333333334, - "Chinese": 0.21333333333333335, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.3466666666666667, - "Indonesian": 0.4266666666666667, - "Malay": 0.3333333333333333 + "Filipino": 0.2866666666666667, + "Vietnamese": 0.24666666666666667, + "Chinese": 0.26666666666666666, + "Spanish": 0.44666666666666666, + "Malay": 0.34, + "Indonesian": 0.37333333333333335, + "English": 0.56 }, - "consistency_score_2": 0.4158730158730159, - "consistency_score_3": 0.21504761904761902, - "consistency_score_4": 0.12685714285714284, - "consistency_score_5": 0.08031746031746029, - "consistency_score_6": 0.052380952380952375, - "consistency_score_7": 0.03333333333333333, + "consistency_score_2": 0.5149206349206349, + "consistency_score_3": 0.3361904761904762, + "consistency_score_4": 0.2432380952380952, + "consistency_score_5": 0.1853968253968254, + "consistency_score_6": 0.1466666666666667, + "consistency_score_7": 0.12, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5933333333333334, - "Spanish,Chinese": 0.25333333333333335, - "Spanish,Vietnamese": 0.34, - "Spanish,Filipino": 0.4066666666666667, - "Spanish,Indonesian": 0.5866666666666667, - "Spanish,Malay": 0.4866666666666667, - "English,Chinese": 0.2733333333333333, - "English,Vietnamese": 0.32666666666666666, - "English,Filipino": 0.44, - "English,Indonesian": 0.5266666666666666, - "English,Malay": 0.4266666666666667, - "Chinese,Vietnamese": 0.5, - "Chinese,Filipino": 0.31333333333333335, - "Chinese,Indonesian": 0.32, - "Chinese,Malay": 0.2866666666666667, - "Vietnamese,Filipino": 0.4, - "Vietnamese,Indonesian": 0.36, - "Vietnamese,Malay": 0.32, - "Filipino,Indonesian": 0.48, - "Filipino,Malay": 0.47333333333333333, - "Indonesian,Malay": 0.62 + "Filipino,Vietnamese": 0.5066666666666667, + "Filipino,Chinese": 0.5066666666666667, + "Filipino,Spanish": 0.49333333333333335, + "Filipino,Malay": 0.5866666666666667, + "Filipino,Indonesian": 0.5266666666666666, + "Filipino,English": 0.5133333333333333, + "Vietnamese,Chinese": 0.66, + "Vietnamese,Spanish": 0.44, + "Vietnamese,Malay": 0.5266666666666666, + "Vietnamese,Indonesian": 0.5266666666666666, + "Vietnamese,English": 0.36666666666666664, + "Chinese,Spanish": 0.4266666666666667, + "Chinese,Malay": 0.5533333333333333, + "Chinese,Indonesian": 0.5666666666666667, + "Chinese,English": 0.3466666666666667, + "Spanish,Malay": 0.5, + "Spanish,Indonesian": 0.5466666666666666, + "Spanish,English": 0.5866666666666667, + "Malay,Indonesian": 0.6666666666666666, + "Malay,English": 0.48, + "Indonesian,English": 0.4866666666666667 }, "3_combine": { - "Spanish,English,Chinese": 0.16, - "Spanish,English,Vietnamese": 0.21333333333333335, - "Spanish,English,Filipino": 0.2866666666666667, - "Spanish,English,Indonesian": 0.3933333333333333, - "Spanish,English,Malay": 0.32, - "Spanish,Chinese,Vietnamese": 0.14, - "Spanish,Chinese,Filipino": 0.13333333333333333, - "Spanish,Chinese,Indonesian": 0.18, - "Spanish,Chinese,Malay": 0.15333333333333332, - "Spanish,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,Vietnamese,Malay": 0.18666666666666668, - "Spanish,Filipino,Indonesian": 0.3, - "Spanish,Filipino,Malay": 0.2866666666666667, - "Spanish,Indonesian,Malay": 0.4, - "English,Chinese,Vietnamese": 0.13333333333333333, - "English,Chinese,Filipino": 0.15333333333333332, - "English,Chinese,Indonesian": 0.16666666666666666, - "English,Chinese,Malay": 0.12, - "English,Vietnamese,Filipino": 0.19333333333333333, - "English,Vietnamese,Indonesian": 0.2, - "English,Vietnamese,Malay": 0.15333333333333332, - "English,Filipino,Indonesian": 0.28, - "English,Filipino,Malay": 0.26, - "English,Indonesian,Malay": 0.3466666666666667, - "Chinese,Vietnamese,Filipino": 0.17333333333333334, - "Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Chinese,Vietnamese,Malay": 0.14, - "Chinese,Filipino,Indonesian": 0.15333333333333332, - "Chinese,Filipino,Malay": 0.13333333333333333, - "Chinese,Indonesian,Malay": 0.19333333333333333, - "Vietnamese,Filipino,Indonesian": 0.22, - "Vietnamese,Filipino,Malay": 0.2, - "Vietnamese,Indonesian,Malay": 0.22, - "Filipino,Indonesian,Malay": 0.3466666666666667 + "Filipino,Vietnamese,Chinese": 0.38, + "Filipino,Vietnamese,Spanish": 0.2866666666666667, + "Filipino,Vietnamese,Malay": 0.37333333333333335, + "Filipino,Vietnamese,Indonesian": 0.3333333333333333, + "Filipino,Vietnamese,English": 0.26, + "Filipino,Chinese,Spanish": 0.26666666666666666, + "Filipino,Chinese,Malay": 0.38666666666666666, + "Filipino,Chinese,Indonesian": 0.35333333333333333, + "Filipino,Chinese,English": 0.26666666666666666, + "Filipino,Spanish,Malay": 0.36, + "Filipino,Spanish,Indonesian": 0.34, + "Filipino,Spanish,English": 0.36666666666666664, + "Filipino,Malay,Indonesian": 0.4533333333333333, + "Filipino,Malay,English": 0.35333333333333333, + "Filipino,Indonesian,English": 0.32666666666666666, + "Vietnamese,Chinese,Spanish": 0.32, + "Vietnamese,Chinese,Malay": 0.42, + "Vietnamese,Chinese,Indonesian": 0.4266666666666667, + "Vietnamese,Chinese,English": 0.26, + "Vietnamese,Spanish,Malay": 0.31333333333333335, + "Vietnamese,Spanish,Indonesian": 0.3333333333333333, + "Vietnamese,Spanish,English": 0.2733333333333333, + "Vietnamese,Malay,Indonesian": 0.4, + "Vietnamese,Malay,English": 0.26666666666666666, + "Vietnamese,Indonesian,English": 0.2733333333333333, + "Chinese,Spanish,Malay": 0.29333333333333333, + "Chinese,Spanish,Indonesian": 0.32666666666666666, + "Chinese,Spanish,English": 0.26, + "Chinese,Malay,Indonesian": 0.44, + "Chinese,Malay,English": 0.26666666666666666, + "Chinese,Indonesian,English": 0.29333333333333333, + "Spanish,Malay,Indonesian": 0.41333333333333333, + "Spanish,Malay,English": 0.35333333333333333, + "Spanish,Indonesian,English": 0.36, + "Malay,Indonesian,English": 0.36666666666666664 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.08, - "Spanish,English,Chinese,Filipino": 0.08666666666666667, - "Spanish,English,Chinese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Malay": 0.09333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Malay": 0.13333333333333333, - "Spanish,English,Filipino,Indonesian": 0.23333333333333334, - "Spanish,English,Filipino,Malay": 0.22, - "Spanish,English,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.1, - "Spanish,Chinese,Vietnamese,Malay": 0.08, - "Spanish,Chinese,Filipino,Indonesian": 0.1, - "Spanish,Chinese,Filipino,Malay": 0.1, - "Spanish,Chinese,Indonesian,Malay": 0.14, - "Spanish,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.07333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.08, - "English,Chinese,Vietnamese,Malay": 0.06, - "English,Chinese,Filipino,Indonesian": 0.08666666666666667, - "English,Chinese,Filipino,Malay": 0.08, - "English,Chinese,Indonesian,Malay": 0.10666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "English,Vietnamese,Filipino,Malay": 0.11333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "English,Filipino,Indonesian,Malay": 0.22, - "Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.08, - "Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 + "Filipino,Vietnamese,Chinese,Spanish": 0.2, + "Filipino,Vietnamese,Chinese,Malay": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,English": 0.19333333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.24, + "Filipino,Vietnamese,Spanish,Indonesian": 0.22, + "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.3, + "Filipino,Vietnamese,Malay,English": 0.21333333333333335, + "Filipino,Vietnamese,Indonesian,English": 0.20666666666666667, + "Filipino,Chinese,Spanish,Malay": 0.22666666666666666, + "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, + "Filipino,Chinese,Spanish,English": 0.2, + "Filipino,Chinese,Malay,Indonesian": 0.31333333333333335, + "Filipino,Chinese,Malay,English": 0.22666666666666666, + "Filipino,Chinese,Indonesian,English": 0.22666666666666666, + "Filipino,Spanish,Malay,Indonesian": 0.3, + "Filipino,Spanish,Malay,English": 0.2733333333333333, + "Filipino,Spanish,Indonesian,English": 0.26, + "Filipino,Malay,Indonesian,English": 0.28, + "Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,English": 0.2, + "Vietnamese,Chinese,Malay,Indonesian": 0.3333333333333333, + "Vietnamese,Chinese,Malay,English": 0.2, + "Vietnamese,Chinese,Indonesian,English": 0.23333333333333334, + "Vietnamese,Spanish,Malay,Indonesian": 0.2733333333333333, + "Vietnamese,Spanish,Malay,English": 0.21333333333333335, + "Vietnamese,Spanish,Indonesian,English": 0.22, + "Vietnamese,Malay,Indonesian,English": 0.22, + "Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, + "Chinese,Spanish,Malay,English": 0.20666666666666667, + "Chinese,Spanish,Indonesian,English": 0.23333333333333334, + "Chinese,Malay,Indonesian,English": 0.23333333333333334, + "Spanish,Malay,Indonesian,English": 0.29333333333333333 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.04, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,English,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.06, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.05333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.04, - "English,Chinese,Vietnamese,Filipino,Malay": 0.04, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.05333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.24, + "Filipino,Vietnamese,Chinese,Malay,English": 0.16, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.18, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Indonesian,English": 0.18, + "Filipino,Chinese,Malay,Indonesian,English": 0.19333333333333333, + "Filipino,Spanish,Malay,Indonesian,English": 0.22666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22, + "Vietnamese,Chinese,Spanish,Malay,English": 0.16, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.18666666666666668, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.18, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.18666666666666668, + "Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.04, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.03333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.06666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.03333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.03333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 } }, - "AC3_2": 0.39556216396704713, - "AC3_3": 0.2739107659327013, - "AC3_4": 0.1898542273675783, - "AC3_5": 0.13243184293724472, - "AC3_6": 0.0919860626963542, - "AC3_7": 0.06125290021709616 + "AC3_2": 0.4237445572810306, + "AC3_3": 0.3476880984452705, + "AC3_4": 0.2903189137504208, + "AC3_5": 0.24474970891903602, + "AC3_6": 0.20842105259044322, + "AC3_7": 0.1799999999625 }, "prompt_2": { - "overall_acc": 0.38666666666666666, + "overall_acc": 0.35809523809523813, "language_acc": { + "Filipino": 0.2733333333333333, + "Vietnamese": 0.24666666666666667, + "Chinese": 0.2866666666666667, "Spanish": 0.4533333333333333, - "English": 0.5933333333333334, - "Chinese": 0.22666666666666666, - "Vietnamese": 0.2733333333333333, - "Filipino": 0.38666666666666666, - "Indonesian": 0.42, - "Malay": 0.35333333333333333 + "Malay": 0.36, + "Indonesian": 0.38, + "English": 0.5066666666666667 }, - "consistency_score_2": 0.4304761904761905, - "consistency_score_3": 0.23295238095238097, - "consistency_score_4": 0.1426666666666667, - "consistency_score_5": 0.09301587301587301, - "consistency_score_6": 0.0619047619047619, - "consistency_score_7": 0.04, + "consistency_score_2": 0.5292063492063492, + "consistency_score_3": 0.3466666666666667, + "consistency_score_4": 0.24723809523809526, + "consistency_score_5": 0.1841269841269841, + "consistency_score_6": 0.13999999999999999, + "consistency_score_7": 0.10666666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.58, - "Spanish,Chinese": 0.35333333333333333, - "Spanish,Vietnamese": 0.3466666666666667, - "Spanish,Filipino": 0.4266666666666667, - "Spanish,Indonesian": 0.58, - "Spanish,Malay": 0.48, - "English,Chinese": 0.31333333333333335, - "English,Vietnamese": 0.3333333333333333, - "English,Filipino": 0.47333333333333333, - "English,Indonesian": 0.54, - "English,Malay": 0.4266666666666667, - "Chinese,Vietnamese": 0.4, - "Chinese,Filipino": 0.32666666666666666, - "Chinese,Indonesian": 0.38, - "Chinese,Malay": 0.35333333333333333, - "Vietnamese,Filipino": 0.4, - "Vietnamese,Indonesian": 0.37333333333333335, - "Vietnamese,Malay": 0.36666666666666664, - "Filipino,Indonesian": 0.48, - "Filipino,Malay": 0.49333333333333335, - "Indonesian,Malay": 0.6133333333333333 + "Filipino,Vietnamese": 0.52, + "Filipino,Chinese": 0.5533333333333333, + "Filipino,Spanish": 0.5066666666666667, + "Filipino,Malay": 0.62, + "Filipino,Indonesian": 0.5666666666666667, + "Filipino,English": 0.4866666666666667, + "Vietnamese,Chinese": 0.64, + "Vietnamese,Spanish": 0.44666666666666666, + "Vietnamese,Malay": 0.52, + "Vietnamese,Indonesian": 0.54, + "Vietnamese,English": 0.36, + "Chinese,Spanish": 0.43333333333333335, + "Chinese,Malay": 0.5866666666666667, + "Chinese,Indonesian": 0.58, + "Chinese,English": 0.37333333333333335, + "Spanish,Malay": 0.52, + "Spanish,Indonesian": 0.5466666666666666, + "Spanish,English": 0.6266666666666667, + "Malay,Indonesian": 0.6666666666666666, + "Malay,English": 0.5, + "Indonesian,English": 0.52 }, "3_combine": { - "Spanish,English,Chinese": 0.20666666666666667, - "Spanish,English,Vietnamese": 0.22, - "Spanish,English,Filipino": 0.3, - "Spanish,English,Indonesian": 0.3933333333333333, - "Spanish,English,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese": 0.16, - "Spanish,Chinese,Filipino": 0.18, - "Spanish,Chinese,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino": 0.2, - "Spanish,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,Vietnamese,Malay": 0.2, - "Spanish,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Filipino,Malay": 0.2866666666666667, - "Spanish,Indonesian,Malay": 0.3933333333333333, - "English,Chinese,Vietnamese": 0.13333333333333333, - "English,Chinese,Filipino": 0.17333333333333334, - "English,Chinese,Indonesian": 0.2, - "English,Chinese,Malay": 0.16, - "English,Vietnamese,Filipino": 0.20666666666666667, - "English,Vietnamese,Indonesian": 0.22, - "English,Vietnamese,Malay": 0.16666666666666666, - "English,Filipino,Indonesian": 0.30666666666666664, - "English,Filipino,Malay": 0.2866666666666667, - "English,Indonesian,Malay": 0.3466666666666667, - "Chinese,Vietnamese,Filipino": 0.16, - "Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "Chinese,Vietnamese,Malay": 0.15333333333333332, - "Chinese,Filipino,Indonesian": 0.17333333333333334, - "Chinese,Filipino,Malay": 0.16666666666666666, - "Chinese,Indonesian,Malay": 0.24666666666666667, - "Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Vietnamese,Filipino,Malay": 0.22, - "Vietnamese,Indonesian,Malay": 0.24, - "Filipino,Indonesian,Malay": 0.34 + "Filipino,Vietnamese,Chinese": 0.4, + "Filipino,Vietnamese,Spanish": 0.2866666666666667, + "Filipino,Vietnamese,Malay": 0.37333333333333335, + "Filipino,Vietnamese,Indonesian": 0.36, + "Filipino,Vietnamese,English": 0.24666666666666667, + "Filipino,Chinese,Spanish": 0.2866666666666667, + "Filipino,Chinese,Malay": 0.4266666666666667, + "Filipino,Chinese,Indonesian": 0.3933333333333333, + "Filipino,Chinese,English": 0.2733333333333333, + "Filipino,Spanish,Malay": 0.38, + "Filipino,Spanish,Indonesian": 0.3466666666666667, + "Filipino,Spanish,English": 0.36666666666666664, + "Filipino,Malay,Indonesian": 0.47333333333333333, + "Filipino,Malay,English": 0.35333333333333333, + "Filipino,Indonesian,English": 0.3466666666666667, + "Vietnamese,Chinese,Spanish": 0.31333333333333335, + "Vietnamese,Chinese,Malay": 0.41333333333333333, + "Vietnamese,Chinese,Indonesian": 0.4266666666666667, + "Vietnamese,Chinese,English": 0.25333333333333335, + "Vietnamese,Spanish,Malay": 0.30666666666666664, + "Vietnamese,Spanish,Indonesian": 0.32666666666666666, + "Vietnamese,Spanish,English": 0.28, + "Vietnamese,Malay,Indonesian": 0.4066666666666667, + "Vietnamese,Malay,English": 0.26, + "Vietnamese,Indonesian,English": 0.2866666666666667, + "Chinese,Spanish,Malay": 0.31333333333333335, + "Chinese,Spanish,Indonesian": 0.31333333333333335, + "Chinese,Spanish,English": 0.2733333333333333, + "Chinese,Malay,Indonesian": 0.46, + "Chinese,Malay,English": 0.29333333333333333, + "Chinese,Indonesian,English": 0.31333333333333335, + "Spanish,Malay,Indonesian": 0.41333333333333333, + "Spanish,Malay,English": 0.38666666666666666, + "Spanish,Indonesian,English": 0.3933333333333333, + "Malay,Indonesian,English": 0.38666666666666666 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.08666666666666667, - "Spanish,English,Chinese,Filipino": 0.12, - "Spanish,English,Chinese,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,English,Vietnamese,Malay": 0.14666666666666667, - "Spanish,English,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Filipino,Malay": 0.22, - "Spanish,English,Indonesian,Malay": 0.2733333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.1, - "Spanish,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Filipino,Malay": 0.12, - "Spanish,Chinese,Indonesian,Malay": 0.18, - "Spanish,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,Vietnamese,Filipino,Malay": 0.14, - "Spanish,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.07333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.07333333333333333, - "English,Chinese,Vietnamese,Malay": 0.06666666666666667, - "English,Chinese,Filipino,Indonesian": 0.11333333333333333, - "English,Chinese,Filipino,Malay": 0.10666666666666667, - "English,Chinese,Indonesian,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "English,Vietnamese,Filipino,Malay": 0.12666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "English,Filipino,Indonesian,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.12, - "Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666 + "Filipino,Vietnamese,Chinese,Spanish": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,English": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Malay": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,Indonesian": 0.22, + "Filipino,Vietnamese,Spanish,English": 0.19333333333333333, + "Filipino,Vietnamese,Malay,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Malay,English": 0.2, + "Filipino,Vietnamese,Indonesian,English": 0.22, + "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, + "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, + "Filipino,Chinese,Spanish,English": 0.2, + "Filipino,Chinese,Malay,Indonesian": 0.35333333333333333, + "Filipino,Chinese,Malay,English": 0.22666666666666666, + "Filipino,Chinese,Indonesian,English": 0.24, + "Filipino,Spanish,Malay,Indonesian": 0.30666666666666664, + "Filipino,Spanish,Malay,English": 0.28, + "Filipino,Spanish,Indonesian,English": 0.26666666666666666, + "Filipino,Malay,Indonesian,English": 0.29333333333333333, + "Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,English": 0.19333333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.3333333333333333, + "Vietnamese,Chinese,Malay,English": 0.2, + "Vietnamese,Chinese,Indonesian,English": 0.23333333333333334, + "Vietnamese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Vietnamese,Spanish,Malay,English": 0.21333333333333335, + "Vietnamese,Spanish,Indonesian,English": 0.22666666666666666, + "Vietnamese,Malay,Indonesian,English": 0.23333333333333334, + "Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Chinese,Spanish,Malay,English": 0.22666666666666666, + "Chinese,Spanish,Indonesian,English": 0.22666666666666666, + "Chinese,Malay,Indonesian,English": 0.25333333333333335, + "Spanish,Malay,Indonesian,English": 0.31333333333333335 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.05333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.1, - "Spanish,English,Chinese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.06666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.04666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.04666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.06, - "English,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.2, + "Filipino,Vietnamese,Spanish,Malay,English": 0.16, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.18666666666666668, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Indonesian,English": 0.17333333333333334, + "Filipino,Chinese,Malay,Indonesian,English": 0.20666666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.16, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.18666666666666668, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, + "Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.04666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.06, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.12, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, - "AC3_2": 0.4073970473471911, - "AC3_3": 0.29074290393914126, - "AC3_4": 0.20842989080865099, - "AC3_5": 0.14995808512205813, - "AC3_6": 0.10672328376955566, - "AC3_7": 0.07249999998300781 + "AC3_2": 0.4271518868246766, + "AC3_3": 0.3522882882383015, + "AC3_4": 0.2925158080370256, + "AC3_5": 0.24320285486760376, + "AC3_6": 0.20130019116417489, + "AC3_7": 0.16437158466408672 }, "prompt_3": { - "overall_acc": 0.3895238095238095, + "overall_acc": 0.35809523809523813, "language_acc": { - "Spanish": 0.4533333333333333, - "English": 0.5866666666666667, + "Filipino": 0.3, + "Vietnamese": 0.25333333333333335, "Chinese": 0.26666666666666666, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.38666666666666666, - "Indonesian": 0.4, - "Malay": 0.3466666666666667 + "Spanish": 0.44, + "Malay": 0.32666666666666666, + "Indonesian": 0.3933333333333333, + "English": 0.5266666666666666 }, - "consistency_score_2": 0.4422222222222222, - "consistency_score_3": 0.24685714285714283, - "consistency_score_4": 0.15314285714285716, - "consistency_score_5": 0.09936507936507938, - "consistency_score_6": 0.06476190476190476, - "consistency_score_7": 0.04, + "consistency_score_2": 0.540952380952381, + "consistency_score_3": 0.36266666666666664, + "consistency_score_4": 0.26514285714285707, + "consistency_score_5": 0.20412698412698416, + "consistency_score_6": 0.16285714285714287, + "consistency_score_7": 0.13333333333333333, "detailed_consistency_score": { "2_combine": { + "Filipino,Vietnamese": 0.5333333333333333, + "Filipino,Chinese": 0.54, + "Filipino,Spanish": 0.52, + "Filipino,Malay": 0.6666666666666666, + "Filipino,Indonesian": 0.6, + "Filipino,English": 0.5066666666666667, + "Vietnamese,Chinese": 0.66, + "Vietnamese,Spanish": 0.44666666666666666, + "Vietnamese,Malay": 0.56, + "Vietnamese,Indonesian": 0.5666666666666667, + "Vietnamese,English": 0.38666666666666666, + "Chinese,Spanish": 0.43333333333333335, + "Chinese,Malay": 0.5866666666666667, + "Chinese,Indonesian": 0.6, + "Chinese,English": 0.38, + "Spanish,Malay": 0.5133333333333333, + "Spanish,Indonesian": 0.5466666666666666, "Spanish,English": 0.5933333333333334, - "Spanish,Chinese": 0.38, - "Spanish,Vietnamese": 0.35333333333333333, - "Spanish,Filipino": 0.47333333333333333, - "Spanish,Indonesian": 0.5933333333333334, - "Spanish,Malay": 0.46, - "English,Chinese": 0.32666666666666666, - "English,Vietnamese": 0.3333333333333333, - "English,Filipino": 0.47333333333333333, - "English,Indonesian": 0.5333333333333333, - "English,Malay": 0.44666666666666666, - "Chinese,Vietnamese": 0.43333333333333335, - "Chinese,Filipino": 0.3933333333333333, - "Chinese,Indonesian": 0.38, - "Chinese,Malay": 0.37333333333333335, - "Vietnamese,Filipino": 0.44666666666666666, - "Vietnamese,Indonesian": 0.38666666666666666, - "Vietnamese,Malay": 0.36666666666666664, - "Filipino,Indonesian": 0.47333333333333333, - "Filipino,Malay": 0.4666666666666667, - "Indonesian,Malay": 0.6 + "Malay,Indonesian": 0.7, + "Malay,English": 0.4866666666666667, + "Indonesian,English": 0.5333333333333333 }, "3_combine": { - "Spanish,English,Chinese": 0.21333333333333335, - "Spanish,English,Vietnamese": 0.22, - "Spanish,English,Filipino": 0.32, - "Spanish,English,Indonesian": 0.4066666666666667, - "Spanish,English,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese": 0.18666666666666668, - "Spanish,Chinese,Filipino": 0.24, - "Spanish,Chinese,Indonesian": 0.26, - "Spanish,Chinese,Malay": 0.22, - "Spanish,Vietnamese,Filipino": 0.22666666666666666, - "Spanish,Vietnamese,Indonesian": 0.26, - "Spanish,Vietnamese,Malay": 0.19333333333333333, - "Spanish,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Filipino,Malay": 0.29333333333333333, - "Spanish,Indonesian,Malay": 0.38666666666666666, - "English,Chinese,Vietnamese": 0.14, - "English,Chinese,Filipino": 0.2, - "English,Chinese,Indonesian": 0.21333333333333335, - "English,Chinese,Malay": 0.18, - "English,Vietnamese,Filipino": 0.22, - "English,Vietnamese,Indonesian": 0.22666666666666666, - "English,Vietnamese,Malay": 0.18, - "English,Filipino,Indonesian": 0.30666666666666664, - "English,Filipino,Malay": 0.2866666666666667, - "English,Indonesian,Malay": 0.35333333333333333, - "Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "Chinese,Vietnamese,Malay": 0.18666666666666668, - "Chinese,Filipino,Indonesian": 0.2, - "Chinese,Filipino,Malay": 0.2, - "Chinese,Indonesian,Malay": 0.25333333333333335, - "Vietnamese,Filipino,Indonesian": 0.24, - "Vietnamese,Filipino,Malay": 0.22666666666666666, - "Vietnamese,Indonesian,Malay": 0.24, - "Filipino,Indonesian,Malay": 0.32666666666666666 + "Filipino,Vietnamese,Chinese": 0.4, + "Filipino,Vietnamese,Spanish": 0.29333333333333333, + "Filipino,Vietnamese,Malay": 0.42, + "Filipino,Vietnamese,Indonesian": 0.38666666666666666, + "Filipino,Vietnamese,English": 0.28, + "Filipino,Chinese,Spanish": 0.2866666666666667, + "Filipino,Chinese,Malay": 0.44, + "Filipino,Chinese,Indonesian": 0.41333333333333333, + "Filipino,Chinese,English": 0.2866666666666667, + "Filipino,Spanish,Malay": 0.3933333333333333, + "Filipino,Spanish,Indonesian": 0.37333333333333335, + "Filipino,Spanish,English": 0.37333333333333335, + "Filipino,Malay,Indonesian": 0.5266666666666666, + "Filipino,Malay,English": 0.38, + "Filipino,Indonesian,English": 0.37333333333333335, + "Vietnamese,Chinese,Spanish": 0.32666666666666666, + "Vietnamese,Chinese,Malay": 0.44666666666666666, + "Vietnamese,Chinese,Indonesian": 0.46, + "Vietnamese,Chinese,English": 0.2866666666666667, + "Vietnamese,Spanish,Malay": 0.32, + "Vietnamese,Spanish,Indonesian": 0.3333333333333333, + "Vietnamese,Spanish,English": 0.2733333333333333, + "Vietnamese,Malay,Indonesian": 0.4533333333333333, + "Vietnamese,Malay,English": 0.28, + "Vietnamese,Indonesian,English": 0.30666666666666664, + "Chinese,Spanish,Malay": 0.30666666666666664, + "Chinese,Spanish,Indonesian": 0.32666666666666666, + "Chinese,Spanish,English": 0.2733333333333333, + "Chinese,Malay,Indonesian": 0.48, + "Chinese,Malay,English": 0.2866666666666667, + "Chinese,Indonesian,English": 0.3333333333333333, + "Spanish,Malay,Indonesian": 0.42, + "Spanish,Malay,English": 0.36, + "Spanish,Indonesian,English": 0.38666666666666666, + "Malay,Indonesian,English": 0.4066666666666667 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.09333333333333334, - "Spanish,English,Chinese,Filipino": 0.14, - "Spanish,English,Chinese,Indonesian": 0.18, - "Spanish,English,Chinese,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Malay": 0.13333333333333333, - "Spanish,English,Filipino,Indonesian": 0.26, - "Spanish,English,Filipino,Malay": 0.22, - "Spanish,English,Indonesian,Malay": 0.2733333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.12, - "Spanish,Chinese,Vietnamese,Indonesian": 0.12, - "Spanish,Chinese,Vietnamese,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.14, - "Spanish,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.09333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.08, - "English,Chinese,Vietnamese,Malay": 0.08, - "English,Chinese,Filipino,Indonesian": 0.12666666666666668, - "English,Chinese,Filipino,Malay": 0.12666666666666668, - "English,Chinese,Indonesian,Malay": 0.16, - "English,Vietnamese,Filipino,Indonesian": 0.16, - "English,Vietnamese,Filipino,Malay": 0.13333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "Chinese,Vietnamese,Filipino,Malay": 0.12, - "Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.14, - "Vietnamese,Filipino,Indonesian,Malay": 0.16 + "Filipino,Vietnamese,Chinese,Spanish": 0.21333333333333335, + "Filipino,Vietnamese,Chinese,Malay": 0.3333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,English": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,Malay": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian": 0.24, + "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.3466666666666667, + "Filipino,Vietnamese,Malay,English": 0.23333333333333334, + "Filipino,Vietnamese,Indonesian,English": 0.24666666666666667, + "Filipino,Chinese,Spanish,Malay": 0.24, + "Filipino,Chinese,Spanish,Indonesian": 0.25333333333333335, + "Filipino,Chinese,Spanish,English": 0.20666666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.36666666666666664, + "Filipino,Chinese,Malay,English": 0.24, + "Filipino,Chinese,Indonesian,English": 0.26666666666666666, + "Filipino,Spanish,Malay,Indonesian": 0.3333333333333333, + "Filipino,Spanish,Malay,English": 0.2866666666666667, + "Filipino,Spanish,Indonesian,English": 0.2866666666666667, + "Filipino,Malay,Indonesian,English": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Malay": 0.24, + "Vietnamese,Chinese,Spanish,Indonesian": 0.26, + "Vietnamese,Chinese,Spanish,English": 0.21333333333333335, + "Vietnamese,Chinese,Malay,Indonesian": 0.37333333333333335, + "Vietnamese,Chinese,Malay,English": 0.22, + "Vietnamese,Chinese,Indonesian,English": 0.26, + "Vietnamese,Spanish,Malay,Indonesian": 0.2866666666666667, + "Vietnamese,Spanish,Malay,English": 0.21333333333333335, + "Vietnamese,Spanish,Indonesian,English": 0.23333333333333334, + "Vietnamese,Malay,Indonesian,English": 0.26, + "Chinese,Spanish,Malay,Indonesian": 0.28, + "Chinese,Spanish,Malay,English": 0.21333333333333335, + "Chinese,Spanish,Indonesian,English": 0.24666666666666667, + "Chinese,Malay,Indonesian,English": 0.26666666666666666, + "Spanish,Malay,Indonesian,English": 0.31333333333333335 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.06, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.08, - "Spanish,English,Chinese,Vietnamese,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,English,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Filipino,Indonesian,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.12, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.05333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.06, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.06, - "English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.18, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.2, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Malay,English": 0.18, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.22, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, + "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Indonesian,English": 0.2, + "Filipino,Chinese,Malay,Indonesian,English": 0.22666666666666666, + "Filipino,Spanish,Malay,Indonesian,English": 0.25333333333333335, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, + "Vietnamese,Chinese,Spanish,Malay,English": 0.16666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.19333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.21333333333333335, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.05333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.06, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 } }, - "AC3_2": 0.4142035622911676, - "AC3_3": 0.30219865732626755, - "AC3_4": 0.21985057409577544, - "AC3_5": 0.15833889916365457, - "AC3_6": 0.11105919933663916, - "AC3_7": 0.07254988911836421 + "AC3_2": 0.4309281678293888, + "AC3_3": 0.3603664552001782, + "AC3_4": 0.3046873907998722, + "AC3_5": 0.26002850147293466, + "AC3_6": 0.22389135540230864, + "AC3_7": 0.1943152454384953 }, "prompt_4": { - "overall_acc": 0.38952380952380955, + "overall_acc": 0.3590476190476191, "language_acc": { - "Spanish": 0.4666666666666667, - "English": 0.6333333333333333, - "Chinese": 0.22, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.38666666666666666, - "Indonesian": 0.4, - "Malay": 0.3333333333333333 + "Filipino": 0.3, + "Vietnamese": 0.24, + "Chinese": 0.26, + "Spanish": 0.4533333333333333, + "Malay": 0.3466666666666667, + "Indonesian": 0.38666666666666666, + "English": 0.5266666666666666 }, - "consistency_score_2": 0.42253968253968255, - "consistency_score_3": 0.23028571428571434, - "consistency_score_4": 0.14723809523809525, - "consistency_score_5": 0.10253968253968251, - "consistency_score_6": 0.0742857142857143, - "consistency_score_7": 0.05333333333333334, + "consistency_score_2": 0.5333333333333334, + "consistency_score_3": 0.3523809523809524, + "consistency_score_4": 0.25580952380952376, + "consistency_score_5": 0.1955555555555556, + "consistency_score_6": 0.15333333333333332, + "consistency_score_7": 0.12, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6066666666666667, - "Spanish,Chinese": 0.28, - "Spanish,Vietnamese": 0.38, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.5666666666666667, - "Spanish,Malay": 0.46, - "English,Chinese": 0.2733333333333333, - "English,Vietnamese": 0.35333333333333333, - "English,Filipino": 0.48, - "English,Indonesian": 0.5133333333333333, - "English,Malay": 0.42, - "Chinese,Vietnamese": 0.52, - "Chinese,Filipino": 0.31333333333333335, - "Chinese,Indonesian": 0.34, - "Chinese,Malay": 0.26666666666666666, - "Vietnamese,Filipino": 0.4066666666666667, - "Vietnamese,Indonesian": 0.4, - "Vietnamese,Malay": 0.36, - "Filipino,Indonesian": 0.5066666666666667, - "Filipino,Malay": 0.43333333333333335, - "Indonesian,Malay": 0.5533333333333333 + "Filipino,Vietnamese": 0.5333333333333333, + "Filipino,Chinese": 0.5466666666666666, + "Filipino,Spanish": 0.49333333333333335, + "Filipino,Malay": 0.66, + "Filipino,Indonesian": 0.5866666666666667, + "Filipino,English": 0.4866666666666667, + "Vietnamese,Chinese": 0.68, + "Vietnamese,Spanish": 0.42, + "Vietnamese,Malay": 0.5533333333333333, + "Vietnamese,Indonesian": 0.5733333333333334, + "Vietnamese,English": 0.36, + "Chinese,Spanish": 0.44, + "Chinese,Malay": 0.58, + "Chinese,Indonesian": 0.5866666666666667, + "Chinese,English": 0.37333333333333335, + "Spanish,Malay": 0.4866666666666667, + "Spanish,Indonesian": 0.5333333333333333, + "Spanish,English": 0.6, + "Malay,Indonesian": 0.6866666666666666, + "Malay,English": 0.5066666666666667, + "Indonesian,English": 0.5133333333333333 }, "3_combine": { - "Spanish,English,Chinese": 0.18, - "Spanish,English,Vietnamese": 0.26666666666666666, - "Spanish,English,Filipino": 0.32666666666666666, - "Spanish,English,Indonesian": 0.3933333333333333, - "Spanish,English,Malay": 0.32, - "Spanish,Chinese,Vietnamese": 0.18666666666666668, - "Spanish,Chinese,Filipino": 0.16666666666666666, - "Spanish,Chinese,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Malay": 0.14, - "Spanish,Vietnamese,Filipino": 0.24, - "Spanish,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Malay": 0.20666666666666667, - "Spanish,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Filipino,Malay": 0.2866666666666667, - "Spanish,Indonesian,Malay": 0.36, - "English,Chinese,Vietnamese": 0.15333333333333332, - "English,Chinese,Filipino": 0.16666666666666666, - "English,Chinese,Indonesian": 0.17333333333333334, - "English,Chinese,Malay": 0.11333333333333333, - "English,Vietnamese,Filipino": 0.22666666666666666, - "English,Vietnamese,Indonesian": 0.23333333333333334, - "English,Vietnamese,Malay": 0.18, - "English,Filipino,Indonesian": 0.32666666666666666, - "English,Filipino,Malay": 0.28, - "English,Indonesian,Malay": 0.32, - "Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian": 0.2, - "Chinese,Vietnamese,Malay": 0.16, - "Chinese,Filipino,Indonesian": 0.16666666666666666, - "Chinese,Filipino,Malay": 0.11333333333333333, - "Chinese,Indonesian,Malay": 0.16666666666666666, - "Vietnamese,Filipino,Indonesian": 0.26, - "Vietnamese,Filipino,Malay": 0.2, - "Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Filipino,Indonesian,Malay": 0.32 + "Filipino,Vietnamese,Chinese": 0.41333333333333333, + "Filipino,Vietnamese,Spanish": 0.2866666666666667, + "Filipino,Vietnamese,Malay": 0.42, + "Filipino,Vietnamese,Indonesian": 0.38666666666666666, + "Filipino,Vietnamese,English": 0.26, + "Filipino,Chinese,Spanish": 0.28, + "Filipino,Chinese,Malay": 0.44, + "Filipino,Chinese,Indonesian": 0.4066666666666667, + "Filipino,Chinese,English": 0.28, + "Filipino,Spanish,Malay": 0.36666666666666664, + "Filipino,Spanish,Indonesian": 0.3466666666666667, + "Filipino,Spanish,English": 0.36, + "Filipino,Malay,Indonesian": 0.5, + "Filipino,Malay,English": 0.36666666666666664, + "Filipino,Indonesian,English": 0.35333333333333333, + "Vietnamese,Chinese,Spanish": 0.31333333333333335, + "Vietnamese,Chinese,Malay": 0.44666666666666666, + "Vietnamese,Chinese,Indonesian": 0.4666666666666667, + "Vietnamese,Chinese,English": 0.26666666666666666, + "Vietnamese,Spanish,Malay": 0.3, + "Vietnamese,Spanish,Indonesian": 0.32, + "Vietnamese,Spanish,English": 0.26, + "Vietnamese,Malay,Indonesian": 0.43333333333333335, + "Vietnamese,Malay,English": 0.2733333333333333, + "Vietnamese,Indonesian,English": 0.29333333333333333, + "Chinese,Spanish,Malay": 0.29333333333333333, + "Chinese,Spanish,Indonesian": 0.32, + "Chinese,Spanish,English": 0.2733333333333333, + "Chinese,Malay,Indonesian": 0.4666666666666667, + "Chinese,Malay,English": 0.2866666666666667, + "Chinese,Indonesian,English": 0.31333333333333335, + "Spanish,Malay,Indonesian": 0.4, + "Spanish,Malay,English": 0.36666666666666664, + "Spanish,Indonesian,English": 0.37333333333333335, + "Malay,Indonesian,English": 0.4 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.12, - "Spanish,English,Chinese,Filipino": 0.11333333333333333, - "Spanish,English,Chinese,Indonesian": 0.14, - "Spanish,English,Chinese,Malay": 0.09333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,English,Vietnamese,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Malay": 0.16666666666666666, - "Spanish,English,Filipino,Indonesian": 0.2733333333333333, - "Spanish,English,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino": 0.12, - "Spanish,Chinese,Vietnamese,Indonesian": 0.14, - "Spanish,Chinese,Vietnamese,Malay": 0.1, - "Spanish,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Filipino,Malay": 0.1, - "Spanish,Chinese,Indonesian,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,Vietnamese,Filipino,Malay": 0.16, - "Spanish,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino": 0.09333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.10666666666666667, - "English,Chinese,Vietnamese,Malay": 0.07333333333333333, - "English,Chinese,Filipino,Indonesian": 0.10666666666666667, - "English,Chinese,Filipino,Malay": 0.08, - "English,Chinese,Indonesian,Malay": 0.11333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.18, - "English,Vietnamese,Filipino,Malay": 0.14, - "English,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "English,Filipino,Indonesian,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Chinese,Vietnamese,Filipino,Malay": 0.08, - "Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.1, - "Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666 + "Filipino,Vietnamese,Chinese,Spanish": 0.20666666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.34, + "Filipino,Vietnamese,Chinese,Indonesian": 0.32, + "Filipino,Vietnamese,Chinese,English": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Malay": 0.24666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,English": 0.2, + "Filipino,Vietnamese,Malay,Indonesian": 0.3466666666666667, + "Filipino,Vietnamese,Malay,English": 0.22666666666666666, + "Filipino,Vietnamese,Indonesian,English": 0.23333333333333334, + "Filipino,Chinese,Spanish,Malay": 0.22666666666666666, + "Filipino,Chinese,Spanish,Indonesian": 0.23333333333333334, + "Filipino,Chinese,Spanish,English": 0.2, + "Filipino,Chinese,Malay,Indonesian": 0.36666666666666664, + "Filipino,Chinese,Malay,English": 0.24, + "Filipino,Chinese,Indonesian,English": 0.24666666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.3, + "Filipino,Spanish,Malay,English": 0.2733333333333333, + "Filipino,Spanish,Indonesian,English": 0.2733333333333333, + "Filipino,Malay,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish,Malay": 0.22666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian": 0.25333333333333335, + "Vietnamese,Chinese,Spanish,English": 0.2, + "Vietnamese,Chinese,Malay,Indonesian": 0.36, + "Vietnamese,Chinese,Malay,English": 0.21333333333333335, + "Vietnamese,Chinese,Indonesian,English": 0.24666666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Vietnamese,Spanish,Malay,English": 0.20666666666666667, + "Vietnamese,Spanish,Indonesian,English": 0.22, + "Vietnamese,Malay,Indonesian,English": 0.25333333333333335, + "Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, + "Chinese,Spanish,Malay,English": 0.21333333333333335, + "Chinese,Spanish,Indonesian,English": 0.23333333333333334, + "Chinese,Malay,Indonesian,English": 0.26, + "Spanish,Malay,Indonesian,English": 0.30666666666666664 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.08, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.1, - "Spanish,English,Chinese,Vietnamese,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.14, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.05333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.08, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.18, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.18, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.18, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.19333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.21333333333333335, + "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.17333333333333334, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.21333333333333335, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, + "Filipino,Chinese,Spanish,Malay,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Indonesian,English": 0.18, + "Filipino,Chinese,Malay,Indonesian,English": 0.22, + "Filipino,Spanish,Malay,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, + "Vietnamese,Chinese,Spanish,Malay,English": 0.16, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.20666666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, + "Chinese,Spanish,Malay,Indonesian,English": 0.2 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.05333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.16, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 } }, - "AC3_2": 0.4053605867180189, - "AC3_3": 0.28944946873897065, - "AC3_4": 0.21369900972733058, - "AC3_5": 0.16234347154918882, - "AC3_6": 0.12477559398893812, - "AC3_7": 0.09382078850928062 + "AC3_2": 0.42917111343465364, + "AC3_3": 0.35568304960895775, + "AC3_4": 0.2987614326533628, + "AC3_5": 0.2532035870560433, + "AC3_6": 0.21489467158135575, + "AC3_7": 0.17988071566821576 }, "prompt_5": { - "overall_acc": 0.3942857142857143, + "overall_acc": 0.36380952380952386, "language_acc": { - "Spanish": 0.4866666666666667, - "English": 0.6066666666666667, - "Chinese": 0.2733333333333333, - "Vietnamese": 0.2733333333333333, - "Filipino": 0.38, - "Indonesian": 0.4066666666666667, - "Malay": 0.3333333333333333 + "Filipino": 0.30666666666666664, + "Vietnamese": 0.26, + "Chinese": 0.26, + "Spanish": 0.4533333333333333, + "Malay": 0.35333333333333333, + "Indonesian": 0.37333333333333335, + "English": 0.54 }, - "consistency_score_2": 0.45174603174603184, - "consistency_score_3": 0.2601904761904762, - "consistency_score_4": 0.1685714285714286, - "consistency_score_5": 0.11777777777777777, - "consistency_score_6": 0.08666666666666667, + "consistency_score_2": 0.48571428571428565, + "consistency_score_3": 0.29657142857142854, + "consistency_score_4": 0.19504761904761914, + "consistency_score_5": 0.13238095238095238, + "consistency_score_6": 0.09238095238095237, "consistency_score_7": 0.06666666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6533333333333333, - "Spanish,Chinese": 0.3933333333333333, - "Spanish,Vietnamese": 0.36666666666666664, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.5933333333333334, - "Spanish,Malay": 0.5, - "English,Chinese": 0.37333333333333335, - "English,Vietnamese": 0.36, - "English,Filipino": 0.48, - "English,Indonesian": 0.48, - "English,Malay": 0.44, - "Chinese,Vietnamese": 0.34, - "Chinese,Filipino": 0.37333333333333335, - "Chinese,Indonesian": 0.44666666666666666, - "Chinese,Malay": 0.38666666666666666, - "Vietnamese,Filipino": 0.42, - "Vietnamese,Indonesian": 0.4, - "Vietnamese,Malay": 0.4066666666666667, - "Filipino,Indonesian": 0.54, - "Filipino,Malay": 0.5, - "Indonesian,Malay": 0.5933333333333334 + "Filipino,Vietnamese": 0.5266666666666666, + "Filipino,Chinese": 0.4, + "Filipino,Spanish": 0.5066666666666667, + "Filipino,Malay": 0.62, + "Filipino,Indonesian": 0.56, + "Filipino,English": 0.5, + "Vietnamese,Chinese": 0.47333333333333333, + "Vietnamese,Spanish": 0.43333333333333335, + "Vietnamese,Malay": 0.5133333333333333, + "Vietnamese,Indonesian": 0.52, + "Vietnamese,English": 0.3466666666666667, + "Chinese,Spanish": 0.34, + "Chinese,Malay": 0.42, + "Chinese,Indonesian": 0.4866666666666667, + "Chinese,English": 0.31333333333333335, + "Spanish,Malay": 0.49333333333333335, + "Spanish,Indonesian": 0.5133333333333333, + "Spanish,English": 0.6, + "Malay,Indonesian": 0.6733333333333333, + "Malay,English": 0.47333333333333333, + "Indonesian,English": 0.4866666666666667 }, "3_combine": { - "Spanish,English,Chinese": 0.25333333333333335, - "Spanish,English,Vietnamese": 0.25333333333333335, - "Spanish,English,Filipino": 0.34, - "Spanish,English,Indonesian": 0.38666666666666666, - "Spanish,English,Malay": 0.3466666666666667, - "Spanish,Chinese,Vietnamese": 0.18666666666666668, - "Spanish,Chinese,Filipino": 0.21333333333333335, - "Spanish,Chinese,Indonesian": 0.28, - "Spanish,Chinese,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Filipino": 0.22, - "Spanish,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,Vietnamese,Malay": 0.23333333333333334, - "Spanish,Filipino,Indonesian": 0.34, - "Spanish,Filipino,Malay": 0.31333333333333335, - "Spanish,Indonesian,Malay": 0.4066666666666667, - "English,Chinese,Vietnamese": 0.17333333333333334, - "English,Chinese,Filipino": 0.21333333333333335, - "English,Chinese,Indonesian": 0.22, - "English,Chinese,Malay": 0.2, - "English,Vietnamese,Filipino": 0.22666666666666666, - "English,Vietnamese,Indonesian": 0.22666666666666666, - "English,Vietnamese,Malay": 0.20666666666666667, - "English,Filipino,Indonesian": 0.30666666666666664, - "English,Filipino,Malay": 0.30666666666666664, - "English,Indonesian,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Filipino": 0.18, - "Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Chinese,Vietnamese,Malay": 0.18666666666666668, - "Chinese,Filipino,Indonesian": 0.24666666666666667, - "Chinese,Filipino,Malay": 0.23333333333333334, - "Chinese,Indonesian,Malay": 0.2866666666666667, - "Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Vietnamese,Filipino,Malay": 0.24, - "Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Filipino,Indonesian,Malay": 0.37333333333333335 + "Filipino,Vietnamese,Chinese": 0.2733333333333333, + "Filipino,Vietnamese,Spanish": 0.29333333333333333, + "Filipino,Vietnamese,Malay": 0.38, + "Filipino,Vietnamese,Indonesian": 0.36, + "Filipino,Vietnamese,English": 0.25333333333333335, + "Filipino,Chinese,Spanish": 0.19333333333333333, + "Filipino,Chinese,Malay": 0.28, + "Filipino,Chinese,Indonesian": 0.29333333333333333, + "Filipino,Chinese,English": 0.21333333333333335, + "Filipino,Spanish,Malay": 0.37333333333333335, + "Filipino,Spanish,Indonesian": 0.32666666666666666, + "Filipino,Spanish,English": 0.36666666666666664, + "Filipino,Malay,Indonesian": 0.47333333333333333, + "Filipino,Malay,English": 0.3466666666666667, + "Filipino,Indonesian,English": 0.3333333333333333, + "Vietnamese,Chinese,Spanish": 0.22, + "Vietnamese,Chinese,Malay": 0.26, + "Vietnamese,Chinese,Indonesian": 0.30666666666666664, + "Vietnamese,Chinese,English": 0.18, + "Vietnamese,Spanish,Malay": 0.3, + "Vietnamese,Spanish,Indonesian": 0.30666666666666664, + "Vietnamese,Spanish,English": 0.26, + "Vietnamese,Malay,Indonesian": 0.38666666666666666, + "Vietnamese,Malay,English": 0.24, + "Vietnamese,Indonesian,English": 0.25333333333333335, + "Chinese,Spanish,Malay": 0.20666666666666667, + "Chinese,Spanish,Indonesian": 0.23333333333333334, + "Chinese,Spanish,English": 0.22, + "Chinese,Malay,Indonesian": 0.3333333333333333, + "Chinese,Malay,English": 0.20666666666666667, + "Chinese,Indonesian,English": 0.25333333333333335, + "Spanish,Malay,Indonesian": 0.38666666666666666, + "Spanish,Malay,English": 0.36, + "Spanish,Indonesian,English": 0.3466666666666667, + "Malay,Indonesian,English": 0.36 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.13333333333333333, - "Spanish,English,Chinese,Filipino": 0.16666666666666666, - "Spanish,English,Chinese,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,English,Vietnamese,Indonesian": 0.18, - "Spanish,English,Vietnamese,Malay": 0.16666666666666666, - "Spanish,English,Filipino,Indonesian": 0.26, - "Spanish,English,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Indonesian,Malay": 0.2733333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Malay": 0.12666666666666668, - "Spanish,Chinese,Filipino,Indonesian": 0.16, - "Spanish,Chinese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Indonesian,Malay": 0.2, - "Spanish,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.2, - "Spanish,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "English,Chinese,Vietnamese,Indonesian": 0.1, - "English,Chinese,Vietnamese,Malay": 0.12, - "English,Chinese,Filipino,Indonesian": 0.14666666666666667, - "English,Chinese,Filipino,Malay": 0.16, - "English,Chinese,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "English,Vietnamese,Filipino,Malay": 0.16, - "English,Vietnamese,Indonesian,Malay": 0.16, - "English,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.14, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Filipino,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "English,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.08, - "English,Chinese,Filipino,Indonesian,Malay": 0.12, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 + "Filipino,Vietnamese,Chinese,Spanish": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay": 0.18666666666666668, + "Filipino,Vietnamese,Chinese,Indonesian": 0.2, + "Filipino,Vietnamese,Chinese,English": 0.14, + "Filipino,Vietnamese,Spanish,Malay": 0.24, + "Filipino,Vietnamese,Spanish,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,English": 0.2, + "Filipino,Vietnamese,Malay,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Malay,English": 0.19333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.20666666666666667, + "Filipino,Chinese,Spanish,Malay": 0.14, + "Filipino,Chinese,Spanish,Indonesian": 0.14666666666666667, + "Filipino,Chinese,Spanish,English": 0.14666666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.23333333333333334, + "Filipino,Chinese,Malay,English": 0.16, + "Filipino,Chinese,Indonesian,English": 0.18, + "Filipino,Spanish,Malay,Indonesian": 0.3, + "Filipino,Spanish,Malay,English": 0.2733333333333333, + "Filipino,Spanish,Indonesian,English": 0.24, + "Filipino,Malay,Indonesian,English": 0.28, + "Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,English": 0.12666666666666668, + "Vietnamese,Chinese,Malay,Indonesian": 0.20666666666666667, + "Vietnamese,Chinese,Malay,English": 0.12, + "Vietnamese,Chinese,Indonesian,English": 0.15333333333333332, + "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, + "Vietnamese,Spanish,Malay,English": 0.20666666666666667, + "Vietnamese,Spanish,Indonesian,English": 0.2, + "Vietnamese,Malay,Indonesian,English": 0.19333333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.16666666666666666, + "Chinese,Spanish,Malay,English": 0.15333333333333332, + "Chinese,Spanish,Indonesian,English": 0.17333333333333334, + "Chinese,Malay,Indonesian,English": 0.17333333333333334, + "Spanish,Malay,Indonesian,English": 0.2866666666666667 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Malay,English": 0.1, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.12, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12, + "Filipino,Chinese,Spanish,Malay,English": 0.11333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.12, + "Filipino,Chinese,Malay,Indonesian,English": 0.14, + "Filipino,Spanish,Malay,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.09333333333333334, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.1, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.08666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 } }, - "AC3_2": 0.42106459389287343, - "AC3_3": 0.3135007483306122, - "AC3_4": 0.2361711384642045, - "AC3_5": 0.18137631738553406, - "AC3_6": 0.14209900987144478, - "AC3_7": 0.11404958675211735 + "AC3_2": 0.4160153747107989, + "AC3_3": 0.326767481099309, + "AC3_4": 0.2539474794235115, + "AC3_5": 0.19412485143697603, + "AC3_6": 0.14734665470475203, + "AC3_7": 0.11268436575553424 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.37743506493506496, + "overall_acc": 0.34983766233766234, "language_acc": { - "English": 0.6022727272727273, - "Filipino": 0.2897727272727273, - "Spanish": 0.45454545454545453, - "Malay": 0.3522727272727273, + "Indonesian": 0.35795454545454547, + "English": 0.5284090909090909, + "Filipino": 0.30113636363636365, + "Spanish": 0.42045454545454547, "Chinese": 0.24431818181818182, - "Indonesian": 0.4034090909090909, - "Vietnamese": 0.29545454545454547 + "Malay": 0.3352272727272727, + "Vietnamese": 0.26136363636363635 }, - "consistency_score_2": 0.40205627705627706, - "consistency_score_3": 0.20129870129870128, - "consistency_score_4": 0.11347402597402596, - "consistency_score_5": 0.06953463203463205, - "consistency_score_6": 0.04626623376623377, - "consistency_score_7": 0.03409090909090909, + "consistency_score_2": 0.417478354978355, + "consistency_score_3": 0.21948051948051953, + "consistency_score_4": 0.13441558441558446, + "consistency_score_5": 0.09117965367965367, + "consistency_score_6": 0.06655844155844155, + "consistency_score_7": 0.05113636363636364, "detailed_consistency_score": { "2_combine": { + "Indonesian,English": 0.3806818181818182, + "Indonesian,Filipino": 0.4659090909090909, + "Indonesian,Spanish": 0.4602272727272727, + "Indonesian,Chinese": 0.39204545454545453, + "Indonesian,Malay": 0.5568181818181818, + "Indonesian,Vietnamese": 0.42613636363636365, "English,Filipino": 0.3409090909090909, - "English,Spanish": 0.6647727272727273, - "English,Malay": 0.42045454545454547, - "English,Chinese": 0.2215909090909091, - "English,Indonesian": 0.4602272727272727, + "English,Spanish": 0.6079545454545454, + "English,Chinese": 0.2840909090909091, + "English,Malay": 0.3352272727272727, "English,Vietnamese": 0.29545454545454547, - "Filipino,Spanish": 0.4034090909090909, - "Filipino,Malay": 0.48863636363636365, - "Filipino,Chinese": 0.2784090909090909, - "Filipino,Indonesian": 0.48295454545454547, - "Filipino,Vietnamese": 0.3693181818181818, - "Spanish,Malay": 0.4659090909090909, - "Spanish,Chinese": 0.25, - "Spanish,Indonesian": 0.5056818181818182, - "Spanish,Vietnamese": 0.3465909090909091, - "Malay,Chinese": 0.3465909090909091, - "Malay,Indonesian": 0.5397727272727273, - "Malay,Vietnamese": 0.3977272727272727, - "Chinese,Indonesian": 0.3409090909090909, - "Chinese,Vietnamese": 0.4943181818181818, - "Indonesian,Vietnamese": 0.32954545454545453 + "Filipino,Spanish": 0.4090909090909091, + "Filipino,Chinese": 0.39204545454545453, + "Filipino,Malay": 0.4318181818181818, + "Filipino,Vietnamese": 0.4431818181818182, + "Spanish,Chinese": 0.29545454545454547, + "Spanish,Malay": 0.4090909090909091, + "Spanish,Vietnamese": 0.35795454545454547, + "Chinese,Malay": 0.4943181818181818, + "Chinese,Vietnamese": 0.5170454545454546, + "Malay,Vietnamese": 0.4715909090909091 }, "3_combine": { - "English,Filipino,Spanish": 0.2556818181818182, - "English,Filipino,Malay": 0.2159090909090909, - "English,Filipino,Chinese": 0.09659090909090909, - "English,Filipino,Indonesian": 0.23295454545454544, - "English,Filipino,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay": 0.32954545454545453, - "English,Spanish,Chinese": 0.1590909090909091, - "English,Spanish,Indonesian": 0.36363636363636365, - "English,Spanish,Vietnamese": 0.23863636363636365, - "English,Malay,Chinese": 0.10795454545454546, - "English,Malay,Indonesian": 0.2840909090909091, - "English,Malay,Vietnamese": 0.1534090909090909, - "English,Chinese,Indonesian": 0.13636363636363635, - "English,Chinese,Vietnamese": 0.10795454545454546, - "English,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay": 0.26136363636363635, - "Filipino,Spanish,Chinese": 0.11931818181818182, - "Filipino,Spanish,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Chinese": 0.17045454545454544, - "Filipino,Malay,Indonesian": 0.3181818181818182, - "Filipino,Malay,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese": 0.14772727272727273, - "Spanish,Malay,Indonesian": 0.3181818181818182, - "Spanish,Malay,Vietnamese": 0.19886363636363635, - "Spanish,Chinese,Indonesian": 0.1534090909090909, - "Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Spanish,Indonesian,Vietnamese": 0.1875, - "Malay,Chinese,Indonesian": 0.21022727272727273, - "Malay,Chinese,Vietnamese": 0.2215909090909091, - "Malay,Indonesian,Vietnamese": 0.20454545454545456, - "Chinese,Indonesian,Vietnamese": 0.1875 + "Indonesian,English,Filipino": 0.20454545454545456, + "Indonesian,English,Spanish": 0.2840909090909091, + "Indonesian,English,Chinese": 0.17045454545454544, + "Indonesian,English,Malay": 0.22727272727272727, + "Indonesian,English,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Spanish": 0.26136363636363635, + "Indonesian,Filipino,Chinese": 0.23863636363636365, + "Indonesian,Filipino,Malay": 0.2840909090909091, + "Indonesian,Filipino,Vietnamese": 0.23863636363636365, + "Indonesian,Spanish,Chinese": 0.1875, + "Indonesian,Spanish,Malay": 0.26704545454545453, + "Indonesian,Spanish,Vietnamese": 0.21022727272727273, + "Indonesian,Chinese,Malay": 0.30113636363636365, + "Indonesian,Chinese,Vietnamese": 0.25, + "Indonesian,Malay,Vietnamese": 0.29545454545454547, + "English,Filipino,Spanish": 0.24431818181818182, + "English,Filipino,Chinese": 0.14772727272727273, + "English,Filipino,Malay": 0.17613636363636365, + "English,Filipino,Vietnamese": 0.1590909090909091, + "English,Spanish,Chinese": 0.17613636363636365, + "English,Spanish,Malay": 0.24431818181818182, + "English,Spanish,Vietnamese": 0.2159090909090909, + "English,Chinese,Malay": 0.1590909090909091, + "English,Chinese,Vietnamese": 0.16477272727272727, + "English,Malay,Vietnamese": 0.1590909090909091, + "Filipino,Spanish,Chinese": 0.17045454545454544, + "Filipino,Spanish,Malay": 0.2215909090909091, + "Filipino,Spanish,Vietnamese": 0.19886363636363635, + "Filipino,Chinese,Malay": 0.23863636363636365, + "Filipino,Chinese,Vietnamese": 0.25, + "Filipino,Malay,Vietnamese": 0.25, + "Spanish,Chinese,Malay": 0.19318181818181818, + "Spanish,Chinese,Vietnamese": 0.19318181818181818, + "Spanish,Malay,Vietnamese": 0.2159090909090909, + "Chinese,Malay,Vietnamese": 0.3125 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.17045454545454544, - "English,Filipino,Spanish,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Indonesian": 0.19318181818181818, - "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese": 0.0625, - "English,Filipino,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese": 0.09659090909090909, - "English,Spanish,Malay,Indonesian": 0.24431818181818182, - "English,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Spanish,Chinese,Indonesian": 0.11363636363636363, - "English,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "English,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "English,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay,Chinese": 0.08522727272727272, - "Filipino,Spanish,Malay,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.09090909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Malay,Chinese,Indonesian,Vietnamese": 0.125 + "Indonesian,English,Filipino,Spanish": 0.1534090909090909, + "Indonesian,English,Filipino,Chinese": 0.11931818181818182, + "Indonesian,English,Filipino,Malay": 0.13068181818181818, + "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, + "Indonesian,English,Spanish,Chinese": 0.125, + "Indonesian,English,Spanish,Malay": 0.16477272727272727, + "Indonesian,English,Spanish,Vietnamese": 0.13636363636363635, + "Indonesian,English,Chinese,Malay": 0.13068181818181818, + "Indonesian,English,Chinese,Vietnamese": 0.10795454545454546, + "Indonesian,English,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Spanish,Chinese": 0.14204545454545456, + "Indonesian,Filipino,Spanish,Malay": 0.17613636363636365, + "Indonesian,Filipino,Spanish,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Chinese,Malay": 0.17613636363636365, + "Indonesian,Filipino,Chinese,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Malay": 0.14772727272727273, + "Indonesian,Spanish,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,Spanish,Malay,Vietnamese": 0.14772727272727273, + "Indonesian,Chinese,Malay,Vietnamese": 0.21022727272727273, + "English,Filipino,Spanish,Chinese": 0.11363636363636363, + "English,Filipino,Spanish,Malay": 0.1534090909090909, + "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "English,Filipino,Chinese,Malay": 0.10227272727272728, + "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, + "English,Filipino,Malay,Vietnamese": 0.09659090909090909, + "English,Spanish,Chinese,Malay": 0.11931818181818182, + "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, + "English,Spanish,Malay,Vietnamese": 0.13068181818181818, + "English,Chinese,Malay,Vietnamese": 0.09659090909090909, + "Filipino,Spanish,Chinese,Malay": 0.125, + "Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, + "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, + "Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, + "Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 + "Indonesian,English,Filipino,Spanish,Chinese": 0.09090909090909091, + "Indonesian,English,Filipino,Spanish,Malay": 0.10795454545454546, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.09090909090909091, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.0625, + "Indonesian,English,Spanish,Chinese,Malay": 0.10227272727272728, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, + "English,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, + "English,Filipino,Chinese,Malay,Vietnamese": 0.0625, + "English,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.07954545454545454, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.05113636363636364, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364 } }, - "AC3_2": 0.3893568250062849, - "AC3_3": 0.26256352338772, - "AC3_4": 0.17448882958721415, - "AC3_5": 0.11743439669081092, - "AC3_6": 0.08242834749510006, - "AC3_7": 0.06253362021073237 + "AC3_2": 0.38067666637282366, + "AC3_3": 0.26973511228824704, + "AC3_4": 0.19421091810644825, + "AC3_5": 0.14465680022215904, + "AC3_6": 0.11183894075945042, + "AC3_7": 0.08922984907601689 }, "prompt_2": { - "overall_acc": 0.3758116883116883, + "overall_acc": 0.3538961038961039, "language_acc": { - "English": 0.6022727272727273, - "Filipino": 0.29545454545454547, - "Spanish": 0.4602272727272727, - "Malay": 0.3352272727272727, + "Indonesian": 0.32386363636363635, + "English": 0.5340909090909091, + "Filipino": 0.3181818181818182, + "Spanish": 0.4375, "Chinese": 0.24431818181818182, - "Indonesian": 0.4034090909090909, - "Vietnamese": 0.2897727272727273 + "Malay": 0.3522727272727273, + "Vietnamese": 0.26704545454545453 }, - "consistency_score_2": 0.40665584415584427, - "consistency_score_3": 0.20844155844155846, - "consistency_score_4": 0.11866883116883116, - "consistency_score_5": 0.07196969696969696, - "consistency_score_6": 0.04707792207792207, - "consistency_score_7": 0.03409090909090909, + "consistency_score_2": 0.42153679653679665, + "consistency_score_3": 0.2202922077922078, + "consistency_score_4": 0.13555194805194803, + "consistency_score_5": 0.09361471861471862, + "consistency_score_6": 0.0706168831168831, + "consistency_score_7": 0.056818181818181816, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.3409090909090909, - "English,Spanish": 0.6590909090909091, - "English,Malay": 0.4147727272727273, - "English,Chinese": 0.2215909090909091, - "English,Indonesian": 0.4715909090909091, - "English,Vietnamese": 0.2784090909090909, - "Filipino,Spanish": 0.4318181818181818, - "Filipino,Malay": 0.5113636363636364, - "Filipino,Chinese": 0.3068181818181818, - "Filipino,Indonesian": 0.4772727272727273, - "Filipino,Vietnamese": 0.3693181818181818, - "Spanish,Malay": 0.48295454545454547, - "Spanish,Chinese": 0.2556818181818182, - "Spanish,Indonesian": 0.5454545454545454, - "Spanish,Vietnamese": 0.32386363636363635, - "Malay,Chinese": 0.35795454545454547, - "Malay,Indonesian": 0.5454545454545454, - "Malay,Vietnamese": 0.3806818181818182, - "Chinese,Indonesian": 0.3352272727272727, - "Chinese,Vietnamese": 0.4943181818181818, - "Indonesian,Vietnamese": 0.3352272727272727 + "Indonesian,English": 0.38636363636363635, + "Indonesian,Filipino": 0.48863636363636365, + "Indonesian,Spanish": 0.44886363636363635, + "Indonesian,Chinese": 0.3806818181818182, + "Indonesian,Malay": 0.6022727272727273, + "Indonesian,Vietnamese": 0.35795454545454547, + "English,Filipino": 0.3465909090909091, + "English,Spanish": 0.6306818181818182, + "English,Chinese": 0.29545454545454547, + "English,Malay": 0.3522727272727273, + "English,Vietnamese": 0.29545454545454547, + "Filipino,Spanish": 0.4602272727272727, + "Filipino,Chinese": 0.42045454545454547, + "Filipino,Malay": 0.4772727272727273, + "Filipino,Vietnamese": 0.42613636363636365, + "Spanish,Chinese": 0.29545454545454547, + "Spanish,Malay": 0.3977272727272727, + "Spanish,Vietnamese": 0.3522727272727273, + "Chinese,Malay": 0.4772727272727273, + "Chinese,Vietnamese": 0.5, + "Malay,Vietnamese": 0.4602272727272727 }, "3_combine": { - "English,Filipino,Spanish": 0.26136363636363635, - "English,Filipino,Malay": 0.2215909090909091, - "English,Filipino,Chinese": 0.09659090909090909, - "English,Filipino,Indonesian": 0.25, - "English,Filipino,Vietnamese": 0.125, - "English,Spanish,Malay": 0.3352272727272727, - "English,Spanish,Chinese": 0.1590909090909091, - "English,Spanish,Indonesian": 0.3806818181818182, + "Indonesian,English,Filipino": 0.22727272727272727, + "Indonesian,English,Spanish": 0.2897727272727273, + "Indonesian,English,Chinese": 0.18181818181818182, + "Indonesian,English,Malay": 0.2556818181818182, + "Indonesian,English,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Spanish": 0.2727272727272727, + "Indonesian,Filipino,Chinese": 0.23863636363636365, + "Indonesian,Filipino,Malay": 0.32954545454545453, + "Indonesian,Filipino,Vietnamese": 0.21022727272727273, + "Indonesian,Spanish,Chinese": 0.18181818181818182, + "Indonesian,Spanish,Malay": 0.2897727272727273, + "Indonesian,Spanish,Vietnamese": 0.16477272727272727, + "Indonesian,Chinese,Malay": 0.2897727272727273, + "Indonesian,Chinese,Vietnamese": 0.2159090909090909, + "Indonesian,Malay,Vietnamese": 0.26704545454545453, + "English,Filipino,Spanish": 0.2840909090909091, + "English,Filipino,Chinese": 0.1590909090909091, + "English,Filipino,Malay": 0.1875, + "English,Filipino,Vietnamese": 0.14204545454545456, + "English,Spanish,Chinese": 0.19886363636363635, + "English,Spanish,Malay": 0.2556818181818182, "English,Spanish,Vietnamese": 0.2159090909090909, - "English,Malay,Chinese": 0.11363636363636363, - "English,Malay,Indonesian": 0.2897727272727273, + "English,Chinese,Malay": 0.17045454545454544, + "English,Chinese,Vietnamese": 0.1534090909090909, "English,Malay,Vietnamese": 0.14772727272727273, - "English,Chinese,Indonesian": 0.14204545454545456, - "English,Chinese,Vietnamese": 0.10795454545454546, - "English,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.14204545454545456, - "Filipino,Spanish,Indonesian": 0.3068181818181818, - "Filipino,Spanish,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Chinese": 0.19318181818181818, - "Filipino,Malay,Indonesian": 0.3409090909090909, - "Filipino,Malay,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese": 0.16477272727272727, - "Spanish,Malay,Indonesian": 0.3465909090909091, - "Spanish,Malay,Vietnamese": 0.19318181818181818, - "Spanish,Chinese,Indonesian": 0.16477272727272727, - "Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Spanish,Indonesian,Vietnamese": 0.1875, - "Malay,Chinese,Indonesian": 0.21022727272727273, - "Malay,Chinese,Vietnamese": 0.2215909090909091, - "Malay,Indonesian,Vietnamese": 0.2159090909090909, - "Chinese,Indonesian,Vietnamese": 0.19318181818181818 + "Filipino,Spanish,Chinese": 0.19886363636363635, + "Filipino,Spanish,Malay": 0.25, + "Filipino,Spanish,Vietnamese": 0.21022727272727273, + "Filipino,Chinese,Malay": 0.26704545454545453, + "Filipino,Chinese,Vietnamese": 0.23863636363636365, + "Filipino,Malay,Vietnamese": 0.24431818181818182, + "Spanish,Chinese,Malay": 0.1875, + "Spanish,Chinese,Vietnamese": 0.17045454545454544, + "Spanish,Malay,Vietnamese": 0.1875, + "Chinese,Malay,Vietnamese": 0.2897727272727273 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.07954545454545454, - "English,Filipino,Spanish,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese": 0.0625, - "English,Filipino,Malay,Indonesian": 0.17045454545454544, - "English,Filipino,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese": 0.10227272727272728, - "English,Spanish,Malay,Indonesian": 0.26136363636363635, - "English,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian": 0.11931818181818182, - "English,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "English,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "English,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese": 0.10227272727272728, - "Filipino,Spanish,Malay,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.125, - "Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 + "Indonesian,English,Filipino,Spanish": 0.18181818181818182, + "Indonesian,English,Filipino,Chinese": 0.125, + "Indonesian,English,Filipino,Malay": 0.1534090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.09090909090909091, + "Indonesian,English,Spanish,Chinese": 0.13636363636363635, + "Indonesian,English,Spanish,Malay": 0.19318181818181818, + "Indonesian,English,Spanish,Vietnamese": 0.11931818181818182, + "Indonesian,English,Chinese,Malay": 0.14204545454545456, + "Indonesian,English,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,English,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,Filipino,Spanish,Chinese": 0.14204545454545456, + "Indonesian,Filipino,Spanish,Malay": 0.19318181818181818, + "Indonesian,Filipino,Spanish,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.1875, + "Indonesian,Filipino,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Malay": 0.14772727272727273, + "Indonesian,Spanish,Chinese,Vietnamese": 0.10795454545454546, + "Indonesian,Spanish,Malay,Vietnamese": 0.125, + "Indonesian,Chinese,Malay,Vietnamese": 0.1875, + "English,Filipino,Spanish,Chinese": 0.13636363636363635, + "English,Filipino,Spanish,Malay": 0.16477272727272727, + "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, + "English,Filipino,Chinese,Malay": 0.125, + "English,Filipino,Chinese,Vietnamese": 0.09090909090909091, + "English,Filipino,Malay,Vietnamese": 0.09090909090909091, + "English,Spanish,Chinese,Malay": 0.13636363636363635, + "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, + "English,Spanish,Malay,Vietnamese": 0.11363636363636363, + "English,Chinese,Malay,Vietnamese": 0.09659090909090909, + "Filipino,Spanish,Chinese,Malay": 0.1590909090909091, + "Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, + "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, + "Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, + "Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 + "Indonesian,English,Filipino,Spanish,Chinese": 0.10227272727272728, + "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.08522727272727272, + "Indonesian,English,Filipino,Chinese,Malay": 0.10795454545454546, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.06818181818181818, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Spanish,Chinese,Malay": 0.11363636363636363, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.125, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, + "English,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, + "English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, + "English,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.09659090909090909, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0625, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816 } }, - "AC3_2": 0.3906258419504011, - "AC3_3": 0.2681534913640548, - "AC3_4": 0.18037974005271648, - "AC3_5": 0.1208047239499509, - "AC3_6": 0.08367400351986026, - "AC3_7": 0.06251125110986225 + "AC3_2": 0.38476631528194355, + "AC3_3": 0.27155047379325725, + "AC3_4": 0.19602205419095983, + "AC3_5": 0.14806294065379674, + "AC3_6": 0.11773981274075816, + "AC3_7": 0.09791591805018045 }, "prompt_3": { - "overall_acc": 0.3741883116883117, + "overall_acc": 0.3547077922077922, "language_acc": { - "English": 0.6193181818181818, - "Filipino": 0.29545454545454547, - "Spanish": 0.4602272727272727, - "Malay": 0.3352272727272727, - "Chinese": 0.22727272727272727, - "Indonesian": 0.4034090909090909, - "Vietnamese": 0.2784090909090909 + "Indonesian": 0.3409090909090909, + "English": 0.5227272727272727, + "Filipino": 0.3125, + "Spanish": 0.4375, + "Chinese": 0.25, + "Malay": 0.3522727272727273, + "Vietnamese": 0.26704545454545453 }, - "consistency_score_2": 0.4120670995670995, - "consistency_score_3": 0.20941558441558442, - "consistency_score_4": 0.11672077922077924, - "consistency_score_5": 0.06899350649350651, - "consistency_score_6": 0.04301948051948051, - "consistency_score_7": 0.028409090909090908, + "consistency_score_2": 0.4315476190476191, + "consistency_score_3": 0.23344155844155842, + "consistency_score_4": 0.15162337662337666, + "consistency_score_5": 0.1130952380952381, + "consistency_score_6": 0.09253246753246754, + "consistency_score_7": 0.07954545454545454, "detailed_consistency_score": { "2_combine": { + "Indonesian,English": 0.39204545454545453, + "Indonesian,Filipino": 0.4943181818181818, + "Indonesian,Spanish": 0.4431818181818182, + "Indonesian,Chinese": 0.38636363636363635, + "Indonesian,Malay": 0.5965909090909091, + "Indonesian,Vietnamese": 0.38636363636363635, "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.6590909090909091, - "English,Malay": 0.42045454545454547, - "English,Chinese": 0.23295454545454544, - "English,Indonesian": 0.4715909090909091, - "English,Vietnamese": 0.2840909090909091, - "Filipino,Spanish": 0.4090909090909091, - "Filipino,Malay": 0.5340909090909091, - "Filipino,Chinese": 0.3068181818181818, - "Filipino,Indonesian": 0.4715909090909091, - "Filipino,Vietnamese": 0.3806818181818182, - "Spanish,Malay": 0.48863636363636365, - "Spanish,Chinese": 0.2727272727272727, - "Spanish,Indonesian": 0.5340909090909091, - "Spanish,Vietnamese": 0.3125, - "Malay,Chinese": 0.39204545454545453, - "Malay,Indonesian": 0.5397727272727273, - "Malay,Vietnamese": 0.39204545454545453, - "Chinese,Indonesian": 0.3465909090909091, - "Chinese,Vietnamese": 0.5340909090909091, - "Indonesian,Vietnamese": 0.3352272727272727 + "English,Spanish": 0.6420454545454546, + "English,Chinese": 0.32954545454545453, + "English,Malay": 0.375, + "English,Vietnamese": 0.3125, + "Filipino,Spanish": 0.4375, + "Filipino,Chinese": 0.4034090909090909, + "Filipino,Malay": 0.4772727272727273, + "Filipino,Vietnamese": 0.4715909090909091, + "Spanish,Chinese": 0.3125, + "Spanish,Malay": 0.4375, + "Spanish,Vietnamese": 0.36363636363636365, + "Chinese,Malay": 0.48295454545454547, + "Chinese,Vietnamese": 0.5170454545454546, + "Malay,Vietnamese": 0.4659090909090909 }, "3_combine": { - "English,Filipino,Spanish": 0.25, - "English,Filipino,Malay": 0.23295454545454544, - "English,Filipino,Chinese": 0.09090909090909091, - "English,Filipino,Indonesian": 0.23863636363636365, - "English,Filipino,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay": 0.3352272727272727, - "English,Spanish,Chinese": 0.16477272727272727, - "English,Spanish,Indonesian": 0.3806818181818182, - "English,Spanish,Vietnamese": 0.21022727272727273, - "English,Malay,Chinese": 0.125, - "English,Malay,Indonesian": 0.2897727272727273, - "English,Malay,Vietnamese": 0.13636363636363635, - "English,Chinese,Indonesian": 0.14204545454545456, - "English,Chinese,Vietnamese": 0.11363636363636363, - "English,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.13068181818181818, - "Filipino,Spanish,Indonesian": 0.2897727272727273, - "Filipino,Spanish,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Chinese": 0.20454545454545456, - "Filipino,Malay,Indonesian": 0.3522727272727273, - "Filipino,Malay,Vietnamese": 0.23295454545454544, - "Filipino,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Chinese,Vietnamese": 0.21022727272727273, - "Filipino,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese": 0.17613636363636365, - "Spanish,Malay,Indonesian": 0.3465909090909091, - "Spanish,Malay,Vietnamese": 0.18181818181818182, - "Spanish,Chinese,Indonesian": 0.17045454545454544, - "Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "Malay,Chinese,Indonesian": 0.22727272727272727, - "Malay,Chinese,Vietnamese": 0.24431818181818182, - "Malay,Indonesian,Vietnamese": 0.2159090909090909, - "Chinese,Indonesian,Vietnamese": 0.21022727272727273 + "Indonesian,English,Filipino": 0.22727272727272727, + "Indonesian,English,Spanish": 0.29545454545454547, + "Indonesian,English,Chinese": 0.19318181818181818, + "Indonesian,English,Malay": 0.26136363636363635, + "Indonesian,English,Vietnamese": 0.16477272727272727, + "Indonesian,Filipino,Spanish": 0.2727272727272727, + "Indonesian,Filipino,Chinese": 0.23863636363636365, + "Indonesian,Filipino,Malay": 0.32954545454545453, + "Indonesian,Filipino,Vietnamese": 0.25, + "Indonesian,Spanish,Chinese": 0.19886363636363635, + "Indonesian,Spanish,Malay": 0.29545454545454547, + "Indonesian,Spanish,Vietnamese": 0.19318181818181818, + "Indonesian,Chinese,Malay": 0.3068181818181818, + "Indonesian,Chinese,Vietnamese": 0.2215909090909091, + "Indonesian,Malay,Vietnamese": 0.26704545454545453, + "English,Filipino,Spanish": 0.26704545454545453, + "English,Filipino,Chinese": 0.16477272727272727, + "English,Filipino,Malay": 0.19886363636363635, + "English,Filipino,Vietnamese": 0.1534090909090909, + "English,Spanish,Chinese": 0.2215909090909091, + "English,Spanish,Malay": 0.2784090909090909, + "English,Spanish,Vietnamese": 0.2215909090909091, + "English,Chinese,Malay": 0.19318181818181818, + "English,Chinese,Vietnamese": 0.17613636363636365, + "English,Malay,Vietnamese": 0.16477272727272727, + "Filipino,Spanish,Chinese": 0.19318181818181818, + "Filipino,Spanish,Malay": 0.26136363636363635, + "Filipino,Spanish,Vietnamese": 0.22727272727272727, + "Filipino,Chinese,Malay": 0.26704545454545453, + "Filipino,Chinese,Vietnamese": 0.26704545454545453, + "Filipino,Malay,Vietnamese": 0.2727272727272727, + "Spanish,Chinese,Malay": 0.20454545454545456, + "Spanish,Chinese,Vietnamese": 0.19318181818181818, + "Spanish,Malay,Vietnamese": 0.2159090909090909, + "Chinese,Malay,Vietnamese": 0.3125 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.18181818181818182, - "English,Filipino,Spanish,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Indonesian": 0.19886363636363635, - "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese": 0.0625, - "English,Filipino,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Malay,Vietnamese": 0.07386363636363637, - "English,Filipino,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese": 0.10795454545454546, - "English,Spanish,Malay,Indonesian": 0.26136363636363635, - "English,Spanish,Malay,Vietnamese": 0.11931818181818182, - "English,Spanish,Chinese,Indonesian": 0.11931818181818182, - "English,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "English,Spanish,Indonesian,Vietnamese": 0.125, - "English,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Chinese": 0.09659090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.125, - "Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 + "Indonesian,English,Filipino,Spanish": 0.18181818181818182, + "Indonesian,English,Filipino,Chinese": 0.13068181818181818, + "Indonesian,English,Filipino,Malay": 0.1590909090909091, + "Indonesian,English,Filipino,Vietnamese": 0.11363636363636363, + "Indonesian,English,Spanish,Chinese": 0.14772727272727273, + "Indonesian,English,Spanish,Malay": 0.19886363636363635, + "Indonesian,English,Spanish,Vietnamese": 0.14204545454545456, + "Indonesian,English,Chinese,Malay": 0.16477272727272727, + "Indonesian,English,Chinese,Vietnamese": 0.11931818181818182, + "Indonesian,English,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Spanish,Chinese": 0.1590909090909091, + "Indonesian,Filipino,Spanish,Malay": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "Indonesian,Filipino,Chinese,Malay": 0.19886363636363635, + "Indonesian,Filipino,Chinese,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,Spanish,Chinese,Malay": 0.17045454545454544, + "Indonesian,Spanish,Chinese,Vietnamese": 0.125, + "Indonesian,Spanish,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,Chinese,Malay,Vietnamese": 0.19318181818181818, + "English,Filipino,Spanish,Chinese": 0.14204545454545456, + "English,Filipino,Spanish,Malay": 0.17613636363636365, + "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "English,Filipino,Chinese,Malay": 0.13636363636363635, + "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, + "English,Filipino,Malay,Vietnamese": 0.11363636363636363, + "English,Spanish,Chinese,Malay": 0.14772727272727273, + "English,Spanish,Chinese,Vietnamese": 0.13636363636363635, + "English,Spanish,Malay,Vietnamese": 0.13068181818181818, + "English,Chinese,Malay,Vietnamese": 0.11931818181818182, + "Filipino,Spanish,Chinese,Malay": 0.1590909090909091, + "Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, + "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, + "Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, + "Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.0625, - "English,Filipino,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 + "Indonesian,English,Filipino,Spanish,Chinese": 0.11931818181818182, + "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, + "Indonesian,English,Filipino,Chinese,Malay": 0.125, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,English,Spanish,Chinese,Malay": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.125, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363, + "English,Filipino,Spanish,Chinese,Malay": 0.125, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, + "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, + "English,Filipino,Chinese,Malay,Vietnamese": 0.09659090909090909, + "English,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454 } }, - "AC3_2": 0.39221527778996373, - "AC3_3": 0.2685412639750828, - "AC3_4": 0.1779374312493991, - "AC3_5": 0.11650551826494418, - "AC3_6": 0.07716723935691445, - "AC3_7": 0.052808834297733406 + "AC3_2": 0.38937297212627153, + "AC3_3": 0.2815731742247202, + "AC3_4": 0.2124380108090074, + "AC3_5": 0.17150706432754498, + "AC3_6": 0.1467756381221304, + "AC3_7": 0.12994902290975244 }, "prompt_4": { - "overall_acc": 0.37905844155844154, + "overall_acc": 0.35227272727272724, "language_acc": { - "English": 0.6022727272727273, - "Filipino": 0.30113636363636365, - "Spanish": 0.4602272727272727, - "Malay": 0.3409090909090909, - "Chinese": 0.23295454545454544, - "Indonesian": 0.4090909090909091, - "Vietnamese": 0.3068181818181818 + "Indonesian": 0.3352272727272727, + "English": 0.5340909090909091, + "Filipino": 0.3068181818181818, + "Spanish": 0.42613636363636365, + "Chinese": 0.24431818181818182, + "Malay": 0.3465909090909091, + "Vietnamese": 0.2727272727272727 }, - "consistency_score_2": 0.41017316017316013, - "consistency_score_3": 0.2089285714285715, - "consistency_score_4": 0.11704545454545452, - "consistency_score_5": 0.0698051948051948, - "consistency_score_6": 0.04383116883116883, - "consistency_score_7": 0.028409090909090908, + "consistency_score_2": 0.43181818181818166, + "consistency_score_3": 0.23522727272727276, + "consistency_score_4": 0.1525974025974026, + "consistency_score_5": 0.11282467532467534, + "consistency_score_6": 0.09172077922077922, + "consistency_score_7": 0.07954545454545454, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.3465909090909091, - "English,Spanish": 0.6477272727272727, - "English,Malay": 0.42613636363636365, - "English,Chinese": 0.2159090909090909, - "English,Indonesian": 0.4659090909090909, - "English,Vietnamese": 0.2897727272727273, - "Filipino,Spanish": 0.42045454545454547, - "Filipino,Malay": 0.5284090909090909, - "Filipino,Chinese": 0.29545454545454547, - "Filipino,Indonesian": 0.4715909090909091, - "Filipino,Vietnamese": 0.36363636363636365, - "Spanish,Malay": 0.4772727272727273, - "Spanish,Chinese": 0.2556818181818182, - "Spanish,Indonesian": 0.5454545454545454, - "Spanish,Vietnamese": 0.3409090909090909, - "Malay,Chinese": 0.375, - "Malay,Indonesian": 0.5454545454545454, - "Malay,Vietnamese": 0.3977272727272727, - "Chinese,Indonesian": 0.3352272727272727, - "Chinese,Vietnamese": 0.5113636363636364, - "Indonesian,Vietnamese": 0.35795454545454547 + "Indonesian,English": 0.38636363636363635, + "Indonesian,Filipino": 0.5, + "Indonesian,Spanish": 0.45454545454545453, + "Indonesian,Chinese": 0.3806818181818182, + "Indonesian,Malay": 0.5909090909090909, + "Indonesian,Vietnamese": 0.39204545454545453, + "English,Filipino": 0.3409090909090909, + "English,Spanish": 0.6363636363636364, + "English,Chinese": 0.30113636363636365, + "English,Malay": 0.375, + "English,Vietnamese": 0.3068181818181818, + "Filipino,Spanish": 0.4602272727272727, + "Filipino,Chinese": 0.42045454545454547, + "Filipino,Malay": 0.4659090909090909, + "Filipino,Vietnamese": 0.4602272727272727, + "Spanish,Chinese": 0.3068181818181818, + "Spanish,Malay": 0.4147727272727273, + "Spanish,Vietnamese": 0.36363636363636365, + "Chinese,Malay": 0.5, + "Chinese,Vietnamese": 0.5340909090909091, + "Malay,Vietnamese": 0.4772727272727273 }, "3_combine": { - "English,Filipino,Spanish": 0.2556818181818182, - "English,Filipino,Malay": 0.24431818181818182, - "English,Filipino,Chinese": 0.08522727272727272, - "English,Filipino,Indonesian": 0.23863636363636365, - "English,Filipino,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay": 0.3352272727272727, - "English,Spanish,Chinese": 0.1534090909090909, - "English,Spanish,Indonesian": 0.375, + "Indonesian,English,Filipino": 0.22727272727272727, + "Indonesian,English,Spanish": 0.2897727272727273, + "Indonesian,English,Chinese": 0.18181818181818182, + "Indonesian,English,Malay": 0.25, + "Indonesian,English,Vietnamese": 0.16477272727272727, + "Indonesian,Filipino,Spanish": 0.2840909090909091, + "Indonesian,Filipino,Chinese": 0.25, + "Indonesian,Filipino,Malay": 0.32954545454545453, + "Indonesian,Filipino,Vietnamese": 0.25, + "Indonesian,Spanish,Chinese": 0.19318181818181818, + "Indonesian,Spanish,Malay": 0.2897727272727273, + "Indonesian,Spanish,Vietnamese": 0.19318181818181818, + "Indonesian,Chinese,Malay": 0.30113636363636365, + "Indonesian,Chinese,Vietnamese": 0.23863636363636365, + "Indonesian,Malay,Vietnamese": 0.2784090909090909, + "English,Filipino,Spanish": 0.2727272727272727, + "English,Filipino,Chinese": 0.16477272727272727, + "English,Filipino,Malay": 0.19318181818181818, + "English,Filipino,Vietnamese": 0.1590909090909091, + "English,Spanish,Chinese": 0.2159090909090909, + "English,Spanish,Malay": 0.2727272727272727, "English,Spanish,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese": 0.11363636363636363, - "English,Malay,Indonesian": 0.29545454545454547, - "English,Malay,Vietnamese": 0.14772727272727273, - "English,Chinese,Indonesian": 0.13068181818181818, - "English,Chinese,Vietnamese": 0.10795454545454546, - "English,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.125, - "Filipino,Spanish,Indonesian": 0.2897727272727273, - "Filipino,Spanish,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Chinese": 0.19318181818181818, - "Filipino,Malay,Indonesian": 0.3522727272727273, - "Filipino,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Chinese,Vietnamese": 0.1875, - "Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese": 0.1590909090909091, - "Spanish,Malay,Indonesian": 0.3465909090909091, - "Spanish,Malay,Vietnamese": 0.19318181818181818, - "Spanish,Chinese,Indonesian": 0.16477272727272727, - "Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Spanish,Indonesian,Vietnamese": 0.19886363636363635, - "Malay,Chinese,Indonesian": 0.2215909090909091, - "Malay,Chinese,Vietnamese": 0.24431818181818182, - "Malay,Indonesian,Vietnamese": 0.22727272727272727, - "Chinese,Indonesian,Vietnamese": 0.20454545454545456 + "English,Chinese,Malay": 0.19318181818181818, + "English,Chinese,Vietnamese": 0.17045454545454544, + "English,Malay,Vietnamese": 0.17613636363636365, + "Filipino,Spanish,Chinese": 0.20454545454545456, + "Filipino,Spanish,Malay": 0.2556818181818182, + "Filipino,Spanish,Vietnamese": 0.22727272727272727, + "Filipino,Chinese,Malay": 0.2784090909090909, + "Filipino,Chinese,Vietnamese": 0.2727272727272727, + "Filipino,Malay,Vietnamese": 0.2727272727272727, + "Spanish,Chinese,Malay": 0.19886363636363635, + "Spanish,Chinese,Vietnamese": 0.19886363636363635, + "Spanish,Malay,Vietnamese": 0.2215909090909091, + "Chinese,Malay,Vietnamese": 0.3352272727272727 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.18181818181818182, - "English,Filipino,Spanish,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Indonesian": 0.19886363636363635, - "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese": 0.0625, - "English,Filipino,Malay,Indonesian": 0.18181818181818182, - "English,Filipino,Malay,Vietnamese": 0.07386363636363637, - "English,Filipino,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese": 0.10227272727272728, - "English,Spanish,Malay,Indonesian": 0.26136363636363635, - "English,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian": 0.10795454545454546, - "English,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "English,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "English,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese": 0.09659090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 + "Indonesian,English,Filipino,Spanish": 0.18181818181818182, + "Indonesian,English,Filipino,Chinese": 0.13068181818181818, + "Indonesian,English,Filipino,Malay": 0.1534090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.11363636363636363, + "Indonesian,English,Spanish,Chinese": 0.14772727272727273, + "Indonesian,English,Spanish,Malay": 0.19318181818181818, + "Indonesian,English,Spanish,Vietnamese": 0.14204545454545456, + "Indonesian,English,Chinese,Malay": 0.1534090909090909, + "Indonesian,English,Chinese,Vietnamese": 0.11931818181818182, + "Indonesian,English,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Spanish,Chinese": 0.1590909090909091, + "Indonesian,Filipino,Spanish,Malay": 0.19886363636363635, + "Indonesian,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "Indonesian,Filipino,Chinese,Malay": 0.20454545454545456, + "Indonesian,Filipino,Chinese,Vietnamese": 0.1590909090909091, + "Indonesian,Filipino,Malay,Vietnamese": 0.1875, + "Indonesian,Spanish,Chinese,Malay": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Vietnamese": 0.125, + "Indonesian,Spanish,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,Chinese,Malay,Vietnamese": 0.21022727272727273, + "English,Filipino,Spanish,Chinese": 0.14772727272727273, + "English,Filipino,Spanish,Malay": 0.17045454545454544, + "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "English,Filipino,Chinese,Malay": 0.13068181818181818, + "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, + "English,Filipino,Malay,Vietnamese": 0.11363636363636363, + "English,Spanish,Chinese,Malay": 0.14772727272727273, + "English,Spanish,Chinese,Vietnamese": 0.13636363636363635, + "English,Spanish,Malay,Vietnamese": 0.14204545454545456, + "English,Chinese,Malay,Vietnamese": 0.125, + "Filipino,Spanish,Chinese,Malay": 0.1590909090909091, + "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, + "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, + "Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, + "Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.0625, - "English,Filipino,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 + "Indonesian,English,Filipino,Spanish,Chinese": 0.11931818181818182, + "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, + "Indonesian,English,Filipino,Chinese,Malay": 0.11931818181818182, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,English,Spanish,Chinese,Malay": 0.125, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.13636363636363635, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.125, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363, + "English,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, + "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, + "English,Filipino,Chinese,Malay,Vietnamese": 0.09659090909090909, + "English,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.10795454545454546, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454 } }, - "AC3_2": 0.3940024664585435, - "AC3_3": 0.26938057106722535, - "AC3_4": 0.17886200020192855, - "AC3_5": 0.11789882808956023, - "AC3_6": 0.07857641397050111, - "AC3_7": 0.05285675478601386 + "AC3_2": 0.3880105401349677, + "AC3_3": 0.2820907332032594, + "AC3_4": 0.21294942994905158, + "AC3_5": 0.1709106774184328, + "AC3_6": 0.14554595310000934, + "AC3_7": 0.1297846889651599 }, "prompt_5": { - "overall_acc": 0.36363636363636365, + "overall_acc": 0.3538961038961039, "language_acc": { - "English": 0.6022727272727273, - "Filipino": 0.26704545454545453, - "Spanish": 0.42045454545454547, - "Malay": 0.3409090909090909, - "Chinese": 0.22727272727272727, - "Indonesian": 0.4034090909090909, - "Vietnamese": 0.2840909090909091 + "Indonesian": 0.32954545454545453, + "English": 0.5284090909090909, + "Filipino": 0.3125, + "Spanish": 0.4318181818181818, + "Chinese": 0.26136363636363635, + "Malay": 0.3522727272727273, + "Vietnamese": 0.26136363636363635 }, - "consistency_score_2": 0.39123376623376627, - "consistency_score_3": 0.19237012987012983, - "consistency_score_4": 0.10389610389610388, - "consistency_score_5": 0.05925324675324676, - "consistency_score_6": 0.03571428571428571, - "consistency_score_7": 0.022727272727272728, + "consistency_score_2": 0.4218073593073593, + "consistency_score_3": 0.22532467532467537, + "consistency_score_4": 0.1430194805194805, + "consistency_score_5": 0.10173160173160173, + "consistency_score_6": 0.07792207792207792, + "consistency_score_7": 0.0625, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.6477272727272727, - "English,Malay": 0.4147727272727273, - "English,Chinese": 0.2897727272727273, - "English,Indonesian": 0.4772727272727273, - "English,Vietnamese": 0.2840909090909091, - "Filipino,Spanish": 0.4090909090909091, - "Filipino,Malay": 0.4772727272727273, - "Filipino,Chinese": 0.2727272727272727, - "Filipino,Indonesian": 0.4715909090909091, - "Filipino,Vietnamese": 0.3806818181818182, - "Spanish,Malay": 0.5113636363636364, - "Spanish,Chinese": 0.2840909090909091, - "Spanish,Indonesian": 0.5681818181818182, - "Spanish,Vietnamese": 0.3181818181818182, - "Malay,Chinese": 0.2784090909090909, - "Malay,Indonesian": 0.5511363636363636, - "Malay,Vietnamese": 0.4147727272727273, - "Chinese,Indonesian": 0.25, - "Chinese,Vietnamese": 0.25, - "Indonesian,Vietnamese": 0.32954545454545453 + "Indonesian,English": 0.39204545454545453, + "Indonesian,Filipino": 0.4602272727272727, + "Indonesian,Spanish": 0.4431818181818182, + "Indonesian,Chinese": 0.36363636363636365, + "Indonesian,Malay": 0.5795454545454546, + "Indonesian,Vietnamese": 0.42045454545454547, + "English,Filipino": 0.32386363636363635, + "English,Spanish": 0.6363636363636364, + "English,Chinese": 0.32954545454545453, + "English,Malay": 0.36363636363636365, + "English,Vietnamese": 0.3068181818181818, + "Filipino,Spanish": 0.4431818181818182, + "Filipino,Chinese": 0.38636363636363635, + "Filipino,Malay": 0.5, + "Filipino,Vietnamese": 0.4147727272727273, + "Spanish,Chinese": 0.32954545454545453, + "Spanish,Malay": 0.4034090909090909, + "Spanish,Vietnamese": 0.3352272727272727, + "Chinese,Malay": 0.48863636363636365, + "Chinese,Vietnamese": 0.4602272727272727, + "Malay,Vietnamese": 0.4772727272727273 }, "3_combine": { - "English,Filipino,Spanish": 0.2556818181818182, + "Indonesian,English,Filipino": 0.21022727272727273, + "Indonesian,English,Spanish": 0.3068181818181818, + "Indonesian,English,Chinese": 0.18181818181818182, + "Indonesian,English,Malay": 0.2556818181818182, + "Indonesian,English,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Spanish": 0.2556818181818182, + "Indonesian,Filipino,Chinese": 0.2215909090909091, + "Indonesian,Filipino,Malay": 0.3181818181818182, + "Indonesian,Filipino,Vietnamese": 0.23863636363636365, + "Indonesian,Spanish,Chinese": 0.19318181818181818, + "Indonesian,Spanish,Malay": 0.2840909090909091, + "Indonesian,Spanish,Vietnamese": 0.18181818181818182, + "Indonesian,Chinese,Malay": 0.2897727272727273, + "Indonesian,Chinese,Vietnamese": 0.21022727272727273, + "Indonesian,Malay,Vietnamese": 0.2840909090909091, + "English,Filipino,Spanish": 0.26136363636363635, + "English,Filipino,Chinese": 0.16477272727272727, "English,Filipino,Malay": 0.20454545454545456, - "English,Filipino,Chinese": 0.10795454545454546, - "English,Filipino,Indonesian": 0.23295454545454544, - "English,Filipino,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay": 0.3409090909090909, - "English,Spanish,Chinese": 0.19886363636363635, - "English,Spanish,Indonesian": 0.3806818181818182, - "English,Spanish,Vietnamese": 0.20454545454545456, - "English,Malay,Chinese": 0.10795454545454546, - "English,Malay,Indonesian": 0.30113636363636365, - "English,Malay,Vietnamese": 0.14204545454545456, - "English,Chinese,Indonesian": 0.1590909090909091, - "English,Chinese,Vietnamese": 0.09659090909090909, - "English,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay": 0.26136363636363635, - "Filipino,Spanish,Chinese": 0.11931818181818182, - "Filipino,Spanish,Indonesian": 0.29545454545454547, - "Filipino,Spanish,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Chinese": 0.13068181818181818, - "Filipino,Malay,Indonesian": 0.32954545454545453, - "Filipino,Malay,Vietnamese": 0.2159090909090909, - "Filipino,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese": 0.14772727272727273, - "Spanish,Malay,Indonesian": 0.375, + "English,Filipino,Vietnamese": 0.13636363636363635, + "English,Spanish,Chinese": 0.23295454545454544, + "English,Spanish,Malay": 0.2840909090909091, + "English,Spanish,Vietnamese": 0.22727272727272727, + "English,Chinese,Malay": 0.19318181818181818, + "English,Chinese,Vietnamese": 0.1590909090909091, + "English,Malay,Vietnamese": 0.17045454545454544, + "Filipino,Spanish,Chinese": 0.20454545454545456, + "Filipino,Spanish,Malay": 0.2556818181818182, + "Filipino,Spanish,Vietnamese": 0.1875, + "Filipino,Chinese,Malay": 0.2727272727272727, + "Filipino,Chinese,Vietnamese": 0.21022727272727273, + "Filipino,Malay,Vietnamese": 0.26136363636363635, + "Spanish,Chinese,Malay": 0.20454545454545456, + "Spanish,Chinese,Vietnamese": 0.16477272727272727, "Spanish,Malay,Vietnamese": 0.20454545454545456, - "Spanish,Chinese,Indonesian": 0.17613636363636365, - "Spanish,Chinese,Vietnamese": 0.09090909090909091, - "Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "Malay,Chinese,Indonesian": 0.1590909090909091, - "Malay,Chinese,Vietnamese": 0.10227272727272728, - "Malay,Indonesian,Vietnamese": 0.23295454545454544, - "Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.16477272727272727, - "English,Filipino,Spanish,Chinese": 0.08522727272727272, - "English,Filipino,Spanish,Indonesian": 0.19318181818181818, - "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese": 0.056818181818181816, - "English,Filipino,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese": 0.09659090909090909, - "English,Spanish,Malay,Indonesian": 0.26704545454545453, - "English,Spanish,Malay,Vietnamese": 0.125, - "English,Spanish,Chinese,Indonesian": 0.14204545454545456, - "English,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "English,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "English,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Chinese": 0.07386363636363637, - "Filipino,Spanish,Malay,Indonesian": 0.21022727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Chinese,Indonesian": 0.09090909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.045454545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Malay,Chinese,Indonesian,Vietnamese": 0.0625 + "Chinese,Malay,Vietnamese": 0.2840909090909091 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.17045454545454544, + "Indonesian,English,Filipino,Chinese": 0.11931818181818182, + "Indonesian,English,Filipino,Malay": 0.1534090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, + "Indonesian,English,Spanish,Chinese": 0.1534090909090909, + "Indonesian,English,Spanish,Malay": 0.21022727272727273, + "Indonesian,English,Spanish,Vietnamese": 0.14772727272727273, + "Indonesian,English,Chinese,Malay": 0.1534090909090909, + "Indonesian,English,Chinese,Vietnamese": 0.11363636363636363, + "Indonesian,English,Malay,Vietnamese": 0.125, + "Indonesian,Filipino,Spanish,Chinese": 0.14772727272727273, + "Indonesian,Filipino,Spanish,Malay": 0.19886363636363635, + "Indonesian,Filipino,Spanish,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.18181818181818182, + "Indonesian,Filipino,Chinese,Vietnamese": 0.125, + "Indonesian,Filipino,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Spanish,Chinese,Malay": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Vietnamese": 0.10795454545454546, + "Indonesian,Spanish,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,Chinese,Malay,Vietnamese": 0.1875, + "English,Filipino,Spanish,Chinese": 0.1534090909090909, + "English,Filipino,Spanish,Malay": 0.17613636363636365, + "English,Filipino,Spanish,Vietnamese": 0.11931818181818182, + "English,Filipino,Chinese,Malay": 0.13068181818181818, + "English,Filipino,Chinese,Vietnamese": 0.09090909090909091, + "English,Filipino,Malay,Vietnamese": 0.10795454545454546, + "English,Spanish,Chinese,Malay": 0.1534090909090909, + "English,Spanish,Chinese,Vietnamese": 0.11931818181818182, + "English,Spanish,Malay,Vietnamese": 0.14204545454545456, + "English,Chinese,Malay,Vietnamese": 0.11363636363636363, + "Filipino,Spanish,Chinese,Malay": 0.16477272727272727, + "Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, + "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, + "Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, + "Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.0625, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 + "Indonesian,English,Filipino,Spanish,Chinese": 0.11363636363636363, + "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.10795454545454546, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,English,Spanish,Chinese,Malay": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.09659090909090909, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, + "English,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, + "English,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, + "English,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 } }, - "AC3_2": 0.3769305962355019, - "AC3_3": 0.25162574647100155, - "AC3_4": 0.16161616158159367, - "AC3_5": 0.10190193681066176, - "AC3_6": 0.06504065039021746, - "AC3_7": 0.04278074865202894 + "AC3_2": 0.3848789856316012, + "AC3_3": 0.2753406906518987, + "AC3_4": 0.20371281768685, + "AC3_5": 0.15803436465051104, + "AC3_6": 0.12772190212841467, + "AC3_7": 0.1062378167386189 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.5436893203883495 + "accuracy": 0.6116504854368932 }, "prompt_2": { - "accuracy": 0.5728155339805825 + "accuracy": 0.6310679611650486 }, "prompt_3": { - "accuracy": 0.5631067961165048 + "accuracy": 0.5533980582524272 }, "prompt_4": { - "accuracy": 0.5631067961165048 + "accuracy": 0.6019417475728155 }, "prompt_5": { - "accuracy": 0.5631067961165048 + "accuracy": 0.5533980582524272 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.23809523809523808 + "accuracy": 0.2571428571428571 }, "prompt_2": { - "accuracy": 0.23809523809523808 + "accuracy": 0.29523809523809524 }, "prompt_3": { - "accuracy": 0.23809523809523808 + "accuracy": 0.3047619047619048 }, "prompt_4": { - "accuracy": 0.23809523809523808 + "accuracy": 0.2761904761904762 }, "prompt_5": { - "accuracy": 0.23809523809523808 + "accuracy": 0.2857142857142857 } }, "us_eval": { "prompt_1": { - "accuracy": 0.6448598130841121 + "accuracy": 0.5607476635514018 }, "prompt_2": { - "accuracy": 0.6355140186915887 + "accuracy": 0.5233644859813084 }, "prompt_3": { - "accuracy": 0.6542056074766355 + "accuracy": 0.5794392523364486 }, "prompt_4": { - "accuracy": 0.6355140186915887 + "accuracy": 0.5607476635514018 }, "prompt_5": { - "accuracy": 0.6542056074766355 + "accuracy": 0.5794392523364486 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.33, + "accuracy": 0.3, "category_acc": { - "brand": 0.3, + "brand": 0.2, "demographics": 0.2, - "biology": 0.4, - "history": 0.3333333333333333, - "literature": 0.3, - "politics": 0.3, - "culture": 0.4, - "film": 0.3, + "biology": 0.5, + "history": 0.26666666666666666, + "literature": 0.4, + "politics": 0.2, + "culture": 0.3, + "film": 0.1, "law": 0.3, - "geography": 0.4 + "geography": 0.5 } }, "prompt_2": { - "accuracy": 0.3, + "accuracy": 0.28, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.4, - "history": 0.4, - "literature": 0.3, - "politics": 0.3, + "history": 0.26666666666666666, + "literature": 0.4, + "politics": 0.2, "culture": 0.3, "film": 0.1, - "law": 0.3, - "geography": 0.4 + "law": 0.2, + "geography": 0.5 } }, "prompt_3": { "accuracy": 0.32, "category_acc": { - "brand": 0.2, + "brand": 0.3, "demographics": 0.2, - "biology": 0.5, - "history": 0.2, - "literature": 0.3, - "politics": 0.3, - "culture": 0.3, - "film": 0.3, + "biology": 0.4, + "history": 0.3333333333333333, + "literature": 0.4, + "politics": 0.2, + "culture": 0.4, + "film": 0.2, "law": 0.3, - "geography": 0.6 + "geography": 0.4 } }, "prompt_4": { - "accuracy": 0.33, + "accuracy": 0.28, "category_acc": { "brand": 0.3, "demographics": 0.2, - "biology": 0.5, + "biology": 0.4, "history": 0.2, - "literature": 0.3, - "politics": 0.3, - "culture": 0.5, - "film": 0.2, - "law": 0.3, - "geography": 0.5 + "literature": 0.5, + "politics": 0.2, + "culture": 0.3, + "film": 0.1, + "law": 0.2, + "geography": 0.4 } }, "prompt_5": { - "accuracy": 0.31, + "accuracy": 0.32, "category_acc": { "brand": 0.2, "demographics": 0.2, "biology": 0.5, "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.3, - "culture": 0.3, - "film": 0.2, + "literature": 0.4, + "politics": 0.2, + "culture": 0.4, + "film": 0.3, "law": 0.3, - "geography": 0.5 + "geography": 0.4 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.17111295690470238 + "bleu_score": 0.1757121671640427 }, "prompt_2": { - "bleu_score": 0.17578262863245286 + "bleu_score": 0.1713078295205753 }, "prompt_3": { - "bleu_score": 0.181061792829869 + "bleu_score": 0.1784941784513121 }, "prompt_4": { - "bleu_score": 0.17795860280445558 + "bleu_score": 0.18529617182140173 }, "prompt_5": { - "bleu_score": 0.17554492731604188 + "bleu_score": 0.16808729137211453 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "prompt_1": { + "accuracy": 0.29601442018826357, + "category_acc": { + "History": 0.285140562248996, + "Geography": 0.21020408163265306, + "Lampungic": 0.2789115646258503, + "Social science": 0.3639398998330551, + "Balinese": 0.3184713375796178, + "Makassarese": 0.24193548387096775, + "Banjarese": 0.2361111111111111, + "Chemistry": 0.21897810218978103, + "Biology": 0.2828402366863905, + "Science": 0.3013415892672859, + "Christian religion": 0.3781094527363184, + "Art": 0.2895174708818636, + "Islam religion": 0.310099573257468, + "Hindu religion": 0.29333333333333333, + "Madurese": 0.29491525423728815, + "Sport": 0.32432432432432434, + "Indonesian language": 0.3297011207970112, + "Physics": 0.26464646464646463, + "Minangkabau culture": 0.271356783919598, + "Dayak language": 0.30275229357798167, + "Sociology": 0.28830645161290325, + "Economy": 0.2848360655737705, + "Sundanese": 0.28867761452031115, + "Javanese": 0.2590725806451613, + "Civic education": 0.3190271816881259 + } + }, + "prompt_2": { + "accuracy": 0.2902730489351759, + "category_acc": { + "History": 0.29116465863453816, + "Geography": 0.21224489795918366, + "Lampungic": 0.32653061224489793, + "Social science": 0.34557595993322204, + "Balinese": 0.3099787685774947, + "Makassarese": 0.24193548387096775, + "Banjarese": 0.2152777777777778, + "Chemistry": 0.23357664233576642, + "Biology": 0.27928994082840236, + "Science": 0.2899896800825593, + "Christian religion": 0.3383084577114428, + "Art": 0.26788685524126454, + "Islam religion": 0.3157894736842105, + "Hindu religion": 0.2733333333333333, + "Madurese": 0.2847457627118644, + "Sport": 0.3108108108108108, + "Indonesian language": 0.31787048567870485, + "Physics": 0.25656565656565655, + "Minangkabau culture": 0.271356783919598, + "Dayak language": 0.30275229357798167, + "Sociology": 0.26814516129032256, + "Economy": 0.26024590163934425, + "Sundanese": 0.29472774416594644, + "Javanese": 0.26310483870967744, + "Civic education": 0.32331902718168815 + } + }, + "prompt_3": { + "accuracy": 0.2936778156085186, + "category_acc": { + "History": 0.30120481927710846, + "Geography": 0.20408163265306123, + "Lampungic": 0.3129251700680272, + "Social science": 0.3606010016694491, + "Balinese": 0.3099787685774947, + "Makassarese": 0.20967741935483872, + "Banjarese": 0.2222222222222222, + "Chemistry": 0.22481751824817517, + "Biology": 0.2781065088757396, + "Science": 0.30030959752321984, + "Christian religion": 0.3383084577114428, + "Art": 0.26955074875207985, + "Islam religion": 0.29445234708392604, + "Hindu religion": 0.2733333333333333, + "Madurese": 0.2677966101694915, + "Sport": 0.3108108108108108, + "Indonesian language": 0.3234744707347447, + "Physics": 0.25252525252525254, + "Minangkabau culture": 0.2864321608040201, + "Dayak language": 0.30275229357798167, + "Sociology": 0.3004032258064516, + "Economy": 0.27049180327868855, + "Sundanese": 0.31028522039757994, + "Javanese": 0.2782258064516129, + "Civic education": 0.3104434907010014 + } + }, + "prompt_4": { + "accuracy": 0.29127445089792375, + "category_acc": { + "History": 0.27710843373493976, + "Geography": 0.23061224489795917, + "Lampungic": 0.29931972789115646, + "Social science": 0.35392320534223703, + "Balinese": 0.3248407643312102, + "Makassarese": 0.26344086021505375, + "Banjarese": 0.22916666666666666, + "Chemistry": 0.21313868613138687, + "Biology": 0.28402366863905326, + "Science": 0.2786377708978328, + "Christian religion": 0.373134328358209, + "Art": 0.2995008319467554, + "Islam religion": 0.29445234708392604, + "Hindu religion": 0.28, + "Madurese": 0.288135593220339, + "Sport": 0.2972972972972973, + "Indonesian language": 0.31880448318804483, + "Physics": 0.25656565656565655, + "Minangkabau culture": 0.27638190954773867, + "Dayak language": 0.29357798165137616, + "Sociology": 0.29435483870967744, + "Economy": 0.2786885245901639, + "Sundanese": 0.2878133102852204, + "Javanese": 0.2610887096774194, + "Civic education": 0.3147353361945637 + } + }, + "prompt_5": { + "accuracy": 0.2872688430469324, + "category_acc": { + "History": 0.27309236947791166, + "Geography": 0.20204081632653062, + "Lampungic": 0.30612244897959184, + "Social science": 0.34390651085141904, + "Balinese": 0.3036093418259023, + "Makassarese": 0.27419354838709675, + "Banjarese": 0.2013888888888889, + "Chemistry": 0.22043795620437956, + "Biology": 0.263905325443787, + "Science": 0.2786377708978328, + "Christian religion": 0.34328358208955223, + "Art": 0.28452579034941766, + "Islam religion": 0.2958748221906117, + "Hindu religion": 0.29333333333333333, + "Madurese": 0.2983050847457627, + "Sport": 0.30405405405405406, + "Indonesian language": 0.3234744707347447, + "Physics": 0.23636363636363636, + "Minangkabau culture": 0.2663316582914573, + "Dayak language": 0.25688073394495414, + "Sociology": 0.28830645161290325, + "Economy": 0.2725409836065574, + "Sundanese": 0.2938634399308557, + "Javanese": 0.2540322580645161, + "Civic education": 0.3147353361945637 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.298729720423162 + "bleu_score": 0.23519019047974593 }, "prompt_2": { - "bleu_score": 0.29714403066664635 + "bleu_score": 0.23081721810773512 }, "prompt_3": { - "bleu_score": 0.29688089941244467 + "bleu_score": 0.2317036111734395 }, "prompt_4": { - "bleu_score": 0.29657094327603384 + "bleu_score": 0.23574032794862365 }, "prompt_5": { - "bleu_score": 0.2891515350076281 + "bleu_score": 0.22734515454295717 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.06332150290827988 + "bleu_score": 0.05726857609979452 }, "prompt_2": { - "bleu_score": 0.06237270430229427 + "bleu_score": 0.056030313148321104 }, "prompt_3": { - "bleu_score": 0.06261245548230009 + "bleu_score": 0.056220586684100834 }, "prompt_4": { - "bleu_score": 0.0638050751721597 + "bleu_score": 0.05707255304827606 }, "prompt_5": { - "bleu_score": 0.06116577273766147 + "bleu_score": 0.05453166476110994 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.015709981480553 + "bleu_score": 0.01556584838007841 }, "prompt_2": { - "bleu_score": 0.015737669889031625 + "bleu_score": 0.016250348764526006 }, "prompt_3": { - "bleu_score": 0.018161111996093608 + "bleu_score": 0.015726636553525738 }, "prompt_4": { - "bleu_score": 0.015298302651452203 + "bleu_score": 0.018325681250757293 }, "prompt_5": { - "bleu_score": 0.01808485717283886 + "bleu_score": 0.014647484001582058 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.27795819511180947 + "bleu_score": 0.21964506743805942 }, "prompt_2": { - "bleu_score": 0.28022199889007265 + "bleu_score": 0.21688082199907668 }, "prompt_3": { - "bleu_score": 0.2797980730259455 + "bleu_score": 0.21728844545846515 }, "prompt_4": { - "bleu_score": 0.2800502337161776 + "bleu_score": 0.22147378722039016 }, "prompt_5": { - "bleu_score": 0.26872088931620774 + "bleu_score": 0.20762462597989922 } }, "mmlu": { "prompt_1": { - "accuracy": 0.5542590431738623 + "accuracy": 0.5029171528588098 }, "prompt_2": { - "accuracy": 0.5484247374562428 + "accuracy": 0.5040840140023337 }, "prompt_3": { - "accuracy": 0.5425904317386231 + "accuracy": 0.5122520420070011 }, "prompt_4": { - "accuracy": 0.543757292882147 + "accuracy": 0.5099183197199533 }, "prompt_5": { - "accuracy": 0.5484247374562428 + "accuracy": 0.5134189031505251 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.5319985698963174, + "accuracy": 0.4889524490525563, "category_acc": { - "high_school_european_history": 0.7439024390243902, + "high_school_european_history": 0.7012195121951219, "business_ethics": 0.6464646464646465, - "clinical_knowledge": 0.6060606060606061, - "medical_genetics": 0.5757575757575758, - "high_school_us_history": 0.7684729064039408, - "high_school_physics": 0.24666666666666667, - "high_school_world_history": 0.7330508474576272, - "virology": 0.4666666666666667, - "high_school_microeconomics": 0.5864978902953587, - "econometrics": 0.2743362831858407, - "college_computer_science": 0.45454545454545453, - "high_school_biology": 0.6763754045307443, - "abstract_algebra": 0.25252525252525254, - "professional_accounting": 0.398576512455516, - "philosophy": 0.5580645161290323, - "professional_medicine": 0.5166051660516605, - "nutrition": 0.5770491803278689, - "global_facts": 0.2828282828282828, + "clinical_knowledge": 0.5606060606060606, + "medical_genetics": 0.46464646464646464, + "high_school_us_history": 0.6847290640394089, + "high_school_physics": 0.2733333333333333, + "high_school_world_history": 0.6694915254237288, + "virology": 0.43636363636363634, + "high_school_microeconomics": 0.510548523206751, + "econometrics": 0.30973451327433627, + "college_computer_science": 0.40404040404040403, + "high_school_biology": 0.5792880258899676, + "abstract_algebra": 0.2222222222222222, + "professional_accounting": 0.3736654804270463, + "philosophy": 0.5225806451612903, + "professional_medicine": 0.4575645756457565, + "nutrition": 0.4786885245901639, + "global_facts": 0.31313131313131315, "machine_learning": 0.35135135135135137, - "security_studies": 0.6557377049180327, - "public_relations": 0.6422018348623854, - "professional_psychology": 0.5433715220949263, - "prehistory": 0.5758513931888545, + "security_studies": 0.5737704918032787, + "public_relations": 0.5871559633027523, + "professional_psychology": 0.4795417348608838, + "prehistory": 0.47678018575851394, "anatomy": 0.4552238805970149, - "human_sexuality": 0.5615384615384615, - "college_medicine": 0.5465116279069767, - "high_school_government_and_politics": 0.7864583333333334, - "college_chemistry": 0.41414141414141414, - "logical_fallacies": 0.654320987654321, - "high_school_geography": 0.6802030456852792, - "elementary_mathematics": 0.3793103448275862, - "human_aging": 0.5495495495495496, - "college_mathematics": 0.36363636363636365, - "high_school_psychology": 0.78125, - "formal_logic": 0.392, - "high_school_statistics": 0.3953488372093023, - "international_law": 0.6666666666666666, - "high_school_mathematics": 0.2862453531598513, - "high_school_computer_science": 0.5050505050505051, - "conceptual_physics": 0.44017094017094016, - "miscellaneous": 0.7161125319693095, - "high_school_chemistry": 0.3613861386138614, - "marketing": 0.8154506437768241, - "professional_law": 0.40378343118069143, - "management": 0.7549019607843137, - "college_physics": 0.38613861386138615, - "jurisprudence": 0.6728971962616822, - "world_religions": 0.6823529411764706, - "sociology": 0.7, - "us_foreign_policy": 0.7070707070707071, - "high_school_macroeconomics": 0.5578406169665809, - "computer_security": 0.6565656565656566, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.6376811594202898, - "electrical_engineering": 0.5138888888888888, - "astronomy": 0.5364238410596026, - "college_biology": 0.5104895104895105 - } - }, - "prompt_2": { - "accuracy": 0.5324991061851985, - "category_acc": { - "high_school_european_history": 0.75, - "business_ethics": 0.6363636363636364, - "clinical_knowledge": 0.6098484848484849, - "medical_genetics": 0.5757575757575758, - "high_school_us_history": 0.7684729064039408, - "high_school_physics": 0.26666666666666666, - "high_school_world_history": 0.7330508474576272, - "virology": 0.4727272727272727, - "high_school_microeconomics": 0.6160337552742616, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.41414141414141414, - "high_school_biology": 0.6634304207119741, - "abstract_algebra": 0.20202020202020202, - "professional_accounting": 0.4128113879003559, - "philosophy": 0.5774193548387097, - "professional_medicine": 0.5239852398523985, - "nutrition": 0.5639344262295082, - "global_facts": 0.30303030303030304, - "machine_learning": 0.35135135135135137, - "security_studies": 0.6557377049180327, - "public_relations": 0.6055045871559633, - "professional_psychology": 0.5384615384615384, - "prehistory": 0.5820433436532507, - "anatomy": 0.4253731343283582, - "human_sexuality": 0.6, - "college_medicine": 0.5523255813953488, - "high_school_government_and_politics": 0.7760416666666666, + "human_sexuality": 0.5769230769230769, + "college_medicine": 0.47093023255813954, + "high_school_government_and_politics": 0.6875, "college_chemistry": 0.40404040404040403, - "logical_fallacies": 0.6666666666666666, - "high_school_geography": 0.6802030456852792, - "elementary_mathematics": 0.3819628647214854, + "logical_fallacies": 0.5555555555555556, + "high_school_geography": 0.6700507614213198, + "elementary_mathematics": 0.34748010610079577, "human_aging": 0.536036036036036, - "college_mathematics": 0.35353535353535354, - "high_school_psychology": 0.7757352941176471, + "college_mathematics": 0.25252525252525254, + "high_school_psychology": 0.6911764705882353, "formal_logic": 0.352, - "high_school_statistics": 0.40930232558139534, - "international_law": 0.6666666666666666, - "high_school_mathematics": 0.2788104089219331, + "high_school_statistics": 0.4046511627906977, + "international_law": 0.65, + "high_school_mathematics": 0.2862453531598513, "high_school_computer_science": 0.494949494949495, - "conceptual_physics": 0.4188034188034188, - "miscellaneous": 0.7199488491048593, - "high_school_chemistry": 0.36633663366336633, - "marketing": 0.8111587982832618, - "professional_law": 0.4050880626223092, - "management": 0.7647058823529411, - "college_physics": 0.38613861386138615, - "jurisprudence": 0.6915887850467289, - "world_religions": 0.6764705882352942, - "sociology": 0.695, - "us_foreign_policy": 0.7070707070707071, - "high_school_macroeconomics": 0.5758354755784062, - "computer_security": 0.6666666666666666, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.6405797101449275, - "electrical_engineering": 0.5208333333333334, - "astronomy": 0.5629139072847682, - "college_biology": 0.48951048951048953 + "conceptual_physics": 0.4444444444444444, + "miscellaneous": 0.659846547314578, + "high_school_chemistry": 0.30198019801980197, + "marketing": 0.7811158798283262, + "professional_law": 0.3842139595564253, + "management": 0.6764705882352942, + "college_physics": 0.3465346534653465, + "jurisprudence": 0.5794392523364486, + "world_religions": 0.5764705882352941, + "sociology": 0.73, + "us_foreign_policy": 0.7272727272727273, + "high_school_macroeconomics": 0.5244215938303342, + "computer_security": 0.5858585858585859, + "moral_scenarios": 0.23154362416107382, + "moral_disputes": 0.5797101449275363, + "electrical_engineering": 0.4027777777777778, + "astronomy": 0.5099337748344371, + "college_biology": 0.45454545454545453 } }, - "prompt_3": { - "accuracy": 0.5335001787629603, + "prompt_2": { + "accuracy": 0.48859492313192704, "category_acc": { - "high_school_european_history": 0.7439024390243902, - "business_ethics": 0.6363636363636364, - "clinical_knowledge": 0.5984848484848485, - "medical_genetics": 0.5757575757575758, - "high_school_us_history": 0.7635467980295566, - "high_school_physics": 0.26, - "high_school_world_history": 0.7288135593220338, - "virology": 0.4727272727272727, - "high_school_microeconomics": 0.6118143459915611, - "econometrics": 0.24778761061946902, - "college_computer_science": 0.40404040404040403, - "high_school_biology": 0.6666666666666666, - "abstract_algebra": 0.21212121212121213, - "professional_accounting": 0.398576512455516, - "philosophy": 0.5870967741935483, - "professional_medicine": 0.5239852398523985, - "nutrition": 0.5737704918032787, - "global_facts": 0.29292929292929293, - "machine_learning": 0.34234234234234234, - "security_studies": 0.6639344262295082, - "public_relations": 0.6238532110091743, - "professional_psychology": 0.5384615384615384, - "prehistory": 0.5603715170278638, - "anatomy": 0.43283582089552236, - "human_sexuality": 0.6, - "college_medicine": 0.5523255813953488, - "high_school_government_and_politics": 0.8020833333333334, - "college_chemistry": 0.40404040404040403, - "logical_fallacies": 0.6728395061728395, - "high_school_geography": 0.7157360406091371, - "elementary_mathematics": 0.3740053050397878, - "human_aging": 0.5630630630630631, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.7904411764705882, + "high_school_european_history": 0.7073170731707317, + "business_ethics": 0.6262626262626263, + "clinical_knowledge": 0.5681818181818182, + "medical_genetics": 0.494949494949495, + "high_school_us_history": 0.6847290640394089, + "high_school_physics": 0.3, + "high_school_world_history": 0.6864406779661016, + "virology": 0.4303030303030303, + "high_school_microeconomics": 0.510548523206751, + "econometrics": 0.336283185840708, + "college_computer_science": 0.3939393939393939, + "high_school_biology": 0.56957928802589, + "abstract_algebra": 0.25252525252525254, + "professional_accounting": 0.37722419928825623, + "philosophy": 0.5225806451612903, + "professional_medicine": 0.44649446494464945, + "nutrition": 0.4721311475409836, + "global_facts": 0.3333333333333333, + "machine_learning": 0.36036036036036034, + "security_studies": 0.5860655737704918, + "public_relations": 0.5688073394495413, + "professional_psychology": 0.46317512274959083, + "prehistory": 0.49226006191950467, + "anatomy": 0.44029850746268656, + "human_sexuality": 0.5538461538461539, + "college_medicine": 0.48255813953488375, + "high_school_government_and_politics": 0.6822916666666666, + "college_chemistry": 0.43434343434343436, + "logical_fallacies": 0.5802469135802469, + "high_school_geography": 0.6700507614213198, + "elementary_mathematics": 0.3209549071618037, + "human_aging": 0.5135135135135135, + "college_mathematics": 0.2727272727272727, + "high_school_psychology": 0.6930147058823529, "formal_logic": 0.392, - "high_school_statistics": 0.3767441860465116, - "international_law": 0.6666666666666666, - "high_school_mathematics": 0.26394052044609667, - "high_school_computer_science": 0.41414141414141414, - "conceptual_physics": 0.4230769230769231, - "miscellaneous": 0.7225063938618926, - "high_school_chemistry": 0.35148514851485146, - "marketing": 0.8412017167381974, - "professional_law": 0.4044357469015003, - "management": 0.7549019607843137, - "college_physics": 0.40594059405940597, - "jurisprudence": 0.6915887850467289, - "world_religions": 0.6823529411764706, - "sociology": 0.705, - "us_foreign_policy": 0.7373737373737373, - "high_school_macroeconomics": 0.5681233933161953, - "computer_security": 0.6565656565656566, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.6492753623188405, - "electrical_engineering": 0.5625, - "astronomy": 0.5827814569536424, - "college_biology": 0.4755244755244755 + "high_school_statistics": 0.4186046511627907, + "international_law": 0.65, + "high_school_mathematics": 0.3048327137546468, + "high_school_computer_science": 0.46464646464646464, + "conceptual_physics": 0.42735042735042733, + "miscellaneous": 0.6483375959079284, + "high_school_chemistry": 0.3069306930693069, + "marketing": 0.7467811158798283, + "professional_law": 0.38290932811480755, + "management": 0.6764705882352942, + "college_physics": 0.31683168316831684, + "jurisprudence": 0.5887850467289719, + "world_religions": 0.5941176470588235, + "sociology": 0.74, + "us_foreign_policy": 0.7171717171717171, + "high_school_macroeconomics": 0.5244215938303342, + "computer_security": 0.5959595959595959, + "moral_scenarios": 0.2203579418344519, + "moral_disputes": 0.5855072463768116, + "electrical_engineering": 0.4166666666666667, + "astronomy": 0.543046357615894, + "college_biology": 0.4965034965034965 } }, - "prompt_4": { - "accuracy": 0.5308544869503039, + "prompt_3": { + "accuracy": 0.488165892027172, "category_acc": { - "high_school_european_history": 0.7560975609756098, - "business_ethics": 0.6363636363636364, - "clinical_knowledge": 0.5946969696969697, - "medical_genetics": 0.5858585858585859, - "high_school_us_history": 0.7586206896551724, - "high_school_physics": 0.23333333333333334, - "high_school_world_history": 0.7245762711864406, - "virology": 0.4727272727272727, - "high_school_microeconomics": 0.5991561181434599, - "econometrics": 0.2743362831858407, - "college_computer_science": 0.40404040404040403, - "high_school_biology": 0.6699029126213593, + "high_school_european_history": 0.7073170731707317, + "business_ethics": 0.6161616161616161, + "clinical_knowledge": 0.5454545454545454, + "medical_genetics": 0.494949494949495, + "high_school_us_history": 0.6847290640394089, + "high_school_physics": 0.2866666666666667, + "high_school_world_history": 0.6694915254237288, + "virology": 0.44242424242424244, + "high_school_microeconomics": 0.5147679324894515, + "econometrics": 0.336283185840708, + "college_computer_science": 0.41414141414141414, + "high_school_biology": 0.56957928802589, "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.398576512455516, - "philosophy": 0.567741935483871, - "professional_medicine": 0.5202952029520295, - "nutrition": 0.5672131147540984, - "global_facts": 0.31313131313131315, - "machine_learning": 0.35135135135135137, - "security_studies": 0.6680327868852459, - "public_relations": 0.6146788990825688, - "professional_psychology": 0.5482815057283142, - "prehistory": 0.56656346749226, - "anatomy": 0.44029850746268656, - "human_sexuality": 0.6, - "college_medicine": 0.5406976744186046, - "high_school_government_and_politics": 0.7708333333333334, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.6604938271604939, - "high_school_geography": 0.7258883248730964, - "elementary_mathematics": 0.32625994694960214, - "human_aging": 0.5585585585585585, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.7830882352941176, - "formal_logic": 0.384, - "high_school_statistics": 0.4, - "international_law": 0.6583333333333333, - "high_school_mathematics": 0.27137546468401486, - "high_school_computer_science": 0.45454545454545453, - "conceptual_physics": 0.42735042735042733, - "miscellaneous": 0.7225063938618926, - "high_school_chemistry": 0.3465346534653465, - "marketing": 0.8326180257510729, - "professional_law": 0.39856490541422046, - "management": 0.7647058823529411, - "college_physics": 0.40594059405940597, - "jurisprudence": 0.6635514018691588, - "world_religions": 0.7176470588235294, - "sociology": 0.695, - "us_foreign_policy": 0.7575757575757576, - "high_school_macroeconomics": 0.570694087403599, - "computer_security": 0.6363636363636364, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.6405797101449275, - "electrical_engineering": 0.5763888888888888, - "astronomy": 0.5761589403973509, + "professional_accounting": 0.3879003558718861, + "philosophy": 0.5225806451612903, + "professional_medicine": 0.45018450184501846, + "nutrition": 0.4819672131147541, + "global_facts": 0.3434343434343434, + "machine_learning": 0.32432432432432434, + "security_studies": 0.5901639344262295, + "public_relations": 0.5871559633027523, + "professional_psychology": 0.4664484451718494, + "prehistory": 0.47368421052631576, + "anatomy": 0.44776119402985076, + "human_sexuality": 0.5769230769230769, + "college_medicine": 0.47674418604651164, + "high_school_government_and_politics": 0.6822916666666666, + "college_chemistry": 0.42424242424242425, + "logical_fallacies": 0.5864197530864198, + "high_school_geography": 0.6751269035532995, + "elementary_mathematics": 0.33421750663129973, + "human_aging": 0.5225225225225225, + "college_mathematics": 0.24242424242424243, + "high_school_psychology": 0.6911764705882353, + "formal_logic": 0.376, + "high_school_statistics": 0.413953488372093, + "international_law": 0.65, + "high_school_mathematics": 0.30111524163568776, + "high_school_computer_science": 0.4444444444444444, + "conceptual_physics": 0.43162393162393164, + "miscellaneous": 0.6547314578005116, + "high_school_chemistry": 0.30198019801980197, + "marketing": 0.759656652360515, + "professional_law": 0.3796477495107632, + "management": 0.6764705882352942, + "college_physics": 0.32673267326732675, + "jurisprudence": 0.5887850467289719, + "world_religions": 0.5764705882352941, + "sociology": 0.735, + "us_foreign_policy": 0.7070707070707071, + "high_school_macroeconomics": 0.532133676092545, + "computer_security": 0.6161616161616161, + "moral_scenarios": 0.2225950782997763, + "moral_disputes": 0.5855072463768116, + "electrical_engineering": 0.4166666666666667, + "astronomy": 0.5364238410596026, "college_biology": 0.46853146853146854 } }, - "prompt_5": { - "accuracy": 0.5338577046835895, + "prompt_4": { + "accuracy": 0.48623525205577406, "category_acc": { - "high_school_european_history": 0.7560975609756098, - "business_ethics": 0.6767676767676768, - "clinical_knowledge": 0.5757575757575758, - "medical_genetics": 0.5353535353535354, - "high_school_us_history": 0.7733990147783252, - "high_school_physics": 0.24, - "high_school_world_history": 0.7330508474576272, - "virology": 0.4727272727272727, - "high_school_microeconomics": 0.5907172995780591, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.4444444444444444, - "high_school_biology": 0.6634304207119741, + "high_school_european_history": 0.7073170731707317, + "business_ethics": 0.6363636363636364, + "clinical_knowledge": 0.553030303030303, + "medical_genetics": 0.48484848484848486, + "high_school_us_history": 0.6748768472906403, + "high_school_physics": 0.3, + "high_school_world_history": 0.6610169491525424, + "virology": 0.4484848484848485, + "high_school_microeconomics": 0.5189873417721519, + "econometrics": 0.3274336283185841, + "college_computer_science": 0.45454545454545453, + "high_school_biology": 0.5631067961165048, "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.4092526690391459, - "philosophy": 0.5806451612903226, - "professional_medicine": 0.5166051660516605, - "nutrition": 0.5868852459016394, - "global_facts": 0.29292929292929293, - "machine_learning": 0.3333333333333333, - "security_studies": 0.680327868852459, - "public_relations": 0.6055045871559633, - "professional_psychology": 0.5417348608837971, - "prehistory": 0.5603715170278638, - "anatomy": 0.4253731343283582, - "human_sexuality": 0.6230769230769231, - "college_medicine": 0.5406976744186046, - "high_school_government_and_politics": 0.78125, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.6604938271604939, - "high_school_geography": 0.6954314720812182, - "elementary_mathematics": 0.35013262599469497, - "human_aging": 0.5495495495495496, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.78125, - "formal_logic": 0.384, + "professional_accounting": 0.3807829181494662, + "philosophy": 0.5225806451612903, + "professional_medicine": 0.44280442804428044, + "nutrition": 0.47540983606557374, + "global_facts": 0.3434343434343434, + "machine_learning": 0.36036036036036034, + "security_studies": 0.5778688524590164, + "public_relations": 0.5779816513761468, + "professional_psychology": 0.46153846153846156, + "prehistory": 0.47368421052631576, + "anatomy": 0.44029850746268656, + "human_sexuality": 0.5461538461538461, + "college_medicine": 0.48255813953488375, + "high_school_government_and_politics": 0.6770833333333334, + "college_chemistry": 0.42424242424242425, + "logical_fallacies": 0.5555555555555556, + "high_school_geography": 0.6802030456852792, + "elementary_mathematics": 0.3183023872679045, + "human_aging": 0.509009009009009, + "college_mathematics": 0.26262626262626265, + "high_school_psychology": 0.6875, + "formal_logic": 0.368, "high_school_statistics": 0.413953488372093, "international_law": 0.6583333333333333, - "high_school_mathematics": 0.27137546468401486, - "high_school_computer_science": 0.48484848484848486, + "high_school_mathematics": 0.2899628252788104, + "high_school_computer_science": 0.47474747474747475, "conceptual_physics": 0.43162393162393164, - "miscellaneous": 0.7250639386189258, - "high_school_chemistry": 0.37623762376237624, - "marketing": 0.8369098712446352, - "professional_law": 0.4050880626223092, - "management": 0.7745098039215687, - "college_physics": 0.40594059405940597, - "jurisprudence": 0.6915887850467289, - "world_religions": 0.6941176470588235, - "sociology": 0.705, - "us_foreign_policy": 0.7272727272727273, - "high_school_macroeconomics": 0.5784061696658098, - "computer_security": 0.6464646464646465, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.6579710144927536, - "electrical_engineering": 0.5416666666666666, - "astronomy": 0.5761589403973509, - "college_biology": 0.5034965034965035 + "miscellaneous": 0.6483375959079284, + "high_school_chemistry": 0.297029702970297, + "marketing": 0.7424892703862661, + "professional_law": 0.3822570123939987, + "management": 0.6862745098039216, + "college_physics": 0.3465346534653465, + "jurisprudence": 0.5794392523364486, + "world_religions": 0.5705882352941176, + "sociology": 0.745, + "us_foreign_policy": 0.6767676767676768, + "high_school_macroeconomics": 0.5167095115681234, + "computer_security": 0.6060606060606061, + "moral_scenarios": 0.23154362416107382, + "moral_disputes": 0.5826086956521739, + "electrical_engineering": 0.4236111111111111, + "astronomy": 0.5298013245033113, + "college_biology": 0.4755244755244755 } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.24888558692421991 - }, - "prompt_2": { - "accuracy": 0.24888558692421991 - }, - "prompt_3": { - "accuracy": 0.24888558692421991 - }, - "prompt_4": { - "accuracy": 0.24665676077265974 }, "prompt_5": { - "accuracy": 0.24962852897473997 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.23848069738480698, + "accuracy": 0.4868787987129067, "category_acc": { - "computer_network": 0.125, - "operating_system": 0.125, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.21428571428571427, - "college_physics": 0.125, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.125, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.36666666666666664, - "business_administration": 0.18421052631578946, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.27586206896551724, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.25, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.11764705882352941, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.18518518518518517, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.25, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.2, - "middle_school_history": 0.18518518518518517, - "civil_servant": 0.25, - "sports_science": 0.08333333333333333, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.125, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.16666666666666666, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.2222222222222222, - "physician": 0.2777777777777778 + "high_school_european_history": 0.7134146341463414, + "business_ethics": 0.6262626262626263, + "clinical_knowledge": 0.5643939393939394, + "medical_genetics": 0.5050505050505051, + "high_school_us_history": 0.6650246305418719, + "high_school_physics": 0.3, + "high_school_world_history": 0.6779661016949152, + "virology": 0.45454545454545453, + "high_school_microeconomics": 0.5189873417721519, + "econometrics": 0.3274336283185841, + "college_computer_science": 0.42424242424242425, + "high_school_biology": 0.5598705501618123, + "abstract_algebra": 0.2727272727272727, + "professional_accounting": 0.3665480427046263, + "philosophy": 0.5, + "professional_medicine": 0.45018450184501846, + "nutrition": 0.4786885245901639, + "global_facts": 0.31313131313131315, + "machine_learning": 0.34234234234234234, + "security_studies": 0.5860655737704918, + "public_relations": 0.5688073394495413, + "professional_psychology": 0.46153846153846156, + "prehistory": 0.47678018575851394, + "anatomy": 0.44029850746268656, + "human_sexuality": 0.5538461538461539, + "college_medicine": 0.4883720930232558, + "high_school_government_and_politics": 0.6770833333333334, + "college_chemistry": 0.41414141414141414, + "logical_fallacies": 0.5555555555555556, + "high_school_geography": 0.6700507614213198, + "elementary_mathematics": 0.32625994694960214, + "human_aging": 0.5180180180180181, + "college_mathematics": 0.26262626262626265, + "high_school_psychology": 0.6893382352941176, + "formal_logic": 0.352, + "high_school_statistics": 0.4232558139534884, + "international_law": 0.6666666666666666, + "high_school_mathematics": 0.31226765799256506, + "high_school_computer_science": 0.47474747474747475, + "conceptual_physics": 0.42735042735042733, + "miscellaneous": 0.6508951406649617, + "high_school_chemistry": 0.3217821782178218, + "marketing": 0.759656652360515, + "professional_law": 0.37638617090671883, + "management": 0.696078431372549, + "college_physics": 0.32673267326732675, + "jurisprudence": 0.5887850467289719, + "world_religions": 0.5764705882352941, + "sociology": 0.74, + "us_foreign_policy": 0.6767676767676768, + "high_school_macroeconomics": 0.5347043701799485, + "computer_security": 0.6161616161616161, + "moral_scenarios": 0.22706935123042504, + "moral_disputes": 0.5768115942028985, + "electrical_engineering": 0.4166666666666667, + "astronomy": 0.5165562913907285, + "college_biology": 0.4755244755244755 } + } + }, + "c_eval": { + "prompt_1": { + "accuracy": 0.25705794947994054 }, "prompt_2": { - "accuracy": 0.23910336239103364, + "accuracy": 0.2473997028231798 + }, + "prompt_3": { + "accuracy": 0.25185735512630014 + }, + "prompt_4": { + "accuracy": 0.2473997028231798 + }, + "prompt_5": { + "accuracy": 0.24665676077265974 + } + }, + "c_eval_full": { + "prompt_1": { + "accuracy": 0.26214196762141967, "category_acc": { - "computer_network": 0.125, - "operating_system": 0.125, + "computer_network": 0.20833333333333334, + "operating_system": 0.4583333333333333, "computer_architecture": 0.23076923076923078, - "college_programming": 0.23809523809523808, + "college_programming": 0.3333333333333333, "college_physics": 0.125, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.125, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.36666666666666664, - "business_administration": 0.18421052631578946, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.3333333333333333, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.19047619047619047, + "metrology_engineer": 0.1724137931034483, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.2916666666666667, + "high_school_biology": 0.2916666666666667, + "middle_school_mathematics": 0.16666666666666666, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.24, + "veterinary_medicine": 0.10714285714285714, + "college_economics": 0.21666666666666667, + "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.27586206896551724, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.25, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.11764705882352941, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.18518518518518517, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.25, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.2, - "middle_school_history": 0.18518518518518517, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.22448979591836735, + "high_school_politics": 0.08333333333333333, + "high_school_geography": 0.25, + "middle_school_politics": 0.38461538461538464, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.3333333333333333, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.08, + "middle_school_history": 0.2962962962962963, "civil_servant": 0.25, - "sports_science": 0.08333333333333333, + "sports_science": 0.2916666666666667, "plant_protection": 0.18518518518518517, - "basic_medicine": 0.125, + "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.16666666666666666, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.2222222222222222, - "physician": 0.2777777777777778 + "urban_and_rural_planner": 0.27450980392156865, + "accountant": 0.3148148148148148, + "fire_engineer": 0.2777777777777778, + "environmental_impact_assessment_engineer": 0.25, + "tax_accountant": 0.3148148148148148, + "physician": 0.25925925925925924 } }, - "prompt_3": { - "accuracy": 0.23910336239103364, + "prompt_2": { + "accuracy": 0.27210460772104605, "category_acc": { - "computer_network": 0.125, - "operating_system": 0.125, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.23809523809523808, - "college_physics": 0.125, + "computer_network": 0.16666666666666666, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.2692307692307692, + "college_programming": 0.3333333333333333, + "college_physics": 0.16666666666666666, "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.2619047619047619, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.34782608695652173, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.21428571428571427, "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.20833333333333334, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.125, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.36666666666666664, - "business_administration": 0.18421052631578946, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.17857142857142858, + "college_economics": 0.26666666666666666, + "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.27586206896551724, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.25, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.11764705882352941, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.18518518518518517, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.25, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.2, - "middle_school_history": 0.18518518518518517, - "civil_servant": 0.25, - "sports_science": 0.08333333333333333, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.24489795918367346, + "high_school_politics": 0.08333333333333333, + "high_school_geography": 0.25, + "middle_school_politics": 0.38461538461538464, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.3333333333333333, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.08, + "middle_school_history": 0.2962962962962963, + "civil_servant": 0.2692307692307692, + "sports_science": 0.2916666666666667, "plant_protection": 0.18518518518518517, - "basic_medicine": 0.125, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.16666666666666666, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.2222222222222222, - "physician": 0.2777777777777778 + "basic_medicine": 0.375, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.29411764705882354, + "accountant": 0.3148148148148148, + "fire_engineer": 0.2777777777777778, + "environmental_impact_assessment_engineer": 0.25, + "tax_accountant": 0.3148148148148148, + "physician": 0.2962962962962963 } }, - "prompt_4": { - "accuracy": 0.24097135740971357, + "prompt_3": { + "accuracy": 0.26899128268991285, "category_acc": { "computer_network": 0.20833333333333334, - "operating_system": 0.08333333333333333, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.23809523809523808, - "college_physics": 0.125, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.3333333333333333, + "operating_system": 0.5, + "computer_architecture": 0.19230769230769232, + "college_programming": 0.30952380952380953, + "college_physics": 0.16666666666666666, + "college_chemistry": 0.3103448275862069, + "advanced_mathematics": 0.3333333333333333, + "probability_and_statistics": 0.34782608695652173, + "discrete_mathematics": 0.19047619047619047, + "electrical_engineer": 0.21428571428571427, + "metrology_engineer": 0.20689655172413793, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.2916666666666667, "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.08333333333333333, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, "middle_school_chemistry": 0.24, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.31666666666666665, - "business_administration": 0.18421052631578946, + "veterinary_medicine": 0.21428571428571427, + "college_economics": 0.25, + "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.27586206896551724, + "mao_zedong_thought": 0.1724137931034483, "education_science": 0.23529411764705882, - "teacher_qualification": 0.2857142857142857, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.11764705882352941, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.18518518518518517, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.25, - "high_school_chinese": 0.25, - "high_school_history": 0.2, - "middle_school_history": 0.18518518518518517, - "civil_servant": 0.25, - "sports_science": 0.08333333333333333, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.125, + "teacher_qualification": 0.24489795918367346, + "high_school_politics": 0.08333333333333333, + "high_school_geography": 0.25, + "middle_school_politics": 0.38461538461538464, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.3333333333333333, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.08, + "middle_school_history": 0.2962962962962963, + "civil_servant": 0.2692307692307692, + "sports_science": 0.2916666666666667, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.18518518518518517, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.25925925925925924, + "urban_and_rural_planner": 0.29411764705882354, + "accountant": 0.2962962962962963, + "fire_engineer": 0.2777777777777778, + "environmental_impact_assessment_engineer": 0.2777777777777778, + "tax_accountant": 0.3148148148148148, "physician": 0.2777777777777778 } }, - "prompt_5": { - "accuracy": 0.23848069738480698, + "prompt_4": { + "accuracy": 0.2683686176836862, "category_acc": { - "computer_network": 0.125, - "operating_system": 0.125, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.21428571428571427, + "computer_network": 0.3333333333333333, + "operating_system": 0.375, + "computer_architecture": 0.19230769230769232, + "college_programming": 0.30952380952380953, "college_physics": 0.125, "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.20689655172413793, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.21739130434782608, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.23809523809523808, + "metrology_engineer": 0.1724137931034483, "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.125, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.36666666666666664, - "business_administration": 0.18421052631578946, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.25, + "high_school_biology": 0.2916666666666667, + "middle_school_mathematics": 0.16666666666666666, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.28, + "veterinary_medicine": 0.14285714285714285, + "college_economics": 0.21666666666666667, + "business_administration": 0.15789473684210525, "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.27586206896551724, + "mao_zedong_thought": 0.1724137931034483, "education_science": 0.2647058823529412, - "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.25, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.11764705882352941, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.18518518518518517, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.25, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.2, - "middle_school_history": 0.18518518518518517, + "teacher_qualification": 0.24489795918367346, + "high_school_politics": 0.08333333333333333, + "high_school_geography": 0.25, + "middle_school_politics": 0.38461538461538464, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.3333333333333333, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.35714285714285715, + "high_school_chinese": 0.25, + "high_school_history": 0.12, + "middle_school_history": 0.2962962962962963, "civil_servant": 0.25, - "sports_science": 0.08333333333333333, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.125, + "sports_science": 0.375, + "plant_protection": 0.2222222222222222, + "basic_medicine": 0.3333333333333333, "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.16666666666666666, + "urban_and_rural_planner": 0.3137254901960784, + "accountant": 0.2962962962962963, "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.2222222222222222, - "physician": 0.2777777777777778 + "environmental_impact_assessment_engineer": 0.3055555555555556, + "tax_accountant": 0.3148148148148148, + "physician": 0.2962962962962963 } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.2903225806451613 - }, - "prompt_2": { - "accuracy": 0.27598566308243727 - }, - "prompt_3": { - "accuracy": 0.2867383512544803 - }, - "prompt_4": { - "accuracy": 0.2831541218637993 }, "prompt_5": { - "accuracy": 0.2831541218637993 + "accuracy": 0.26650062266500624, + "category_acc": { + "computer_network": 0.20833333333333334, + "operating_system": 0.4583333333333333, + "computer_architecture": 0.3076923076923077, + "college_programming": 0.2619047619047619, + "college_physics": 0.16666666666666666, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.21739130434782608, + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.16666666666666666, + "metrology_engineer": 0.2413793103448276, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.25, + "high_school_chemistry": 0.375, + "high_school_biology": 0.2916666666666667, + "middle_school_mathematics": 0.20833333333333334, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.24, + "veterinary_medicine": 0.17857142857142858, + "college_economics": 0.21666666666666667, + "business_administration": 0.15789473684210525, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.24489795918367346, + "high_school_politics": 0.08333333333333333, + "high_school_geography": 0.25, + "middle_school_politics": 0.38461538461538464, + "middle_school_geography": 0.4117647058823529, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.3333333333333333, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.08, + "middle_school_history": 0.2962962962962963, + "civil_servant": 0.25, + "sports_science": 0.2916666666666667, + "plant_protection": 0.18518518518518517, + "basic_medicine": 0.3333333333333333, + "clinical_medicine": 0.3333333333333333, + "urban_and_rural_planner": 0.29411764705882354, + "accountant": 0.3148148148148148, + "fire_engineer": 0.3055555555555556, + "environmental_impact_assessment_engineer": 0.2777777777777778, + "tax_accountant": 0.3148148148148148, + "physician": 0.2777777777777778 + } + } + }, + "cmmlu": { + "prompt_1": { + "accuracy": 0.2616487455197133 + }, + "prompt_2": { + "accuracy": 0.23297491039426524 + }, + "prompt_3": { + "accuracy": 0.25806451612903225 + }, + "prompt_4": { + "accuracy": 0.2724014336917563 + }, + "prompt_5": { + "accuracy": 0.27956989247311825 } }, "cmmlu_full": { "prompt_1": { - "accuracy": 0.25254705577620445, + "accuracy": 0.24563978587463306, "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.2621951219512195, - "arts": 0.25625, - "astronomy": 0.21212121212121213, - "business_ethics": 0.2535885167464115, - "chinese_civil_service_exam": 0.24375, + "agronomy": 0.2485207100591716, + "anatomy": 0.21621621621621623, + "ancient_chinese": 0.24390243902439024, + "arts": 0.225, + "astronomy": 0.3090909090909091, + "business_ethics": 0.2583732057416268, + "chinese_civil_service_exam": 0.26875, "chinese_driving_rule": 0.25190839694656486, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.24299065420560748, - "chinese_history": 0.24458204334365324, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.24472573839662448, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.308411214953271, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.2222222222222222, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.2358490566037736, + "chinese_food_culture": 0.27205882352941174, + "chinese_foreign_policy": 0.22429906542056074, + "chinese_history": 0.23219814241486067, + "chinese_literature": 0.20098039215686275, + "chinese_teacher_qualification": 0.21787709497206703, + "clinical_knowledge": 0.24050632911392406, + "college_actuarial_science": 0.2169811320754717, + "college_education": 0.21495327102803738, + "college_engineering_hydrology": 0.32075471698113206, + "college_law": 0.23148148148148148, + "college_mathematics": 0.2571428571428571, + "college_medical_statistics": 0.27358490566037735, "college_medicine": 0.25274725274725274, - "computer_science": 0.2696078431372549, - "computer_security": 0.25146198830409355, - "conceptual_physics": 0.23809523809523808, - "construction_project_management": 0.2446043165467626, - "economics": 0.2389937106918239, - "education": 0.25153374233128833, - "electrical_engineering": 0.23255813953488372, - "elementary_chinese": 0.28174603174603174, - "elementary_commonsense": 0.24242424242424243, - "elementary_information_and_technology": 0.2773109243697479, - "elementary_mathematics": 0.2565217391304348, - "ethnology": 0.24444444444444444, - "food_science": 0.24475524475524477, - "genetics": 0.26704545454545453, - "global_facts": 0.24161073825503357, - "high_school_biology": 0.21893491124260356, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.2711864406779661, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.2727272727272727, - "human_sexuality": 0.25396825396825395, - "international_law": 0.25405405405405407, - "journalism": 0.2558139534883721, - "jurisprudence": 0.24817518248175183, - "legal_and_moral_basis": 0.2523364485981308, - "logical": 0.2682926829268293, + "computer_science": 0.22058823529411764, + "computer_security": 0.2631578947368421, + "conceptual_physics": 0.20408163265306123, + "construction_project_management": 0.30935251798561153, + "economics": 0.18238993710691823, + "education": 0.25766871165644173, + "electrical_engineering": 0.22674418604651161, + "elementary_chinese": 0.1865079365079365, + "elementary_commonsense": 0.2676767676767677, + "elementary_information_and_technology": 0.28991596638655465, + "elementary_mathematics": 0.23043478260869565, + "ethnology": 0.3037037037037037, + "food_science": 0.26573426573426573, + "genetics": 0.3125, + "global_facts": 0.2348993288590604, + "high_school_biology": 0.20710059171597633, + "high_school_chemistry": 0.2727272727272727, + "high_school_geography": 0.2288135593220339, + "high_school_mathematics": 0.21341463414634146, + "high_school_physics": 0.24545454545454545, + "high_school_politics": 0.1958041958041958, + "human_sexuality": 0.1984126984126984, + "international_law": 0.2594594594594595, + "journalism": 0.20348837209302326, + "jurisprudence": 0.25060827250608275, + "legal_and_moral_basis": 0.27102803738317754, + "logical": 0.24390243902439024, "machine_learning": 0.2540983606557377, - "management": 0.24761904761904763, - "marketing": 0.24444444444444444, - "marxist_theory": 0.23809523809523808, - "modern_chinese": 0.27586206896551724, + "management": 0.22380952380952382, + "marketing": 0.2111111111111111, + "marxist_theory": 0.21693121693121692, + "modern_chinese": 0.31896551724137934, "nutrition": 0.23448275862068965, "philosophy": 0.2571428571428571, - "professional_accounting": 0.25142857142857145, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.2553191489361702, + "professional_accounting": 0.2857142857142857, + "professional_law": 0.22748815165876776, + "professional_medicine": 0.25, "professional_psychology": 0.24568965517241378, - "public_relations": 0.2471264367816092, - "security_study": 0.25925925925925924, - "sociology": 0.25663716814159293, - "sports_science": 0.24242424242424243, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.2485207100591716, - "world_history": 0.2360248447204969, - "world_religions": 0.26875 + "public_relations": 0.28735632183908044, + "security_study": 0.2222222222222222, + "sociology": 0.19911504424778761, + "sports_science": 0.3393939393939394, + "traditional_chinese_medicine": 0.2648648648648649, + "virology": 0.22485207100591717, + "world_history": 0.2732919254658385, + "world_religions": 0.275 } }, "prompt_2": { - "accuracy": 0.25254705577620445, + "accuracy": 0.2487480573303402, "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.2682926829268293, - "arts": 0.25625, - "astronomy": 0.22424242424242424, - "business_ethics": 0.2535885167464115, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.25190839694656486, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.24299065420560748, - "chinese_history": 0.24458204334365324, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.25316455696202533, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.308411214953271, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.2222222222222222, - "college_mathematics": 0.22857142857142856, + "agronomy": 0.2958579881656805, + "anatomy": 0.25, + "ancient_chinese": 0.21341463414634146, + "arts": 0.225, + "astronomy": 0.296969696969697, + "business_ethics": 0.27751196172248804, + "chinese_civil_service_exam": 0.25625, + "chinese_driving_rule": 0.24427480916030533, + "chinese_food_culture": 0.3014705882352941, + "chinese_foreign_policy": 0.2616822429906542, + "chinese_history": 0.24148606811145512, + "chinese_literature": 0.2696078431372549, + "chinese_teacher_qualification": 0.29608938547486036, + "clinical_knowledge": 0.25738396624472576, + "college_actuarial_science": 0.2169811320754717, + "college_education": 0.24299065420560748, + "college_engineering_hydrology": 0.3018867924528302, + "college_law": 0.16666666666666666, + "college_mathematics": 0.3333333333333333, "college_medical_statistics": 0.2358490566037736, - "college_medicine": 0.25274725274725274, - "computer_science": 0.25980392156862747, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.23809523809523808, - "construction_project_management": 0.2446043165467626, - "economics": 0.2389937106918239, - "education": 0.25153374233128833, - "electrical_engineering": 0.23255813953488372, - "elementary_chinese": 0.28174603174603174, - "elementary_commonsense": 0.23737373737373738, - "elementary_information_and_technology": 0.2773109243697479, - "elementary_mathematics": 0.2565217391304348, - "ethnology": 0.24444444444444444, - "food_science": 0.23776223776223776, - "genetics": 0.2784090909090909, - "global_facts": 0.24161073825503357, - "high_school_biology": 0.21893491124260356, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.2711864406779661, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.2818181818181818, - "high_school_politics": 0.2727272727272727, - "human_sexuality": 0.25396825396825395, - "international_law": 0.25405405405405407, - "journalism": 0.25, - "jurisprudence": 0.24817518248175183, - "legal_and_moral_basis": 0.2523364485981308, - "logical": 0.2682926829268293, - "machine_learning": 0.2459016393442623, - "management": 0.24285714285714285, - "marketing": 0.24444444444444444, - "marxist_theory": 0.23809523809523808, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.23448275862068965, - "philosophy": 0.2571428571428571, - "professional_accounting": 0.25142857142857145, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.2553191489361702, - "professional_psychology": 0.24568965517241378, - "public_relations": 0.2471264367816092, - "security_study": 0.26666666666666666, - "sociology": 0.252212389380531, - "sports_science": 0.24242424242424243, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.2485207100591716, - "world_history": 0.2360248447204969, - "world_religions": 0.26875 + "college_medicine": 0.20512820512820512, + "computer_science": 0.27941176470588236, + "computer_security": 0.25146198830409355, + "conceptual_physics": 0.20408163265306123, + "construction_project_management": 0.26618705035971224, + "economics": 0.23270440251572327, + "education": 0.24539877300613497, + "electrical_engineering": 0.3023255813953488, + "elementary_chinese": 0.21825396825396826, + "elementary_commonsense": 0.23232323232323232, + "elementary_information_and_technology": 0.24369747899159663, + "elementary_mathematics": 0.26521739130434785, + "ethnology": 0.22962962962962963, + "food_science": 0.25874125874125875, + "genetics": 0.2727272727272727, + "global_facts": 0.2348993288590604, + "high_school_biology": 0.20710059171597633, + "high_school_chemistry": 0.24242424242424243, + "high_school_geography": 0.211864406779661, + "high_school_mathematics": 0.20121951219512196, + "high_school_physics": 0.2, + "high_school_politics": 0.24475524475524477, + "human_sexuality": 0.2857142857142857, + "international_law": 0.2918918918918919, + "journalism": 0.23255813953488372, + "jurisprudence": 0.26520681265206814, + "legal_and_moral_basis": 0.205607476635514, + "logical": 0.22764227642276422, + "machine_learning": 0.22131147540983606, + "management": 0.19523809523809524, + "marketing": 0.2388888888888889, + "marxist_theory": 0.1693121693121693, + "modern_chinese": 0.23275862068965517, + "nutrition": 0.2689655172413793, + "philosophy": 0.26666666666666666, + "professional_accounting": 0.21142857142857144, + "professional_law": 0.2559241706161137, + "professional_medicine": 0.2579787234042553, + "professional_psychology": 0.2672413793103448, + "public_relations": 0.3160919540229885, + "security_study": 0.22962962962962963, + "sociology": 0.22566371681415928, + "sports_science": 0.2727272727272727, + "traditional_chinese_medicine": 0.25405405405405407, + "virology": 0.25443786982248523, + "world_history": 0.2484472049689441, + "world_religions": 0.33125 } }, "prompt_3": { - "accuracy": 0.25254705577620445, + "accuracy": 0.25755482645484373, "category_acc": { - "agronomy": 0.2485207100591716, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.2682926829268293, - "arts": 0.25625, - "astronomy": 0.20606060606060606, - "business_ethics": 0.2535885167464115, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.25190839694656486, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.24299065420560748, - "chinese_history": 0.24458204334365324, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.25139664804469275, + "agronomy": 0.27218934911242604, + "anatomy": 0.2702702702702703, + "ancient_chinese": 0.2073170731707317, + "arts": 0.26875, + "astronomy": 0.2606060606060606, + "business_ethics": 0.2822966507177033, + "chinese_civil_service_exam": 0.2625, + "chinese_driving_rule": 0.29770992366412213, + "chinese_food_culture": 0.29411764705882354, + "chinese_foreign_policy": 0.205607476635514, + "chinese_history": 0.25077399380804954, + "chinese_literature": 0.2696078431372549, + "chinese_teacher_qualification": 0.20670391061452514, "clinical_knowledge": 0.2489451476793249, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.308411214953271, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.2222222222222222, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.2358490566037736, - "college_medicine": 0.2454212454212454, - "computer_science": 0.2647058823529412, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.23809523809523808, - "construction_project_management": 0.2446043165467626, + "college_actuarial_science": 0.25471698113207547, + "college_education": 0.27102803738317754, + "college_engineering_hydrology": 0.32075471698113206, + "college_law": 0.28703703703703703, + "college_mathematics": 0.3333333333333333, + "college_medical_statistics": 0.3018867924528302, + "college_medicine": 0.2490842490842491, + "computer_science": 0.27450980392156865, + "computer_security": 0.30994152046783624, + "conceptual_physics": 0.272108843537415, + "construction_project_management": 0.26618705035971224, "economics": 0.2389937106918239, - "education": 0.25153374233128833, - "electrical_engineering": 0.23255813953488372, - "elementary_chinese": 0.28174603174603174, - "elementary_commonsense": 0.23737373737373738, - "elementary_information_and_technology": 0.2773109243697479, - "elementary_mathematics": 0.2608695652173913, - "ethnology": 0.24444444444444444, - "food_science": 0.2517482517482518, - "genetics": 0.26704545454545453, - "global_facts": 0.2483221476510067, - "high_school_biology": 0.21893491124260356, + "education": 0.2392638036809816, + "electrical_engineering": 0.27906976744186046, + "elementary_chinese": 0.26587301587301587, + "elementary_commonsense": 0.2474747474747475, + "elementary_information_and_technology": 0.29411764705882354, + "elementary_mathematics": 0.29130434782608694, + "ethnology": 0.22962962962962963, + "food_science": 0.27972027972027974, + "genetics": 0.23295454545454544, + "global_facts": 0.22818791946308725, + "high_school_biology": 0.23076923076923078, "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.2711864406779661, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.27972027972027974, - "human_sexuality": 0.25396825396825395, - "international_law": 0.25405405405405407, - "journalism": 0.2558139534883721, - "jurisprudence": 0.24817518248175183, + "high_school_geography": 0.22033898305084745, + "high_school_mathematics": 0.23780487804878048, + "high_school_physics": 0.24545454545454545, + "high_school_politics": 0.23776223776223776, + "human_sexuality": 0.20634920634920634, + "international_law": 0.1891891891891892, + "journalism": 0.20930232558139536, + "jurisprudence": 0.25304136253041365, "legal_and_moral_basis": 0.2523364485981308, - "logical": 0.2682926829268293, - "machine_learning": 0.2540983606557377, - "management": 0.24285714285714285, - "marketing": 0.24444444444444444, - "marxist_theory": 0.23809523809523808, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.23448275862068965, - "philosophy": 0.2571428571428571, - "professional_accounting": 0.25142857142857145, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.2553191489361702, - "professional_psychology": 0.24568965517241378, - "public_relations": 0.2471264367816092, - "security_study": 0.25925925925925924, - "sociology": 0.252212389380531, - "sports_science": 0.24848484848484848, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.25443786982248523, - "world_history": 0.2360248447204969, - "world_religions": 0.26875 + "logical": 0.25203252032520324, + "machine_learning": 0.23770491803278687, + "management": 0.2714285714285714, + "marketing": 0.2833333333333333, + "marxist_theory": 0.24338624338624337, + "modern_chinese": 0.2672413793103448, + "nutrition": 0.2413793103448276, + "philosophy": 0.22857142857142856, + "professional_accounting": 0.28, + "professional_law": 0.21800947867298578, + "professional_medicine": 0.2526595744680851, + "professional_psychology": 0.22413793103448276, + "public_relations": 0.25862068965517243, + "security_study": 0.3037037037037037, + "sociology": 0.22123893805309736, + "sports_science": 0.2606060606060606, + "traditional_chinese_medicine": 0.33513513513513515, + "virology": 0.2781065088757396, + "world_history": 0.2857142857142857, + "world_religions": 0.2625 } }, "prompt_4": { - "accuracy": 0.2536694871352098, + "accuracy": 0.2485753755828009, "category_acc": { - "agronomy": 0.23668639053254437, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.27439024390243905, - "arts": 0.25625, - "astronomy": 0.22424242424242424, - "business_ethics": 0.24880382775119617, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.25190839694656486, - "chinese_food_culture": 0.25735294117647056, - "chinese_foreign_policy": 0.24299065420560748, - "chinese_history": 0.25077399380804954, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.2489451476793249, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.29906542056074764, - "college_engineering_hydrology": 0.2830188679245283, - "college_law": 0.21296296296296297, - "college_mathematics": 0.21904761904761905, + "agronomy": 0.24260355029585798, + "anatomy": 0.23648648648648649, + "ancient_chinese": 0.21951219512195122, + "arts": 0.25, + "astronomy": 0.24242424242424243, + "business_ethics": 0.3062200956937799, + "chinese_civil_service_exam": 0.3125, + "chinese_driving_rule": 0.21374045801526717, + "chinese_food_culture": 0.2647058823529412, + "chinese_foreign_policy": 0.29906542056074764, + "chinese_history": 0.2260061919504644, + "chinese_literature": 0.23039215686274508, + "chinese_teacher_qualification": 0.22905027932960895, + "clinical_knowledge": 0.270042194092827, + "college_actuarial_science": 0.20754716981132076, + "college_education": 0.17757009345794392, + "college_engineering_hydrology": 0.2358490566037736, + "college_law": 0.23148148148148148, + "college_mathematics": 0.29523809523809524, "college_medical_statistics": 0.2641509433962264, - "college_medicine": 0.2564102564102564, - "computer_science": 0.2647058823529412, - "computer_security": 0.2631578947368421, - "conceptual_physics": 0.25170068027210885, - "construction_project_management": 0.23741007194244604, - "economics": 0.23270440251572327, - "education": 0.25153374233128833, - "electrical_engineering": 0.25, - "elementary_chinese": 0.2777777777777778, - "elementary_commonsense": 0.24242424242424243, - "elementary_information_and_technology": 0.29831932773109243, - "elementary_mathematics": 0.2782608695652174, - "ethnology": 0.24444444444444444, - "food_science": 0.24475524475524477, - "genetics": 0.24431818181818182, - "global_facts": 0.2483221476510067, - "high_school_biology": 0.23668639053254437, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.2542372881355932, - "high_school_mathematics": 0.27439024390243905, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.2727272727272727, - "human_sexuality": 0.25396825396825395, - "international_law": 0.24864864864864866, - "journalism": 0.25, - "jurisprudence": 0.25060827250608275, - "legal_and_moral_basis": 0.2523364485981308, - "logical": 0.2601626016260163, - "machine_learning": 0.2459016393442623, - "management": 0.24285714285714285, - "marketing": 0.25, - "marxist_theory": 0.24338624338624337, - "modern_chinese": 0.25862068965517243, + "college_medicine": 0.21611721611721613, + "computer_science": 0.25, + "computer_security": 0.29239766081871343, + "conceptual_physics": 0.30612244897959184, + "construction_project_management": 0.26618705035971224, + "economics": 0.25157232704402516, + "education": 0.2883435582822086, + "electrical_engineering": 0.23837209302325582, + "elementary_chinese": 0.27380952380952384, + "elementary_commonsense": 0.22727272727272727, + "elementary_information_and_technology": 0.18067226890756302, + "elementary_mathematics": 0.24782608695652175, + "ethnology": 0.26666666666666666, + "food_science": 0.27972027972027974, + "genetics": 0.29545454545454547, + "global_facts": 0.2080536912751678, + "high_school_biology": 0.28402366863905326, + "high_school_chemistry": 0.25757575757575757, + "high_school_geography": 0.22033898305084745, + "high_school_mathematics": 0.2073170731707317, + "high_school_physics": 0.2909090909090909, + "high_school_politics": 0.25874125874125875, + "human_sexuality": 0.2698412698412698, + "international_law": 0.2972972972972973, + "journalism": 0.20930232558139536, + "jurisprudence": 0.25547445255474455, + "legal_and_moral_basis": 0.22429906542056074, + "logical": 0.21138211382113822, + "machine_learning": 0.26229508196721313, + "management": 0.24285714285714285, + "marketing": 0.2777777777777778, + "marxist_theory": 0.24867724867724866, + "modern_chinese": 0.21551724137931033, "nutrition": 0.2482758620689655, - "philosophy": 0.23809523809523808, - "professional_accounting": 0.25142857142857145, - "professional_law": 0.26066350710900477, - "professional_medicine": 0.25, - "professional_psychology": 0.25, - "public_relations": 0.2471264367816092, - "security_study": 0.2740740740740741, - "sociology": 0.24778761061946902, - "sports_science": 0.2545454545454545, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.2485207100591716, + "philosophy": 0.2857142857142857, + "professional_accounting": 0.16, + "professional_law": 0.22274881516587677, + "professional_medicine": 0.23138297872340424, + "professional_psychology": 0.28879310344827586, + "public_relations": 0.3218390804597701, + "security_study": 0.2074074074074074, + "sociology": 0.22123893805309736, + "sports_science": 0.2, + "traditional_chinese_medicine": 0.25405405405405407, + "virology": 0.23668639053254437, "world_history": 0.2422360248447205, - "world_religions": 0.25625 + "world_religions": 0.31875 } }, "prompt_5": { - "accuracy": 0.2526333966499741, + "accuracy": 0.25392850975651876, "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.2682926829268293, - "arts": 0.25625, - "astronomy": 0.22424242424242424, + "agronomy": 0.28994082840236685, + "anatomy": 0.2702702702702703, + "ancient_chinese": 0.2621951219512195, + "arts": 0.25, + "astronomy": 0.2727272727272727, "business_ethics": 0.2535885167464115, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.25190839694656486, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.24299065420560748, - "chinese_history": 0.24458204334365324, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.24472573839662448, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.308411214953271, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.2222222222222222, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.2358490566037736, - "college_medicine": 0.2454212454212454, - "computer_science": 0.2696078431372549, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.23809523809523808, - "construction_project_management": 0.2517985611510791, - "economics": 0.2389937106918239, - "education": 0.25153374233128833, - "electrical_engineering": 0.23255813953488372, - "elementary_chinese": 0.28174603174603174, - "elementary_commonsense": 0.23737373737373738, - "elementary_information_and_technology": 0.2773109243697479, - "elementary_mathematics": 0.2608695652173913, - "ethnology": 0.24444444444444444, - "food_science": 0.24475524475524477, - "genetics": 0.2784090909090909, - "global_facts": 0.24161073825503357, - "high_school_biology": 0.21893491124260356, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.2711864406779661, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.2727272727272727, + "chinese_civil_service_exam": 0.28125, + "chinese_driving_rule": 0.2748091603053435, + "chinese_food_culture": 0.27941176470588236, + "chinese_foreign_policy": 0.205607476635514, + "chinese_history": 0.24148606811145512, + "chinese_literature": 0.28431372549019607, + "chinese_teacher_qualification": 0.2122905027932961, + "clinical_knowledge": 0.22784810126582278, + "college_actuarial_science": 0.19811320754716982, + "college_education": 0.18691588785046728, + "college_engineering_hydrology": 0.27358490566037735, + "college_law": 0.2777777777777778, + "college_mathematics": 0.29523809523809524, + "college_medical_statistics": 0.19811320754716982, + "college_medicine": 0.2564102564102564, + "computer_science": 0.21568627450980393, + "computer_security": 0.23391812865497075, + "conceptual_physics": 0.2925170068027211, + "construction_project_management": 0.23741007194244604, + "economics": 0.22641509433962265, + "education": 0.26993865030674846, + "electrical_engineering": 0.2558139534883721, + "elementary_chinese": 0.2222222222222222, + "elementary_commonsense": 0.2828282828282828, + "elementary_information_and_technology": 0.25210084033613445, + "elementary_mathematics": 0.25217391304347825, + "ethnology": 0.28888888888888886, + "food_science": 0.2727272727272727, + "genetics": 0.3068181818181818, + "global_facts": 0.2214765100671141, + "high_school_biology": 0.23668639053254437, + "high_school_chemistry": 0.20454545454545456, + "high_school_geography": 0.2796610169491525, + "high_school_mathematics": 0.23170731707317074, + "high_school_physics": 0.22727272727272727, + "high_school_politics": 0.21678321678321677, "human_sexuality": 0.25396825396825395, - "international_law": 0.25405405405405407, - "journalism": 0.2558139534883721, - "jurisprudence": 0.24817518248175183, - "legal_and_moral_basis": 0.2523364485981308, - "logical": 0.2682926829268293, - "machine_learning": 0.2459016393442623, - "management": 0.24285714285714285, - "marketing": 0.24444444444444444, - "marxist_theory": 0.23809523809523808, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.2413793103448276, - "philosophy": 0.2571428571428571, - "professional_accounting": 0.25142857142857145, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.2553191489361702, - "professional_psychology": 0.24568965517241378, - "public_relations": 0.2471264367816092, - "security_study": 0.25925925925925924, - "sociology": 0.25663716814159293, - "sports_science": 0.24242424242424243, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.2485207100591716, - "world_history": 0.2360248447204969, - "world_religions": 0.26875 + "international_law": 0.21621621621621623, + "journalism": 0.22093023255813954, + "jurisprudence": 0.25547445255474455, + "legal_and_moral_basis": 0.24766355140186916, + "logical": 0.2845528455284553, + "machine_learning": 0.27049180327868855, + "management": 0.26666666666666666, + "marketing": 0.22777777777777777, + "marxist_theory": 0.20105820105820105, + "modern_chinese": 0.22413793103448276, + "nutrition": 0.22758620689655173, + "philosophy": 0.23809523809523808, + "professional_accounting": 0.2857142857142857, + "professional_law": 0.26540284360189575, + "professional_medicine": 0.26595744680851063, + "professional_psychology": 0.27586206896551724, + "public_relations": 0.27011494252873564, + "security_study": 0.2814814814814815, + "sociology": 0.30973451327433627, + "sports_science": 0.23636363636363636, + "traditional_chinese_medicine": 0.25405405405405407, + "virology": 0.3076923076923077, + "world_history": 0.2670807453416149, + "world_religions": 0.2875 } } }, "zbench": { "prompt_1": { - "accuracy": 0.18181818181818182 + "accuracy": 0.5454545454545454 }, "prompt_2": { - "accuracy": 0.18181818181818182 + "accuracy": 0.3939393939393939 }, "prompt_3": { - "accuracy": 0.18181818181818182 + "accuracy": 0.42424242424242425 }, "prompt_4": { - "accuracy": 0.15151515151515152 + "accuracy": 0.36363636363636365 }, "prompt_5": { - "accuracy": 0.18181818181818182 + "accuracy": 0.42424242424242425 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.5 + "accuracy": 0.36818181818181817 }, "prompt_2": { - "accuracy": 0.47045454545454546 + "accuracy": 0.425 }, "prompt_3": { - "accuracy": 0.4863636363636364 + "accuracy": 0.4090909090909091 }, "prompt_4": { - "accuracy": 0.41363636363636364 + "accuracy": 0.4431818181818182 }, "prompt_5": { - "accuracy": 0.46136363636363636 + "accuracy": 0.4590909090909091 } }, "ocnli": { "prompt_1": { - "accuracy": 0.3440677966101695 + "accuracy": 0.3257627118644068 }, "prompt_2": { - "accuracy": 0.34372881355932206 + "accuracy": 0.31966101694915255 }, "prompt_3": { - "accuracy": 0.3440677966101695 + "accuracy": 0.31559322033898307 }, "prompt_4": { - "accuracy": 0.3440677966101695 + "accuracy": 0.32610169491525426 }, "prompt_5": { - "accuracy": 0.3440677966101695 + "accuracy": 0.3247457627118644 } }, "c3": { "prompt_1": { - "accuracy": 0.2707554225878833 + "accuracy": 0.275243081525804 }, "prompt_2": { - "accuracy": 0.2647718773373224 + "accuracy": 0.27599102468212416 }, "prompt_3": { - "accuracy": 0.2673896783844428 + "accuracy": 0.27898279730740466 }, "prompt_4": { - "accuracy": 0.2643979057591623 + "accuracy": 0.27225130890052357 }, "prompt_5": { - "accuracy": 0.26327599102468213 + "accuracy": 0.2801047120418848 } }, "dream": { "prompt_1": { - "accuracy": 0.9363057324840764 + "accuracy": 0.9216070553650172 }, "prompt_2": { - "accuracy": 0.9318961293483586 + "accuracy": 0.924546790788829 }, "prompt_3": { - "accuracy": 0.9353258206761391 + "accuracy": 0.9220970112689858 }, "prompt_4": { - "accuracy": 0.9372856442920137 + "accuracy": 0.9230769230769231 }, "prompt_5": { - "accuracy": 0.9348358647721705 + "accuracy": 0.9240568348848603 } }, "samsum": { "prompt_1": { - "rouge1": 0.5392068754143731, - "rouge2": 0.2950280260068732, - "rougeL": 0.4506368895080014, - "avg_rouge": 0.4282905969764159 + "rouge1": 0.5283994525332824, + "rouge2": 0.28407900462604924, + "rougeL": 0.4438398405240641, + "avg_rouge": 0.41877276589446516 }, "prompt_2": { - "rouge1": 0.5359945124407804, - "rouge2": 0.29276027430028645, - "rougeL": 0.44971572354652717, - "avg_rouge": 0.4261568367625314 + "rouge1": 0.5310040504588704, + "rouge2": 0.2889554535263174, + "rougeL": 0.4448795824256869, + "avg_rouge": 0.4216130288036249 }, "prompt_3": { - "rouge1": 0.5369216789882587, - "rouge2": 0.2932230928399554, - "rougeL": 0.4508405893944396, - "avg_rouge": 0.4269951204075513 + "rouge1": 0.5301678345319392, + "rouge2": 0.2888796755768626, + "rougeL": 0.44474721577330006, + "avg_rouge": 0.42126490862736726 }, "prompt_4": { - "rouge1": 0.538447184048574, - "rouge2": 0.29528202636477136, - "rougeL": 0.4516316980089774, - "avg_rouge": 0.42845363614077425 + "rouge1": 0.5289238690020692, + "rouge2": 0.286647554142563, + "rougeL": 0.4426701857145179, + "avg_rouge": 0.4194138696197167 }, "prompt_5": { - "rouge1": 0.5365832097389195, - "rouge2": 0.2954036639870932, - "rougeL": 0.45167227299325957, - "avg_rouge": 0.42788638223975745 + "rouge1": 0.5286747119788637, + "rouge2": 0.28676322097611173, + "rougeL": 0.44334451795455654, + "avg_rouge": 0.4195941503031773 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.23098825839708775, - "rouge2": 0.06235340689809025, - "rougeL": 0.1710693119069414, - "avg_rouge": 0.15480365906737314 + "rouge1": 0.2348990714241612, + "rouge2": 0.0654692982269716, + "rougeL": 0.17545716448016013, + "avg_rouge": 0.15860851137709764 }, "prompt_2": { - "rouge1": 0.23057479562213193, - "rouge2": 0.0626605453865275, - "rougeL": 0.16987328159127832, - "avg_rouge": 0.15436954086664592 + "rouge1": 0.23235043239609107, + "rouge2": 0.06516831879832731, + "rougeL": 0.17253374137518865, + "avg_rouge": 0.156684164189869 }, "prompt_3": { - "rouge1": 0.23134441905824937, - "rouge2": 0.06285173990336988, - "rougeL": 0.17136941481315845, - "avg_rouge": 0.15518852459159257 + "rouge1": 0.2318659794631441, + "rouge2": 0.064963463287211, + "rougeL": 0.1716820969020561, + "avg_rouge": 0.1561705132174704 }, "prompt_4": { - "rouge1": 0.23060303444666233, - "rouge2": 0.06339967025872248, - "rougeL": 0.16971858763937736, - "avg_rouge": 0.15457376411492071 + "rouge1": 0.2347594030365569, + "rouge2": 0.06517399918190821, + "rougeL": 0.17496958526054637, + "avg_rouge": 0.15830099582633717 }, "prompt_5": { - "rouge1": 0.23634127906884056, - "rouge2": 0.06468803693978939, - "rougeL": 0.17715576683843468, - "avg_rouge": 0.15939502761568822 + "rouge1": 0.23476352815147197, + "rouge2": 0.06338707364815505, + "rougeL": 0.18093639777893736, + "avg_rouge": 0.15969566652618813 } }, "sst2": { "prompt_1": { - "accuracy": 0.9529816513761468 + "accuracy": 0.9598623853211009 }, "prompt_2": { - "accuracy": 0.9506880733944955 + "accuracy": 0.9598623853211009 }, "prompt_3": { - "accuracy": 0.9564220183486238 + "accuracy": 0.9541284403669725 }, "prompt_4": { - "accuracy": 0.9518348623853211 + "accuracy": 0.9587155963302753 }, "prompt_5": { - "accuracy": 0.9575688073394495 + "accuracy": 0.9541284403669725 } }, "cola": { "prompt_1": { - "accuracy": 0.8235858101629914 + "accuracy": 0.8072866730584851 }, "prompt_2": { - "accuracy": 0.8235858101629914 + "accuracy": 0.8072866730584851 }, "prompt_3": { - "accuracy": 0.8245445829338447 + "accuracy": 0.8015340364333653 }, "prompt_4": { - "accuracy": 0.8216682646212847 + "accuracy": 0.8015340364333653 }, "prompt_5": { - "accuracy": 0.8235858101629914 + "accuracy": 0.8034515819750719 } }, "qqp": { "prompt_1": { - "accuracy": 0.8415 + "accuracy": 0.844 }, "prompt_2": { - "accuracy": 0.8435 + "accuracy": 0.8665 }, "prompt_3": { - "accuracy": 0.838 + "accuracy": 0.8565 }, "prompt_4": { - "accuracy": 0.849 + "accuracy": 0.863 }, "prompt_5": { - "accuracy": 0.843 + "accuracy": 0.864 } }, "mnli": { "prompt_1": { - "accuracy": 0.8925 + "accuracy": 1.0 }, "prompt_2": { - "accuracy": 0.8915 + "accuracy": 1.0 }, "prompt_3": { - "accuracy": 0.893 + "accuracy": 1.0 }, "prompt_4": { - "accuracy": 0.895 + "accuracy": 1.0 }, "prompt_5": { - "accuracy": 0.888 + "accuracy": 1.0 } }, "qnli": { "prompt_1": { - "accuracy": 0.948 + "accuracy": 1.0 }, "prompt_2": { - "accuracy": 0.9475 + "accuracy": 1.0 }, "prompt_3": { - "accuracy": 0.946 + "accuracy": 1.0 }, "prompt_4": { - "accuracy": 0.947 + "accuracy": 1.0 }, "prompt_5": { - "accuracy": 0.947 + "accuracy": 1.0 } }, "wnli": { "prompt_1": { - "accuracy": 0.704225352112676 + "accuracy": 0.8 }, "prompt_2": { - "accuracy": 0.704225352112676 + "accuracy": 0.8 }, "prompt_3": { - "accuracy": 0.676056338028169 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.6901408450704225 + "accuracy": 0.8 }, "prompt_5": { - "accuracy": 0.704225352112676 + "accuracy": 0.8 } }, "rte": { "prompt_1": { - "accuracy": 0.924187725631769 + "accuracy": 1.0 }, "prompt_2": { - "accuracy": 0.9205776173285198 + "accuracy": 1.0 }, "prompt_3": { - "accuracy": 0.924187725631769 + "accuracy": 1.0 }, "prompt_4": { - "accuracy": 0.9133574007220217 + "accuracy": 1.0 }, "prompt_5": { - "accuracy": 0.9169675090252708 + "accuracy": 1.0 } }, "mrpc": { "prompt_1": { - "accuracy": 0.7647058823529411 + "accuracy": 0.8 }, "prompt_2": { - "accuracy": 0.7647058823529411 + "accuracy": 0.8 }, "prompt_3": { - "accuracy": 0.7769607843137255 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.7818627450980392 + "accuracy": 0.8 }, "prompt_5": { - "accuracy": 0.7622549019607843 + "accuracy": 0.8 } } }, @@ -57573,79731 +44230,19303 @@ "prompt_1": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.4066666666666666, - "language_acc": { - "Spanish": 0.4666666666666667, - "English": 0.6266666666666667, - "Chinese": 0.26, - "Vietnamese": 0.30666666666666664, - "Filipino": 0.4066666666666667, - "Indonesian": 0.42, - "Malay": 0.36 - }, - "consistency_score_2": 0.4574603174603175, - "consistency_score_3": 0.2666666666666667, - "consistency_score_4": 0.1780952380952381, - "consistency_score_5": 0.1301587301587301, - "consistency_score_6": 0.10285714285714287, - "consistency_score_7": 0.08666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6066666666666667, - "Spanish,Chinese": 0.3, - "Spanish,Vietnamese": 0.4, - "Spanish,Filipino": 0.43333333333333335, - "Spanish,Indonesian": 0.6133333333333333, - "Spanish,Malay": 0.4866666666666667, - "English,Chinese": 0.34, - "English,Vietnamese": 0.35333333333333333, - "English,Filipino": 0.5066666666666667, - "English,Indonesian": 0.54, - "English,Malay": 0.44, - "Chinese,Vietnamese": 0.5466666666666666, - "Chinese,Filipino": 0.38, - "Chinese,Indonesian": 0.38666666666666666, - "Chinese,Malay": 0.32666666666666666, - "Vietnamese,Filipino": 0.43333333333333335, - "Vietnamese,Indonesian": 0.44, - "Vietnamese,Malay": 0.43333333333333335, - "Filipino,Indonesian": 0.54, - "Filipino,Malay": 0.4866666666666667, - "Indonesian,Malay": 0.6133333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.2, - "Spanish,English,Vietnamese": 0.26, - "Spanish,English,Filipino": 0.3466666666666667, - "Spanish,English,Indonesian": 0.4266666666666667, - "Spanish,English,Malay": 0.34, - "Spanish,Chinese,Vietnamese": 0.22, - "Spanish,Chinese,Filipino": 0.17333333333333334, - "Spanish,Chinese,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Malay": 0.16, - "Spanish,Vietnamese,Filipino": 0.24, - "Spanish,Vietnamese,Indonesian": 0.29333333333333333, - "Spanish,Vietnamese,Malay": 0.24666666666666667, - "Spanish,Filipino,Indonesian": 0.36666666666666664, - "Spanish,Filipino,Malay": 0.29333333333333333, - "Spanish,Indonesian,Malay": 0.4, - "English,Chinese,Vietnamese": 0.20666666666666667, - "English,Chinese,Filipino": 0.20666666666666667, - "English,Chinese,Indonesian": 0.22, - "English,Chinese,Malay": 0.16666666666666666, - "English,Vietnamese,Filipino": 0.24, - "English,Vietnamese,Indonesian": 0.24666666666666667, - "English,Vietnamese,Malay": 0.23333333333333334, - "English,Filipino,Indonesian": 0.36, - "English,Filipino,Malay": 0.30666666666666664, - "English,Indonesian,Malay": 0.37333333333333335, - "Chinese,Vietnamese,Filipino": 0.24, - "Chinese,Vietnamese,Indonesian": 0.26, - "Chinese,Vietnamese,Malay": 0.22, - "Chinese,Filipino,Indonesian": 0.24666666666666667, - "Chinese,Filipino,Malay": 0.2, - "Chinese,Indonesian,Malay": 0.23333333333333334, - "Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Vietnamese,Filipino,Malay": 0.26, - "Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Filipino,Indonesian,Malay": 0.35333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.14, - "Spanish,English,Chinese,Filipino": 0.14, - "Spanish,English,Chinese,Indonesian": 0.16, - "Spanish,English,Chinese,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,English,Vietnamese,Indonesian": 0.2, - "Spanish,English,Vietnamese,Malay": 0.18666666666666668, - "Spanish,English,Filipino,Indonesian": 0.30666666666666664, - "Spanish,English,Filipino,Malay": 0.24, - "Spanish,English,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Malay": 0.13333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.12666666666666668, - "Spanish,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.2, - "Spanish,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.14, - "English,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian": 0.16, - "English,Chinese,Filipino,Malay": 0.14, - "English,Chinese,Indonesian,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "English,Vietnamese,Filipino,Malay": 0.17333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "English,Filipino,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.1, - "Spanish,English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.12, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 - } - }, - "AC3_2": 0.430570658779465, - "AC3_3": 0.3221122111732837, - "AC3_4": 0.24770901190117897, - "AC3_5": 0.19720086730014683, - "AC3_6": 0.1641869158556269, - "AC3_7": 0.14288288285392012 - } + "prompt_1": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.3871753246753246, - "language_acc": { - "English": 0.6136363636363636, - "Filipino": 0.3181818181818182, - "Spanish": 0.4772727272727273, - "Malay": 0.3806818181818182, - "Chinese": 0.24431818181818182, - "Indonesian": 0.38636363636363635, - "Vietnamese": 0.2897727272727273 - }, - "consistency_score_2": 0.41991341991341985, - "consistency_score_3": 0.21899350649350646, - "consistency_score_4": 0.12792207792207794, - "consistency_score_5": 0.08170995670995669, - "consistency_score_6": 0.05600649350649351, - "consistency_score_7": 0.03977272727272727, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3977272727272727, - "English,Spanish": 0.6420454545454546, - "English,Malay": 0.4715909090909091, - "English,Chinese": 0.23295454545454544, - "English,Indonesian": 0.5113636363636364, - "English,Vietnamese": 0.30113636363636365, - "Filipino,Spanish": 0.4147727272727273, - "Filipino,Malay": 0.5113636363636364, - "Filipino,Chinese": 0.36363636363636365, - "Filipino,Indonesian": 0.4715909090909091, - "Filipino,Vietnamese": 0.3693181818181818, - "Spanish,Malay": 0.48863636363636365, - "Spanish,Chinese": 0.26704545454545453, - "Spanish,Indonesian": 0.5, - "Spanish,Vietnamese": 0.3125, - "Malay,Chinese": 0.375, - "Malay,Indonesian": 0.5795454545454546, - "Malay,Vietnamese": 0.4375, - "Chinese,Indonesian": 0.30113636363636365, - "Chinese,Vietnamese": 0.5397727272727273, - "Indonesian,Vietnamese": 0.32954545454545453 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Malay": 0.26704545454545453, - "English,Filipino,Chinese": 0.11931818181818182, - "English,Filipino,Indonesian": 0.2897727272727273, - "English,Filipino,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay": 0.3522727272727273, - "English,Spanish,Chinese": 0.1590909090909091, - "English,Spanish,Indonesian": 0.375, - "English,Spanish,Vietnamese": 0.20454545454545456, - "English,Malay,Chinese": 0.14204545454545456, - "English,Malay,Indonesian": 0.3465909090909091, - "English,Malay,Vietnamese": 0.18181818181818182, - "English,Chinese,Indonesian": 0.14772727272727273, - "English,Chinese,Vietnamese": 0.14772727272727273, - "English,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Malay": 0.2784090909090909, - "Filipino,Spanish,Chinese": 0.1534090909090909, - "Filipino,Spanish,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese": 0.20454545454545456, - "Filipino,Malay,Indonesian": 0.35795454545454547, - "Filipino,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Chinese,Vietnamese": 0.22727272727272727, - "Filipino,Indonesian,Vietnamese": 0.19886363636363635, - "Spanish,Malay,Chinese": 0.1534090909090909, - "Spanish,Malay,Indonesian": 0.32954545454545453, - "Spanish,Malay,Vietnamese": 0.1875, - "Spanish,Chinese,Indonesian": 0.14204545454545456, - "Spanish,Chinese,Vietnamese": 0.17613636363636365, - "Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "Malay,Chinese,Indonesian": 0.21022727272727273, - "Malay,Chinese,Vietnamese": 0.25, - "Malay,Indonesian,Vietnamese": 0.23863636363636365, - "Chinese,Indonesian,Vietnamese": 0.18181818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.21022727272727273, - "English,Filipino,Spanish,Chinese": 0.09090909090909091, - "English,Filipino,Spanish,Indonesian": 0.22727272727272727, - "English,Filipino,Spanish,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese": 0.09659090909090909, - "English,Filipino,Malay,Indonesian": 0.21022727272727273, - "English,Filipino,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese": 0.10227272727272728, - "English,Spanish,Malay,Indonesian": 0.26704545454545453, - "English,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Spanish,Chinese,Indonesian": 0.11363636363636363, - "English,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "English,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "English,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese": 0.10227272727272728, - "Filipino,Spanish,Malay,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.0625, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727 - } - }, - "AC3_2": 0.40288039143255217, - "AC3_3": 0.27975335452643213, - "AC3_4": 0.19230643289311408, - "AC3_5": 0.13494165958108575, - "AC3_6": 0.09785749962113327, - "AC3_7": 0.0721353266336402 - } + "prompt_1": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.5436893203883495 - } + "prompt_1": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.21904761904761905 - } + "prompt_1": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.6448598130841121 - } + "prompt_1": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.3, - "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.4, - "history": 0.3333333333333333, - "literature": 0.2, - "politics": 0.3, - "culture": 0.4, - "film": 0.3, - "law": 0.3, - "geography": 0.4 - } - } + "prompt_1": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.1772916760247971 - } + "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.30111791137077243 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.06350768708766931 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.011889840671548057 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.2824783492618919 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.5554259043173863 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.5376474794422595, - "category_acc": { - "high_school_european_history": 0.774390243902439, - "business_ethics": 0.6666666666666666, - "clinical_knowledge": 0.5984848484848485, - "medical_genetics": 0.5858585858585859, - "high_school_us_history": 0.7881773399014779, - "high_school_physics": 0.22666666666666666, - "high_school_world_history": 0.7161016949152542, - "virology": 0.4666666666666667, - "high_school_microeconomics": 0.5907172995780591, - "econometrics": 0.2743362831858407, - "college_computer_science": 0.3838383838383838, - "high_school_biology": 0.6537216828478964, - "abstract_algebra": 0.1919191919191919, - "professional_accounting": 0.39501779359430605, - "philosophy": 0.5548387096774193, - "professional_medicine": 0.5313653136531366, - "nutrition": 0.5770491803278689, - "global_facts": 0.3333333333333333, - "machine_learning": 0.35135135135135137, - "security_studies": 0.6680327868852459, - "public_relations": 0.6055045871559633, - "professional_psychology": 0.5662847790507365, - "prehistory": 0.5603715170278638, - "anatomy": 0.43283582089552236, - "human_sexuality": 0.6461538461538462, - "college_medicine": 0.4941860465116279, - "high_school_government_and_politics": 0.7760416666666666, - "college_chemistry": 0.36363636363636365, - "logical_fallacies": 0.6358024691358025, - "high_school_geography": 0.751269035532995, - "elementary_mathematics": 0.3687002652519894, - "human_aging": 0.5540540540540541, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.8143382352941176, - "formal_logic": 0.352, - "high_school_statistics": 0.3953488372093023, - "international_law": 0.675, - "high_school_mathematics": 0.24535315985130113, - "high_school_computer_science": 0.46464646464646464, - "conceptual_physics": 0.4444444444444444, - "miscellaneous": 0.7378516624040921, - "high_school_chemistry": 0.3811881188118812, - "marketing": 0.8369098712446352, - "professional_law": 0.4142204827136334, - "management": 0.7549019607843137, - "college_physics": 0.39603960396039606, - "jurisprudence": 0.6635514018691588, - "world_religions": 0.7, - "sociology": 0.73, - "us_foreign_policy": 0.7575757575757576, - "high_school_macroeconomics": 0.570694087403599, - "computer_security": 0.6868686868686869, - "moral_scenarios": 0.24272930648769575, - "moral_disputes": 0.6608695652173913, - "electrical_engineering": 0.5347222222222222, - "astronomy": 0.5761589403973509, - "college_biology": 0.4965034965034965 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.24145616641901932 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.24782067247820672, - "category_acc": { - "computer_network": 0.125, - "operating_system": 0.16666666666666666, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.2619047619047619, - "college_physics": 0.041666666666666664, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.16666666666666666, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.08333333333333333, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.08333333333333333, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.38333333333333336, - "business_administration": 0.13157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.2413793103448276, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.11764705882352941, - "modern_chinese_history": 0.25, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.3333333333333333, - "law": 0.41379310344827586, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.2, - "middle_school_history": 0.25925925925925924, - "civil_servant": 0.2692307692307692, - "sports_science": 0.08333333333333333, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.16666666666666666, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.16666666666666666, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.2037037037037037, - "physician": 0.2777777777777778 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.2939068100358423 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.24909342082541874, - "category_acc": { - "agronomy": 0.22485207100591717, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.2621951219512195, - "arts": 0.26875, - "astronomy": 0.2, - "business_ethics": 0.24401913875598086, - "chinese_civil_service_exam": 0.20625, - "chinese_driving_rule": 0.31297709923664124, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.24299065420560748, - "chinese_history": 0.25696594427244585, - "chinese_literature": 0.24019607843137256, - "chinese_teacher_qualification": 0.2122905027932961, - "clinical_knowledge": 0.25738396624472576, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.2897196261682243, - "college_engineering_hydrology": 0.3113207547169811, - "college_law": 0.21296296296296297, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.25471698113207547, - "college_medicine": 0.2490842490842491, - "computer_science": 0.27941176470588236, - "computer_security": 0.2573099415204678, - "conceptual_physics": 0.2857142857142857, - "construction_project_management": 0.2302158273381295, - "economics": 0.22012578616352202, - "education": 0.24539877300613497, - "electrical_engineering": 0.22093023255813954, - "elementary_chinese": 0.26587301587301587, - "elementary_commonsense": 0.2727272727272727, - "elementary_information_and_technology": 0.29411764705882354, - "elementary_mathematics": 0.23043478260869565, - "ethnology": 0.23703703703703705, - "food_science": 0.21678321678321677, - "genetics": 0.26136363636363635, - "global_facts": 0.2214765100671141, - "high_school_biology": 0.21301775147928995, - "high_school_chemistry": 0.1893939393939394, - "high_school_geography": 0.2627118644067797, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.3, - "high_school_politics": 0.23076923076923078, - "human_sexuality": 0.25396825396825395, - "international_law": 0.24324324324324326, - "journalism": 0.25, - "jurisprudence": 0.25790754257907544, - "legal_and_moral_basis": 0.24299065420560748, - "logical": 0.24390243902439024, - "machine_learning": 0.22131147540983606, - "management": 0.24761904761904763, - "marketing": 0.25, - "marxist_theory": 0.24867724867724866, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.2689655172413793, - "philosophy": 0.24761904761904763, - "professional_accounting": 0.2571428571428571, - "professional_law": 0.27488151658767773, - "professional_medicine": 0.2526595744680851, - "professional_psychology": 0.25862068965517243, - "public_relations": 0.23563218390804597, - "security_study": 0.25925925925925924, - "sociology": 0.24336283185840707, - "sports_science": 0.23030303030303031, - "traditional_chinese_medicine": 0.2648648648648649, - "virology": 0.2603550295857988, - "world_history": 0.2236024844720497, - "world_religions": 0.25 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.15151515151515152 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.5022727272727273 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.33491525423728813 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.2733732236350037 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.9343459088682019 - } + "prompt_1": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.5436837371496774, - "rouge2": 0.3005498456698427, - "rougeL": 0.45771684782014865, - "avg_rouge": 0.43398347687988964 - } + "prompt_1": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.405717578042725, - "rouge2": 0.16979164386539405, - "rougeL": 0.33332768257046824, - "avg_rouge": 0.3029456348261958 - } + "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.9587155963302753 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.8302972195589645 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.86 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.891 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.945 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.7183098591549296 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.9205776173285198 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.7818627450980392 - } + "prompt_1": -1 } } }, - "random": { - "model_size": "NA", - "model_link": "https://huggingface.co/spaces/SeaEval/SeaEval_Leaderboard", + "flan_t5_xxl": { + "model_size": "11B", + "model_link": "https://huggingface.co/google/flan-t5-xxl", "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.25399159663865545, + "overall_acc": 0.7771008403361345, "language_acc": { - "English": 0.253781512605042, - "Chinese": 0.2504201680672269, - "Spanish": 0.25882352941176473, - "Vietnamese": 0.2529411764705882 + "Spanish": 0.9613445378151261, + "English": 0.9697478991596639, + "Chinese": 0.3445378151260504, + "Vietnamese": 0.8327731092436975 }, - "consistency_score_2": 0.24971988795518207, - "consistency_score_3": 0.0596638655462185, - "consistency_score_4": 0.01764705882352941, + "consistency_score_2": 0.6126050420168068, + "consistency_score_3": 0.44054621848739495, + "consistency_score_4": 0.3025210084033613, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.23865546218487396, - "English,Spanish": 0.2495798319327731, - "English,Vietnamese": 0.2638655462184874, - "Chinese,Spanish": 0.25126050420168067, - "Chinese,Vietnamese": 0.246218487394958, - "Spanish,Vietnamese": 0.24873949579831933 + "Spanish,English": 0.973109243697479, + "Spanish,Chinese": 0.33949579831932775, + "Spanish,Vietnamese": 0.8243697478991596, + "English,Chinese": 0.33865546218487397, + "English,Vietnamese": 0.8294117647058824, + "Chinese,Vietnamese": 0.37058823529411766 }, "3_combine": { - "English,Chinese,Spanish": 0.05966386554621849, - "English,Chinese,Vietnamese": 0.05378151260504202, - "English,Spanish,Vietnamese": 0.06554621848739496, - "Chinese,Spanish,Vietnamese": 0.05966386554621849 + "Spanish,English,Chinese": 0.3319327731092437, + "Spanish,English,Vietnamese": 0.8184873949579832, + "Spanish,Chinese,Vietnamese": 0.3050420168067227, + "English,Chinese,Vietnamese": 0.3067226890756303 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.01764705882352941 + "Spanish,English,Chinese,Vietnamese": 0.3025210084033613 } }, - "AC3_2": 0.25183762920440245, - "AC3_3": 0.09662908696902939, - "AC3_4": 0.033001228321411856 + "AC3_2": 0.6851174755123539, + "AC3_3": 0.5623120987063642, + "AC3_4": 0.43550309787133057 }, "prompt_2": { - "overall_acc": 0.24285714285714285, + "overall_acc": 0.7785714285714286, "language_acc": { - "English": 0.23781512605042016, - "Chinese": 0.25126050420168067, - "Spanish": 0.2361344537815126, - "Vietnamese": 0.246218487394958 + "Spanish": 0.9621848739495799, + "English": 0.9689075630252101, + "Chinese": 0.34705882352941175, + "Vietnamese": 0.8361344537815126 }, - "consistency_score_2": 0.24299719887955182, - "consistency_score_3": 0.05798319327731093, - "consistency_score_4": 0.015126050420168067, + "consistency_score_2": 0.6145658263305322, + "consistency_score_3": 0.4430672268907563, + "consistency_score_4": 0.3042016806722689, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.2546218487394958, - "English,Spanish": 0.22436974789915967, - "English,Vietnamese": 0.22941176470588234, - "Chinese,Spanish": 0.253781512605042, - "Chinese,Vietnamese": 0.25126050420168067, - "Spanish,Vietnamese": 0.2445378151260504 + "Spanish,English": 0.9747899159663865, + "Spanish,Chinese": 0.3420168067226891, + "Spanish,Vietnamese": 0.8319327731092437, + "English,Chinese": 0.3420168067226891, + "English,Vietnamese": 0.8302521008403362, + "Chinese,Vietnamese": 0.3663865546218487 }, "3_combine": { - "English,Chinese,Spanish": 0.05798319327731093, - "English,Chinese,Vietnamese": 0.06134453781512605, - "English,Spanish,Vietnamese": 0.0546218487394958, - "Chinese,Spanish,Vietnamese": 0.05798319327731093 + "Spanish,English,Chinese": 0.3352941176470588, + "Spanish,English,Vietnamese": 0.8226890756302521, + "Spanish,Chinese,Vietnamese": 0.30756302521008405, + "English,Chinese,Vietnamese": 0.3067226890756303 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.015126050420168067 + "Spanish,English,Chinese,Vietnamese": 0.3042016806722689 } }, - "AC3_2": 0.24292715063154674, - "AC3_3": 0.09361532319314383, - "AC3_4": 0.028478362017811783 + "AC3_2": 0.6869149348197849, + "AC3_3": 0.5647488023122037, + "AC3_4": 0.43747436106608834 }, "prompt_3": { - "overall_acc": 0.24474789915966388, + "overall_acc": 0.7779411764705882, "language_acc": { - "English": 0.23529411764705882, - "Chinese": 0.23865546218487396, - "Spanish": 0.26134453781512607, - "Vietnamese": 0.24369747899159663 + "Spanish": 0.9605042016806723, + "English": 0.9689075630252101, + "Chinese": 0.3487394957983193, + "Vietnamese": 0.8336134453781513 }, - "consistency_score_2": 0.24789915966386555, - "consistency_score_3": 0.06071428571428572, - "consistency_score_4": 0.014285714285714285, + "consistency_score_2": 0.6162464985994397, + "consistency_score_3": 0.4445378151260504, + "consistency_score_4": 0.30840336134453783, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.2529411764705882, - "English,Spanish": 0.23025210084033612, - "English,Vietnamese": 0.2453781512605042, - "Chinese,Spanish": 0.2453781512605042, - "Chinese,Vietnamese": 0.2630252100840336, - "Spanish,Vietnamese": 0.2504201680672269 + "Spanish,English": 0.9739495798319328, + "Spanish,Chinese": 0.34369747899159664, + "Spanish,Vietnamese": 0.8260504201680672, + "English,Chinese": 0.3445378151260504, + "English,Vietnamese": 0.8277310924369747, + "Chinese,Vietnamese": 0.3815126050420168 }, "3_combine": { - "English,Chinese,Spanish": 0.052100840336134456, - "English,Chinese,Vietnamese": 0.06134453781512605, - "English,Spanish,Vietnamese": 0.05630252100840336, - "Chinese,Spanish,Vietnamese": 0.073109243697479 + "Spanish,English,Chinese": 0.3369747899159664, + "Spanish,English,Vietnamese": 0.8184873949579832, + "Spanish,Chinese,Vietnamese": 0.31176470588235294, + "English,Chinese,Vietnamese": 0.31092436974789917 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.014285714285714285 + "Spanish,English,Chinese,Vietnamese": 0.30840336134453783 } }, - "AC3_2": 0.24631345070343777, - "AC3_3": 0.09729318133973465, - "AC3_4": 0.026995713116678286 + "AC3_2": 0.6877173491838008, + "AC3_3": 0.5657754010232376, + "AC3_4": 0.441700888398259 }, "prompt_4": { - "overall_acc": 0.25630252100840334, + "overall_acc": 0.7779411764705882, "language_acc": { - "English": 0.27563025210084036, - "Chinese": 0.2445378151260504, - "Spanish": 0.2529411764705882, - "Vietnamese": 0.25210084033613445 + "Spanish": 0.9613445378151261, + "English": 0.9705882352941176, + "Chinese": 0.3487394957983193, + "Vietnamese": 0.83109243697479 }, - "consistency_score_2": 0.24677871148459385, - "consistency_score_3": 0.06323529411764706, - "consistency_score_4": 0.015966386554621848, + "consistency_score_2": 0.6147058823529411, + "consistency_score_3": 0.4432773109243697, + "consistency_score_4": 0.3058823529411765, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.23865546218487396, - "English,Spanish": 0.24789915966386555, - "English,Vietnamese": 0.2689075630252101, - "Chinese,Spanish": 0.23361344537815126, - "Chinese,Vietnamese": 0.2647058823529412, - "Spanish,Vietnamese": 0.226890756302521 + "Spanish,English": 0.9747899159663865, + "Spanish,Chinese": 0.3453781512605042, + "Spanish,Vietnamese": 0.8243697478991596, + "English,Chinese": 0.34369747899159664, + "English,Vietnamese": 0.826890756302521, + "Chinese,Vietnamese": 0.373109243697479 }, "3_combine": { - "English,Chinese,Spanish": 0.06134453781512605, - "English,Chinese,Vietnamese": 0.06722689075630252, - "English,Spanish,Vietnamese": 0.06890756302521009, - "Chinese,Spanish,Vietnamese": 0.05546218487394958 + "Spanish,English,Chinese": 0.3378151260504202, + "Spanish,English,Vietnamese": 0.8168067226890756, + "Spanish,Chinese,Vietnamese": 0.3092436974789916, + "English,Chinese,Vietnamese": 0.3092436974789916 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.015966386554621848 + "Spanish,English,Chinese,Vietnamese": 0.3058823529411765 } }, - "AC3_2": 0.25145046877897487, - "AC3_3": 0.10144254937460365, - "AC3_4": 0.03006017220599434 + "AC3_2": 0.6867569413756202, + "AC3_3": 0.564753443047834, + "AC3_4": 0.4391092664612012 }, "prompt_5": { - "overall_acc": 0.261344537815126, + "overall_acc": 0.7655462184873949, "language_acc": { - "English": 0.25882352941176473, - "Chinese": 0.2495798319327731, - "Spanish": 0.27310924369747897, - "Vietnamese": 0.2638655462184874 - }, - "consistency_score_2": 0.2400560224089636, - "consistency_score_3": 0.05525210084033613, - "consistency_score_4": 0.011764705882352941, + "Spanish": 0.9487394957983193, + "English": 0.9554621848739496, + "Chinese": 0.346218487394958, + "Vietnamese": 0.8117647058823529 + }, + "consistency_score_2": 0.5976190476190476, + "consistency_score_3": 0.4262605042016807, + "consistency_score_4": 0.28823529411764703, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.26134453781512607, - "English,Spanish": 0.2184873949579832, - "English,Vietnamese": 0.22268907563025211, - "Chinese,Spanish": 0.23361344537815126, - "Chinese,Vietnamese": 0.2495798319327731, - "Spanish,Vietnamese": 0.2546218487394958 + "Spanish,English": 0.9512605042016806, + "Spanish,Chinese": 0.3420168067226891, + "Spanish,Vietnamese": 0.8042016806722689, + "English,Chinese": 0.34285714285714286, + "English,Vietnamese": 0.8033613445378152, + "Chinese,Vietnamese": 0.3420168067226891 }, "3_combine": { - "English,Chinese,Spanish": 0.05714285714285714, - "English,Chinese,Vietnamese": 0.0546218487394958, - "English,Spanish,Vietnamese": 0.05042016806722689, - "Chinese,Spanish,Vietnamese": 0.058823529411764705 + "Spanish,English,Chinese": 0.3302521008403361, + "Spanish,English,Vietnamese": 0.7865546218487395, + "Spanish,Chinese,Vietnamese": 0.29327731092436976, + "English,Chinese,Vietnamese": 0.2949579831932773 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.011764705882352941 + "Spanish,English,Chinese,Vietnamese": 0.28823529411764703 } }, - "AC3_2": 0.25024834509820526, - "AC3_3": 0.09121912863127803, - "AC3_4": 0.022515837095828134 + "AC3_2": 0.6712392302570803, + "AC3_3": 0.5476091229263507, + "AC3_4": 0.41879163144538817 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.25333333333333335, + "overall_acc": 0.3771428571428571, "language_acc": { - "Spanish": 0.21333333333333335, - "English": 0.26, - "Chinese": 0.26, - "Vietnamese": 0.26666666666666666, - "Filipino": 0.25333333333333335, - "Indonesian": 0.24666666666666667, - "Malay": 0.2733333333333333 + "Filipino": 0.3466666666666667, + "Vietnamese": 0.2866666666666667, + "Chinese": 0.22, + "Spanish": 0.44, + "Malay": 0.3333333333333333, + "Indonesian": 0.42, + "English": 0.5933333333333334 }, - "consistency_score_2": 0.2558730158730159, - "consistency_score_3": 0.06857142857142857, - "consistency_score_4": 0.02057142857142857, - "consistency_score_5": 0.006666666666666666, - "consistency_score_6": 0.001904761904761905, - "consistency_score_7": 0.0, + "consistency_score_2": 0.41587301587301584, + "consistency_score_3": 0.21504761904761907, + "consistency_score_4": 0.12685714285714284, + "consistency_score_5": 0.0803174603174603, + "consistency_score_6": 0.05238095238095238, + "consistency_score_7": 0.03333333333333333, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.26666666666666666, - "Spanish,Chinese": 0.2733333333333333, - "Spanish,Vietnamese": 0.2733333333333333, - "Spanish,Filipino": 0.26, - "Spanish,Indonesian": 0.29333333333333333, - "Spanish,Malay": 0.16666666666666666, - "English,Chinese": 0.20666666666666667, - "English,Vietnamese": 0.28, - "English,Filipino": 0.25333333333333335, - "English,Indonesian": 0.26, - "English,Malay": 0.25333333333333335, - "Chinese,Vietnamese": 0.29333333333333333, - "Chinese,Filipino": 0.24, - "Chinese,Indonesian": 0.34, - "Chinese,Malay": 0.22666666666666666, - "Vietnamese,Filipino": 0.26666666666666666, - "Vietnamese,Indonesian": 0.24666666666666667, - "Vietnamese,Malay": 0.25333333333333335, - "Filipino,Indonesian": 0.23333333333333334, - "Filipino,Malay": 0.26666666666666666, - "Indonesian,Malay": 0.22 - }, - "3_combine": { - "Spanish,English,Chinese": 0.04666666666666667, - "Spanish,English,Vietnamese": 0.06666666666666667, - "Spanish,English,Filipino": 0.06, - "Spanish,English,Indonesian": 0.08666666666666667, - "Spanish,English,Malay": 0.04666666666666667, - "Spanish,Chinese,Vietnamese": 0.08666666666666667, - "Spanish,Chinese,Filipino": 0.06666666666666667, - "Spanish,Chinese,Indonesian": 0.12, - "Spanish,Chinese,Malay": 0.05333333333333334, - "Spanish,Vietnamese,Filipino": 0.05333333333333334, - "Spanish,Vietnamese,Indonesian": 0.08, - "Spanish,Vietnamese,Malay": 0.05333333333333334, - "Spanish,Filipino,Indonesian": 0.04666666666666667, - "Spanish,Filipino,Malay": 0.05333333333333334, - "Spanish,Indonesian,Malay": 0.04666666666666667, - "English,Chinese,Vietnamese": 0.08, - "English,Chinese,Filipino": 0.04666666666666667, - "English,Chinese,Indonesian": 0.07333333333333333, - "English,Chinese,Malay": 0.04, - "English,Vietnamese,Filipino": 0.08, - "English,Vietnamese,Indonesian": 0.07333333333333333, - "English,Vietnamese,Malay": 0.06, - "English,Filipino,Indonesian": 0.05333333333333334, - "English,Filipino,Malay": 0.07333333333333333, - "English,Indonesian,Malay": 0.08, - "Chinese,Vietnamese,Filipino": 0.07333333333333333, - "Chinese,Vietnamese,Indonesian": 0.10666666666666667, - "Chinese,Vietnamese,Malay": 0.09333333333333334, - "Chinese,Filipino,Indonesian": 0.08666666666666667, - "Chinese,Filipino,Malay": 0.08, - "Chinese,Indonesian,Malay": 0.08666666666666667, - "Vietnamese,Filipino,Indonesian": 0.05333333333333334, - "Vietnamese,Filipino,Malay": 0.08, - "Vietnamese,Indonesian,Malay": 0.05333333333333334, - "Filipino,Indonesian,Malay": 0.06 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.02666666666666667, - "Spanish,English,Chinese,Filipino": 0.006666666666666667, - "Spanish,English,Chinese,Indonesian": 0.02666666666666667, - "Spanish,English,Chinese,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.013333333333333334, - "Spanish,English,Vietnamese,Indonesian": 0.02666666666666667, - "Spanish,English,Vietnamese,Malay": 0.006666666666666667, - "Spanish,English,Filipino,Indonesian": 0.013333333333333334, - "Spanish,English,Filipino,Malay": 0.013333333333333334, - "Spanish,English,Indonesian,Malay": 0.02, - "Spanish,Chinese,Vietnamese,Filipino": 0.02666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.04, - "Spanish,Chinese,Vietnamese,Malay": 0.02666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.02666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.02666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.02, - "Spanish,Vietnamese,Indonesian,Malay": 0.02, - "Spanish,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino": 0.02, - "English,Chinese,Vietnamese,Indonesian": 0.03333333333333333, - "English,Chinese,Vietnamese,Malay": 0.02, - "English,Chinese,Filipino,Indonesian": 0.02, - "English,Chinese,Filipino,Malay": 0.02, - "English,Chinese,Indonesian,Malay": 0.02, - "English,Vietnamese,Filipino,Indonesian": 0.02, - "English,Vietnamese,Filipino,Malay": 0.02666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "English,Filipino,Indonesian,Malay": 0.02, - "Chinese,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.03333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.04, - "Chinese,Filipino,Indonesian,Malay": 0.02666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.02, - "Spanish,English,Chinese,Vietnamese,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.013333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese": 0.3933333333333333, + "Filipino,Chinese": 0.30666666666666664, + "Filipino,Spanish": 0.41333333333333333, + "Filipino,Malay": 0.4866666666666667, + "Filipino,Indonesian": 0.49333333333333335, + "Filipino,English": 0.4266666666666667, + "Vietnamese,Chinese": 0.5066666666666667, + "Vietnamese,Spanish": 0.34, + "Vietnamese,Malay": 0.32, + "Vietnamese,Indonesian": 0.36, + "Vietnamese,English": 0.32666666666666666, + "Chinese,Spanish": 0.26, + "Chinese,Malay": 0.28, + "Chinese,Indonesian": 0.32666666666666666, + "Chinese,English": 0.2733333333333333, + "Spanish,Malay": 0.4866666666666667, + "Spanish,Indonesian": 0.58, + "Spanish,English": 0.5933333333333334, + "Malay,Indonesian": 0.6133333333333333, + "Malay,English": 0.4266666666666667, + "Indonesian,English": 0.52 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.16666666666666666, + "Filipino,Vietnamese,Spanish": 0.19333333333333333, + "Filipino,Vietnamese,Malay": 0.2, + "Filipino,Vietnamese,Indonesian": 0.22, + "Filipino,Vietnamese,English": 0.18666666666666668, + "Filipino,Chinese,Spanish": 0.13333333333333333, + "Filipino,Chinese,Malay": 0.13333333333333333, + "Filipino,Chinese,Indonesian": 0.15333333333333332, + "Filipino,Chinese,English": 0.14666666666666667, + "Filipino,Spanish,Malay": 0.29333333333333333, + "Filipino,Spanish,Indonesian": 0.30666666666666664, + "Filipino,Spanish,English": 0.2866666666666667, + "Filipino,Malay,Indonesian": 0.35333333333333333, + "Filipino,Malay,English": 0.26, + "Filipino,Indonesian,English": 0.28, + "Vietnamese,Chinese,Spanish": 0.14666666666666667, + "Vietnamese,Chinese,Malay": 0.14, + "Vietnamese,Chinese,Indonesian": 0.17333333333333334, + "Vietnamese,Chinese,English": 0.13333333333333333, + "Vietnamese,Spanish,Malay": 0.18666666666666668, + "Vietnamese,Spanish,Indonesian": 0.22666666666666666, + "Vietnamese,Spanish,English": 0.21333333333333335, + "Vietnamese,Malay,Indonesian": 0.22, + "Vietnamese,Malay,English": 0.15333333333333332, + "Vietnamese,Indonesian,English": 0.2, + "Chinese,Spanish,Malay": 0.15333333333333332, + "Chinese,Spanish,Indonesian": 0.18666666666666668, + "Chinese,Spanish,English": 0.16, + "Chinese,Malay,Indonesian": 0.19333333333333333, + "Chinese,Malay,English": 0.12, + "Chinese,Indonesian,English": 0.16666666666666666, + "Spanish,Malay,Indonesian": 0.3933333333333333, + "Spanish,Malay,English": 0.32, + "Spanish,Indonesian,English": 0.38666666666666666, + "Malay,Indonesian,English": 0.34 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Malay": 0.08, + "Filipino,Vietnamese,Chinese,Indonesian": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,English": 0.06666666666666667, + "Filipino,Vietnamese,Spanish,Malay": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,English": 0.13333333333333333, + "Filipino,Vietnamese,Malay,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Malay,English": 0.11333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.13333333333333333, + "Filipino,Chinese,Spanish,Malay": 0.1, + "Filipino,Chinese,Spanish,Indonesian": 0.1, + "Filipino,Chinese,Spanish,English": 0.08666666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.11333333333333333, + "Filipino,Chinese,Malay,English": 0.08, + "Filipino,Chinese,Indonesian,English": 0.08666666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.24, + "Filipino,Spanish,Malay,English": 0.22, + "Filipino,Spanish,Indonesian,English": 0.23333333333333334, + "Filipino,Malay,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish,Malay": 0.08, + "Vietnamese,Chinese,Spanish,Indonesian": 0.10666666666666667, + "Vietnamese,Chinese,Spanish,English": 0.08, + "Vietnamese,Chinese,Malay,Indonesian": 0.10666666666666667, + "Vietnamese,Chinese,Malay,English": 0.06, + "Vietnamese,Chinese,Indonesian,English": 0.08, + "Vietnamese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Vietnamese,Spanish,Malay,English": 0.13333333333333333, + "Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, + "Vietnamese,Malay,Indonesian,English": 0.12666666666666668, + "Chinese,Spanish,Malay,Indonesian": 0.14, + "Chinese,Spanish,Malay,English": 0.09333333333333334, + "Chinese,Spanish,Indonesian,English": 0.13333333333333333, + "Chinese,Malay,Indonesian,English": 0.10666666666666667, + "Spanish,Malay,Indonesian,English": 0.2733333333333333 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.05333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.06, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.04, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.06666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.04, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.04, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.11333333333333333, + "Filipino,Vietnamese,Spanish,Malay,English": 0.10666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.09333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, + "Filipino,Chinese,Spanish,Malay,English": 0.06666666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.07333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.07333333333333333, + "Filipino,Spanish,Malay,Indonesian,English": 0.19333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, + "Vietnamese,Chinese,Spanish,Malay,English": 0.04666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.07333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.05333333333333334, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.11333333333333333, + "Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.05333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.03333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.04, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.03333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.09333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.04666666666666667 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.03333333333333333 } }, - "AC3_2": 0.254596841180259, - "AC3_3": 0.107928994049312, - "AC3_4": 0.038052851168304765, - "AC3_5": 0.012991452986456278, - "AC3_6": 0.003781094525881785, - "AC3_7": 0.0 + "AC3_2": 0.395562163967047, + "AC3_3": 0.2739107659327013, + "AC3_4": 0.18985422736757826, + "AC3_5": 0.13243184293724472, + "AC3_6": 0.09198606269635422, + "AC3_7": 0.06125290021709616 }, "prompt_2": { - "overall_acc": 0.24666666666666667, + "overall_acc": 0.3857142857142857, "language_acc": { - "Spanish": 0.19333333333333333, - "English": 0.25333333333333335, - "Chinese": 0.28, - "Vietnamese": 0.26666666666666666, - "Filipino": 0.23333333333333334, - "Indonesian": 0.21333333333333335, - "Malay": 0.2866666666666667 + "Filipino": 0.38666666666666666, + "Vietnamese": 0.2733333333333333, + "Chinese": 0.22666666666666666, + "Spanish": 0.4533333333333333, + "Malay": 0.3466666666666667, + "Indonesian": 0.42, + "English": 0.5933333333333334 }, - "consistency_score_2": 0.25460317460317466, - "consistency_score_3": 0.06666666666666667, - "consistency_score_4": 0.018857142857142857, - "consistency_score_5": 0.005714285714285715, - "consistency_score_6": 0.001904761904761905, - "consistency_score_7": 0.0, + "consistency_score_2": 0.43015873015873013, + "consistency_score_3": 0.2327619047619048, + "consistency_score_4": 0.14266666666666664, + "consistency_score_5": 0.09301587301587302, + "consistency_score_6": 0.06190476190476191, + "consistency_score_7": 0.04, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.25333333333333335, - "Spanish,Chinese": 0.2733333333333333, - "Spanish,Vietnamese": 0.2733333333333333, - "Spanish,Filipino": 0.28, - "Spanish,Indonesian": 0.22, - "Spanish,Malay": 0.24666666666666667, - "English,Chinese": 0.25333333333333335, - "English,Vietnamese": 0.24666666666666667, - "English,Filipino": 0.26, - "English,Indonesian": 0.24666666666666667, - "English,Malay": 0.28, - "Chinese,Vietnamese": 0.21333333333333335, - "Chinese,Filipino": 0.2, - "Chinese,Indonesian": 0.24666666666666667, - "Chinese,Malay": 0.24, - "Vietnamese,Filipino": 0.23333333333333334, - "Vietnamese,Indonesian": 0.28, - "Vietnamese,Malay": 0.2866666666666667, - "Filipino,Indonesian": 0.2733333333333333, - "Filipino,Malay": 0.2733333333333333, - "Indonesian,Malay": 0.26666666666666666 + "Filipino,Vietnamese": 0.4, + "Filipino,Chinese": 0.3333333333333333, + "Filipino,Spanish": 0.4266666666666667, + "Filipino,Malay": 0.49333333333333335, + "Filipino,Indonesian": 0.48, + "Filipino,English": 0.4666666666666667, + "Vietnamese,Chinese": 0.4, + "Vietnamese,Spanish": 0.3466666666666667, + "Vietnamese,Malay": 0.36, + "Vietnamese,Indonesian": 0.37333333333333335, + "Vietnamese,English": 0.3333333333333333, + "Chinese,Spanish": 0.35333333333333333, + "Chinese,Malay": 0.35333333333333333, + "Chinese,Indonesian": 0.38666666666666666, + "Chinese,English": 0.31333333333333335, + "Spanish,Malay": 0.47333333333333333, + "Spanish,Indonesian": 0.58, + "Spanish,English": 0.58, + "Malay,Indonesian": 0.6133333333333333, + "Malay,English": 0.43333333333333335, + "Indonesian,English": 0.5333333333333333 }, "3_combine": { - "Spanish,English,Chinese": 0.07333333333333333, - "Spanish,English,Vietnamese": 0.07333333333333333, - "Spanish,English,Filipino": 0.06, - "Spanish,English,Indonesian": 0.05333333333333334, - "Spanish,English,Malay": 0.06666666666666667, - "Spanish,Chinese,Vietnamese": 0.08666666666666667, - "Spanish,Chinese,Filipino": 0.06666666666666667, - "Spanish,Chinese,Indonesian": 0.04666666666666667, - "Spanish,Chinese,Malay": 0.06, - "Spanish,Vietnamese,Filipino": 0.08, - "Spanish,Vietnamese,Indonesian": 0.06666666666666667, - "Spanish,Vietnamese,Malay": 0.08, - "Spanish,Filipino,Indonesian": 0.08666666666666667, - "Spanish,Filipino,Malay": 0.07333333333333333, - "Spanish,Indonesian,Malay": 0.04666666666666667, - "English,Chinese,Vietnamese": 0.07333333333333333, - "English,Chinese,Filipino": 0.04666666666666667, - "English,Chinese,Indonesian": 0.07333333333333333, - "English,Chinese,Malay": 0.08, - "English,Vietnamese,Filipino": 0.06, - "English,Vietnamese,Indonesian": 0.06, - "English,Vietnamese,Malay": 0.06, - "English,Filipino,Indonesian": 0.06666666666666667, - "English,Filipino,Malay": 0.08, - "English,Indonesian,Malay": 0.06666666666666667, - "Chinese,Vietnamese,Filipino": 0.02666666666666667, - "Chinese,Vietnamese,Indonesian": 0.04666666666666667, - "Chinese,Vietnamese,Malay": 0.06, - "Chinese,Filipino,Indonesian": 0.04666666666666667, - "Chinese,Filipino,Malay": 0.04, - "Chinese,Indonesian,Malay": 0.08, - "Vietnamese,Filipino,Indonesian": 0.08, - "Vietnamese,Filipino,Malay": 0.08, - "Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Filipino,Indonesian,Malay": 0.07333333333333333 + "Filipino,Vietnamese,Chinese": 0.16, + "Filipino,Vietnamese,Spanish": 0.2, + "Filipino,Vietnamese,Malay": 0.22, + "Filipino,Vietnamese,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,English": 0.20666666666666667, + "Filipino,Chinese,Spanish": 0.18, + "Filipino,Chinese,Malay": 0.16666666666666666, + "Filipino,Chinese,Indonesian": 0.18, + "Filipino,Chinese,English": 0.17333333333333334, + "Filipino,Spanish,Malay": 0.2866666666666667, + "Filipino,Spanish,Indonesian": 0.30666666666666664, + "Filipino,Spanish,English": 0.3, + "Filipino,Malay,Indonesian": 0.34, + "Filipino,Malay,English": 0.2866666666666667, + "Filipino,Indonesian,English": 0.3, + "Vietnamese,Chinese,Spanish": 0.16, + "Vietnamese,Chinese,Malay": 0.15333333333333332, + "Vietnamese,Chinese,Indonesian": 0.15333333333333332, + "Vietnamese,Chinese,English": 0.13333333333333333, + "Vietnamese,Spanish,Malay": 0.19333333333333333, + "Vietnamese,Spanish,Indonesian": 0.25333333333333335, + "Vietnamese,Spanish,English": 0.22, + "Vietnamese,Malay,Indonesian": 0.24, + "Vietnamese,Malay,English": 0.16666666666666666, + "Vietnamese,Indonesian,English": 0.22, + "Chinese,Spanish,Malay": 0.20666666666666667, + "Chinese,Spanish,Indonesian": 0.24666666666666667, + "Chinese,Spanish,English": 0.20666666666666667, + "Chinese,Malay,Indonesian": 0.24666666666666667, + "Chinese,Malay,English": 0.16, + "Chinese,Indonesian,English": 0.2, + "Spanish,Malay,Indonesian": 0.3933333333333333, + "Spanish,Malay,English": 0.31333333333333335, + "Spanish,Indonesian,English": 0.3933333333333333, + "Malay,Indonesian,English": 0.3466666666666667 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.04666666666666667, - "Spanish,English,Chinese,Filipino": 0.006666666666666667, - "Spanish,English,Chinese,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Malay": 0.02666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.02, - "Spanish,English,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,English,Vietnamese,Malay": 0.02, - "Spanish,English,Filipino,Indonesian": 0.02, - "Spanish,English,Filipino,Malay": 0.02, - "Spanish,English,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Malay": 0.02, - "Spanish,Chinese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.02, - "Spanish,Chinese,Indonesian,Malay": 0.02, - "Spanish,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.02666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.02666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.013333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.013333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.02, - "English,Chinese,Vietnamese,Malay": 0.02666666666666667, - "English,Chinese,Filipino,Indonesian": 0.02, - "English,Chinese,Filipino,Malay": 0.013333333333333334, - "English,Chinese,Indonesian,Malay": 0.02666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "English,Vietnamese,Filipino,Malay": 0.013333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.03333333333333333, - "English,Filipino,Indonesian,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.02666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.013333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.04 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.02, - "Spanish,English,Chinese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Filipino,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "English,Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.0, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Malay": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Indonesian": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,English": 0.07333333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.14, + "Filipino,Vietnamese,Spanish,Indonesian": 0.16, + "Filipino,Vietnamese,Spanish,English": 0.14666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.16666666666666666, + "Filipino,Vietnamese,Malay,English": 0.12666666666666668, + "Filipino,Vietnamese,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Malay": 0.12, + "Filipino,Chinese,Spanish,Indonesian": 0.12666666666666668, + "Filipino,Chinese,Spanish,English": 0.12, + "Filipino,Chinese,Malay,Indonesian": 0.12, + "Filipino,Chinese,Malay,English": 0.10666666666666667, + "Filipino,Chinese,Indonesian,English": 0.11333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.23333333333333334, + "Filipino,Spanish,Malay,English": 0.22, + "Filipino,Spanish,Indonesian,English": 0.24666666666666667, + "Filipino,Malay,Indonesian,English": 0.22666666666666666, + "Vietnamese,Chinese,Spanish,Malay": 0.1, + "Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.08666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.11333333333333333, + "Vietnamese,Chinese,Malay,English": 0.06666666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.07333333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, + "Vietnamese,Spanish,Malay,English": 0.14666666666666667, + "Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, + "Vietnamese,Malay,Indonesian,English": 0.14666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.18, + "Chinese,Spanish,Malay,English": 0.14, + "Chinese,Spanish,Indonesian,English": 0.17333333333333334, + "Chinese,Malay,Indonesian,English": 0.14666666666666667, + "Spanish,Malay,Indonesian,English": 0.2733333333333333 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.06666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.06666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.05333333333333334, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.06666666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.04666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.04666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.12, + "Filipino,Vietnamese,Spanish,Malay,English": 0.11333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.10666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.1, + "Filipino,Chinese,Spanish,Malay,English": 0.09333333333333334, + "Filipino,Chinese,Spanish,Indonesian,English": 0.1, + "Filipino,Chinese,Malay,Indonesian,English": 0.09333333333333334, + "Filipino,Spanish,Malay,Indonesian,English": 0.19333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08666666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.06666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.07333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.06, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.13333333333333333, + "Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.05333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.04666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.04666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.04, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.1, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.04 } }, - "AC3_2": 0.25057209199138897, - "AC3_3": 0.10496453897359287, - "AC3_4": 0.03503586799254387, - "AC3_5": 0.011169811316328944, - "AC3_6": 0.0037803320546733023, - "AC3_7": 0.0 + "AC3_2": 0.40672595881618495, + "AC3_3": 0.290325135242776, + "AC3_4": 0.20829127609612366, + "AC3_5": 0.1498863205446666, + "AC3_6": 0.1066869300673511, + "AC3_7": 0.07248322145948381 }, "prompt_3": { - "overall_acc": 0.23047619047619047, + "overall_acc": 0.3876190476190477, "language_acc": { - "Spanish": 0.2733333333333333, - "English": 0.18, - "Chinese": 0.18666666666666668, - "Vietnamese": 0.26666666666666666, - "Filipino": 0.22666666666666666, - "Indonesian": 0.22666666666666666, - "Malay": 0.25333333333333335 + "Filipino": 0.38666666666666666, + "Vietnamese": 0.2866666666666667, + "Chinese": 0.25333333333333335, + "Spanish": 0.4533333333333333, + "Malay": 0.3466666666666667, + "Indonesian": 0.4, + "English": 0.5866666666666667 }, - "consistency_score_2": 0.25492063492063494, - "consistency_score_3": 0.06228571428571428, - "consistency_score_4": 0.012380952380952376, - "consistency_score_5": 0.0015873015873015873, - "consistency_score_6": 0.0, - "consistency_score_7": 0.0, + "consistency_score_2": 0.4428571428571428, + "consistency_score_3": 0.2472380952380952, + "consistency_score_4": 0.15333333333333332, + "consistency_score_5": 0.09936507936507938, + "consistency_score_6": 0.06476190476190476, + "consistency_score_7": 0.04, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.31333333333333335, - "Spanish,Chinese": 0.22666666666666666, - "Spanish,Vietnamese": 0.2733333333333333, - "Spanish,Filipino": 0.21333333333333335, - "Spanish,Indonesian": 0.19333333333333333, - "Spanish,Malay": 0.28, - "English,Chinese": 0.34, - "English,Vietnamese": 0.23333333333333334, - "English,Filipino": 0.22666666666666666, - "English,Indonesian": 0.28, - "English,Malay": 0.25333333333333335, - "Chinese,Vietnamese": 0.24, - "Chinese,Filipino": 0.26, - "Chinese,Indonesian": 0.24666666666666667, - "Chinese,Malay": 0.2733333333333333, - "Vietnamese,Filipino": 0.26666666666666666, - "Vietnamese,Indonesian": 0.21333333333333335, - "Vietnamese,Malay": 0.26, - "Filipino,Indonesian": 0.2866666666666667, - "Filipino,Malay": 0.22666666666666666, - "Indonesian,Malay": 0.24666666666666667 + "Filipino,Vietnamese": 0.44666666666666666, + "Filipino,Chinese": 0.3933333333333333, + "Filipino,Spanish": 0.47333333333333333, + "Filipino,Malay": 0.4666666666666667, + "Filipino,Indonesian": 0.47333333333333333, + "Filipino,English": 0.48, + "Vietnamese,Chinese": 0.43333333333333335, + "Vietnamese,Spanish": 0.35333333333333333, + "Vietnamese,Malay": 0.36666666666666664, + "Vietnamese,Indonesian": 0.38666666666666666, + "Vietnamese,English": 0.34, + "Chinese,Spanish": 0.37333333333333335, + "Chinese,Malay": 0.38, + "Chinese,Indonesian": 0.38, + "Chinese,English": 0.3333333333333333, + "Spanish,Malay": 0.46, + "Spanish,Indonesian": 0.5933333333333334, + "Spanish,English": 0.6066666666666667, + "Malay,Indonesian": 0.5933333333333334, + "Malay,English": 0.44, + "Indonesian,English": 0.5266666666666666 }, "3_combine": { - "Spanish,English,Chinese": 0.09333333333333334, - "Spanish,English,Vietnamese": 0.07333333333333333, - "Spanish,English,Filipino": 0.06666666666666667, - "Spanish,English,Indonesian": 0.06666666666666667, - "Spanish,English,Malay": 0.08, - "Spanish,Chinese,Vietnamese": 0.05333333333333334, - "Spanish,Chinese,Filipino": 0.06666666666666667, - "Spanish,Chinese,Indonesian": 0.02, - "Spanish,Chinese,Malay": 0.06666666666666667, - "Spanish,Vietnamese,Filipino": 0.04, - "Spanish,Vietnamese,Indonesian": 0.04, - "Spanish,Vietnamese,Malay": 0.07333333333333333, - "Spanish,Filipino,Indonesian": 0.06, - "Spanish,Filipino,Malay": 0.02666666666666667, - "Spanish,Indonesian,Malay": 0.02666666666666667, - "English,Chinese,Vietnamese": 0.06, - "English,Chinese,Filipino": 0.06666666666666667, - "English,Chinese,Indonesian": 0.11333333333333333, - "English,Chinese,Malay": 0.09333333333333334, - "English,Vietnamese,Filipino": 0.06, - "English,Vietnamese,Indonesian": 0.06, - "English,Vietnamese,Malay": 0.04666666666666667, - "English,Filipino,Indonesian": 0.09333333333333334, - "English,Filipino,Malay": 0.04666666666666667, - "English,Indonesian,Malay": 0.06666666666666667, - "Chinese,Vietnamese,Filipino": 0.05333333333333334, - "Chinese,Vietnamese,Indonesian": 0.05333333333333334, - "Chinese,Vietnamese,Malay": 0.05333333333333334, - "Chinese,Filipino,Indonesian": 0.08, - "Chinese,Filipino,Malay": 0.05333333333333334, - "Chinese,Indonesian,Malay": 0.08, - "Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "Vietnamese,Filipino,Malay": 0.07333333333333333, - "Vietnamese,Indonesian,Malay": 0.06, - "Filipino,Indonesian,Malay": 0.04666666666666667 + "Filipino,Vietnamese,Chinese": 0.21333333333333335, + "Filipino,Vietnamese,Spanish": 0.22666666666666666, + "Filipino,Vietnamese,Malay": 0.22666666666666666, + "Filipino,Vietnamese,Indonesian": 0.24, + "Filipino,Vietnamese,English": 0.22, + "Filipino,Chinese,Spanish": 0.24, + "Filipino,Chinese,Malay": 0.2, + "Filipino,Chinese,Indonesian": 0.2, + "Filipino,Chinese,English": 0.20666666666666667, + "Filipino,Spanish,Malay": 0.29333333333333333, + "Filipino,Spanish,Indonesian": 0.3333333333333333, + "Filipino,Spanish,English": 0.32666666666666666, + "Filipino,Malay,Indonesian": 0.32666666666666666, + "Filipino,Malay,English": 0.2866666666666667, + "Filipino,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish": 0.18, + "Vietnamese,Chinese,Malay": 0.18666666666666668, + "Vietnamese,Chinese,Indonesian": 0.17333333333333334, + "Vietnamese,Chinese,English": 0.14, + "Vietnamese,Spanish,Malay": 0.19333333333333333, + "Vietnamese,Spanish,Indonesian": 0.26, + "Vietnamese,Spanish,English": 0.22666666666666666, + "Vietnamese,Malay,Indonesian": 0.24, + "Vietnamese,Malay,English": 0.18, + "Vietnamese,Indonesian,English": 0.22666666666666666, + "Chinese,Spanish,Malay": 0.22, + "Chinese,Spanish,Indonesian": 0.26, + "Chinese,Spanish,English": 0.22, + "Chinese,Malay,Indonesian": 0.25333333333333335, + "Chinese,Malay,English": 0.18, + "Chinese,Indonesian,English": 0.21333333333333335, + "Spanish,Malay,Indonesian": 0.38666666666666666, + "Spanish,Malay,English": 0.31333333333333335, + "Spanish,Indonesian,English": 0.4066666666666667, + "Malay,Indonesian,English": 0.3466666666666667 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.02, - "Spanish,English,Chinese,Filipino": 0.02666666666666667, - "Spanish,English,Chinese,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Malay": 0.013333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.013333333333333334, - "Spanish,English,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,English,Vietnamese,Malay": 0.013333333333333334, - "Spanish,English,Filipino,Indonesian": 0.03333333333333333, - "Spanish,English,Filipino,Malay": 0.013333333333333334, - "Spanish,English,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.0, - "Spanish,Chinese,Vietnamese,Malay": 0.013333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.02, - "Spanish,Chinese,Filipino,Malay": 0.006666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.0, - "Spanish,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.0, - "Spanish,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino": 0.0, - "English,Chinese,Vietnamese,Indonesian": 0.006666666666666667, - "English,Chinese,Vietnamese,Malay": 0.013333333333333334, - "English,Chinese,Filipino,Indonesian": 0.03333333333333333, - "English,Chinese,Filipino,Malay": 0.013333333333333334, - "English,Chinese,Indonesian,Malay": 0.03333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.02, - "English,Vietnamese,Filipino,Malay": 0.006666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "English,Filipino,Indonesian,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.013333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.02 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.0, - "Spanish,English,Chinese,Vietnamese,Malay": 0.0, - "Spanish,English,Chinese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "English,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667 + "Filipino,Vietnamese,Chinese,Spanish": 0.12, + "Filipino,Vietnamese,Chinese,Malay": 0.12, + "Filipino,Vietnamese,Chinese,Indonesian": 0.1, + "Filipino,Vietnamese,Chinese,English": 0.09333333333333334, + "Filipino,Vietnamese,Spanish,Malay": 0.14, + "Filipino,Vietnamese,Spanish,Indonesian": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,English": 0.14666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.16, + "Filipino,Vietnamese,Malay,English": 0.13333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.16, + "Filipino,Chinese,Spanish,Malay": 0.14666666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.15333333333333332, + "Filipino,Chinese,Spanish,English": 0.14666666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.14, + "Filipino,Chinese,Malay,English": 0.12666666666666668, + "Filipino,Chinese,Indonesian,English": 0.12666666666666668, + "Filipino,Spanish,Malay,Indonesian": 0.24666666666666667, + "Filipino,Spanish,Malay,English": 0.22, + "Filipino,Spanish,Indonesian,English": 0.26, + "Filipino,Malay,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Malay": 0.12, + "Vietnamese,Chinese,Spanish,Indonesian": 0.12, + "Vietnamese,Chinese,Spanish,English": 0.09333333333333334, + "Vietnamese,Chinese,Malay,Indonesian": 0.12666666666666668, + "Vietnamese,Chinese,Malay,English": 0.08, + "Vietnamese,Chinese,Indonesian,English": 0.08, + "Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, + "Vietnamese,Spanish,Malay,English": 0.13333333333333333, + "Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, + "Vietnamese,Malay,Indonesian,English": 0.14666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Chinese,Spanish,Malay,English": 0.14666666666666667, + "Chinese,Spanish,Indonesian,English": 0.18, + "Chinese,Malay,Indonesian,English": 0.16, + "Spanish,Malay,Indonesian,English": 0.2733333333333333 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.08, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.06, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.06, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.05333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.12, + "Filipino,Vietnamese,Spanish,Malay,English": 0.1, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.10666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12, + "Filipino,Chinese,Spanish,Malay,English": 0.10666666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.10666666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.2, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, + "Vietnamese,Chinese,Spanish,Malay,English": 0.06666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.06, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.12666666666666668, + "Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.06, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.04666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.05333333333333334, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.04, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.09333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.04 } }, - "AC3_2": 0.24208290505461463, - "AC3_3": 0.09806859370197953, - "AC3_4": 0.02349953313691568, - "AC3_5": 0.0031528890612431528, - "AC3_6": 0.0, - "AC3_7": 0.0 + "AC3_2": 0.41340104844301284, + "AC3_3": 0.30190790502866666, + "AC3_4": 0.21974178399693736, + "AC3_5": 0.15818091509758594, + "AC3_6": 0.11098145360955247, + "AC3_7": 0.07251670376923329 }, "prompt_4": { - "overall_acc": 0.24, + "overall_acc": 0.3904761904761905, "language_acc": { - "Spanish": 0.2733333333333333, - "English": 0.24, + "Filipino": 0.38666666666666666, + "Vietnamese": 0.2866666666666667, "Chinese": 0.22, - "Vietnamese": 0.25333333333333335, - "Filipino": 0.22, - "Indonesian": 0.23333333333333334, - "Malay": 0.24 + "Spanish": 0.4666666666666667, + "Malay": 0.34, + "Indonesian": 0.4, + "English": 0.6333333333333333 }, - "consistency_score_2": 0.24888888888888888, - "consistency_score_3": 0.05904761904761905, - "consistency_score_4": 0.012571428571428568, - "consistency_score_5": 0.0022222222222222222, - "consistency_score_6": 0.0, - "consistency_score_7": 0.0, + "consistency_score_2": 0.4238095238095238, + "consistency_score_3": 0.23142857142857143, + "consistency_score_4": 0.148, + "consistency_score_5": 0.10285714285714284, + "consistency_score_6": 0.07428571428571429, + "consistency_score_7": 0.05333333333333334, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.32, - "Spanish,Chinese": 0.23333333333333334, - "Spanish,Vietnamese": 0.20666666666666667, - "Spanish,Filipino": 0.24, - "Spanish,Indonesian": 0.24666666666666667, - "Spanish,Malay": 0.26666666666666666, - "English,Chinese": 0.3333333333333333, - "English,Vietnamese": 0.26666666666666666, - "English,Filipino": 0.23333333333333334, - "English,Indonesian": 0.25333333333333335, - "English,Malay": 0.22666666666666666, - "Chinese,Vietnamese": 0.26, - "Chinese,Filipino": 0.18666666666666668, - "Chinese,Indonesian": 0.2, - "Chinese,Malay": 0.25333333333333335, - "Vietnamese,Filipino": 0.22666666666666666, - "Vietnamese,Indonesian": 0.24666666666666667, - "Vietnamese,Malay": 0.29333333333333333, - "Filipino,Indonesian": 0.26666666666666666, - "Filipino,Malay": 0.24, - "Indonesian,Malay": 0.22666666666666666 + "Filipino,Vietnamese": 0.4066666666666667, + "Filipino,Chinese": 0.31333333333333335, + "Filipino,Spanish": 0.44, + "Filipino,Malay": 0.4266666666666667, + "Filipino,Indonesian": 0.5133333333333333, + "Filipino,English": 0.48, + "Vietnamese,Chinese": 0.52, + "Vietnamese,Spanish": 0.38, + "Vietnamese,Malay": 0.36666666666666664, + "Vietnamese,Indonesian": 0.4, + "Vietnamese,English": 0.35333333333333333, + "Chinese,Spanish": 0.28, + "Chinese,Malay": 0.2733333333333333, + "Chinese,Indonesian": 0.34, + "Chinese,English": 0.2733333333333333, + "Spanish,Malay": 0.4666666666666667, + "Spanish,Indonesian": 0.5666666666666667, + "Spanish,English": 0.6066666666666667, + "Malay,Indonesian": 0.5533333333333333, + "Malay,English": 0.4266666666666667, + "Indonesian,English": 0.5133333333333333 }, "3_combine": { - "Spanish,English,Chinese": 0.08666666666666667, - "Spanish,English,Vietnamese": 0.08666666666666667, - "Spanish,English,Filipino": 0.04666666666666667, - "Spanish,English,Indonesian": 0.06666666666666667, - "Spanish,English,Malay": 0.06, - "Spanish,Chinese,Vietnamese": 0.06, - "Spanish,Chinese,Filipino": 0.03333333333333333, - "Spanish,Chinese,Indonesian": 0.03333333333333333, - "Spanish,Chinese,Malay": 0.09333333333333334, - "Spanish,Vietnamese,Filipino": 0.04666666666666667, - "Spanish,Vietnamese,Indonesian": 0.04, - "Spanish,Vietnamese,Malay": 0.05333333333333334, - "Spanish,Filipino,Indonesian": 0.07333333333333333, - "Spanish,Filipino,Malay": 0.06, - "Spanish,Indonesian,Malay": 0.06, - "English,Chinese,Vietnamese": 0.09333333333333334, - "English,Chinese,Filipino": 0.05333333333333334, - "English,Chinese,Indonesian": 0.05333333333333334, - "English,Chinese,Malay": 0.07333333333333333, - "English,Vietnamese,Filipino": 0.05333333333333334, - "English,Vietnamese,Indonesian": 0.06666666666666667, - "English,Vietnamese,Malay": 0.06666666666666667, - "English,Filipino,Indonesian": 0.08, - "English,Filipino,Malay": 0.06, - "English,Indonesian,Malay": 0.06, - "Chinese,Vietnamese,Filipino": 0.04, - "Chinese,Vietnamese,Indonesian": 0.06666666666666667, - "Chinese,Vietnamese,Malay": 0.06666666666666667, - "Chinese,Filipino,Indonesian": 0.04, - "Chinese,Filipino,Malay": 0.04666666666666667, - "Chinese,Indonesian,Malay": 0.02, - "Vietnamese,Filipino,Indonesian": 0.04666666666666667, - "Vietnamese,Filipino,Malay": 0.05333333333333334, - "Vietnamese,Indonesian,Malay": 0.06, - "Filipino,Indonesian,Malay": 0.06666666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.03333333333333333, - "Spanish,English,Chinese,Filipino": 0.0, - "Spanish,English,Chinese,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Malay": 0.013333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.006666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,English,Vietnamese,Malay": 0.02666666666666667, - "Spanish,English,Filipino,Indonesian": 0.02, - "Spanish,English,Filipino,Malay": 0.013333333333333334, - "Spanish,English,Indonesian,Malay": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Filipino": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.02666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.013333333333333334, - "Spanish,Chinese,Indonesian,Malay": 0.006666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,Filipino,Indonesian,Malay": 0.02, - "English,Chinese,Vietnamese,Filipino": 0.013333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.02, - "English,Chinese,Vietnamese,Malay": 0.02, - "English,Chinese,Filipino,Indonesian": 0.013333333333333334, - "English,Chinese,Filipino,Malay": 0.02, - "English,Chinese,Indonesian,Malay": 0.0, - "English,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "English,Vietnamese,Filipino,Malay": 0.006666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.02, - "English,Filipino,Indonesian,Malay": 0.02, - "Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.0, - "Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334 + "Filipino,Vietnamese,Chinese": 0.19333333333333333, + "Filipino,Vietnamese,Spanish": 0.24, + "Filipino,Vietnamese,Malay": 0.2, + "Filipino,Vietnamese,Indonesian": 0.26, + "Filipino,Vietnamese,English": 0.22666666666666666, + "Filipino,Chinese,Spanish": 0.16666666666666666, + "Filipino,Chinese,Malay": 0.11333333333333333, + "Filipino,Chinese,Indonesian": 0.16666666666666666, + "Filipino,Chinese,English": 0.16666666666666666, + "Filipino,Spanish,Malay": 0.2866666666666667, + "Filipino,Spanish,Indonesian": 0.3333333333333333, + "Filipino,Spanish,English": 0.32666666666666666, + "Filipino,Malay,Indonesian": 0.32, + "Filipino,Malay,English": 0.28, + "Filipino,Indonesian,English": 0.32666666666666666, + "Vietnamese,Chinese,Spanish": 0.18666666666666668, + "Vietnamese,Chinese,Malay": 0.16666666666666666, + "Vietnamese,Chinese,Indonesian": 0.2, + "Vietnamese,Chinese,English": 0.15333333333333332, + "Vietnamese,Spanish,Malay": 0.21333333333333335, + "Vietnamese,Spanish,Indonesian": 0.2733333333333333, + "Vietnamese,Spanish,English": 0.26666666666666666, + "Vietnamese,Malay,Indonesian": 0.23333333333333334, + "Vietnamese,Malay,English": 0.18666666666666668, + "Vietnamese,Indonesian,English": 0.23333333333333334, + "Chinese,Spanish,Malay": 0.14666666666666667, + "Chinese,Spanish,Indonesian": 0.19333333333333333, + "Chinese,Spanish,English": 0.18, + "Chinese,Malay,Indonesian": 0.16666666666666666, + "Chinese,Malay,English": 0.12, + "Chinese,Indonesian,English": 0.17333333333333334, + "Spanish,Malay,Indonesian": 0.36, + "Spanish,Malay,English": 0.32666666666666666, + "Spanish,Indonesian,English": 0.3933333333333333, + "Malay,Indonesian,English": 0.32 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.12, + "Filipino,Vietnamese,Chinese,Malay": 0.08, + "Filipino,Vietnamese,Chinese,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,English": 0.09333333333333334, + "Filipino,Vietnamese,Spanish,Malay": 0.16, + "Filipino,Vietnamese,Spanish,Indonesian": 0.2, + "Filipino,Vietnamese,Spanish,English": 0.18666666666666668, + "Filipino,Vietnamese,Malay,Indonesian": 0.16666666666666666, + "Filipino,Vietnamese,Malay,English": 0.14, + "Filipino,Vietnamese,Indonesian,English": 0.18, + "Filipino,Chinese,Spanish,Malay": 0.1, + "Filipino,Chinese,Spanish,Indonesian": 0.12666666666666668, + "Filipino,Chinese,Spanish,English": 0.11333333333333333, + "Filipino,Chinese,Malay,Indonesian": 0.1, + "Filipino,Chinese,Malay,English": 0.08, + "Filipino,Chinese,Indonesian,English": 0.10666666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.24, + "Filipino,Spanish,Malay,English": 0.23333333333333334, + "Filipino,Spanish,Indonesian,English": 0.2733333333333333, + "Filipino,Malay,Indonesian,English": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.10666666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.14, + "Vietnamese,Chinese,Spanish,English": 0.12, + "Vietnamese,Chinese,Malay,Indonesian": 0.11333333333333333, + "Vietnamese,Chinese,Malay,English": 0.08, + "Vietnamese,Chinese,Indonesian,English": 0.10666666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, + "Vietnamese,Spanish,Malay,English": 0.17333333333333334, + "Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, + "Vietnamese,Malay,Indonesian,English": 0.15333333333333332, + "Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, + "Chinese,Spanish,Malay,English": 0.1, + "Chinese,Spanish,Indonesian,English": 0.14, + "Chinese,Malay,Indonesian,English": 0.11333333333333333, + "Spanish,Malay,Indonesian,English": 0.26 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.013333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "English,Chinese,Filipino,Indonesian,Malay": 0.0, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.08, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.05333333333333334, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.07333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.14, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, + "Filipino,Chinese,Spanish,Malay,English": 0.07333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.09333333333333334, + "Filipino,Chinese,Malay,Indonesian,English": 0.08, + "Filipino,Spanish,Malay,Indonesian,English": 0.21333333333333335, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, + "Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.1, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.07333333333333333, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.14, + "Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.05333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.05333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334 } }, - "AC3_2": 0.2443636363136529, - "AC3_3": 0.09477707003200128, - "AC3_4": 0.023891402705472853, - "AC3_5": 0.004403669722952614, - "AC3_6": 0.0, - "AC3_7": 0.0 + "AC3_2": 0.40646059588436395, + "AC3_3": 0.29061474508562635, + "AC3_4": 0.21464449942954075, + "AC3_5": 0.16282404850532936, + "AC3_6": 0.12482435594503927, + "AC3_7": 0.09384835477141472 }, "prompt_5": { - "overall_acc": 0.2580952380952381, + "overall_acc": 0.39333333333333326, "language_acc": { - "Spanish": 0.22666666666666666, - "English": 0.26666666666666666, - "Chinese": 0.26666666666666666, - "Vietnamese": 0.22666666666666666, - "Filipino": 0.28, - "Indonesian": 0.26666666666666666, - "Malay": 0.2733333333333333 + "Filipino": 0.38, + "Vietnamese": 0.28, + "Chinese": 0.26, + "Spanish": 0.4866666666666667, + "Malay": 0.3333333333333333, + "Indonesian": 0.4066666666666667, + "English": 0.6066666666666667 }, - "consistency_score_2": 0.2511111111111111, - "consistency_score_3": 0.06342857142857143, - "consistency_score_4": 0.015428571428571427, - "consistency_score_5": 0.0038095238095238095, - "consistency_score_6": 0.0009523809523809525, - "consistency_score_7": 0.0, + "consistency_score_2": 0.45142857142857135, + "consistency_score_3": 0.2594285714285714, + "consistency_score_4": 0.1678095238095238, + "consistency_score_5": 0.11746031746031749, + "consistency_score_6": 0.08666666666666667, + "consistency_score_7": 0.06666666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.26666666666666666, - "Spanish,Chinese": 0.2733333333333333, - "Spanish,Vietnamese": 0.30666666666666664, - "Spanish,Filipino": 0.25333333333333335, - "Spanish,Indonesian": 0.2, - "Spanish,Malay": 0.23333333333333334, - "English,Chinese": 0.28, - "English,Vietnamese": 0.24666666666666667, - "English,Filipino": 0.2733333333333333, - "English,Indonesian": 0.23333333333333334, - "English,Malay": 0.21333333333333335, - "Chinese,Vietnamese": 0.20666666666666667, - "Chinese,Filipino": 0.23333333333333334, - "Chinese,Indonesian": 0.23333333333333334, - "Chinese,Malay": 0.26666666666666666, - "Vietnamese,Filipino": 0.2733333333333333, - "Vietnamese,Indonesian": 0.26, - "Vietnamese,Malay": 0.17333333333333334, - "Filipino,Indonesian": 0.34, - "Filipino,Malay": 0.25333333333333335, - "Indonesian,Malay": 0.25333333333333335 + "Filipino,Vietnamese": 0.42, + "Filipino,Chinese": 0.36666666666666664, + "Filipino,Spanish": 0.44666666666666666, + "Filipino,Malay": 0.5, + "Filipino,Indonesian": 0.54, + "Filipino,English": 0.48, + "Vietnamese,Chinese": 0.34, + "Vietnamese,Spanish": 0.36666666666666664, + "Vietnamese,Malay": 0.4066666666666667, + "Vietnamese,Indonesian": 0.3933333333333333, + "Vietnamese,English": 0.35333333333333333, + "Chinese,Spanish": 0.38, + "Chinese,Malay": 0.3933333333333333, + "Chinese,Indonesian": 0.4666666666666667, + "Chinese,English": 0.36666666666666664, + "Spanish,Malay": 0.5066666666666667, + "Spanish,Indonesian": 0.5933333333333334, + "Spanish,English": 0.6533333333333333, + "Malay,Indonesian": 0.5933333333333334, + "Malay,English": 0.44, + "Indonesian,English": 0.47333333333333333 }, "3_combine": { - "Spanish,English,Chinese": 0.08666666666666667, - "Spanish,English,Vietnamese": 0.06666666666666667, - "Spanish,English,Filipino": 0.09333333333333334, - "Spanish,English,Indonesian": 0.03333333333333333, - "Spanish,English,Malay": 0.06666666666666667, - "Spanish,Chinese,Vietnamese": 0.06666666666666667, - "Spanish,Chinese,Filipino": 0.08, - "Spanish,Chinese,Indonesian": 0.04, - "Spanish,Chinese,Malay": 0.08, - "Spanish,Vietnamese,Filipino": 0.09333333333333334, - "Spanish,Vietnamese,Indonesian": 0.08, - "Spanish,Vietnamese,Malay": 0.04666666666666667, - "Spanish,Filipino,Indonesian": 0.08666666666666667, - "Spanish,Filipino,Malay": 0.06, - "Spanish,Indonesian,Malay": 0.02666666666666667, - "English,Chinese,Vietnamese": 0.06, - "English,Chinese,Filipino": 0.08666666666666667, - "English,Chinese,Indonesian": 0.05333333333333334, - "English,Chinese,Malay": 0.04, - "English,Vietnamese,Filipino": 0.07333333333333333, - "English,Vietnamese,Indonesian": 0.06, - "English,Vietnamese,Malay": 0.03333333333333333, - "English,Filipino,Indonesian": 0.1, - "English,Filipino,Malay": 0.04666666666666667, - "English,Indonesian,Malay": 0.04666666666666667, - "Chinese,Vietnamese,Filipino": 0.06, - "Chinese,Vietnamese,Indonesian": 0.05333333333333334, - "Chinese,Vietnamese,Malay": 0.02666666666666667, - "Chinese,Filipino,Indonesian": 0.07333333333333333, - "Chinese,Filipino,Malay": 0.06666666666666667, - "Chinese,Indonesian,Malay": 0.08, - "Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Vietnamese,Filipino,Malay": 0.02, - "Vietnamese,Indonesian,Malay": 0.04666666666666667, - "Filipino,Indonesian,Malay": 0.07333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.02, - "Spanish,English,Chinese,Filipino": 0.04, - "Spanish,English,Chinese,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Malay": 0.02666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.02, - "Spanish,English,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,English,Vietnamese,Malay": 0.006666666666666667, - "Spanish,English,Filipino,Indonesian": 0.02, - "Spanish,English,Filipino,Malay": 0.013333333333333334, - "Spanish,English,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino": 0.02, - "Spanish,Chinese,Vietnamese,Indonesian": 0.02, - "Spanish,Chinese,Vietnamese,Malay": 0.006666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.02, - "Spanish,Chinese,Filipino,Malay": 0.02666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.013333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian": 0.04, - "Spanish,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.02666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.02, - "English,Chinese,Vietnamese,Malay": 0.0, - "English,Chinese,Filipino,Indonesian": 0.02666666666666667, - "English,Chinese,Filipino,Malay": 0.006666666666666667, - "English,Chinese,Indonesian,Malay": 0.006666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "English,Vietnamese,Filipino,Malay": 0.0, - "English,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "English,Filipino,Indonesian,Malay": 0.013333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.0, - "Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.02, - "Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.0, - "Spanish,English,Chinese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.02, - "English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "English,Chinese,Filipino,Indonesian,Malay": 0.0, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese": 0.17333333333333334, + "Filipino,Vietnamese,Spanish": 0.22, + "Filipino,Vietnamese,Malay": 0.24, + "Filipino,Vietnamese,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,English": 0.22666666666666666, + "Filipino,Chinese,Spanish": 0.20666666666666667, + "Filipino,Chinese,Malay": 0.22666666666666666, + "Filipino,Chinese,Indonesian": 0.24666666666666667, + "Filipino,Chinese,English": 0.20666666666666667, + "Filipino,Spanish,Malay": 0.32, + "Filipino,Spanish,Indonesian": 0.34, + "Filipino,Spanish,English": 0.34, + "Filipino,Malay,Indonesian": 0.37333333333333335, + "Filipino,Malay,English": 0.30666666666666664, + "Filipino,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish": 0.18, + "Vietnamese,Chinese,Malay": 0.19333333333333333, + "Vietnamese,Chinese,Indonesian": 0.17333333333333334, + "Vietnamese,Chinese,English": 0.16666666666666666, + "Vietnamese,Spanish,Malay": 0.23333333333333334, + "Vietnamese,Spanish,Indonesian": 0.26666666666666666, + "Vietnamese,Spanish,English": 0.25333333333333335, + "Vietnamese,Malay,Indonesian": 0.2733333333333333, + "Vietnamese,Malay,English": 0.20666666666666667, + "Vietnamese,Indonesian,English": 0.22, + "Chinese,Spanish,Malay": 0.23333333333333334, + "Chinese,Spanish,Indonesian": 0.28, + "Chinese,Spanish,English": 0.24666666666666667, + "Chinese,Malay,Indonesian": 0.29333333333333333, + "Chinese,Malay,English": 0.2, + "Chinese,Indonesian,English": 0.22, + "Spanish,Malay,Indonesian": 0.4066666666666667, + "Spanish,Malay,English": 0.3466666666666667, + "Spanish,Indonesian,English": 0.38666666666666666, + "Malay,Indonesian,English": 0.31333333333333335 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.12, + "Filipino,Vietnamese,Chinese,Malay": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Indonesian": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,English": 0.12, + "Filipino,Vietnamese,Spanish,Malay": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,English": 0.17333333333333334, + "Filipino,Vietnamese,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Malay,English": 0.16, + "Filipino,Vietnamese,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Malay": 0.15333333333333332, + "Filipino,Chinese,Spanish,Indonesian": 0.16, + "Filipino,Chinese,Spanish,English": 0.16, + "Filipino,Chinese,Malay,Indonesian": 0.16666666666666666, + "Filipino,Chinese,Malay,English": 0.16, + "Filipino,Chinese,Indonesian,English": 0.14666666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.2733333333333333, + "Filipino,Spanish,Malay,English": 0.25333333333333335, + "Filipino,Spanish,Indonesian,English": 0.26, + "Filipino,Malay,Indonesian,English": 0.24, + "Vietnamese,Chinese,Spanish,Malay": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,Indonesian": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,English": 0.12666666666666668, + "Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, + "Vietnamese,Chinese,Malay,English": 0.12, + "Vietnamese,Chinese,Indonesian,English": 0.1, + "Vietnamese,Spanish,Malay,Indonesian": 0.2, + "Vietnamese,Spanish,Malay,English": 0.16666666666666666, + "Vietnamese,Spanish,Indonesian,English": 0.18, + "Vietnamese,Malay,Indonesian,English": 0.16, + "Chinese,Spanish,Malay,Indonesian": 0.2, + "Chinese,Spanish,Malay,English": 0.15333333333333332, + "Chinese,Spanish,Indonesian,English": 0.18666666666666668, + "Chinese,Malay,Indonesian,English": 0.15333333333333332, + "Spanish,Malay,Indonesian,English": 0.2733333333333333 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.1, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Malay,English": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.08, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.14, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,English": 0.12666666666666668, + "Filipino,Chinese,Spanish,Indonesian,English": 0.12666666666666668, + "Filipino,Chinese,Malay,Indonesian,English": 0.12, + "Filipino,Spanish,Malay,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, + "Vietnamese,Chinese,Spanish,Malay,English": 0.09333333333333334, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.09333333333333334, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.08, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.14, + "Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.07333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 } }, - "AC3_2": 0.25455527842049985, - "AC3_3": 0.10183141499879565, - "AC3_4": 0.02911659370204899, - "AC3_5": 0.007508225105358331, - "AC3_6": 0.0018977591029088663, - "AC3_7": 0.0 + "AC3_2": 0.42038331449364114, + "AC3_3": 0.3126466296573856, + "AC3_4": 0.23525231948738146, + "AC3_5": 0.18089910914212767, + "AC3_6": 0.142037037007446, + "AC3_7": 0.11400966181096407 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.2402597402597403, + "overall_acc": 0.37743506493506496, "language_acc": { - "English": 0.2897727272727273, - "Filipino": 0.22727272727272727, - "Spanish": 0.23863636363636365, - "Malay": 0.2215909090909091, - "Chinese": 0.22727272727272727, - "Indonesian": 0.23863636363636365, - "Vietnamese": 0.23863636363636365 + "Indonesian": 0.4034090909090909, + "English": 0.6022727272727273, + "Filipino": 0.2897727272727273, + "Spanish": 0.45454545454545453, + "Chinese": 0.24431818181818182, + "Malay": 0.3522727272727273, + "Vietnamese": 0.29545454545454547 }, - "consistency_score_2": 0.2602813852813853, - "consistency_score_3": 0.07272727272727275, - "consistency_score_4": 0.022077922077922085, - "consistency_score_5": 0.007034632034632034, - "consistency_score_6": 0.0016233766233766235, - "consistency_score_7": 0.0, + "consistency_score_2": 0.40205627705627706, + "consistency_score_3": 0.20129870129870134, + "consistency_score_4": 0.11347402597402598, + "consistency_score_5": 0.06953463203463203, + "consistency_score_6": 0.04626623376623377, + "consistency_score_7": 0.03409090909090909, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.23863636363636365, - "English,Spanish": 0.2840909090909091, - "English,Malay": 0.26136363636363635, - "English,Chinese": 0.26136363636363635, - "English,Indonesian": 0.3352272727272727, - "English,Vietnamese": 0.2840909090909091, - "Filipino,Spanish": 0.25, - "Filipino,Malay": 0.26704545454545453, - "Filipino,Chinese": 0.3409090909090909, - "Filipino,Indonesian": 0.18181818181818182, - "Filipino,Vietnamese": 0.18181818181818182, - "Spanish,Malay": 0.23295454545454544, - "Spanish,Chinese": 0.32386363636363635, - "Spanish,Indonesian": 0.2784090909090909, - "Spanish,Vietnamese": 0.2556818181818182, - "Malay,Chinese": 0.23295454545454544, - "Malay,Indonesian": 0.2215909090909091, - "Malay,Vietnamese": 0.22727272727272727, - "Chinese,Indonesian": 0.2784090909090909, - "Chinese,Vietnamese": 0.23863636363636365, - "Indonesian,Vietnamese": 0.2897727272727273 + "Indonesian,English": 0.4602272727272727, + "Indonesian,Filipino": 0.48295454545454547, + "Indonesian,Spanish": 0.5056818181818182, + "Indonesian,Chinese": 0.3409090909090909, + "Indonesian,Malay": 0.5397727272727273, + "Indonesian,Vietnamese": 0.32954545454545453, + "English,Filipino": 0.3409090909090909, + "English,Spanish": 0.6647727272727273, + "English,Chinese": 0.2215909090909091, + "English,Malay": 0.42045454545454547, + "English,Vietnamese": 0.29545454545454547, + "Filipino,Spanish": 0.4034090909090909, + "Filipino,Chinese": 0.2784090909090909, + "Filipino,Malay": 0.48863636363636365, + "Filipino,Vietnamese": 0.3693181818181818, + "Spanish,Chinese": 0.25, + "Spanish,Malay": 0.4659090909090909, + "Spanish,Vietnamese": 0.3465909090909091, + "Chinese,Malay": 0.3465909090909091, + "Chinese,Vietnamese": 0.4943181818181818, + "Malay,Vietnamese": 0.3977272727272727 }, "3_combine": { - "English,Filipino,Spanish": 0.056818181818181816, - "English,Filipino,Malay": 0.05113636363636364, - "English,Filipino,Chinese": 0.07954545454545454, - "English,Filipino,Indonesian": 0.06818181818181818, - "English,Filipino,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay": 0.07954545454545454, - "English,Spanish,Chinese": 0.08522727272727272, - "English,Spanish,Indonesian": 0.11931818181818182, - "English,Spanish,Vietnamese": 0.08522727272727272, - "English,Malay,Chinese": 0.05113636363636364, - "English,Malay,Indonesian": 0.09090909090909091, - "English,Malay,Vietnamese": 0.07954545454545454, - "English,Chinese,Indonesian": 0.11363636363636363, - "English,Chinese,Vietnamese": 0.0625, - "English,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay": 0.09659090909090909, - "Filipino,Spanish,Chinese": 0.09659090909090909, - "Filipino,Spanish,Indonesian": 0.056818181818181816, - "Filipino,Spanish,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Chinese": 0.10227272727272728, - "Filipino,Malay,Indonesian": 0.03409090909090909, - "Filipino,Malay,Vietnamese": 0.03409090909090909, - "Filipino,Chinese,Indonesian": 0.06818181818181818, - "Filipino,Chinese,Vietnamese": 0.056818181818181816, - "Filipino,Indonesian,Vietnamese": 0.022727272727272728, - "Spanish,Malay,Chinese": 0.07954545454545454, - "Spanish,Malay,Indonesian": 0.07386363636363637, - "Spanish,Malay,Vietnamese": 0.06818181818181818, - "Spanish,Chinese,Indonesian": 0.08522727272727272, - "Spanish,Chinese,Vietnamese": 0.07954545454545454, - "Spanish,Indonesian,Vietnamese": 0.07386363636363637, - "Malay,Chinese,Indonesian": 0.0625, - "Malay,Chinese,Vietnamese": 0.0625, - "Malay,Indonesian,Vietnamese": 0.06818181818181818, - "Chinese,Indonesian,Vietnamese": 0.08522727272727272 - }, + "Indonesian,English,Filipino": 0.23295454545454544, + "Indonesian,English,Spanish": 0.36363636363636365, + "Indonesian,English,Chinese": 0.13636363636363635, + "Indonesian,English,Malay": 0.2840909090909091, + "Indonesian,English,Vietnamese": 0.16477272727272727, + "Indonesian,Filipino,Spanish": 0.2727272727272727, + "Indonesian,Filipino,Chinese": 0.17045454545454544, + "Indonesian,Filipino,Malay": 0.3181818181818182, + "Indonesian,Filipino,Vietnamese": 0.2159090909090909, + "Indonesian,Spanish,Chinese": 0.1534090909090909, + "Indonesian,Spanish,Malay": 0.3181818181818182, + "Indonesian,Spanish,Vietnamese": 0.1875, + "Indonesian,Chinese,Malay": 0.21022727272727273, + "Indonesian,Chinese,Vietnamese": 0.1875, + "Indonesian,Malay,Vietnamese": 0.20454545454545456, + "English,Filipino,Spanish": 0.2556818181818182, + "English,Filipino,Chinese": 0.09659090909090909, + "English,Filipino,Malay": 0.2159090909090909, + "English,Filipino,Vietnamese": 0.13636363636363635, + "English,Spanish,Chinese": 0.1590909090909091, + "English,Spanish,Malay": 0.32954545454545453, + "English,Spanish,Vietnamese": 0.23863636363636365, + "English,Chinese,Malay": 0.10795454545454546, + "English,Chinese,Vietnamese": 0.10795454545454546, + "English,Malay,Vietnamese": 0.1534090909090909, + "Filipino,Spanish,Chinese": 0.11931818181818182, + "Filipino,Spanish,Malay": 0.26136363636363635, + "Filipino,Spanish,Vietnamese": 0.17613636363636365, + "Filipino,Chinese,Malay": 0.17045454545454544, + "Filipino,Chinese,Vietnamese": 0.16477272727272727, + "Filipino,Malay,Vietnamese": 0.2215909090909091, + "Spanish,Chinese,Malay": 0.14772727272727273, + "Spanish,Chinese,Vietnamese": 0.14204545454545456, + "Spanish,Malay,Vietnamese": 0.19886363636363635, + "Chinese,Malay,Vietnamese": 0.2215909090909091 + }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.022727272727272728, - "English,Filipino,Spanish,Chinese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese": 0.011363636363636364, - "English,Filipino,Malay,Indonesian": 0.011363636363636364, - "English,Filipino,Malay,Vietnamese": 0.011363636363636364, - "English,Filipino,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese": 0.022727272727272728, - "English,Spanish,Malay,Indonesian": 0.03409090909090909, - "English,Spanish,Malay,Vietnamese": 0.03409090909090909, - "English,Spanish,Chinese,Indonesian": 0.045454545454545456, - "English,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Spanish,Indonesian,Vietnamese": 0.03409090909090909, - "English,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Malay,Indonesian,Vietnamese": 0.03409090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese": 0.03977272727272727, - "Filipino,Spanish,Malay,Indonesian": 0.017045454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Chinese,Indonesian": 0.03409090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.011363636363636364, - "Filipino,Malay,Chinese,Vietnamese": 0.017045454545454544, - "Filipino,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Spanish,Malay,Indonesian,Vietnamese": 0.03977272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 + "Indonesian,English,Filipino,Spanish": 0.19318181818181818, + "Indonesian,English,Filipino,Chinese": 0.08522727272727272, + "Indonesian,English,Filipino,Malay": 0.14772727272727273, + "Indonesian,English,Filipino,Vietnamese": 0.10795454545454546, + "Indonesian,English,Spanish,Chinese": 0.11363636363636363, + "Indonesian,English,Spanish,Malay": 0.24431818181818182, + "Indonesian,English,Spanish,Vietnamese": 0.14204545454545456, + "Indonesian,English,Chinese,Malay": 0.08522727272727272, + "Indonesian,English,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,English,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Chinese": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Malay": 0.19318181818181818, + "Indonesian,Filipino,Spanish,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Chinese,Malay": 0.13636363636363635, + "Indonesian,Filipino,Chinese,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Malay": 0.10795454545454546, + "Indonesian,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,Spanish,Malay,Vietnamese": 0.125, + "Indonesian,Chinese,Malay,Vietnamese": 0.125, + "English,Filipino,Spanish,Chinese": 0.07386363636363637, + "English,Filipino,Spanish,Malay": 0.17045454545454544, + "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, + "English,Filipino,Chinese,Malay": 0.0625, + "English,Filipino,Chinese,Vietnamese": 0.045454545454545456, + "English,Filipino,Malay,Vietnamese": 0.08522727272727272, + "English,Spanish,Chinese,Malay": 0.09659090909090909, + "English,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "English,Spanish,Malay,Vietnamese": 0.13636363636363635, + "English,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Filipino,Spanish,Chinese,Malay": 0.08522727272727272, + "Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, + "Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, + "Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 + "Indonesian,English,Filipino,Spanish,Chinese": 0.06818181818181818, + "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Spanish,Chinese,Malay": 0.07954545454545454, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.0625, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.07386363636363637, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0625, + "English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, + "English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, + "English,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 } }, - "AC3_2": 0.24987012982020987, - "AC3_3": 0.11165597884022162, - "AC3_4": 0.04043975824612529, - "AC3_5": 0.013669044298090167, - "AC3_6": 0.0032249629551733077, - "AC3_7": 0.0 + "AC3_2": 0.3893568250062849, + "AC3_3": 0.26256352338772004, + "AC3_4": 0.17448882958721418, + "AC3_5": 0.1174343966908109, + "AC3_6": 0.08242834749510006, + "AC3_7": 0.06253362021073237 }, "prompt_2": { - "overall_acc": 0.23457792207792208, + "overall_acc": 0.3758116883116883, "language_acc": { - "English": 0.25, - "Filipino": 0.23295454545454544, - "Spanish": 0.22727272727272727, - "Malay": 0.2215909090909091, - "Chinese": 0.23863636363636365, - "Indonesian": 0.23863636363636365, - "Vietnamese": 0.23295454545454544 + "Indonesian": 0.4034090909090909, + "English": 0.6022727272727273, + "Filipino": 0.29545454545454547, + "Spanish": 0.4602272727272727, + "Chinese": 0.24431818181818182, + "Malay": 0.3352272727272727, + "Vietnamese": 0.2897727272727273 }, - "consistency_score_2": 0.2532467532467533, - "consistency_score_3": 0.0672077922077922, - "consistency_score_4": 0.01834415584415584, - "consistency_score_5": 0.004599567099567099, - "consistency_score_6": 0.0008116883116883117, - "consistency_score_7": 0.0, + "consistency_score_2": 0.4066558441558442, + "consistency_score_3": 0.20844155844155843, + "consistency_score_4": 0.11866883116883116, + "consistency_score_5": 0.07196969696969698, + "consistency_score_6": 0.04707792207792209, + "consistency_score_7": 0.03409090909090909, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.20454545454545456, - "English,Spanish": 0.26136363636363635, - "English,Malay": 0.25, - "English,Chinese": 0.23295454545454544, - "English,Indonesian": 0.2556818181818182, - "English,Vietnamese": 0.23295454545454544, - "Filipino,Spanish": 0.2556818181818182, - "Filipino,Malay": 0.19886363636363635, - "Filipino,Chinese": 0.2215909090909091, - "Filipino,Indonesian": 0.22727272727272727, - "Filipino,Vietnamese": 0.2556818181818182, - "Spanish,Malay": 0.25, - "Spanish,Chinese": 0.23863636363636365, - "Spanish,Indonesian": 0.3125, - "Spanish,Vietnamese": 0.29545454545454547, - "Malay,Chinese": 0.2897727272727273, - "Malay,Indonesian": 0.24431818181818182, - "Malay,Vietnamese": 0.25, - "Chinese,Indonesian": 0.2897727272727273, - "Chinese,Vietnamese": 0.2897727272727273, - "Indonesian,Vietnamese": 0.26136363636363635 + "Indonesian,English": 0.4715909090909091, + "Indonesian,Filipino": 0.4772727272727273, + "Indonesian,Spanish": 0.5454545454545454, + "Indonesian,Chinese": 0.3352272727272727, + "Indonesian,Malay": 0.5454545454545454, + "Indonesian,Vietnamese": 0.3352272727272727, + "English,Filipino": 0.3409090909090909, + "English,Spanish": 0.6590909090909091, + "English,Chinese": 0.2215909090909091, + "English,Malay": 0.4147727272727273, + "English,Vietnamese": 0.2784090909090909, + "Filipino,Spanish": 0.4318181818181818, + "Filipino,Chinese": 0.3068181818181818, + "Filipino,Malay": 0.5113636363636364, + "Filipino,Vietnamese": 0.3693181818181818, + "Spanish,Chinese": 0.2556818181818182, + "Spanish,Malay": 0.48295454545454547, + "Spanish,Vietnamese": 0.32386363636363635, + "Chinese,Malay": 0.35795454545454547, + "Chinese,Vietnamese": 0.4943181818181818, + "Malay,Vietnamese": 0.3806818181818182 }, "3_combine": { - "English,Filipino,Spanish": 0.03409090909090909, - "English,Filipino,Malay": 0.045454545454545456, - "English,Filipino,Chinese": 0.03977272727272727, - "English,Filipino,Indonesian": 0.045454545454545456, - "English,Filipino,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay": 0.056818181818181816, - "English,Spanish,Chinese": 0.08522727272727272, - "English,Spanish,Indonesian": 0.10227272727272728, - "English,Spanish,Vietnamese": 0.056818181818181816, - "English,Malay,Chinese": 0.07954545454545454, - "English,Malay,Indonesian": 0.045454545454545456, - "English,Malay,Vietnamese": 0.0625, - "English,Chinese,Indonesian": 0.07386363636363637, - "English,Chinese,Vietnamese": 0.056818181818181816, - "English,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay": 0.06818181818181818, - "Filipino,Spanish,Chinese": 0.045454545454545456, - "Filipino,Spanish,Indonesian": 0.07954545454545454, - "Filipino,Spanish,Vietnamese": 0.09090909090909091, - "Filipino,Malay,Chinese": 0.06818181818181818, - "Filipino,Malay,Indonesian": 0.045454545454545456, - "Filipino,Malay,Vietnamese": 0.03977272727272727, - "Filipino,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Indonesian,Vietnamese": 0.0625, - "Spanish,Malay,Chinese": 0.056818181818181816, - "Spanish,Malay,Indonesian": 0.06818181818181818, - "Spanish,Malay,Vietnamese": 0.0625, - "Spanish,Chinese,Indonesian": 0.11363636363636363, - "Spanish,Chinese,Vietnamese": 0.07954545454545454, - "Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "Malay,Chinese,Indonesian": 0.07954545454545454, - "Malay,Chinese,Vietnamese": 0.07954545454545454, - "Malay,Indonesian,Vietnamese": 0.0625, - "Chinese,Indonesian,Vietnamese": 0.07386363636363637 + "Indonesian,English,Filipino": 0.25, + "Indonesian,English,Spanish": 0.3806818181818182, + "Indonesian,English,Chinese": 0.14204545454545456, + "Indonesian,English,Malay": 0.2897727272727273, + "Indonesian,English,Vietnamese": 0.1534090909090909, + "Indonesian,Filipino,Spanish": 0.3068181818181818, + "Indonesian,Filipino,Chinese": 0.18181818181818182, + "Indonesian,Filipino,Malay": 0.3409090909090909, + "Indonesian,Filipino,Vietnamese": 0.2159090909090909, + "Indonesian,Spanish,Chinese": 0.16477272727272727, + "Indonesian,Spanish,Malay": 0.3465909090909091, + "Indonesian,Spanish,Vietnamese": 0.1875, + "Indonesian,Chinese,Malay": 0.21022727272727273, + "Indonesian,Chinese,Vietnamese": 0.19318181818181818, + "Indonesian,Malay,Vietnamese": 0.2159090909090909, + "English,Filipino,Spanish": 0.26136363636363635, + "English,Filipino,Chinese": 0.09659090909090909, + "English,Filipino,Malay": 0.2215909090909091, + "English,Filipino,Vietnamese": 0.125, + "English,Spanish,Chinese": 0.1590909090909091, + "English,Spanish,Malay": 0.3352272727272727, + "English,Spanish,Vietnamese": 0.2159090909090909, + "English,Chinese,Malay": 0.11363636363636363, + "English,Chinese,Vietnamese": 0.10795454545454546, + "English,Malay,Vietnamese": 0.14772727272727273, + "Filipino,Spanish,Chinese": 0.14204545454545456, + "Filipino,Spanish,Malay": 0.2840909090909091, + "Filipino,Spanish,Vietnamese": 0.17613636363636365, + "Filipino,Chinese,Malay": 0.19318181818181818, + "Filipino,Chinese,Vietnamese": 0.19318181818181818, + "Filipino,Malay,Vietnamese": 0.2215909090909091, + "Spanish,Chinese,Malay": 0.16477272727272727, + "Spanish,Chinese,Vietnamese": 0.14204545454545456, + "Spanish,Malay,Vietnamese": 0.19318181818181818, + "Chinese,Malay,Vietnamese": 0.2215909090909091 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.005681818181818182, - "English,Filipino,Spanish,Chinese": 0.011363636363636364, - "English,Filipino,Spanish,Indonesian": 0.022727272727272728, - "English,Filipino,Spanish,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian": 0.011363636363636364, - "English,Filipino,Malay,Vietnamese": 0.011363636363636364, - "English,Filipino,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese": 0.022727272727272728, - "English,Spanish,Malay,Indonesian": 0.017045454545454544, - "English,Spanish,Malay,Vietnamese": 0.011363636363636364, - "English,Spanish,Chinese,Indonesian": 0.045454545454545456, - "English,Spanish,Chinese,Vietnamese": 0.011363636363636364, - "English,Spanish,Indonesian,Vietnamese": 0.028409090909090908, - "English,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Malay,Indonesian,Vietnamese": 0.017045454545454544, - "English,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese": 0.011363636363636364, - "Filipino,Spanish,Malay,Indonesian": 0.017045454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "Spanish,Malay,Indonesian,Vietnamese": 0.028409090909090908, - "Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 + "Indonesian,English,Filipino,Spanish": 0.21022727272727273, + "Indonesian,English,Filipino,Chinese": 0.08522727272727272, + "Indonesian,English,Filipino,Malay": 0.17045454545454544, + "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, + "Indonesian,English,Spanish,Chinese": 0.11931818181818182, + "Indonesian,English,Spanish,Malay": 0.26136363636363635, + "Indonesian,English,Spanish,Vietnamese": 0.13068181818181818, + "Indonesian,English,Chinese,Malay": 0.09090909090909091, + "Indonesian,English,Chinese,Vietnamese": 0.06818181818181818, + "Indonesian,English,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Chinese": 0.10795454545454546, + "Indonesian,Filipino,Spanish,Malay": 0.2215909090909091, + "Indonesian,Filipino,Spanish,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Chinese,Malay": 0.14204545454545456, + "Indonesian,Filipino,Chinese,Vietnamese": 0.125, + "Indonesian,Filipino,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Spanish,Chinese,Malay": 0.11363636363636363, + "Indonesian,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,Spanish,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,Chinese,Malay,Vietnamese": 0.13068181818181818, + "English,Filipino,Spanish,Chinese": 0.07954545454545454, + "English,Filipino,Spanish,Malay": 0.17613636363636365, + "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, + "English,Filipino,Chinese,Malay": 0.0625, + "English,Filipino,Chinese,Vietnamese": 0.045454545454545456, + "English,Filipino,Malay,Vietnamese": 0.07954545454545454, + "English,Spanish,Chinese,Malay": 0.10227272727272728, + "English,Spanish,Chinese,Vietnamese": 0.07954545454545454, + "English,Spanish,Malay,Vietnamese": 0.13068181818181818, + "English,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Filipino,Spanish,Chinese,Malay": 0.10227272727272728, + "Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, + "Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, + "Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.0, - "English,Filipino,Spanish,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 + "Indonesian,English,Filipino,Spanish,Chinese": 0.07386363636363637, + "Indonesian,English,Filipino,Spanish,Malay": 0.1534090909090909, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.0625, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0625, + "English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, + "English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, + "English,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 } }, - "AC3_2": 0.24355511372149316, - "AC3_3": 0.10448118311789083, - "AC3_4": 0.03402734940221407, - "AC3_5": 0.009022227768455583, - "AC3_6": 0.0016177787722639098, - "AC3_7": 0.0 + "AC3_2": 0.39062584195040106, + "AC3_3": 0.26815349136405475, + "AC3_4": 0.18037974005271648, + "AC3_5": 0.12080472394995091, + "AC3_6": 0.08367400351986029, + "AC3_7": 0.06251125110986225 }, "prompt_3": { - "overall_acc": 0.26055194805194803, + "overall_acc": 0.3741883116883117, "language_acc": { - "English": 0.26704545454545453, - "Filipino": 0.26136363636363635, - "Spanish": 0.23863636363636365, - "Malay": 0.23863636363636365, - "Chinese": 0.23863636363636365, - "Indonesian": 0.3409090909090909, - "Vietnamese": 0.23863636363636365 + "Indonesian": 0.4034090909090909, + "English": 0.6193181818181818, + "Filipino": 0.29545454545454547, + "Spanish": 0.4602272727272727, + "Chinese": 0.22727272727272727, + "Malay": 0.3352272727272727, + "Vietnamese": 0.2784090909090909 }, - "consistency_score_2": 0.26406926406926406, - "consistency_score_3": 0.0719155844155844, - "consistency_score_4": 0.019480519480519477, - "consistency_score_5": 0.005140692640692639, - "consistency_score_6": 0.0008116883116883117, - "consistency_score_7": 0.0, + "consistency_score_2": 0.4120670995670995, + "consistency_score_3": 0.20941558441558442, + "consistency_score_4": 0.11672077922077921, + "consistency_score_5": 0.06899350649350648, + "consistency_score_6": 0.04301948051948051, + "consistency_score_7": 0.028409090909090908, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.26136363636363635, - "English,Spanish": 0.2556818181818182, - "English,Malay": 0.25, - "English,Chinese": 0.23863636363636365, - "English,Indonesian": 0.2159090909090909, - "English,Vietnamese": 0.2556818181818182, - "Filipino,Spanish": 0.26704545454545453, - "Filipino,Malay": 0.3181818181818182, - "Filipino,Chinese": 0.26704545454545453, - "Filipino,Indonesian": 0.29545454545454547, - "Filipino,Vietnamese": 0.25, - "Spanish,Malay": 0.3068181818181818, - "Spanish,Chinese": 0.2556818181818182, - "Spanish,Indonesian": 0.2897727272727273, - "Spanish,Vietnamese": 0.19886363636363635, - "Malay,Chinese": 0.2727272727272727, - "Malay,Indonesian": 0.2897727272727273, - "Malay,Vietnamese": 0.2784090909090909, - "Chinese,Indonesian": 0.2784090909090909, - "Chinese,Vietnamese": 0.23295454545454544, - "Indonesian,Vietnamese": 0.26704545454545453 + "Indonesian,English": 0.4715909090909091, + "Indonesian,Filipino": 0.4715909090909091, + "Indonesian,Spanish": 0.5340909090909091, + "Indonesian,Chinese": 0.3465909090909091, + "Indonesian,Malay": 0.5397727272727273, + "Indonesian,Vietnamese": 0.3352272727272727, + "English,Filipino": 0.3352272727272727, + "English,Spanish": 0.6590909090909091, + "English,Chinese": 0.23295454545454544, + "English,Malay": 0.42045454545454547, + "English,Vietnamese": 0.2840909090909091, + "Filipino,Spanish": 0.4090909090909091, + "Filipino,Chinese": 0.3068181818181818, + "Filipino,Malay": 0.5340909090909091, + "Filipino,Vietnamese": 0.3806818181818182, + "Spanish,Chinese": 0.2727272727272727, + "Spanish,Malay": 0.48863636363636365, + "Spanish,Vietnamese": 0.3125, + "Chinese,Malay": 0.39204545454545453, + "Chinese,Vietnamese": 0.5340909090909091, + "Malay,Vietnamese": 0.39204545454545453 }, "3_combine": { - "English,Filipino,Spanish": 0.06818181818181818, - "English,Filipino,Malay": 0.09090909090909091, - "English,Filipino,Chinese": 0.07954545454545454, - "English,Filipino,Indonesian": 0.09090909090909091, - "English,Filipino,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay": 0.0625, - "English,Spanish,Chinese": 0.03409090909090909, - "English,Spanish,Indonesian": 0.06818181818181818, - "English,Spanish,Vietnamese": 0.028409090909090908, - "English,Malay,Chinese": 0.0625, - "English,Malay,Indonesian": 0.0625, - "English,Malay,Vietnamese": 0.09090909090909091, - "English,Chinese,Indonesian": 0.05113636363636364, - "English,Chinese,Vietnamese": 0.03977272727272727, - "English,Indonesian,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Malay": 0.09090909090909091, - "Filipino,Spanish,Chinese": 0.06818181818181818, - "Filipino,Spanish,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Vietnamese": 0.03977272727272727, - "Filipino,Malay,Chinese": 0.09090909090909091, - "Filipino,Malay,Indonesian": 0.10795454545454546, - "Filipino,Malay,Vietnamese": 0.07954545454545454, - "Filipino,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Chinese,Vietnamese": 0.0625, - "Filipino,Indonesian,Vietnamese": 0.08522727272727272, - "Spanish,Malay,Chinese": 0.10227272727272728, - "Spanish,Malay,Indonesian": 0.09090909090909091, - "Spanish,Malay,Vietnamese": 0.07386363636363637, - "Spanish,Chinese,Indonesian": 0.09090909090909091, - "Spanish,Chinese,Vietnamese": 0.045454545454545456, - "Spanish,Indonesian,Vietnamese": 0.05113636363636364, - "Malay,Chinese,Indonesian": 0.09090909090909091, - "Malay,Chinese,Vietnamese": 0.07386363636363637, - "Malay,Indonesian,Vietnamese": 0.07954545454545454, - "Chinese,Indonesian,Vietnamese": 0.056818181818181816 + "Indonesian,English,Filipino": 0.23863636363636365, + "Indonesian,English,Spanish": 0.3806818181818182, + "Indonesian,English,Chinese": 0.14204545454545456, + "Indonesian,English,Malay": 0.2897727272727273, + "Indonesian,English,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Spanish": 0.2897727272727273, + "Indonesian,Filipino,Chinese": 0.18181818181818182, + "Indonesian,Filipino,Malay": 0.3522727272727273, + "Indonesian,Filipino,Vietnamese": 0.20454545454545456, + "Indonesian,Spanish,Chinese": 0.17045454545454544, + "Indonesian,Spanish,Malay": 0.3465909090909091, + "Indonesian,Spanish,Vietnamese": 0.17613636363636365, + "Indonesian,Chinese,Malay": 0.22727272727272727, + "Indonesian,Chinese,Vietnamese": 0.21022727272727273, + "Indonesian,Malay,Vietnamese": 0.2159090909090909, + "English,Filipino,Spanish": 0.25, + "English,Filipino,Chinese": 0.09090909090909091, + "English,Filipino,Malay": 0.23295454545454544, + "English,Filipino,Vietnamese": 0.11931818181818182, + "English,Spanish,Chinese": 0.16477272727272727, + "English,Spanish,Malay": 0.3352272727272727, + "English,Spanish,Vietnamese": 0.21022727272727273, + "English,Chinese,Malay": 0.125, + "English,Chinese,Vietnamese": 0.11363636363636363, + "English,Malay,Vietnamese": 0.13636363636363635, + "Filipino,Spanish,Chinese": 0.13068181818181818, + "Filipino,Spanish,Malay": 0.2840909090909091, + "Filipino,Spanish,Vietnamese": 0.1590909090909091, + "Filipino,Chinese,Malay": 0.20454545454545456, + "Filipino,Chinese,Vietnamese": 0.21022727272727273, + "Filipino,Malay,Vietnamese": 0.23295454545454544, + "Spanish,Chinese,Malay": 0.17613636363636365, + "Spanish,Chinese,Vietnamese": 0.1534090909090909, + "Spanish,Malay,Vietnamese": 0.18181818181818182, + "Chinese,Malay,Vietnamese": 0.24431818181818182 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.022727272727272728, - "English,Filipino,Spanish,Chinese": 0.005681818181818182, - "English,Filipino,Spanish,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian": 0.045454545454545456, - "English,Filipino,Malay,Vietnamese": 0.03409090909090909, - "English,Filipino,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Indonesian,Vietnamese": 0.022727272727272728, - "English,Spanish,Malay,Chinese": 0.0, - "English,Spanish,Malay,Indonesian": 0.017045454545454544, - "English,Spanish,Malay,Vietnamese": 0.017045454545454544, - "English,Spanish,Chinese,Indonesian": 0.011363636363636364, - "English,Spanish,Chinese,Vietnamese": 0.005681818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.005681818181818182, - "English,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Malay,Chinese,Vietnamese": 0.0, - "English,Malay,Indonesian,Vietnamese": 0.022727272727272728, - "English,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese": 0.022727272727272728, - "Filipino,Spanish,Malay,Indonesian": 0.022727272727272728, - "Filipino,Spanish,Malay,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Chinese,Indonesian": 0.022727272727272728, - "Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Malay,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Malay,Chinese,Vietnamese": 0.017045454545454544, - "Filipino,Malay,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Spanish,Malay,Chinese,Indonesian": 0.03977272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.022727272727272728, - "Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544 + "Indonesian,English,Filipino,Spanish": 0.19886363636363635, + "Indonesian,English,Filipino,Chinese": 0.07954545454545454, + "Indonesian,English,Filipino,Malay": 0.17613636363636365, + "Indonesian,English,Filipino,Vietnamese": 0.09090909090909091, + "Indonesian,English,Spanish,Chinese": 0.11931818181818182, + "Indonesian,English,Spanish,Malay": 0.26136363636363635, + "Indonesian,English,Spanish,Vietnamese": 0.125, + "Indonesian,English,Chinese,Malay": 0.09659090909090909, + "Indonesian,English,Chinese,Vietnamese": 0.0625, + "Indonesian,English,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Chinese": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Malay": 0.2215909090909091, + "Indonesian,Filipino,Spanish,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.14772727272727273, + "Indonesian,Filipino,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Spanish,Chinese,Malay": 0.125, + "Indonesian,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,Spanish,Malay,Vietnamese": 0.125, + "Indonesian,Chinese,Malay,Vietnamese": 0.14204545454545456, + "English,Filipino,Spanish,Chinese": 0.07386363636363637, + "English,Filipino,Spanish,Malay": 0.18181818181818182, + "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, + "English,Filipino,Chinese,Malay": 0.0625, + "English,Filipino,Chinese,Vietnamese": 0.03977272727272727, + "English,Filipino,Malay,Vietnamese": 0.07386363636363637, + "English,Spanish,Chinese,Malay": 0.10795454545454546, + "English,Spanish,Chinese,Vietnamese": 0.07954545454545454, + "English,Spanish,Malay,Vietnamese": 0.11931818181818182, + "English,Chinese,Malay,Vietnamese": 0.05113636363636364, + "Filipino,Spanish,Chinese,Malay": 0.09659090909090909, + "Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, + "Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, + "Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, + "Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.0, - "English,Filipino,Spanish,Malay,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.0, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.022727272727272728, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian": 0.0, - "English,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 + "Indonesian,English,Filipino,Spanish,Chinese": 0.06818181818181818, + "Indonesian,English,Filipino,Spanish,Malay": 0.1590909090909091, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.03977272727272727, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Spanish,Chinese,Malay": 0.09090909090909091, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.0625, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.0625, + "English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "English,Filipino,Spanish,Malay,Vietnamese": 0.0625, + "English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 } }, - "AC3_2": 0.26229881511111747, - "AC3_3": 0.112719250929976, - "AC3_4": 0.03625070580297802, - "AC3_5": 0.01008245827112785, - "AC3_6": 0.0016183350804488113, - "AC3_7": 0.0 + "AC3_2": 0.39221527778996373, + "AC3_3": 0.2685412639750828, + "AC3_4": 0.17793743124939904, + "AC3_5": 0.11650551826494414, + "AC3_6": 0.07716723935691445, + "AC3_7": 0.052808834297733406 }, "prompt_4": { - "overall_acc": 0.27110389610389607, + "overall_acc": 0.3774350649350649, "language_acc": { - "English": 0.2727272727272727, - "Filipino": 0.2556818181818182, - "Spanish": 0.2784090909090909, - "Malay": 0.26136363636363635, - "Chinese": 0.22727272727272727, - "Indonesian": 0.30113636363636365, - "Vietnamese": 0.30113636363636365 + "Indonesian": 0.4090909090909091, + "English": 0.6022727272727273, + "Filipino": 0.30113636363636365, + "Spanish": 0.45454545454545453, + "Chinese": 0.23295454545454544, + "Malay": 0.3352272727272727, + "Vietnamese": 0.3068181818181818 }, - "consistency_score_2": 0.251082251082251, - "consistency_score_3": 0.06428571428571428, - "consistency_score_4": 0.018181818181818184, - "consistency_score_5": 0.006493506493506493, - "consistency_score_6": 0.002435064935064935, - "consistency_score_7": 0.0, + "consistency_score_2": 0.40882034632034636, + "consistency_score_3": 0.2073051948051948, + "consistency_score_4": 0.11542207792207793, + "consistency_score_5": 0.06845238095238095, + "consistency_score_6": 0.04301948051948052, + "consistency_score_7": 0.028409090909090908, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.2556818181818182, - "English,Spanish": 0.26704545454545453, - "English,Malay": 0.22727272727272727, - "English,Chinese": 0.23295454545454544, - "English,Indonesian": 0.25, - "English,Vietnamese": 0.2784090909090909, - "Filipino,Spanish": 0.26136363636363635, - "Filipino,Malay": 0.26136363636363635, - "Filipino,Chinese": 0.3352272727272727, - "Filipino,Indonesian": 0.2159090909090909, - "Filipino,Vietnamese": 0.2159090909090909, - "Spanish,Malay": 0.2159090909090909, - "Spanish,Chinese": 0.2556818181818182, - "Spanish,Indonesian": 0.26136363636363635, - "Spanish,Vietnamese": 0.22727272727272727, - "Malay,Chinese": 0.3068181818181818, - "Malay,Indonesian": 0.1875, - "Malay,Vietnamese": 0.25, - "Chinese,Indonesian": 0.23863636363636365, - "Chinese,Vietnamese": 0.2840909090909091, - "Indonesian,Vietnamese": 0.24431818181818182 + "Indonesian,English": 0.4659090909090909, + "Indonesian,Filipino": 0.4715909090909091, + "Indonesian,Spanish": 0.5397727272727273, + "Indonesian,Chinese": 0.3352272727272727, + "Indonesian,Malay": 0.5397727272727273, + "Indonesian,Vietnamese": 0.35795454545454547, + "English,Filipino": 0.3465909090909091, + "English,Spanish": 0.6420454545454546, + "English,Chinese": 0.2159090909090909, + "English,Malay": 0.42613636363636365, + "English,Vietnamese": 0.2897727272727273, + "Filipino,Spanish": 0.4147727272727273, + "Filipino,Chinese": 0.29545454545454547, + "Filipino,Malay": 0.5284090909090909, + "Filipino,Vietnamese": 0.36363636363636365, + "Spanish,Chinese": 0.25, + "Spanish,Malay": 0.4715909090909091, + "Spanish,Vietnamese": 0.3409090909090909, + "Chinese,Malay": 0.375, + "Chinese,Vietnamese": 0.5113636363636364, + "Malay,Vietnamese": 0.4034090909090909 }, "3_combine": { - "English,Filipino,Spanish": 0.07954545454545454, - "English,Filipino,Malay": 0.07954545454545454, + "Indonesian,English,Filipino": 0.23863636363636365, + "Indonesian,English,Spanish": 0.3693181818181818, + "Indonesian,English,Chinese": 0.13068181818181818, + "Indonesian,English,Malay": 0.29545454545454547, + "Indonesian,English,Vietnamese": 0.16477272727272727, + "Indonesian,Filipino,Spanish": 0.2840909090909091, + "Indonesian,Filipino,Chinese": 0.17045454545454544, + "Indonesian,Filipino,Malay": 0.3522727272727273, + "Indonesian,Filipino,Vietnamese": 0.2159090909090909, + "Indonesian,Spanish,Chinese": 0.1590909090909091, + "Indonesian,Spanish,Malay": 0.3409090909090909, + "Indonesian,Spanish,Vietnamese": 0.19886363636363635, + "Indonesian,Chinese,Malay": 0.2215909090909091, + "Indonesian,Chinese,Vietnamese": 0.20454545454545456, + "Indonesian,Malay,Vietnamese": 0.22727272727272727, + "English,Filipino,Spanish": 0.25, "English,Filipino,Chinese": 0.08522727272727272, - "English,Filipino,Indonesian": 0.056818181818181816, - "English,Filipino,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay": 0.06818181818181818, - "English,Spanish,Chinese": 0.05113636363636364, - "English,Spanish,Indonesian": 0.0625, - "English,Spanish,Vietnamese": 0.09090909090909091, - "English,Malay,Chinese": 0.056818181818181816, - "English,Malay,Indonesian": 0.05113636363636364, - "English,Malay,Vietnamese": 0.056818181818181816, - "English,Chinese,Indonesian": 0.08522727272727272, - "English,Chinese,Vietnamese": 0.06818181818181818, - "English,Indonesian,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Malay": 0.056818181818181816, - "Filipino,Spanish,Chinese": 0.09090909090909091, - "Filipino,Spanish,Indonesian": 0.0625, - "Filipino,Spanish,Vietnamese": 0.05113636363636364, - "Filipino,Malay,Chinese": 0.10227272727272728, - "Filipino,Malay,Indonesian": 0.056818181818181816, - "Filipino,Malay,Vietnamese": 0.06818181818181818, - "Filipino,Chinese,Indonesian": 0.06818181818181818, - "Filipino,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Indonesian,Vietnamese": 0.028409090909090908, - "Spanish,Malay,Chinese": 0.07386363636363637, - "Spanish,Malay,Indonesian": 0.03409090909090909, - "Spanish,Malay,Vietnamese": 0.03409090909090909, - "Spanish,Chinese,Indonesian": 0.06818181818181818, - "Spanish,Chinese,Vietnamese": 0.07386363636363637, - "Spanish,Indonesian,Vietnamese": 0.045454545454545456, - "Malay,Chinese,Indonesian": 0.05113636363636364, - "Malay,Chinese,Vietnamese": 0.07954545454545454, - "Malay,Indonesian,Vietnamese": 0.045454545454545456, - "Chinese,Indonesian,Vietnamese": 0.05113636363636364 + "English,Filipino,Malay": 0.24431818181818182, + "English,Filipino,Vietnamese": 0.11931818181818182, + "English,Spanish,Chinese": 0.14772727272727273, + "English,Spanish,Malay": 0.32954545454545453, + "English,Spanish,Vietnamese": 0.22727272727272727, + "English,Chinese,Malay": 0.11363636363636363, + "English,Chinese,Vietnamese": 0.10795454545454546, + "English,Malay,Vietnamese": 0.14772727272727273, + "Filipino,Spanish,Chinese": 0.11931818181818182, + "Filipino,Spanish,Malay": 0.2784090909090909, + "Filipino,Spanish,Vietnamese": 0.1590909090909091, + "Filipino,Chinese,Malay": 0.19318181818181818, + "Filipino,Chinese,Vietnamese": 0.1875, + "Filipino,Malay,Vietnamese": 0.22727272727272727, + "Spanish,Chinese,Malay": 0.1534090909090909, + "Spanish,Chinese,Vietnamese": 0.1534090909090909, + "Spanish,Malay,Vietnamese": 0.19318181818181818, + "Chinese,Malay,Vietnamese": 0.24431818181818182 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.028409090909090908, - "English,Filipino,Spanish,Chinese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian": 0.017045454545454544, - "English,Filipino,Malay,Vietnamese": 0.017045454545454544, - "English,Filipino,Chinese,Indonesian": 0.022727272727272728, + "Indonesian,English,Filipino,Spanish": 0.19318181818181818, + "Indonesian,English,Filipino,Chinese": 0.07386363636363637, + "Indonesian,English,Filipino,Malay": 0.18181818181818182, + "Indonesian,English,Filipino,Vietnamese": 0.09090909090909091, + "Indonesian,English,Spanish,Chinese": 0.10227272727272728, + "Indonesian,English,Spanish,Malay": 0.2556818181818182, + "Indonesian,English,Spanish,Vietnamese": 0.14204545454545456, + "Indonesian,English,Chinese,Malay": 0.09090909090909091, + "Indonesian,English,Chinese,Vietnamese": 0.06818181818181818, + "Indonesian,English,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Chinese": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Malay": 0.2159090909090909, + "Indonesian,Filipino,Spanish,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.14772727272727273, + "Indonesian,Filipino,Chinese,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Malay,Vietnamese": 0.17045454545454544, + "Indonesian,Spanish,Chinese,Malay": 0.11363636363636363, + "Indonesian,Spanish,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,Spanish,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,Chinese,Malay,Vietnamese": 0.14204545454545456, + "English,Filipino,Spanish,Chinese": 0.06818181818181818, + "English,Filipino,Spanish,Malay": 0.17613636363636365, + "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, + "English,Filipino,Chinese,Malay": 0.0625, "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Chinese": 0.022727272727272728, - "English,Spanish,Malay,Indonesian": 0.005681818181818182, - "English,Spanish,Malay,Vietnamese": 0.017045454545454544, - "English,Spanish,Chinese,Indonesian": 0.022727272727272728, - "English,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Spanish,Indonesian,Vietnamese": 0.022727272727272728, - "English,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese": 0.022727272727272728, - "Filipino,Spanish,Malay,Indonesian": 0.005681818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.011363636363636364, - "Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Spanish,Malay,Chinese,Indonesian": 0.017045454545454544, - "Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 + "English,Filipino,Malay,Vietnamese": 0.07386363636363637, + "English,Spanish,Chinese,Malay": 0.09659090909090909, + "English,Spanish,Chinese,Vietnamese": 0.07954545454545454, + "English,Spanish,Malay,Vietnamese": 0.13068181818181818, + "English,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Filipino,Spanish,Chinese,Malay": 0.09090909090909091, + "Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, + "Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, + "Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian": 0.0, - "English,Filipino,Spanish,Malay,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.0, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese": 0.0625, + "Indonesian,English,Filipino,Spanish,Malay": 0.1534090909090909, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Spanish,Chinese,Malay": 0.07954545454545454, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.07954545454545454, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, + "English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "English,Filipino,Spanish,Malay,Vietnamese": 0.0625, + "English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 } }, - "AC3_2": 0.26070923889765435, - "AC3_3": 0.10392753419664391, - "AC3_4": 0.034078155278496405, - "AC3_5": 0.01268322320497007, - "AC3_6": 0.004826775596524385, - "AC3_7": 0.0 + "AC3_2": 0.392501295969958, + "AC3_3": 0.2676205318376739, + "AC3_4": 0.17678282682106855, + "AC3_5": 0.11588722258149935, + "AC3_6": 0.07723574686023443, + "AC3_7": 0.052840909077889096 }, "prompt_5": { - "overall_acc": 0.22808441558441556, + "overall_acc": 0.3628246753246754, "language_acc": { - "English": 0.2215909090909091, - "Filipino": 0.3125, - "Spanish": 0.25, - "Malay": 0.18181818181818182, - "Chinese": 0.20454545454545456, - "Indonesian": 0.2556818181818182, - "Vietnamese": 0.17045454545454544 + "Indonesian": 0.4034090909090909, + "English": 0.6022727272727273, + "Filipino": 0.26704545454545453, + "Spanish": 0.42045454545454547, + "Chinese": 0.2215909090909091, + "Malay": 0.3409090909090909, + "Vietnamese": 0.2840909090909091 }, - "consistency_score_2": 0.25487012987012986, - "consistency_score_3": 0.06493506493506494, - "consistency_score_4": 0.016071428571428566, - "consistency_score_5": 0.003787878787878788, - "consistency_score_6": 0.0008116883116883117, - "consistency_score_7": 0.0, + "consistency_score_2": 0.38906926406926406, + "consistency_score_3": 0.1912337662337662, + "consistency_score_4": 0.10357142857142858, + "consistency_score_5": 0.059253246753246745, + "consistency_score_6": 0.03571428571428571, + "consistency_score_7": 0.022727272727272728, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.2840909090909091, - "English,Spanish": 0.2215909090909091, - "English,Malay": 0.2215909090909091, - "English,Chinese": 0.3181818181818182, - "English,Indonesian": 0.22727272727272727, - "English,Vietnamese": 0.23863636363636365, - "Filipino,Spanish": 0.3181818181818182, - "Filipino,Malay": 0.18181818181818182, - "Filipino,Chinese": 0.24431818181818182, - "Filipino,Indonesian": 0.22727272727272727, - "Filipino,Vietnamese": 0.2159090909090909, - "Spanish,Malay": 0.2897727272727273, - "Spanish,Chinese": 0.26704545454545453, - "Spanish,Indonesian": 0.24431818181818182, - "Spanish,Vietnamese": 0.24431818181818182, - "Malay,Chinese": 0.23863636363636365, - "Malay,Indonesian": 0.24431818181818182, - "Malay,Vietnamese": 0.29545454545454547, - "Chinese,Indonesian": 0.2897727272727273, - "Chinese,Vietnamese": 0.2556818181818182, - "Indonesian,Vietnamese": 0.2840909090909091 + "Indonesian,English": 0.4772727272727273, + "Indonesian,Filipino": 0.4715909090909091, + "Indonesian,Spanish": 0.5681818181818182, + "Indonesian,Chinese": 0.24431818181818182, + "Indonesian,Malay": 0.5511363636363636, + "Indonesian,Vietnamese": 0.32954545454545453, + "English,Filipino": 0.3352272727272727, + "English,Spanish": 0.6477272727272727, + "English,Chinese": 0.2727272727272727, + "English,Malay": 0.4147727272727273, + "English,Vietnamese": 0.2840909090909091, + "Filipino,Spanish": 0.4090909090909091, + "Filipino,Chinese": 0.26704545454545453, + "Filipino,Malay": 0.4772727272727273, + "Filipino,Vietnamese": 0.3806818181818182, + "Spanish,Chinese": 0.2727272727272727, + "Spanish,Malay": 0.5113636363636364, + "Spanish,Vietnamese": 0.3181818181818182, + "Chinese,Malay": 0.2784090909090909, + "Chinese,Vietnamese": 0.24431818181818182, + "Malay,Vietnamese": 0.4147727272727273 }, "3_combine": { - "English,Filipino,Spanish": 0.09090909090909091, - "English,Filipino,Malay": 0.056818181818181816, - "English,Filipino,Chinese": 0.07954545454545454, - "English,Filipino,Indonesian": 0.045454545454545456, - "English,Filipino,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay": 0.06818181818181818, - "English,Spanish,Chinese": 0.06818181818181818, - "English,Spanish,Indonesian": 0.028409090909090908, - "English,Spanish,Vietnamese": 0.03409090909090909, - "English,Malay,Chinese": 0.07954545454545454, - "English,Malay,Indonesian": 0.045454545454545456, - "English,Malay,Vietnamese": 0.06818181818181818, - "English,Chinese,Indonesian": 0.0625, + "Indonesian,English,Filipino": 0.23295454545454544, + "Indonesian,English,Spanish": 0.3806818181818182, + "Indonesian,English,Chinese": 0.1534090909090909, + "Indonesian,English,Malay": 0.30113636363636365, + "Indonesian,English,Vietnamese": 0.1534090909090909, + "Indonesian,Filipino,Spanish": 0.29545454545454547, + "Indonesian,Filipino,Chinese": 0.125, + "Indonesian,Filipino,Malay": 0.32954545454545453, + "Indonesian,Filipino,Vietnamese": 0.2159090909090909, + "Indonesian,Spanish,Chinese": 0.17613636363636365, + "Indonesian,Spanish,Malay": 0.375, + "Indonesian,Spanish,Vietnamese": 0.20454545454545456, + "Indonesian,Chinese,Malay": 0.1590909090909091, + "Indonesian,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,Malay,Vietnamese": 0.23295454545454544, + "English,Filipino,Spanish": 0.2556818181818182, + "English,Filipino,Chinese": 0.10227272727272728, + "English,Filipino,Malay": 0.20454545454545456, + "English,Filipino,Vietnamese": 0.11931818181818182, + "English,Spanish,Chinese": 0.1875, + "English,Spanish,Malay": 0.3409090909090909, + "English,Spanish,Vietnamese": 0.20454545454545456, + "English,Chinese,Malay": 0.10795454545454546, "English,Chinese,Vietnamese": 0.08522727272727272, - "English,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay": 0.0625, - "Filipino,Spanish,Chinese": 0.07386363636363637, - "Filipino,Spanish,Indonesian": 0.0625, - "Filipino,Spanish,Vietnamese": 0.06818181818181818, - "Filipino,Malay,Chinese": 0.03409090909090909, - "Filipino,Malay,Indonesian": 0.03977272727272727, - "Filipino,Malay,Vietnamese": 0.03409090909090909, - "Filipino,Chinese,Indonesian": 0.0625, - "Filipino,Chinese,Vietnamese": 0.045454545454545456, - "Filipino,Indonesian,Vietnamese": 0.0625, - "Spanish,Malay,Chinese": 0.07954545454545454, - "Spanish,Malay,Indonesian": 0.08522727272727272, - "Spanish,Malay,Vietnamese": 0.07954545454545454, - "Spanish,Chinese,Indonesian": 0.056818181818181816, - "Spanish,Chinese,Vietnamese": 0.056818181818181816, - "Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "Malay,Chinese,Indonesian": 0.056818181818181816, - "Malay,Chinese,Vietnamese": 0.07954545454545454, - "Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Chinese,Indonesian,Vietnamese": 0.10227272727272728 + "English,Malay,Vietnamese": 0.14204545454545456, + "Filipino,Spanish,Chinese": 0.11931818181818182, + "Filipino,Spanish,Malay": 0.26136363636363635, + "Filipino,Spanish,Vietnamese": 0.1534090909090909, + "Filipino,Chinese,Malay": 0.13068181818181818, + "Filipino,Chinese,Vietnamese": 0.09090909090909091, + "Filipino,Malay,Vietnamese": 0.2159090909090909, + "Spanish,Chinese,Malay": 0.14772727272727273, + "Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Spanish,Malay,Vietnamese": 0.20454545454545456, + "Chinese,Malay,Vietnamese": 0.10795454545454546 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.022727272727272728, - "English,Filipino,Spanish,Chinese": 0.028409090909090908, - "English,Filipino,Spanish,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese": 0.011363636363636364, - "English,Filipino,Malay,Indonesian": 0.005681818181818182, - "English,Filipino,Malay,Vietnamese": 0.011363636363636364, - "English,Filipino,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese": 0.022727272727272728, - "English,Spanish,Malay,Indonesian": 0.005681818181818182, - "English,Spanish,Malay,Vietnamese": 0.011363636363636364, - "English,Spanish,Chinese,Indonesian": 0.0, - "English,Spanish,Chinese,Vietnamese": 0.0, - "English,Spanish,Indonesian,Vietnamese": 0.005681818181818182, - "English,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Malay,Indonesian,Vietnamese": 0.022727272727272728, - "English,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese": 0.017045454545454544, - "Filipino,Spanish,Malay,Indonesian": 0.017045454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Chinese,Indonesian": 0.017045454545454544, - "Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Malay,Chinese,Indonesian": 0.011363636363636364, - "Filipino,Malay,Chinese,Vietnamese": 0.005681818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.028409090909090908, - "Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 + "Indonesian,English,Filipino,Spanish": 0.19318181818181818, + "Indonesian,English,Filipino,Chinese": 0.07954545454545454, + "Indonesian,English,Filipino,Malay": 0.1534090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.09090909090909091, + "Indonesian,English,Spanish,Chinese": 0.14204545454545456, + "Indonesian,English,Spanish,Malay": 0.26704545454545453, + "Indonesian,English,Spanish,Vietnamese": 0.13068181818181818, + "Indonesian,English,Chinese,Malay": 0.09090909090909091, + "Indonesian,English,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,English,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Spanish,Chinese": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Malay": 0.21022727272727273, + "Indonesian,Filipino,Spanish,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Chinese,Malay": 0.09659090909090909, + "Indonesian,Filipino,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,Filipino,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Spanish,Chinese,Malay": 0.10795454545454546, + "Indonesian,Spanish,Chinese,Vietnamese": 0.06818181818181818, + "Indonesian,Spanish,Malay,Vietnamese": 0.1534090909090909, + "Indonesian,Chinese,Malay,Vietnamese": 0.06818181818181818, + "English,Filipino,Spanish,Chinese": 0.08522727272727272, + "English,Filipino,Spanish,Malay": 0.16477272727272727, + "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, + "English,Filipino,Chinese,Malay": 0.056818181818181816, + "English,Filipino,Chinese,Vietnamese": 0.03977272727272727, + "English,Filipino,Malay,Vietnamese": 0.06818181818181818, + "English,Spanish,Chinese,Malay": 0.09659090909090909, + "English,Spanish,Chinese,Vietnamese": 0.06818181818181818, + "English,Spanish,Malay,Vietnamese": 0.125, + "English,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Filipino,Spanish,Chinese,Malay": 0.07386363636363637, + "Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Filipino,Chinese,Malay,Vietnamese": 0.05113636363636364, + "Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian": 0.0, - "English,Filipino,Spanish,Malay,Vietnamese": 0.0, - "English,Filipino,Spanish,Chinese,Indonesian": 0.0, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian": 0.0, - "English,Filipino,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian": 0.0, - "English,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 + "Indonesian,English,Filipino,Spanish,Chinese": 0.07386363636363637, + "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Chinese,Malay": 0.05113636363636364, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.0625, + "Indonesian,English,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0625, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, + "English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, + "English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 } }, - "AC3_2": 0.24073447555857896, - "AC3_3": 0.1010900456538842, - "AC3_4": 0.030027070656394678, - "AC3_5": 0.007451999572473019, - "AC3_6": 0.0016176199679770937, - "AC3_7": 0.0 + "AC3_2": 0.3754889406603043, + "AC3_3": 0.25045852180212497, + "AC3_4": 0.1611431554524962, + "AC3_5": 0.10187000497086962, + "AC3_6": 0.06502764036773935, + "AC3_7": 0.04277511960613037 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.20388349514563106 + "accuracy": 0.5436893203883495 }, "prompt_2": { - "accuracy": 0.23300970873786409 + "accuracy": 0.5728155339805825 }, "prompt_3": { - "accuracy": 0.2815533980582524 + "accuracy": 0.5631067961165048 }, "prompt_4": { - "accuracy": 0.32038834951456313 + "accuracy": 0.5631067961165048 }, "prompt_5": { - "accuracy": 0.22330097087378642 + "accuracy": 0.5631067961165048 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.2857142857142857 + "accuracy": 0.23809523809523808 }, "prompt_2": { - "accuracy": 0.20952380952380953 + "accuracy": 0.23809523809523808 }, "prompt_3": { - "accuracy": 0.26666666666666666 + "accuracy": 0.23809523809523808 }, "prompt_4": { - "accuracy": 0.22857142857142856 + "accuracy": 0.23809523809523808 }, "prompt_5": { - "accuracy": 0.21904761904761905 + "accuracy": 0.23809523809523808 } }, "us_eval": { "prompt_1": { - "accuracy": 0.2897196261682243 + "accuracy": 0.6448598130841121 }, "prompt_2": { - "accuracy": 0.22429906542056074 + "accuracy": 0.6355140186915887 }, "prompt_3": { - "accuracy": 0.27102803738317754 + "accuracy": 0.6542056074766355 }, "prompt_4": { - "accuracy": 0.21495327102803738 + "accuracy": 0.6355140186915887 }, "prompt_5": { - "accuracy": 0.27102803738317754 + "accuracy": 0.6542056074766355 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.35, + "accuracy": 0.33, "category_acc": { "brand": 0.3, - "demographics": 0.6, + "demographics": 0.2, "biology": 0.4, - "history": 0.26666666666666666, + "history": 0.3333333333333333, "literature": 0.3, - "politics": 0.2, - "culture": 0.5, - "film": 0.2, - "law": 0.5, + "politics": 0.3, + "culture": 0.4, + "film": 0.3, + "law": 0.3, "geography": 0.4 } }, "prompt_2": { - "accuracy": 0.24, + "accuracy": 0.3, "category_acc": { "brand": 0.2, "demographics": 0.2, - "biology": 0.1, - "history": 0.3333333333333333, - "literature": 0.2, - "politics": 0.4, - "culture": 0.2, + "biology": 0.4, + "history": 0.4, + "literature": 0.3, + "politics": 0.3, + "culture": 0.3, "film": 0.1, "law": 0.3, - "geography": 0.3 + "geography": 0.4 } }, "prompt_3": { - "accuracy": 0.24, + "accuracy": 0.32, "category_acc": { - "brand": 0.1, + "brand": 0.2, "demographics": 0.2, - "biology": 0.1, + "biology": 0.5, "history": 0.2, - "literature": 0.2, - "politics": 0.7, - "culture": 0.1, - "film": 0.4, + "literature": 0.3, + "politics": 0.3, + "culture": 0.3, + "film": 0.3, "law": 0.3, - "geography": 0.1 + "geography": 0.6 } }, "prompt_4": { - "accuracy": 0.25, + "accuracy": 0.33, "category_acc": { - "brand": 0.0, - "demographics": 0.4, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.0, - "politics": 0.2, - "culture": 0.4, - "film": 0.3, - "law": 0.1, + "brand": 0.3, + "demographics": 0.2, + "biology": 0.5, + "history": 0.2, + "literature": 0.3, + "politics": 0.3, + "culture": 0.5, + "film": 0.2, + "law": 0.3, "geography": 0.5 } }, "prompt_5": { - "accuracy": 0.26, + "accuracy": 0.31, "category_acc": { - "brand": 0.3, + "brand": 0.2, "demographics": 0.2, - "biology": 0.2, - "history": 0.13333333333333333, - "literature": 0.2, - "politics": 0.2, - "culture": 0.5, - "film": 0.4, - "law": 0.2, - "geography": 0.3 + "biology": 0.5, + "history": 0.26666666666666666, + "literature": 0.3, + "politics": 0.3, + "culture": 0.3, + "film": 0.2, + "law": 0.3, + "geography": 0.5 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.008788487658497914 + "bleu_score": 0.17111295690470238 }, "prompt_2": { - "bleu_score": 0.008538730610136495 + "bleu_score": 0.17578262863245286 }, "prompt_3": { - "bleu_score": 0.008544640652726861 + "bleu_score": 0.181061792829869 }, "prompt_4": { - "bleu_score": 0.008509776386455043 + "bleu_score": 0.17795860280445558 }, "prompt_5": { - "bleu_score": 0.00865220715555374 + "bleu_score": 0.17554492731604188 } }, "indommlu": { "prompt_1": { - "accuracy": 0.23285933640429934, + "accuracy": 0.34328059282996193, "category_acc": { - "History": 0.18875502008032127, - "Geography": 0.19591836734693877, - "Lampungic": 0.2789115646258503, - "Social science": 0.25041736227045075, - "Balinese": 0.2484076433121019, - "Makassarese": 0.25268817204301075, - "Banjarese": 0.2777777777777778, - "Chemistry": 0.1854014598540146, - "Biology": 0.21420118343195266, - "Science": 0.26006191950464397, - "Christian religion": 0.17412935323383086, - "Art": 0.26455906821963393, - "Islam religion": 0.22190611664295876, - "Hindu religion": 0.26666666666666666, - "Madurese": 0.20677966101694914, - "Sport": 0.23648648648648649, - "Indonesian language": 0.2328767123287671, - "Physics": 0.24444444444444444, - "Minangkabau culture": 0.24120603015075376, - "Dayak language": 0.29357798165137616, - "Sociology": 0.21774193548387097, - "Economy": 0.2069672131147541, - "Sundanese": 0.25324114088159033, - "Javanese": 0.24092741935483872, - "Civic education": 0.23891273247496422 + "History": 0.3293172690763052, + "Geography": 0.2693877551020408, + "Lampungic": 0.35374149659863946, + "Social science": 0.4340567612687813, + "Balinese": 0.3099787685774947, + "Makassarese": 0.3279569892473118, + "Banjarese": 0.2569444444444444, + "Chemistry": 0.23941605839416058, + "Biology": 0.30177514792899407, + "Science": 0.35810113519091846, + "Christian religion": 0.4079601990049751, + "Art": 0.34608985024958405, + "Islam religion": 0.3627311522048364, + "Hindu religion": 0.29333333333333333, + "Madurese": 0.3050847457627119, + "Sport": 0.40540540540540543, + "Indonesian language": 0.39632627646326274, + "Physics": 0.30707070707070705, + "Minangkabau culture": 0.2562814070351759, + "Dayak language": 0.27522935779816515, + "Sociology": 0.3548387096774194, + "Economy": 0.33401639344262296, + "Sundanese": 0.33189282627484873, + "Javanese": 0.28024193548387094, + "Civic education": 0.3977110157367668 } }, "prompt_2": { - "accuracy": 0.23192469457240136, + "accuracy": 0.33994258628746915, "category_acc": { - "History": 0.2289156626506024, - "Geography": 0.22448979591836735, - "Lampungic": 0.2925170068027211, - "Social science": 0.3021702838063439, - "Balinese": 0.25902335456475584, - "Makassarese": 0.23655913978494625, - "Banjarese": 0.2708333333333333, - "Chemistry": 0.19124087591240876, - "Biology": 0.2106508875739645, - "Science": 0.22703818369453044, - "Christian religion": 0.2537313432835821, - "Art": 0.2113144758735441, - "Islam religion": 0.2119487908961593, - "Hindu religion": 0.25333333333333335, - "Madurese": 0.23389830508474577, - "Sport": 0.3108108108108108, - "Indonesian language": 0.22789539227895392, - "Physics": 0.26262626262626265, - "Minangkabau culture": 0.21105527638190955, - "Dayak language": 0.2018348623853211, - "Sociology": 0.1935483870967742, - "Economy": 0.19057377049180327, - "Sundanese": 0.2610198789974071, - "Javanese": 0.22580645161290322, - "Civic education": 0.2446351931330472 + "History": 0.3253012048192771, + "Geography": 0.2714285714285714, + "Lampungic": 0.36054421768707484, + "Social science": 0.44240400667779634, + "Balinese": 0.3036093418259023, + "Makassarese": 0.2903225806451613, + "Banjarese": 0.2569444444444444, + "Chemistry": 0.25547445255474455, + "Biology": 0.30414201183431955, + "Science": 0.35500515995872034, + "Christian religion": 0.39800995024875624, + "Art": 0.3410981697171381, + "Islam religion": 0.35704125177809387, + "Hindu religion": 0.31333333333333335, + "Madurese": 0.2847457627118644, + "Sport": 0.3716216216216216, + "Indonesian language": 0.39352428393524286, + "Physics": 0.301010101010101, + "Minangkabau culture": 0.2663316582914573, + "Dayak language": 0.26605504587155965, + "Sociology": 0.3528225806451613, + "Economy": 0.32991803278688525, + "Sundanese": 0.31892826274848746, + "Javanese": 0.27318548387096775, + "Civic education": 0.3948497854077253 } }, "prompt_3": { - "accuracy": 0.2318579344415515, + "accuracy": 0.3363375392215769, "category_acc": { - "History": 0.19678714859437751, - "Geography": 0.20204081632653062, - "Lampungic": 0.2857142857142857, - "Social science": 0.25041736227045075, - "Balinese": 0.25477707006369427, - "Makassarese": 0.25806451612903225, - "Banjarese": 0.2638888888888889, - "Chemistry": 0.1927007299270073, - "Biology": 0.19171597633136095, - "Science": 0.2528379772961816, - "Christian religion": 0.18407960199004975, - "Art": 0.21797004991680533, - "Islam religion": 0.22475106685633, - "Hindu religion": 0.26666666666666666, - "Madurese": 0.2745762711864407, - "Sport": 0.22297297297297297, - "Indonesian language": 0.24190535491905354, - "Physics": 0.21616161616161617, - "Minangkabau culture": 0.2663316582914573, - "Dayak language": 0.25688073394495414, - "Sociology": 0.22782258064516128, - "Economy": 0.1864754098360656, - "Sundanese": 0.25064822817631804, - "Javanese": 0.23387096774193547, - "Civic education": 0.24034334763948498 + "History": 0.321285140562249, + "Geography": 0.24897959183673468, + "Lampungic": 0.35374149659863946, + "Social science": 0.44574290484140233, + "Balinese": 0.3205944798301486, + "Makassarese": 0.3118279569892473, + "Banjarese": 0.2569444444444444, + "Chemistry": 0.24087591240875914, + "Biology": 0.3076923076923077, + "Science": 0.33539731682146545, + "Christian religion": 0.4129353233830846, + "Art": 0.33777038269550747, + "Islam religion": 0.34992887624466573, + "Hindu religion": 0.30666666666666664, + "Madurese": 0.2847457627118644, + "Sport": 0.35135135135135137, + "Indonesian language": 0.3894769613947696, + "Physics": 0.28484848484848485, + "Minangkabau culture": 0.271356783919598, + "Dayak language": 0.27522935779816515, + "Sociology": 0.3487903225806452, + "Economy": 0.32991803278688525, + "Sundanese": 0.32238547968885045, + "Javanese": 0.2721774193548387, + "Civic education": 0.39198855507868385 } }, "prompt_4": { - "accuracy": 0.2347953801989452, + "accuracy": 0.3396755457640697, "category_acc": { - "History": 0.1646586345381526, - "Geography": 0.17755102040816326, - "Lampungic": 0.2789115646258503, - "Social science": 0.2988313856427379, - "Balinese": 0.25902335456475584, - "Makassarese": 0.25268817204301075, - "Banjarese": 0.20833333333333334, - "Chemistry": 0.2029197080291971, - "Biology": 0.19053254437869824, - "Science": 0.2518059855521156, - "Christian religion": 0.23383084577114427, - "Art": 0.23793677204658903, - "Islam religion": 0.2460881934566145, - "Hindu religion": 0.16, - "Madurese": 0.23389830508474577, - "Sport": 0.23648648648648649, - "Indonesian language": 0.25, - "Physics": 0.19595959595959597, - "Minangkabau culture": 0.20100502512562815, - "Dayak language": 0.3211009174311927, - "Sociology": 0.1814516129032258, - "Economy": 0.26024590163934425, - "Sundanese": 0.26534140017286084, - "Javanese": 0.23387096774193547, - "Civic education": 0.23319027181688126 + "History": 0.3273092369477912, + "Geography": 0.25510204081632654, + "Lampungic": 0.3333333333333333, + "Social science": 0.44741235392320533, + "Balinese": 0.3227176220806794, + "Makassarese": 0.3118279569892473, + "Banjarese": 0.2569444444444444, + "Chemistry": 0.23795620437956205, + "Biology": 0.3136094674556213, + "Science": 0.34674922600619196, + "Christian religion": 0.40298507462686567, + "Art": 0.34775374376039936, + "Islam religion": 0.3613086770981508, + "Hindu religion": 0.32, + "Madurese": 0.29491525423728815, + "Sport": 0.3783783783783784, + "Indonesian language": 0.3894769613947696, + "Physics": 0.28888888888888886, + "Minangkabau culture": 0.2663316582914573, + "Dayak language": 0.25688073394495414, + "Sociology": 0.3487903225806452, + "Economy": 0.33401639344262296, + "Sundanese": 0.33016421780466726, + "Javanese": 0.2721774193548387, + "Civic education": 0.39198855507868385 } }, "prompt_5": { - "accuracy": 0.23459509980639562, + "accuracy": 0.3384738634087723, "category_acc": { - "History": 0.20481927710843373, - "Geography": 0.19591836734693877, - "Lampungic": 0.2653061224489796, - "Social science": 0.27545909849749584, - "Balinese": 0.2208067940552017, - "Makassarese": 0.25806451612903225, - "Banjarese": 0.2847222222222222, - "Chemistry": 0.19854014598540146, - "Biology": 0.2165680473372781, - "Science": 0.239422084623323, - "Christian religion": 0.19900497512437812, - "Art": 0.27454242928452577, - "Islam religion": 0.24182076813655762, - "Hindu religion": 0.16666666666666666, - "Madurese": 0.29491525423728815, - "Sport": 0.21621621621621623, - "Indonesian language": 0.23567870485678705, - "Physics": 0.21212121212121213, - "Minangkabau culture": 0.24120603015075376, - "Dayak language": 0.22018348623853212, - "Sociology": 0.21169354838709678, - "Economy": 0.1885245901639344, - "Sundanese": 0.2644770959377701, - "Javanese": 0.24495967741935484, - "Civic education": 0.24177396280400573 + "History": 0.3232931726907631, + "Geography": 0.2795918367346939, + "Lampungic": 0.2925170068027211, + "Social science": 0.4323873121869783, + "Balinese": 0.3227176220806794, + "Makassarese": 0.2903225806451613, + "Banjarese": 0.2569444444444444, + "Chemistry": 0.24233576642335766, + "Biology": 0.293491124260355, + "Science": 0.34881320949432404, + "Christian religion": 0.3880597014925373, + "Art": 0.3527454242928453, + "Islam religion": 0.35135135135135137, + "Hindu religion": 0.3333333333333333, + "Madurese": 0.3050847457627119, + "Sport": 0.3716216216216216, + "Indonesian language": 0.3891656288916563, + "Physics": 0.29494949494949496, + "Minangkabau culture": 0.2663316582914573, + "Dayak language": 0.26605504587155965, + "Sociology": 0.3487903225806452, + "Economy": 0.32786885245901637, + "Sundanese": 0.3353500432152118, + "Javanese": 0.27923387096774194, + "Civic education": 0.38197424892703863 } } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.009169663025789155 + "bleu_score": 0.298729720423162 }, "prompt_2": { - "bleu_score": 0.009326777133551406 + "bleu_score": 0.29714403066664635 }, "prompt_3": { - "bleu_score": 0.009059776506681142 + "bleu_score": 0.29688089941244467 }, "prompt_4": { - "bleu_score": 0.00927882222642076 + "bleu_score": 0.29657094327603384 }, "prompt_5": { - "bleu_score": 0.009301266909937263 + "bleu_score": 0.2891515350076281 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.00932266731310241 + "bleu_score": 0.06332150290827988 }, "prompt_2": { - "bleu_score": 0.009473804489996178 + "bleu_score": 0.06237270430229427 }, "prompt_3": { - "bleu_score": 0.009295705587283189 + "bleu_score": 0.06261245548230009 }, "prompt_4": { - "bleu_score": 0.009283455001982199 + "bleu_score": 0.0638050751721597 }, "prompt_5": { - "bleu_score": 0.009391155048524823 + "bleu_score": 0.06116577273766147 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.00913420644076412 + "bleu_score": 0.015709981480553 }, "prompt_2": { - "bleu_score": 0.009240576308806264 + "bleu_score": 0.015737669889031625 }, "prompt_3": { - "bleu_score": 0.009263978665907944 + "bleu_score": 0.018161111996093608 }, "prompt_4": { - "bleu_score": 0.009232764287352922 + "bleu_score": 0.015298302651452203 }, "prompt_5": { - "bleu_score": 0.00943882139323631 + "bleu_score": 0.01808485717283886 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.009236803155971238 + "bleu_score": 0.27795819511180947 }, "prompt_2": { - "bleu_score": 0.009307048997750497 + "bleu_score": 0.28022199889007265 }, "prompt_3": { - "bleu_score": 0.009342543746369149 + "bleu_score": 0.2797980730259455 }, "prompt_4": { - "bleu_score": 0.00924068033702277 + "bleu_score": 0.2800502337161776 }, "prompt_5": { - "bleu_score": 0.009500760488368317 + "bleu_score": 0.26872088931620774 } }, "mmlu": { "prompt_1": { - "accuracy": 0.2707117852975496 + "accuracy": 0.5542590431738623 }, "prompt_2": { - "accuracy": 0.26837806301050177 + "accuracy": 0.5484247374562428 }, "prompt_3": { - "accuracy": 0.22520420070011668 + "accuracy": 0.543757292882147 }, "prompt_4": { - "accuracy": 0.25204200700116686 + "accuracy": 0.543757292882147 }, "prompt_5": { - "accuracy": 0.22753792298716452 + "accuracy": 0.5460910151691949 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.2505541651769753, + "accuracy": 0.5319270647121916, "category_acc": { - "high_school_european_history": 0.23780487804878048, - "business_ethics": 0.32323232323232326, - "clinical_knowledge": 0.25757575757575757, - "medical_genetics": 0.2222222222222222, - "high_school_us_history": 0.2315270935960591, - "high_school_physics": 0.2, - "high_school_world_history": 0.2288135593220339, - "virology": 0.26666666666666666, - "high_school_microeconomics": 0.23628691983122363, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.23232323232323232, - "high_school_biology": 0.23300970873786409, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.2669039145907473, - "philosophy": 0.25806451612903225, - "professional_medicine": 0.23616236162361623, - "nutrition": 0.2360655737704918, - "global_facts": 0.24242424242424243, - "machine_learning": 0.2882882882882883, - "security_studies": 0.2540983606557377, - "public_relations": 0.23853211009174313, - "professional_psychology": 0.23895253682487724, - "prehistory": 0.2848297213622291, - "anatomy": 0.2835820895522388, - "human_sexuality": 0.25384615384615383, - "college_medicine": 0.26744186046511625, - "high_school_government_and_politics": 0.25, - "college_chemistry": 0.29292929292929293, - "logical_fallacies": 0.22839506172839505, - "high_school_geography": 0.27918781725888325, - "elementary_mathematics": 0.23342175066312998, - "human_aging": 0.24774774774774774, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.25, - "formal_logic": 0.28, - "high_school_statistics": 0.24651162790697675, - "international_law": 0.24166666666666667, - "high_school_mathematics": 0.241635687732342, - "high_school_computer_science": 0.30303030303030304, - "conceptual_physics": 0.23504273504273504, - "miscellaneous": 0.24936061381074168, - "high_school_chemistry": 0.25742574257425743, - "marketing": 0.23605150214592274, - "professional_law": 0.26353555120678407, - "management": 0.19607843137254902, - "college_physics": 0.36633663366336633, - "jurisprudence": 0.2803738317757009, - "world_religions": 0.2647058823529412, - "sociology": 0.235, - "us_foreign_policy": 0.21212121212121213, - "high_school_macroeconomics": 0.2596401028277635, - "computer_security": 0.2828282828282828, - "moral_scenarios": 0.24608501118568232, - "moral_disputes": 0.20579710144927535, - "electrical_engineering": 0.2361111111111111, - "astronomy": 0.2251655629139073, - "college_biology": 0.24475524475524477 + "high_school_european_history": 0.7439024390243902, + "business_ethics": 0.6363636363636364, + "clinical_knowledge": 0.6060606060606061, + "medical_genetics": 0.5858585858585859, + "high_school_us_history": 0.7635467980295566, + "high_school_physics": 0.24666666666666667, + "high_school_world_history": 0.7330508474576272, + "virology": 0.4666666666666667, + "high_school_microeconomics": 0.5864978902953587, + "econometrics": 0.2831858407079646, + "college_computer_science": 0.46464646464646464, + "high_school_biology": 0.6699029126213593, + "abstract_algebra": 0.25252525252525254, + "professional_accounting": 0.398576512455516, + "philosophy": 0.5548387096774193, + "professional_medicine": 0.5166051660516605, + "nutrition": 0.5770491803278689, + "global_facts": 0.2828282828282828, + "machine_learning": 0.36036036036036034, + "security_studies": 0.6557377049180327, + "public_relations": 0.6422018348623854, + "professional_psychology": 0.5450081833060556, + "prehistory": 0.5727554179566563, + "anatomy": 0.4552238805970149, + "human_sexuality": 0.5615384615384615, + "college_medicine": 0.5523255813953488, + "high_school_government_and_politics": 0.7864583333333334, + "college_chemistry": 0.41414141414141414, + "logical_fallacies": 0.654320987654321, + "high_school_geography": 0.6802030456852792, + "elementary_mathematics": 0.376657824933687, + "human_aging": 0.5495495495495496, + "college_mathematics": 0.35353535353535354, + "high_school_psychology": 0.7830882352941176, + "formal_logic": 0.392, + "high_school_statistics": 0.3953488372093023, + "international_law": 0.6666666666666666, + "high_school_mathematics": 0.2862453531598513, + "high_school_computer_science": 0.48484848484848486, + "conceptual_physics": 0.4444444444444444, + "miscellaneous": 0.7186700767263428, + "high_school_chemistry": 0.3613861386138614, + "marketing": 0.8154506437768241, + "professional_law": 0.40378343118069143, + "management": 0.7549019607843137, + "college_physics": 0.38613861386138615, + "jurisprudence": 0.6728971962616822, + "world_religions": 0.6823529411764706, + "sociology": 0.7, + "us_foreign_policy": 0.7070707070707071, + "high_school_macroeconomics": 0.5578406169665809, + "computer_security": 0.6565656565656566, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.6376811594202898, + "electrical_engineering": 0.5069444444444444, + "astronomy": 0.5364238410596026, + "college_biology": 0.5104895104895105 } }, "prompt_2": { - "accuracy": 0.25470146585627457, + "accuracy": 0.5332856632105828, "category_acc": { - "high_school_european_history": 0.1951219512195122, - "business_ethics": 0.3333333333333333, - "clinical_knowledge": 0.24621212121212122, - "medical_genetics": 0.2222222222222222, - "high_school_us_history": 0.19704433497536947, - "high_school_physics": 0.26, - "high_school_world_history": 0.24152542372881355, - "virology": 0.3333333333333333, - "high_school_microeconomics": 0.28270042194092826, - "econometrics": 0.21238938053097345, - "college_computer_science": 0.21212121212121213, - "high_school_biology": 0.24919093851132687, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.28825622775800713, - "philosophy": 0.26129032258064516, - "professional_medicine": 0.22140221402214022, - "nutrition": 0.2983606557377049, - "global_facts": 0.23232323232323232, - "machine_learning": 0.2972972972972973, - "security_studies": 0.2581967213114754, - "public_relations": 0.1834862385321101, - "professional_psychology": 0.2635024549918167, - "prehistory": 0.2260061919504644, - "anatomy": 0.2835820895522388, - "human_sexuality": 0.16153846153846155, - "college_medicine": 0.28488372093023256, - "high_school_government_and_politics": 0.2552083333333333, - "college_chemistry": 0.24242424242424243, - "logical_fallacies": 0.25925925925925924, - "high_school_geography": 0.2182741116751269, - "elementary_mathematics": 0.27320954907161804, - "human_aging": 0.26126126126126126, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.2610294117647059, - "formal_logic": 0.248, - "high_school_statistics": 0.2558139534883721, - "international_law": 0.31666666666666665, - "high_school_mathematics": 0.2527881040892193, - "high_school_computer_science": 0.23232323232323232, - "conceptual_physics": 0.23931623931623933, - "miscellaneous": 0.2659846547314578, - "high_school_chemistry": 0.33663366336633666, - "marketing": 0.26609442060085836, - "professional_law": 0.253750815394651, - "management": 0.2647058823529412, - "college_physics": 0.22772277227722773, - "jurisprudence": 0.205607476635514, - "world_religions": 0.22941176470588234, - "sociology": 0.245, - "us_foreign_policy": 0.2828282828282828, - "high_school_macroeconomics": 0.2570694087403599, - "computer_security": 0.2727272727272727, - "moral_scenarios": 0.24384787472035793, - "moral_disputes": 0.2579710144927536, - "electrical_engineering": 0.2847222222222222, - "astronomy": 0.26490066225165565, - "college_biology": 0.1958041958041958 + "high_school_european_history": 0.75, + "business_ethics": 0.6262626262626263, + "clinical_knowledge": 0.6060606060606061, + "medical_genetics": 0.5757575757575758, + "high_school_us_history": 0.7684729064039408, + "high_school_physics": 0.2733333333333333, + "high_school_world_history": 0.7330508474576272, + "virology": 0.4727272727272727, + "high_school_microeconomics": 0.6118143459915611, + "econometrics": 0.2831858407079646, + "college_computer_science": 0.41414141414141414, + "high_school_biology": 0.6634304207119741, + "abstract_algebra": 0.20202020202020202, + "professional_accounting": 0.4128113879003559, + "philosophy": 0.5838709677419355, + "professional_medicine": 0.5239852398523985, + "nutrition": 0.5606557377049181, + "global_facts": 0.30303030303030304, + "machine_learning": 0.36036036036036034, + "security_studies": 0.6557377049180327, + "public_relations": 0.6055045871559633, + "professional_psychology": 0.5368248772504092, + "prehistory": 0.5727554179566563, + "anatomy": 0.4253731343283582, + "human_sexuality": 0.6, + "college_medicine": 0.5523255813953488, + "high_school_government_and_politics": 0.78125, + "college_chemistry": 0.40404040404040403, + "logical_fallacies": 0.6666666666666666, + "high_school_geography": 0.6802030456852792, + "elementary_mathematics": 0.38461538461538464, + "human_aging": 0.536036036036036, + "college_mathematics": 0.35353535353535354, + "high_school_psychology": 0.7775735294117647, + "formal_logic": 0.376, + "high_school_statistics": 0.4186046511627907, + "international_law": 0.675, + "high_school_mathematics": 0.2862453531598513, + "high_school_computer_science": 0.494949494949495, + "conceptual_physics": 0.41452991452991456, + "miscellaneous": 0.7225063938618926, + "high_school_chemistry": 0.36633663366336633, + "marketing": 0.8154506437768241, + "professional_law": 0.4050880626223092, + "management": 0.7647058823529411, + "college_physics": 0.38613861386138615, + "jurisprudence": 0.6915887850467289, + "world_religions": 0.6764705882352942, + "sociology": 0.695, + "us_foreign_policy": 0.7171717171717171, + "high_school_macroeconomics": 0.5732647814910026, + "computer_security": 0.6666666666666666, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.6405797101449275, + "electrical_engineering": 0.5347222222222222, + "astronomy": 0.5629139072847682, + "college_biology": 0.48951048951048953 } }, "prompt_3": { - "accuracy": 0.25227028959599573, + "accuracy": 0.5327851269217018, "category_acc": { - "high_school_european_history": 0.23170731707317074, - "business_ethics": 0.2727272727272727, - "clinical_knowledge": 0.23484848484848486, - "medical_genetics": 0.2828282828282828, - "high_school_us_history": 0.270935960591133, - "high_school_physics": 0.3, - "high_school_world_history": 0.2754237288135593, - "virology": 0.3090909090909091, - "high_school_microeconomics": 0.2489451476793249, - "econometrics": 0.3008849557522124, - "college_computer_science": 0.21212121212121213, - "high_school_biology": 0.22330097087378642, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.298932384341637, - "philosophy": 0.2806451612903226, - "professional_medicine": 0.22878228782287824, - "nutrition": 0.23934426229508196, - "global_facts": 0.20202020202020202, - "machine_learning": 0.2702702702702703, - "security_studies": 0.2459016393442623, - "public_relations": 0.3119266055045872, - "professional_psychology": 0.2144026186579378, - "prehistory": 0.25077399380804954, - "anatomy": 0.2835820895522388, - "human_sexuality": 0.2, - "college_medicine": 0.23837209302325582, - "high_school_government_and_politics": 0.2552083333333333, - "college_chemistry": 0.26262626262626265, - "logical_fallacies": 0.25925925925925924, - "high_school_geography": 0.27918781725888325, - "elementary_mathematics": 0.30238726790450926, - "human_aging": 0.25675675675675674, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.2665441176470588, - "formal_logic": 0.208, - "high_school_statistics": 0.29767441860465116, - "international_law": 0.31666666666666665, - "high_school_mathematics": 0.26022304832713755, - "high_school_computer_science": 0.2727272727272727, - "conceptual_physics": 0.3034188034188034, - "miscellaneous": 0.26342710997442453, - "high_school_chemistry": 0.19801980198019803, - "marketing": 0.26180257510729615, - "professional_law": 0.24070450097847357, - "management": 0.2549019607843137, - "college_physics": 0.2871287128712871, - "jurisprudence": 0.24299065420560748, - "world_religions": 0.2, - "sociology": 0.245, - "us_foreign_policy": 0.26262626262626265, - "high_school_macroeconomics": 0.2365038560411311, - "computer_security": 0.21212121212121213, - "moral_scenarios": 0.24161073825503357, - "moral_disputes": 0.22028985507246376, - "electrical_engineering": 0.1736111111111111, - "astronomy": 0.2582781456953642, - "college_biology": 0.2517482517482518 + "high_school_european_history": 0.7439024390243902, + "business_ethics": 0.6363636363636364, + "clinical_knowledge": 0.5909090909090909, + "medical_genetics": 0.5757575757575758, + "high_school_us_history": 0.7635467980295566, + "high_school_physics": 0.26, + "high_school_world_history": 0.7288135593220338, + "virology": 0.4727272727272727, + "high_school_microeconomics": 0.6075949367088608, + "econometrics": 0.25663716814159293, + "college_computer_science": 0.40404040404040403, + "high_school_biology": 0.6666666666666666, + "abstract_algebra": 0.21212121212121213, + "professional_accounting": 0.398576512455516, + "philosophy": 0.5870967741935483, + "professional_medicine": 0.5239852398523985, + "nutrition": 0.5737704918032787, + "global_facts": 0.29292929292929293, + "machine_learning": 0.34234234234234234, + "security_studies": 0.6680327868852459, + "public_relations": 0.6238532110091743, + "professional_psychology": 0.5384615384615384, + "prehistory": 0.5603715170278638, + "anatomy": 0.43283582089552236, + "human_sexuality": 0.6, + "college_medicine": 0.5465116279069767, + "high_school_government_and_politics": 0.796875, + "college_chemistry": 0.41414141414141414, + "logical_fallacies": 0.6728395061728395, + "high_school_geography": 0.7157360406091371, + "elementary_mathematics": 0.35013262599469497, + "human_aging": 0.5630630630630631, + "college_mathematics": 0.3434343434343434, + "high_school_psychology": 0.7904411764705882, + "formal_logic": 0.384, + "high_school_statistics": 0.3767441860465116, + "international_law": 0.6666666666666666, + "high_school_mathematics": 0.2788104089219331, + "high_school_computer_science": 0.42424242424242425, + "conceptual_physics": 0.4230769230769231, + "miscellaneous": 0.7225063938618926, + "high_school_chemistry": 0.35148514851485146, + "marketing": 0.8412017167381974, + "professional_law": 0.4044357469015003, + "management": 0.7549019607843137, + "college_physics": 0.40594059405940597, + "jurisprudence": 0.6822429906542056, + "world_religions": 0.6823529411764706, + "sociology": 0.705, + "us_foreign_policy": 0.7373737373737373, + "high_school_macroeconomics": 0.5681233933161953, + "computer_security": 0.6565656565656566, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.6492753623188405, + "electrical_engineering": 0.5486111111111112, + "astronomy": 0.5761589403973509, + "college_biology": 0.4755244755244755 } }, "prompt_4": { - "accuracy": 0.25227028959599573, - "category_acc": { - "high_school_european_history": 0.21951219512195122, - "business_ethics": 0.21212121212121213, - "clinical_knowledge": 0.24242424242424243, - "medical_genetics": 0.26262626262626265, - "high_school_us_history": 0.2660098522167488, - "high_school_physics": 0.2733333333333333, - "high_school_world_history": 0.2838983050847458, - "virology": 0.20606060606060606, - "high_school_microeconomics": 0.2869198312236287, - "econometrics": 0.24778761061946902, - "college_computer_science": 0.1919191919191919, - "high_school_biology": 0.2588996763754045, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.23843416370106763, - "philosophy": 0.3161290322580645, - "professional_medicine": 0.3210332103321033, - "nutrition": 0.29180327868852457, - "global_facts": 0.21212121212121213, - "machine_learning": 0.26126126126126126, - "security_studies": 0.24180327868852458, - "public_relations": 0.21100917431192662, - "professional_psychology": 0.23404255319148937, - "prehistory": 0.23219814241486067, - "anatomy": 0.22388059701492538, - "human_sexuality": 0.26153846153846155, - "college_medicine": 0.22674418604651161, - "high_school_government_and_politics": 0.2604166666666667, - "college_chemistry": 0.1717171717171717, - "logical_fallacies": 0.22839506172839505, - "high_school_geography": 0.18274111675126903, - "elementary_mathematics": 0.26525198938992045, - "human_aging": 0.24324324324324326, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.23529411764705882, - "formal_logic": 0.264, - "high_school_statistics": 0.22790697674418606, - "international_law": 0.23333333333333334, - "high_school_mathematics": 0.27137546468401486, - "high_school_computer_science": 0.30303030303030304, - "conceptual_physics": 0.2606837606837607, - "miscellaneous": 0.2391304347826087, - "high_school_chemistry": 0.3069306930693069, - "marketing": 0.26180257510729615, - "professional_law": 0.25244618395303325, - "management": 0.30392156862745096, - "college_physics": 0.27722772277227725, - "jurisprudence": 0.24299065420560748, - "world_religions": 0.2529411764705882, - "sociology": 0.25, - "us_foreign_policy": 0.1414141414141414, - "high_school_macroeconomics": 0.2750642673521851, - "computer_security": 0.2727272727272727, - "moral_scenarios": 0.25838926174496646, - "moral_disputes": 0.26666666666666666, - "electrical_engineering": 0.2638888888888889, - "astronomy": 0.2251655629139073, - "college_biology": 0.1958041958041958 - } - }, - "prompt_5": { - "accuracy": 0.24969610296746514, + "accuracy": 0.5308544869503039, "category_acc": { - "high_school_european_history": 0.24390243902439024, - "business_ethics": 0.29292929292929293, - "clinical_knowledge": 0.2727272727272727, - "medical_genetics": 0.21212121212121213, - "high_school_us_history": 0.2561576354679803, - "high_school_physics": 0.25333333333333335, - "high_school_world_history": 0.2796610169491525, - "virology": 0.24848484848484848, - "high_school_microeconomics": 0.19831223628691982, - "econometrics": 0.2920353982300885, - "college_computer_science": 0.29292929292929293, - "high_school_biology": 0.2686084142394822, - "abstract_algebra": 0.31313131313131315, - "professional_accounting": 0.302491103202847, - "philosophy": 0.24193548387096775, - "professional_medicine": 0.23616236162361623, - "nutrition": 0.29180327868852457, - "global_facts": 0.3434343434343434, - "machine_learning": 0.17117117117117117, - "security_studies": 0.22950819672131148, - "public_relations": 0.29357798165137616, - "professional_psychology": 0.22913256955810146, - "prehistory": 0.24148606811145512, - "anatomy": 0.2462686567164179, - "human_sexuality": 0.23846153846153847, - "college_medicine": 0.2616279069767442, - "high_school_government_and_politics": 0.265625, - "college_chemistry": 0.26262626262626265, - "logical_fallacies": 0.22839506172839505, - "high_school_geography": 0.29949238578680204, - "elementary_mathematics": 0.23076923076923078, - "human_aging": 0.23423423423423423, - "college_mathematics": 0.23232323232323232, - "high_school_psychology": 0.25735294117647056, - "formal_logic": 0.232, - "high_school_statistics": 0.3116279069767442, - "international_law": 0.26666666666666666, - "high_school_mathematics": 0.25650557620817843, - "high_school_computer_science": 0.32323232323232326, - "conceptual_physics": 0.2222222222222222, - "miscellaneous": 0.2506393861892583, - "high_school_chemistry": 0.19801980198019803, - "marketing": 0.22317596566523606, - "professional_law": 0.25048923679060664, - "management": 0.24509803921568626, - "college_physics": 0.27722772277227725, - "jurisprudence": 0.24299065420560748, - "world_religions": 0.28823529411764703, - "sociology": 0.225, - "us_foreign_policy": 0.25252525252525254, - "high_school_macroeconomics": 0.2159383033419023, - "computer_security": 0.21212121212121213, - "moral_scenarios": 0.2348993288590604, - "moral_disputes": 0.24347826086956523, - "electrical_engineering": 0.25, - "astronomy": 0.19205298013245034, - "college_biology": 0.2727272727272727 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.2473997028231798 - }, - "prompt_2": { - "accuracy": 0.26300148588410105 - }, - "prompt_3": { - "accuracy": 0.2384843982169391 - }, - "prompt_4": { - "accuracy": 0.24962852897473997 - }, - "prompt_5": { - "accuracy": 0.23476968796433878 - } - }, + "high_school_european_history": 0.7560975609756098, + "business_ethics": 0.6363636363636364, + "clinical_knowledge": 0.5909090909090909, + "medical_genetics": 0.5858585858585859, + "high_school_us_history": 0.7586206896551724, + "high_school_physics": 0.24, + "high_school_world_history": 0.7245762711864406, + "virology": 0.4727272727272727, + "high_school_microeconomics": 0.5991561181434599, + "econometrics": 0.25663716814159293, + "college_computer_science": 0.40404040404040403, + "high_school_biology": 0.6699029126213593, + "abstract_algebra": 0.2222222222222222, + "professional_accounting": 0.398576512455516, + "philosophy": 0.567741935483871, + "professional_medicine": 0.5202952029520295, + "nutrition": 0.5672131147540984, + "global_facts": 0.31313131313131315, + "machine_learning": 0.35135135135135137, + "security_studies": 0.6680327868852459, + "public_relations": 0.6146788990825688, + "professional_psychology": 0.5482815057283142, + "prehistory": 0.56656346749226, + "anatomy": 0.44029850746268656, + "human_sexuality": 0.6, + "college_medicine": 0.5406976744186046, + "high_school_government_and_politics": 0.7708333333333334, + "college_chemistry": 0.37373737373737376, + "logical_fallacies": 0.6604938271604939, + "high_school_geography": 0.7258883248730964, + "elementary_mathematics": 0.3209549071618037, + "human_aging": 0.5585585585585585, + "college_mathematics": 0.31313131313131315, + "high_school_psychology": 0.7830882352941176, + "formal_logic": 0.392, + "high_school_statistics": 0.4, + "international_law": 0.6583333333333333, + "high_school_mathematics": 0.2788104089219331, + "high_school_computer_science": 0.45454545454545453, + "conceptual_physics": 0.42735042735042733, + "miscellaneous": 0.7237851662404092, + "high_school_chemistry": 0.3465346534653465, + "marketing": 0.8326180257510729, + "professional_law": 0.39856490541422046, + "management": 0.7647058823529411, + "college_physics": 0.40594059405940597, + "jurisprudence": 0.6635514018691588, + "world_religions": 0.7176470588235294, + "sociology": 0.695, + "us_foreign_policy": 0.7575757575757576, + "high_school_macroeconomics": 0.570694087403599, + "computer_security": 0.6363636363636364, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.6405797101449275, + "electrical_engineering": 0.5763888888888888, + "astronomy": 0.5761589403973509, + "college_biology": 0.46853146853146854 + } + }, + "prompt_5": { + "accuracy": 0.5337146943153379, + "category_acc": { + "high_school_european_history": 0.7560975609756098, + "business_ethics": 0.6565656565656566, + "clinical_knowledge": 0.5757575757575758, + "medical_genetics": 0.5353535353535354, + "high_school_us_history": 0.7733990147783252, + "high_school_physics": 0.23333333333333334, + "high_school_world_history": 0.7330508474576272, + "virology": 0.4727272727272727, + "high_school_microeconomics": 0.5907172995780591, + "econometrics": 0.2743362831858407, + "college_computer_science": 0.43434343434343436, + "high_school_biology": 0.6634304207119741, + "abstract_algebra": 0.25252525252525254, + "professional_accounting": 0.4092526690391459, + "philosophy": 0.5774193548387097, + "professional_medicine": 0.5166051660516605, + "nutrition": 0.5868852459016394, + "global_facts": 0.29292929292929293, + "machine_learning": 0.34234234234234234, + "security_studies": 0.680327868852459, + "public_relations": 0.6055045871559633, + "professional_psychology": 0.5417348608837971, + "prehistory": 0.5572755417956656, + "anatomy": 0.4253731343283582, + "human_sexuality": 0.6153846153846154, + "college_medicine": 0.5406976744186046, + "high_school_government_and_politics": 0.7864583333333334, + "college_chemistry": 0.36363636363636365, + "logical_fallacies": 0.6604938271604939, + "high_school_geography": 0.6954314720812182, + "elementary_mathematics": 0.34748010610079577, + "human_aging": 0.5495495495495496, + "college_mathematics": 0.32323232323232326, + "high_school_psychology": 0.7830882352941176, + "formal_logic": 0.4, + "high_school_statistics": 0.413953488372093, + "international_law": 0.6583333333333333, + "high_school_mathematics": 0.275092936802974, + "high_school_computer_science": 0.45454545454545453, + "conceptual_physics": 0.4358974358974359, + "miscellaneous": 0.7276214833759591, + "high_school_chemistry": 0.37623762376237624, + "marketing": 0.8283261802575107, + "professional_law": 0.4050880626223092, + "management": 0.7745098039215687, + "college_physics": 0.40594059405940597, + "jurisprudence": 0.6915887850467289, + "world_religions": 0.6941176470588235, + "sociology": 0.705, + "us_foreign_policy": 0.7272727272727273, + "high_school_macroeconomics": 0.5784061696658098, + "computer_security": 0.6464646464646465, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.6579710144927536, + "electrical_engineering": 0.5486111111111112, + "astronomy": 0.5695364238410596, + "college_biology": 0.4965034965034965 + } + } + }, + "c_eval": { + "prompt_1": { + "accuracy": 0.24888558692421991 + }, + "prompt_2": { + "accuracy": 0.24888558692421991 + }, + "prompt_3": { + "accuracy": 0.24888558692421991 + }, + "prompt_4": { + "accuracy": 0.24591381872213966 + }, + "prompt_5": { + "accuracy": 0.24888558692421991 + } + }, "c_eval_full": { "prompt_1": { - "accuracy": 0.24159402241594022, + "accuracy": 0.23848069738480698, "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.11538461538461539, - "college_programming": 0.19047619047619047, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.047619047619047616, - "electrical_engineer": 0.19047619047619047, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.25, + "computer_network": 0.125, + "operating_system": 0.125, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.21428571428571427, + "college_physics": 0.125, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.17391304347826086, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.20689655172413793, + "high_school_mathematics": 0.391304347826087, + "high_school_physics": 0.16666666666666666, "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.08333333333333333, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.16666666666666666, - "business_administration": 0.3684210526315789, - "marxism": 0.16666666666666666, - "mao_zedong_thought": 0.10344827586206896, - "education_science": 0.29411764705882354, - "teacher_qualification": 0.1836734693877551, + "high_school_biology": 0.3333333333333333, + "middle_school_mathematics": 0.20833333333333334, + "middle_school_biology": 0.15384615384615385, + "middle_school_physics": 0.125, + "middle_school_chemistry": 0.16, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.36666666666666664, + "business_administration": 0.18421052631578946, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.27586206896551724, + "education_science": 0.2647058823529412, + "teacher_qualification": 0.2653061224489796, "high_school_politics": 0.25, - "high_school_geography": 0.375, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.07142857142857142, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.2222222222222222, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.23076923076923078, + "middle_school_geography": 0.11764705882352941, + "modern_chinese_history": 0.17857142857142858, + "ideological_and_moral_cultivation": 0.2916666666666667, + "logic": 0.18518518518518517, "law": 0.27586206896551724, - "chinese_language_and_literature": 0.17857142857142858, - "art_studies": 0.23684210526315788, - "professional_tour_guide": 0.20588235294117646, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.4473684210526316, + "professional_tour_guide": 0.29411764705882354, "legal_professional": 0.25, - "high_school_chinese": 0.25, + "high_school_chinese": 0.3333333333333333, "high_school_history": 0.2, "middle_school_history": 0.18518518518518517, - "civil_servant": 0.2692307692307692, - "sports_science": 0.3333333333333333, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.24074074074074073, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.16666666666666666, - "physician": 0.3333333333333333 + "civil_servant": 0.25, + "sports_science": 0.08333333333333333, + "plant_protection": 0.18518518518518517, + "basic_medicine": 0.125, + "clinical_medicine": 0.3333333333333333, + "urban_and_rural_planner": 0.3333333333333333, + "accountant": 0.16666666666666666, + "fire_engineer": 0.25, + "environmental_impact_assessment_engineer": 0.19444444444444445, + "tax_accountant": 0.2222222222222222, + "physician": 0.2777777777777778 } }, "prompt_2": { - "accuracy": 0.26525529265255293, + "accuracy": 0.23910336239103364, "category_acc": { - "computer_network": 0.375, + "computer_network": 0.125, "operating_system": 0.125, - "computer_architecture": 0.15384615384615385, - "college_programming": 0.38095238095238093, - "college_physics": 0.25, - "college_chemistry": 0.13793103448275862, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.23809523809523808, + "college_physics": 0.125, + "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.375, - "high_school_biology": 0.5416666666666666, + "probability_and_statistics": 0.17391304347826086, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.20689655172413793, + "high_school_mathematics": 0.391304347826087, + "high_school_physics": 0.16666666666666666, + "high_school_chemistry": 0.20833333333333334, + "high_school_biology": 0.3333333333333333, "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.3076923076923077, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.14285714285714285, - "college_economics": 0.2833333333333333, + "middle_school_biology": 0.15384615384615385, + "middle_school_physics": 0.125, + "middle_school_chemistry": 0.16, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.36666666666666664, "business_administration": 0.18421052631578946, - "marxism": 0.25, - "mao_zedong_thought": 0.13793103448275862, - "education_science": 0.23529411764705882, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.27586206896551724, + "education_science": 0.2647058823529412, "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.375, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.15384615384615385, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.4166666666666667, - "logic": 0.1111111111111111, - "law": 0.1724137931034483, - "chinese_language_and_literature": 0.25, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.11764705882352941, - "legal_professional": 0.14285714285714285, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.28, + "high_school_politics": 0.25, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.23076923076923078, + "middle_school_geography": 0.11764705882352941, + "modern_chinese_history": 0.17857142857142858, + "ideological_and_moral_cultivation": 0.2916666666666667, + "logic": 0.18518518518518517, + "law": 0.27586206896551724, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.4473684210526316, + "professional_tour_guide": 0.29411764705882354, + "legal_professional": 0.25, + "high_school_chinese": 0.3333333333333333, + "high_school_history": 0.2, "middle_school_history": 0.18518518518518517, - "civil_servant": 0.3076923076923077, - "sports_science": 0.2916666666666667, - "plant_protection": 0.4444444444444444, + "civil_servant": 0.25, + "sports_science": 0.08333333333333333, + "plant_protection": 0.18518518518518517, "basic_medicine": 0.125, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.35185185185185186, - "fire_engineer": 0.3055555555555556, + "clinical_medicine": 0.3333333333333333, + "urban_and_rural_planner": 0.3333333333333333, + "accountant": 0.16666666666666666, + "fire_engineer": 0.25, "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.24074074074074073, - "physician": 0.35185185185185186 + "tax_accountant": 0.2222222222222222, + "physician": 0.2777777777777778 } }, "prompt_3": { - "accuracy": 0.24968866749688667, + "accuracy": 0.23848069738480698, "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.16666666666666666, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.19047619047619047, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.25, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.125, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.11538461538461539, + "computer_network": 0.125, + "operating_system": 0.125, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.21428571428571427, + "college_physics": 0.125, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.17391304347826086, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.20689655172413793, + "high_school_mathematics": 0.391304347826087, + "high_school_physics": 0.16666666666666666, + "high_school_chemistry": 0.20833333333333334, + "high_school_biology": 0.3333333333333333, + "middle_school_mathematics": 0.20833333333333334, + "middle_school_biology": 0.15384615384615385, "middle_school_physics": 0.125, - "middle_school_chemistry": 0.44, + "middle_school_chemistry": 0.16, "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.21666666666666667, - "business_administration": 0.13157894736842105, - "marxism": 0.16666666666666666, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.23529411764705882, + "college_economics": 0.36666666666666664, + "business_administration": 0.18421052631578946, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.27586206896551724, + "education_science": 0.2647058823529412, "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.25, - "middle_school_politics": 0.34615384615384615, - "middle_school_geography": 0.17647058823529413, + "high_school_politics": 0.25, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.23076923076923078, + "middle_school_geography": 0.11764705882352941, "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.14814814814814814, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.17857142857142858, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.2916666666666667, + "ideological_and_moral_cultivation": 0.2916666666666667, + "logic": 0.18518518518518517, + "law": 0.27586206896551724, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.4473684210526316, + "professional_tour_guide": 0.29411764705882354, + "legal_professional": 0.25, + "high_school_chinese": 0.3333333333333333, "high_school_history": 0.2, - "middle_school_history": 0.07407407407407407, - "civil_servant": 0.21153846153846154, - "sports_science": 0.375, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.375, - "clinical_medicine": 0.14814814814814814, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.2777777777777778, + "middle_school_history": 0.18518518518518517, + "civil_servant": 0.25, + "sports_science": 0.08333333333333333, + "plant_protection": 0.18518518518518517, + "basic_medicine": 0.125, + "clinical_medicine": 0.3333333333333333, + "urban_and_rural_planner": 0.3333333333333333, + "accountant": 0.16666666666666666, "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.2962962962962963, - "physician": 0.18518518518518517 + "environmental_impact_assessment_engineer": 0.19444444444444445, + "tax_accountant": 0.2222222222222222, + "physician": 0.2777777777777778 } }, "prompt_4": { - "accuracy": 0.263387297633873, + "accuracy": 0.24159402241594022, "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.38095238095238093, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.40476190476190477, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.17391304347826086, + "computer_network": 0.20833333333333334, + "operating_system": 0.08333333333333333, + "computer_architecture": 0.3076923076923077, + "college_programming": 0.2619047619047619, + "college_physics": 0.125, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.25, + "probability_and_statistics": 0.17391304347826086, + "discrete_mathematics": 0.3333333333333333, + "electrical_engineer": 0.2857142857142857, + "metrology_engineer": 0.13793103448275862, + "high_school_mathematics": 0.34782608695652173, "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.3333333333333333, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.3, - "business_administration": 0.3157894736842105, - "marxism": 0.25, - "mao_zedong_thought": 0.20689655172413793, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.1836734693877551, - "high_school_politics": 0.125, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.2857142857142857, + "high_school_chemistry": 0.20833333333333334, + "high_school_biology": 0.3333333333333333, + "middle_school_mathematics": 0.16666666666666666, + "middle_school_biology": 0.15384615384615385, + "middle_school_physics": 0.08333333333333333, + "middle_school_chemistry": 0.24, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.31666666666666665, + "business_administration": 0.18421052631578946, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.27586206896551724, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.2857142857142857, + "high_school_politics": 0.2916666666666667, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.11764705882352941, + "modern_chinese_history": 0.21428571428571427, "ideological_and_moral_cultivation": 0.2916666666666667, "logic": 0.18518518518518517, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.42857142857142855, - "art_studies": 0.23684210526315788, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.14285714285714285, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.4, - "middle_school_history": 0.1111111111111111, - "civil_servant": 0.36538461538461536, - "sports_science": 0.20833333333333334, - "plant_protection": 0.1111111111111111, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.1568627450980392, - "accountant": 0.25925925925925924, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.16666666666666666, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.4473684210526316, + "professional_tour_guide": 0.29411764705882354, + "legal_professional": 0.25, + "high_school_chinese": 0.25, + "high_school_history": 0.2, + "middle_school_history": 0.18518518518518517, + "civil_servant": 0.25, + "sports_science": 0.08333333333333333, + "plant_protection": 0.2222222222222222, + "basic_medicine": 0.125, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.3333333333333333, + "accountant": 0.18518518518518517, + "fire_engineer": 0.25, + "environmental_impact_assessment_engineer": 0.19444444444444445, "tax_accountant": 0.25925925925925924, - "physician": 0.24074074074074073 + "physician": 0.2777777777777778 } }, "prompt_5": { - "accuracy": 0.2752179327521793, + "accuracy": 0.23910336239103364, "category_acc": { - "computer_network": 0.2916666666666667, + "computer_network": 0.125, "operating_system": 0.125, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.2619047619047619, - "college_physics": 0.25, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.5217391304347826, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.40476190476190477, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.4166666666666667, - "high_school_biology": 0.08333333333333333, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.3076923076923077, - "middle_school_physics": 0.16666666666666666, - "middle_school_chemistry": 0.48, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.3, - "business_administration": 0.34210526315789475, - "marxism": 0.25, - "mao_zedong_thought": 0.2413793103448276, - "education_science": 0.3235294117647059, - "teacher_qualification": 0.16326530612244897, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.25, - "logic": 0.14814814814814814, - "law": 0.13793103448275862, - "chinese_language_and_literature": 0.25, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.23529411764705882, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.36, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.2692307692307692, - "sports_science": 0.375, - "plant_protection": 0.1111111111111111, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.48148148148148145, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.23809523809523808, + "college_physics": 0.125, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.17391304347826086, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.20689655172413793, + "high_school_mathematics": 0.391304347826087, + "high_school_physics": 0.16666666666666666, + "high_school_chemistry": 0.20833333333333334, + "high_school_biology": 0.3333333333333333, + "middle_school_mathematics": 0.20833333333333334, + "middle_school_biology": 0.15384615384615385, + "middle_school_physics": 0.125, + "middle_school_chemistry": 0.16, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.36666666666666664, + "business_administration": 0.18421052631578946, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.27586206896551724, + "education_science": 0.2647058823529412, + "teacher_qualification": 0.2653061224489796, + "high_school_politics": 0.25, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.23076923076923078, + "middle_school_geography": 0.11764705882352941, + "modern_chinese_history": 0.17857142857142858, + "ideological_and_moral_cultivation": 0.2916666666666667, + "logic": 0.18518518518518517, + "law": 0.27586206896551724, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.4473684210526316, + "professional_tour_guide": 0.29411764705882354, + "legal_professional": 0.25, + "high_school_chinese": 0.3333333333333333, + "high_school_history": 0.2, + "middle_school_history": 0.18518518518518517, + "civil_servant": 0.25, + "sports_science": 0.08333333333333333, + "plant_protection": 0.18518518518518517, + "basic_medicine": 0.125, + "clinical_medicine": 0.3333333333333333, "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.2777777777777778, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.2222222222222222, - "tax_accountant": 0.2037037037037037, + "accountant": 0.16666666666666666, + "fire_engineer": 0.25, + "environmental_impact_assessment_engineer": 0.19444444444444445, + "tax_accountant": 0.2222222222222222, "physician": 0.2777777777777778 } } }, "cmmlu": { "prompt_1": { - "accuracy": 0.23297491039426524 + "accuracy": 0.27956989247311825 }, "prompt_2": { - "accuracy": 0.17562724014336917 + "accuracy": 0.2831541218637993 }, "prompt_3": { "accuracy": 0.27598566308243727 }, "prompt_4": { - "accuracy": 0.25089605734767023 + "accuracy": 0.2831541218637993 }, "prompt_5": { - "accuracy": 0.22580645161290322 + "accuracy": 0.2831541218637993 } }, "cmmlu_full": { "prompt_1": { - "accuracy": 0.24702123985494734, + "accuracy": 0.2526333966499741, "category_acc": { - "agronomy": 0.23668639053254437, - "anatomy": 0.1891891891891892, - "ancient_chinese": 0.2804878048780488, - "arts": 0.2125, - "astronomy": 0.24848484848484848, + "agronomy": 0.24260355029585798, + "anatomy": 0.25675675675675674, + "ancient_chinese": 0.2682926829268293, + "arts": 0.25625, + "astronomy": 0.21818181818181817, "business_ethics": 0.2535885167464115, - "chinese_civil_service_exam": 0.2125, - "chinese_driving_rule": 0.1984732824427481, - "chinese_food_culture": 0.2647058823529412, - "chinese_foreign_policy": 0.2336448598130841, - "chinese_history": 0.24148606811145512, - "chinese_literature": 0.29411764705882354, + "chinese_civil_service_exam": 0.24375, + "chinese_driving_rule": 0.25190839694656486, + "chinese_food_culture": 0.25, + "chinese_foreign_policy": 0.24299065420560748, + "chinese_history": 0.24458204334365324, + "chinese_literature": 0.25, "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.21940928270042195, - "college_actuarial_science": 0.3113207547169811, - "college_education": 0.27102803738317754, + "clinical_knowledge": 0.2489451476793249, + "college_actuarial_science": 0.27358490566037735, + "college_education": 0.308411214953271, "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.25, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.18867924528301888, + "college_law": 0.2222222222222222, + "college_mathematics": 0.22857142857142856, + "college_medical_statistics": 0.2358490566037736, "college_medicine": 0.2490842490842491, - "computer_science": 0.2647058823529412, - "computer_security": 0.23976608187134502, + "computer_science": 0.2696078431372549, + "computer_security": 0.24561403508771928, "conceptual_physics": 0.23809523809523808, - "construction_project_management": 0.2158273381294964, - "economics": 0.2893081761006289, - "education": 0.22085889570552147, - "electrical_engineering": 0.2558139534883721, - "elementary_chinese": 0.2777777777777778, - "elementary_commonsense": 0.24242424242424243, - "elementary_information_and_technology": 0.2815126050420168, - "elementary_mathematics": 0.25217391304347825, - "ethnology": 0.2518518518518518, - "food_science": 0.2727272727272727, - "genetics": 0.2727272727272727, - "global_facts": 0.2953020134228188, - "high_school_biology": 0.20118343195266272, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.2542372881355932, - "high_school_mathematics": 0.25, - "high_school_physics": 0.23636363636363636, - "high_school_politics": 0.1888111888111888, - "human_sexuality": 0.23809523809523808, - "international_law": 0.24864864864864866, - "journalism": 0.23255813953488372, + "construction_project_management": 0.2446043165467626, + "economics": 0.2389937106918239, + "education": 0.25153374233128833, + "electrical_engineering": 0.23255813953488372, + "elementary_chinese": 0.28174603174603174, + "elementary_commonsense": 0.25252525252525254, + "elementary_information_and_technology": 0.2773109243697479, + "elementary_mathematics": 0.2565217391304348, + "ethnology": 0.24444444444444444, + "food_science": 0.24475524475524477, + "genetics": 0.26704545454545453, + "global_facts": 0.2483221476510067, + "high_school_biology": 0.21893491124260356, + "high_school_chemistry": 0.26515151515151514, + "high_school_geography": 0.2711864406779661, + "high_school_mathematics": 0.2804878048780488, + "high_school_physics": 0.2818181818181818, + "high_school_politics": 0.2727272727272727, + "human_sexuality": 0.25396825396825395, + "international_law": 0.25405405405405407, + "journalism": 0.2558139534883721, "jurisprudence": 0.24817518248175183, - "legal_and_moral_basis": 0.2897196261682243, - "logical": 0.23577235772357724, - "machine_learning": 0.27049180327868855, - "management": 0.21904761904761905, + "legal_and_moral_basis": 0.2523364485981308, + "logical": 0.2682926829268293, + "machine_learning": 0.2459016393442623, + "management": 0.24761904761904763, "marketing": 0.24444444444444444, - "marxist_theory": 0.26455026455026454, - "modern_chinese": 0.23275862068965517, - "nutrition": 0.19310344827586207, - "philosophy": 0.26666666666666666, - "professional_accounting": 0.22857142857142856, - "professional_law": 0.22748815165876776, - "professional_medicine": 0.2925531914893617, - "professional_psychology": 0.21120689655172414, - "public_relations": 0.27586206896551724, - "security_study": 0.34074074074074073, - "sociology": 0.23893805309734514, - "sports_science": 0.17575757575757575, - "traditional_chinese_medicine": 0.1891891891891892, - "virology": 0.2781065088757396, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.27586206896551724, + "nutrition": 0.23448275862068965, + "philosophy": 0.2571428571428571, + "professional_accounting": 0.25142857142857145, + "professional_law": 0.25118483412322273, + "professional_medicine": 0.2553191489361702, + "professional_psychology": 0.24568965517241378, + "public_relations": 0.2471264367816092, + "security_study": 0.25925925925925924, + "sociology": 0.252212389380531, + "sports_science": 0.24242424242424243, + "traditional_chinese_medicine": 0.24864864864864866, + "virology": 0.2485207100591716, "world_history": 0.2360248447204969, - "world_religions": 0.2375 + "world_religions": 0.26875 } }, "prompt_2": { - "accuracy": 0.2535831462614402, + "accuracy": 0.2526333966499741, "category_acc": { - "agronomy": 0.2485207100591716, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.25, - "arts": 0.2625, - "astronomy": 0.24242424242424243, + "agronomy": 0.24260355029585798, + "anatomy": 0.2635135135135135, + "ancient_chinese": 0.2682926829268293, + "arts": 0.25625, + "astronomy": 0.21212121212121213, "business_ethics": 0.2535885167464115, - "chinese_civil_service_exam": 0.23125, - "chinese_driving_rule": 0.24427480916030533, - "chinese_food_culture": 0.2867647058823529, - "chinese_foreign_policy": 0.17757009345794392, - "chinese_history": 0.22291021671826625, - "chinese_literature": 0.23529411764705882, - "chinese_teacher_qualification": 0.2737430167597765, - "clinical_knowledge": 0.25316455696202533, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.2336448598130841, - "college_engineering_hydrology": 0.25471698113207547, - "college_law": 0.2962962962962963, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.2830188679245283, - "college_medicine": 0.2564102564102564, - "computer_science": 0.23529411764705882, - "computer_security": 0.1695906432748538, - "conceptual_physics": 0.2653061224489796, - "construction_project_management": 0.2517985611510791, - "economics": 0.24528301886792453, - "education": 0.26993865030674846, - "electrical_engineering": 0.25, + "chinese_civil_service_exam": 0.24375, + "chinese_driving_rule": 0.25190839694656486, + "chinese_food_culture": 0.25, + "chinese_foreign_policy": 0.24299065420560748, + "chinese_history": 0.24458204334365324, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.25139664804469275, + "clinical_knowledge": 0.2489451476793249, + "college_actuarial_science": 0.27358490566037735, + "college_education": 0.308411214953271, + "college_engineering_hydrology": 0.29245283018867924, + "college_law": 0.2222222222222222, + "college_mathematics": 0.22857142857142856, + "college_medical_statistics": 0.2358490566037736, + "college_medicine": 0.2490842490842491, + "computer_science": 0.27450980392156865, + "computer_security": 0.24561403508771928, + "conceptual_physics": 0.23809523809523808, + "construction_project_management": 0.2446043165467626, + "economics": 0.2389937106918239, + "education": 0.25153374233128833, + "electrical_engineering": 0.23837209302325582, "elementary_chinese": 0.28174603174603174, - "elementary_commonsense": 0.22727272727272727, - "elementary_information_and_technology": 0.2857142857142857, - "elementary_mathematics": 0.27391304347826084, - "ethnology": 0.2740740740740741, - "food_science": 0.26573426573426573, - "genetics": 0.1875, - "global_facts": 0.2348993288590604, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.2878787878787879, - "high_school_geography": 0.2796610169491525, - "high_school_mathematics": 0.25, - "high_school_physics": 0.21818181818181817, + "elementary_commonsense": 0.23737373737373738, + "elementary_information_and_technology": 0.2773109243697479, + "elementary_mathematics": 0.2565217391304348, + "ethnology": 0.24444444444444444, + "food_science": 0.23776223776223776, + "genetics": 0.2727272727272727, + "global_facts": 0.2483221476510067, + "high_school_biology": 0.21893491124260356, + "high_school_chemistry": 0.2727272727272727, + "high_school_geography": 0.2711864406779661, + "high_school_mathematics": 0.2804878048780488, + "high_school_physics": 0.2727272727272727, "high_school_politics": 0.2727272727272727, - "human_sexuality": 0.2698412698412698, - "international_law": 0.22162162162162163, - "journalism": 0.313953488372093, - "jurisprudence": 0.22871046228710462, - "legal_and_moral_basis": 0.2336448598130841, - "logical": 0.22764227642276422, - "machine_learning": 0.2786885245901639, - "management": 0.24285714285714285, - "marketing": 0.2777777777777778, - "marxist_theory": 0.291005291005291, - "modern_chinese": 0.2672413793103448, - "nutrition": 0.33793103448275863, - "philosophy": 0.34285714285714286, - "professional_accounting": 0.21714285714285714, - "professional_law": 0.21800947867298578, - "professional_medicine": 0.2526595744680851, - "professional_psychology": 0.2974137931034483, - "public_relations": 0.25862068965517243, - "security_study": 0.2740740740740741, + "human_sexuality": 0.25396825396825395, + "international_law": 0.25405405405405407, + "journalism": 0.25, + "jurisprudence": 0.24817518248175183, + "legal_and_moral_basis": 0.2523364485981308, + "logical": 0.2682926829268293, + "machine_learning": 0.2540983606557377, + "management": 0.24761904761904763, + "marketing": 0.24444444444444444, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.27586206896551724, + "nutrition": 0.2413793103448276, + "philosophy": 0.2571428571428571, + "professional_accounting": 0.25142857142857145, + "professional_law": 0.25118483412322273, + "professional_medicine": 0.2553191489361702, + "professional_psychology": 0.24568965517241378, + "public_relations": 0.2471264367816092, + "security_study": 0.25925925925925924, "sociology": 0.252212389380531, - "sports_science": 0.2606060606060606, - "traditional_chinese_medicine": 0.2756756756756757, - "virology": 0.22485207100591717, - "world_history": 0.2422360248447205, - "world_religions": 0.2 + "sports_science": 0.24242424242424243, + "traditional_chinese_medicine": 0.24864864864864866, + "virology": 0.2485207100591716, + "world_history": 0.2360248447204969, + "world_religions": 0.26875 } }, "prompt_3": { - "accuracy": 0.2508202383008116, + "accuracy": 0.2528060783975134, "category_acc": { - "agronomy": 0.28402366863905326, - "anatomy": 0.30405405405405406, - "ancient_chinese": 0.25, - "arts": 0.2625, - "astronomy": 0.2727272727272727, - "business_ethics": 0.23923444976076555, - "chinese_civil_service_exam": 0.23125, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.21323529411764705, - "chinese_foreign_policy": 0.2897196261682243, - "chinese_history": 0.3065015479876161, - "chinese_literature": 0.25980392156862747, - "chinese_teacher_qualification": 0.2681564245810056, - "clinical_knowledge": 0.23628691983122363, - "college_actuarial_science": 0.3490566037735849, - "college_education": 0.29906542056074764, - "college_engineering_hydrology": 0.2830188679245283, - "college_law": 0.16666666666666666, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.18867924528301888, - "college_medicine": 0.2673992673992674, - "computer_science": 0.29901960784313725, - "computer_security": 0.2807017543859649, - "conceptual_physics": 0.30612244897959184, - "construction_project_management": 0.26618705035971224, - "economics": 0.2641509433962264, - "education": 0.2392638036809816, - "electrical_engineering": 0.23837209302325582, - "elementary_chinese": 0.24603174603174602, - "elementary_commonsense": 0.2727272727272727, - "elementary_information_and_technology": 0.2647058823529412, - "elementary_mathematics": 0.26521739130434785, - "ethnology": 0.22962962962962963, - "food_science": 0.1958041958041958, - "genetics": 0.2556818181818182, - "global_facts": 0.28859060402684567, - "high_school_biology": 0.2603550295857988, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.20909090909090908, - "high_school_politics": 0.2517482517482518, - "human_sexuality": 0.19047619047619047, - "international_law": 0.1945945945945946, - "journalism": 0.16279069767441862, - "jurisprudence": 0.25790754257907544, - "legal_and_moral_basis": 0.21962616822429906, - "logical": 0.21138211382113822, - "machine_learning": 0.21311475409836064, - "management": 0.22380952380952382, - "marketing": 0.28888888888888886, - "marxist_theory": 0.21693121693121692, - "modern_chinese": 0.22413793103448276, + "agronomy": 0.24260355029585798, + "anatomy": 0.2635135135135135, + "ancient_chinese": 0.2621951219512195, + "arts": 0.25625, + "astronomy": 0.23030303030303031, + "business_ethics": 0.2535885167464115, + "chinese_civil_service_exam": 0.24375, + "chinese_driving_rule": 0.25190839694656486, + "chinese_food_culture": 0.25, + "chinese_foreign_policy": 0.24299065420560748, + "chinese_history": 0.24458204334365324, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.25139664804469275, + "clinical_knowledge": 0.24472573839662448, + "college_actuarial_science": 0.2830188679245283, + "college_education": 0.308411214953271, + "college_engineering_hydrology": 0.29245283018867924, + "college_law": 0.2222222222222222, + "college_mathematics": 0.23809523809523808, + "college_medical_statistics": 0.2358490566037736, + "college_medicine": 0.2490842490842491, + "computer_science": 0.2696078431372549, + "computer_security": 0.24561403508771928, + "conceptual_physics": 0.23809523809523808, + "construction_project_management": 0.2446043165467626, + "economics": 0.2389937106918239, + "education": 0.25153374233128833, + "electrical_engineering": 0.23255813953488372, + "elementary_chinese": 0.28174603174603174, + "elementary_commonsense": 0.24242424242424243, + "elementary_information_and_technology": 0.2773109243697479, + "elementary_mathematics": 0.2608695652173913, + "ethnology": 0.24444444444444444, + "food_science": 0.23776223776223776, + "genetics": 0.26704545454545453, + "global_facts": 0.24161073825503357, + "high_school_biology": 0.21893491124260356, + "high_school_chemistry": 0.26515151515151514, + "high_school_geography": 0.2711864406779661, + "high_school_mathematics": 0.2804878048780488, + "high_school_physics": 0.2727272727272727, + "high_school_politics": 0.2727272727272727, + "human_sexuality": 0.25396825396825395, + "international_law": 0.25405405405405407, + "journalism": 0.25, + "jurisprudence": 0.24817518248175183, + "legal_and_moral_basis": 0.2523364485981308, + "logical": 0.2682926829268293, + "machine_learning": 0.26229508196721313, + "management": 0.24761904761904763, + "marketing": 0.24444444444444444, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.27586206896551724, "nutrition": 0.2413793103448276, - "philosophy": 0.3047619047619048, - "professional_accounting": 0.21714285714285714, + "philosophy": 0.2571428571428571, + "professional_accounting": 0.25142857142857145, "professional_law": 0.25118483412322273, - "professional_medicine": 0.26063829787234044, + "professional_medicine": 0.2553191489361702, "professional_psychology": 0.24568965517241378, - "public_relations": 0.25287356321839083, - "security_study": 0.2222222222222222, - "sociology": 0.19911504424778761, - "sports_science": 0.2545454545454545, + "public_relations": 0.2471264367816092, + "security_study": 0.26666666666666666, + "sociology": 0.252212389380531, + "sports_science": 0.24242424242424243, "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.3076923076923077, - "world_history": 0.21739130434782608, - "world_religions": 0.25625 + "virology": 0.2485207100591716, + "world_history": 0.2360248447204969, + "world_religions": 0.26875 } }, "prompt_4": { - "accuracy": 0.25116560179589015, + "accuracy": 0.2530651010188223, "category_acc": { - "agronomy": 0.2781065088757396, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.27439024390243905, - "arts": 0.275, - "astronomy": 0.24848484848484848, - "business_ethics": 0.20095693779904306, - "chinese_civil_service_exam": 0.2625, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.25735294117647056, - "chinese_foreign_policy": 0.2803738317757009, - "chinese_history": 0.25386996904024767, - "chinese_literature": 0.23529411764705882, - "chinese_teacher_qualification": 0.20670391061452514, - "clinical_knowledge": 0.2742616033755274, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.27102803738317754, - "college_engineering_hydrology": 0.24528301886792453, - "college_law": 0.25925925925925924, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.19811320754716982, - "college_medicine": 0.24175824175824176, - "computer_science": 0.28431372549019607, - "computer_security": 0.13450292397660818, + "agronomy": 0.23668639053254437, + "anatomy": 0.2635135135135135, + "ancient_chinese": 0.25609756097560976, + "arts": 0.25625, + "astronomy": 0.24242424242424243, + "business_ethics": 0.24401913875598086, + "chinese_civil_service_exam": 0.24375, + "chinese_driving_rule": 0.25190839694656486, + "chinese_food_culture": 0.2647058823529412, + "chinese_foreign_policy": 0.24299065420560748, + "chinese_history": 0.25077399380804954, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.25139664804469275, + "clinical_knowledge": 0.24472573839662448, + "college_actuarial_science": 0.2641509433962264, + "college_education": 0.29906542056074764, + "college_engineering_hydrology": 0.2830188679245283, + "college_law": 0.21296296296296297, + "college_mathematics": 0.22857142857142856, + "college_medical_statistics": 0.2641509433962264, + "college_medicine": 0.2454212454212454, + "computer_science": 0.2647058823529412, + "computer_security": 0.24561403508771928, "conceptual_physics": 0.25170068027210885, - "construction_project_management": 0.2733812949640288, - "economics": 0.18867924528301888, + "construction_project_management": 0.2517985611510791, + "economics": 0.2389937106918239, "education": 0.25153374233128833, - "electrical_engineering": 0.27325581395348836, - "elementary_chinese": 0.20634920634920634, + "electrical_engineering": 0.23837209302325582, + "elementary_chinese": 0.2777777777777778, "elementary_commonsense": 0.25757575757575757, - "elementary_information_and_technology": 0.3235294117647059, - "elementary_mathematics": 0.24782608695652175, - "ethnology": 0.2740740740740741, - "food_science": 0.23776223776223776, - "genetics": 0.18181818181818182, + "elementary_information_and_technology": 0.29831932773109243, + "elementary_mathematics": 0.27391304347826084, + "ethnology": 0.24444444444444444, + "food_science": 0.2517482517482518, + "genetics": 0.23863636363636365, "global_facts": 0.26174496644295303, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.30303030303030304, - "high_school_geography": 0.23728813559322035, - "high_school_mathematics": 0.2926829268292683, - "high_school_physics": 0.2818181818181818, - "high_school_politics": 0.25874125874125875, - "human_sexuality": 0.2619047619047619, - "international_law": 0.1945945945945946, - "journalism": 0.22674418604651161, - "jurisprudence": 0.27007299270072993, - "legal_and_moral_basis": 0.2757009345794392, - "logical": 0.3333333333333333, - "machine_learning": 0.23770491803278687, - "management": 0.23809523809523808, - "marketing": 0.2833333333333333, - "marxist_theory": 0.2857142857142857, - "modern_chinese": 0.28448275862068967, - "nutrition": 0.2, + "high_school_biology": 0.23668639053254437, + "high_school_chemistry": 0.25, + "high_school_geography": 0.2627118644067797, + "high_school_mathematics": 0.27439024390243905, + "high_school_physics": 0.2636363636363636, + "high_school_politics": 0.2727272727272727, + "human_sexuality": 0.25396825396825395, + "international_law": 0.2594594594594595, + "journalism": 0.25, + "jurisprudence": 0.25060827250608275, + "legal_and_moral_basis": 0.2523364485981308, + "logical": 0.2601626016260163, + "machine_learning": 0.21311475409836064, + "management": 0.24761904761904763, + "marketing": 0.25, + "marxist_theory": 0.24338624338624337, + "modern_chinese": 0.25862068965517243, + "nutrition": 0.2413793103448276, "philosophy": 0.23809523809523808, - "professional_accounting": 0.22285714285714286, - "professional_law": 0.1990521327014218, - "professional_medicine": 0.30851063829787234, - "professional_psychology": 0.2801724137931034, - "public_relations": 0.1896551724137931, - "security_study": 0.23703703703703705, - "sociology": 0.2168141592920354, - "sports_science": 0.23030303030303031, - "traditional_chinese_medicine": 0.2648648648648649, - "virology": 0.23076923076923078, - "world_history": 0.2236024844720497, - "world_religions": 0.23125 + "professional_accounting": 0.25142857142857145, + "professional_law": 0.26066350710900477, + "professional_medicine": 0.24202127659574468, + "professional_psychology": 0.24568965517241378, + "public_relations": 0.2471264367816092, + "security_study": 0.25925925925925924, + "sociology": 0.252212389380531, + "sports_science": 0.26666666666666666, + "traditional_chinese_medicine": 0.24864864864864866, + "virology": 0.2485207100591716, + "world_history": 0.2422360248447205, + "world_religions": 0.26875 } }, "prompt_5": { - "accuracy": 0.24995682956311518, + "accuracy": 0.25297876014505266, "category_acc": { - "agronomy": 0.22485207100591717, - "anatomy": 0.23648648648648649, - "ancient_chinese": 0.25609756097560976, - "arts": 0.18125, - "astronomy": 0.26666666666666666, - "business_ethics": 0.22966507177033493, - "chinese_civil_service_exam": 0.275, - "chinese_driving_rule": 0.29770992366412213, - "chinese_food_culture": 0.3088235294117647, - "chinese_foreign_policy": 0.21495327102803738, - "chinese_history": 0.2755417956656347, - "chinese_literature": 0.22058823529411764, + "agronomy": 0.24260355029585798, + "anatomy": 0.2635135135135135, + "ancient_chinese": 0.2682926829268293, + "arts": 0.25625, + "astronomy": 0.21818181818181817, + "business_ethics": 0.2535885167464115, + "chinese_civil_service_exam": 0.24375, + "chinese_driving_rule": 0.25190839694656486, + "chinese_food_culture": 0.25, + "chinese_foreign_policy": 0.24299065420560748, + "chinese_history": 0.24458204334365324, + "chinese_literature": 0.25, "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.270042194092827, - "college_actuarial_science": 0.22641509433962265, - "college_education": 0.2803738317757009, - "college_engineering_hydrology": 0.24528301886792453, - "college_law": 0.3055555555555556, - "college_mathematics": 0.19047619047619047, - "college_medical_statistics": 0.2830188679245283, - "college_medicine": 0.326007326007326, + "clinical_knowledge": 0.2489451476793249, + "college_actuarial_science": 0.2830188679245283, + "college_education": 0.308411214953271, + "college_engineering_hydrology": 0.29245283018867924, + "college_law": 0.2222222222222222, + "college_mathematics": 0.22857142857142856, + "college_medical_statistics": 0.2358490566037736, + "college_medicine": 0.25274725274725274, "computer_science": 0.27450980392156865, - "computer_security": 0.2631578947368421, + "computer_security": 0.24561403508771928, "conceptual_physics": 0.23809523809523808, - "construction_project_management": 0.2517985611510791, - "economics": 0.23270440251572327, + "construction_project_management": 0.2446043165467626, + "economics": 0.2389937106918239, "education": 0.25153374233128833, - "electrical_engineering": 0.22093023255813954, - "elementary_chinese": 0.21825396825396826, - "elementary_commonsense": 0.30303030303030304, - "elementary_information_and_technology": 0.29411764705882354, - "elementary_mathematics": 0.26521739130434785, - "ethnology": 0.2222222222222222, - "food_science": 0.23076923076923078, - "genetics": 0.2556818181818182, - "global_facts": 0.22818791946308725, - "high_school_biology": 0.26627218934911245, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.22033898305084745, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.22727272727272727, - "high_school_politics": 0.1888111888111888, + "electrical_engineering": 0.23837209302325582, + "elementary_chinese": 0.28174603174603174, + "elementary_commonsense": 0.24242424242424243, + "elementary_information_and_technology": 0.2773109243697479, + "elementary_mathematics": 0.2565217391304348, + "ethnology": 0.24444444444444444, + "food_science": 0.23776223776223776, + "genetics": 0.2727272727272727, + "global_facts": 0.2483221476510067, + "high_school_biology": 0.21893491124260356, + "high_school_chemistry": 0.26515151515151514, + "high_school_geography": 0.2711864406779661, + "high_school_mathematics": 0.2804878048780488, + "high_school_physics": 0.2727272727272727, + "high_school_politics": 0.27972027972027974, "human_sexuality": 0.25396825396825395, - "international_law": 0.21621621621621623, - "journalism": 0.20348837209302326, - "jurisprudence": 0.2384428223844282, - "legal_and_moral_basis": 0.2336448598130841, - "logical": 0.2601626016260163, - "machine_learning": 0.29508196721311475, - "management": 0.2571428571428571, - "marketing": 0.2388888888888889, - "marxist_theory": 0.291005291005291, - "modern_chinese": 0.20689655172413793, - "nutrition": 0.21379310344827587, - "philosophy": 0.21904761904761905, - "professional_accounting": 0.26285714285714284, - "professional_law": 0.3080568720379147, - "professional_medicine": 0.21010638297872342, + "international_law": 0.25405405405405407, + "journalism": 0.25, + "jurisprudence": 0.24817518248175183, + "legal_and_moral_basis": 0.2523364485981308, + "logical": 0.2682926829268293, + "machine_learning": 0.2459016393442623, + "management": 0.24761904761904763, + "marketing": 0.24444444444444444, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.27586206896551724, + "nutrition": 0.2413793103448276, + "philosophy": 0.2571428571428571, + "professional_accounting": 0.25142857142857145, + "professional_law": 0.25118483412322273, + "professional_medicine": 0.2553191489361702, "professional_psychology": 0.24568965517241378, - "public_relations": 0.29310344827586204, - "security_study": 0.2962962962962963, - "sociology": 0.2345132743362832, + "public_relations": 0.2471264367816092, + "security_study": 0.25925925925925924, + "sociology": 0.25663716814159293, "sports_science": 0.24242424242424243, - "traditional_chinese_medicine": 0.3027027027027027, - "virology": 0.23076923076923078, - "world_history": 0.19254658385093168, - "world_religions": 0.19375 + "traditional_chinese_medicine": 0.24864864864864866, + "virology": 0.2485207100591716, + "world_history": 0.2360248447204969, + "world_religions": 0.26875 } } }, "zbench": { "prompt_1": { - "accuracy": 0.2727272727272727 + "accuracy": 0.18181818181818182 }, "prompt_2": { - "accuracy": 0.2727272727272727 + "accuracy": 0.18181818181818182 }, "prompt_3": { - "accuracy": 0.21212121212121213 + "accuracy": 0.18181818181818182 }, "prompt_4": { - "accuracy": 0.2727272727272727 + "accuracy": 0.15151515151515152 }, "prompt_5": { - "accuracy": 0.30303030303030304 + "accuracy": 0.18181818181818182 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.17045454545454544 + "accuracy": 0.5 }, "prompt_2": { - "accuracy": 0.20227272727272727 + "accuracy": 0.47045454545454546 }, "prompt_3": { - "accuracy": 0.19772727272727272 + "accuracy": 0.4863636363636364 }, "prompt_4": { - "accuracy": 0.19318181818181818 + "accuracy": 0.41363636363636364 }, "prompt_5": { - "accuracy": 0.15454545454545454 + "accuracy": 0.46136363636363636 } }, "ocnli": { "prompt_1": { - "accuracy": 0.32949152542372884 + "accuracy": 0.34372881355932206 }, "prompt_2": { - "accuracy": 0.3311864406779661 + "accuracy": 0.34372881355932206 }, "prompt_3": { - "accuracy": 0.3257627118644068 + "accuracy": 0.34440677966101696 }, "prompt_4": { - "accuracy": 0.34135593220338983 + "accuracy": 0.3440677966101695 }, "prompt_5": { - "accuracy": 0.3247457627118644 + "accuracy": 0.3440677966101695 } }, "c3": { "prompt_1": { - "accuracy": 0.2801047120418848 + "accuracy": 0.26626776364996263 }, "prompt_2": { - "accuracy": 0.275243081525804 + "accuracy": 0.26626776364996263 }, "prompt_3": { - "accuracy": 0.287584143605086 + "accuracy": 0.2666417352281227 }, "prompt_4": { - "accuracy": 0.2700074794315632 + "accuracy": 0.26514584891548243 }, "prompt_5": { - "accuracy": 0.27860882572924456 + "accuracy": 0.2673896783844428 } }, "dream": { "prompt_1": { - "accuracy": 0.315531602155806 + "accuracy": 0.9363057324840764 }, "prompt_2": { - "accuracy": 0.34296913277804997 + "accuracy": 0.932876041156296 }, "prompt_3": { - "accuracy": 0.3189612934835865 + "accuracy": 0.9353258206761391 }, "prompt_4": { - "accuracy": 0.3444390004899559 + "accuracy": 0.9372856442920137 }, "prompt_5": { - "accuracy": 0.3341499265066144 + "accuracy": 0.9338559529642332 } }, "samsum": { "prompt_1": { - "rouge1": 0.08958104634587742, - "rouge2": 0.0019002524715458382, - "rougeL": 0.07320320134083368, - "avg_rouge": 0.05489483338608564 + "rouge1": 0.5392068754143731, + "rouge2": 0.2950280260068732, + "rougeL": 0.4506368895080014, + "avg_rouge": 0.4282905969764159 }, "prompt_2": { - "rouge1": 0.08977760919607912, - "rouge2": 0.0015791665642652633, - "rougeL": 0.07186157996971398, - "avg_rouge": 0.054406118576686126 + "rouge1": 0.5359945124407804, + "rouge2": 0.29276027430028645, + "rougeL": 0.44971572354652717, + "avg_rouge": 0.4261568367625314 }, "prompt_3": { - "rouge1": 0.09236732274454623, - "rouge2": 0.001398147826765026, - "rougeL": 0.07427019366878998, - "avg_rouge": 0.056011888080033746 + "rouge1": 0.5369216789882587, + "rouge2": 0.2932230928399554, + "rougeL": 0.4508405893944396, + "avg_rouge": 0.4269951204075513 }, "prompt_4": { - "rouge1": 0.0910082077091497, - "rouge2": 0.002078066457309221, - "rougeL": 0.07423796465096685, - "avg_rouge": 0.05577474627247526 + "rouge1": 0.538447184048574, + "rouge2": 0.29528202636477136, + "rougeL": 0.4516316980089774, + "avg_rouge": 0.42845363614077425 }, "prompt_5": { - "rouge1": 0.09169248714676971, - "rouge2": 0.001673140894516994, - "rougeL": 0.07379820921481818, - "avg_rouge": 0.0557212790853683 + "rouge1": 0.5365832097389195, + "rouge2": 0.2954036639870932, + "rougeL": 0.45167227299325957, + "avg_rouge": 0.42788638223975745 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.10701860776134962, - "rouge2": 0.001869868141395071, - "rougeL": 0.0879918014611341, - "avg_rouge": 0.06562675912129294 + "rouge1": 0.23098825839708775, + "rouge2": 0.06235340689809025, + "rougeL": 0.1710693119069414, + "avg_rouge": 0.15480365906737314 }, "prompt_2": { - "rouge1": 0.10644756063376731, - "rouge2": 0.0016236797161060781, - "rougeL": 0.08687928419405898, - "avg_rouge": 0.06498350818131078 + "rouge1": 0.23057479562213193, + "rouge2": 0.0626605453865275, + "rougeL": 0.16987328159127832, + "avg_rouge": 0.15436954086664592 }, "prompt_3": { - "rouge1": 0.10597014335771661, - "rouge2": 0.0013866341052049605, - "rougeL": 0.08566916194433068, - "avg_rouge": 0.06434197980241742 + "rouge1": 0.23134441905824937, + "rouge2": 0.06285173990336988, + "rougeL": 0.17136941481315845, + "avg_rouge": 0.15518852459159257 }, "prompt_4": { - "rouge1": 0.10532446741097351, - "rouge2": 0.0013105666311989338, - "rougeL": 0.0863256575889469, - "avg_rouge": 0.06432023054370645 + "rouge1": 0.23060303444666233, + "rouge2": 0.06339967025872248, + "rougeL": 0.16971858763937736, + "avg_rouge": 0.15457376411492071 }, "prompt_5": { - "rouge1": 0.10351188190447065, - "rouge2": 0.00187120797020834, - "rougeL": 0.08409764656393401, - "avg_rouge": 0.06316024547953768 + "rouge1": 0.23634127906884056, + "rouge2": 0.06468803693978939, + "rougeL": 0.17715576683843468, + "avg_rouge": 0.15939502761568822 } }, "sst2": { "prompt_1": { - "accuracy": 0.47591743119266056 + "accuracy": 0.9529816513761468 }, "prompt_2": { - "accuracy": 0.5252293577981652 + "accuracy": 0.9506880733944955 }, "prompt_3": { - "accuracy": 0.5068807339449541 + "accuracy": 0.9564220183486238 }, "prompt_4": { - "accuracy": 0.5126146788990825 + "accuracy": 0.9518348623853211 }, "prompt_5": { - "accuracy": 0.4988532110091743 + "accuracy": 0.9575688073394495 } }, "cola": { "prompt_1": { - "accuracy": 0.48897411313518696 + "accuracy": 0.8235858101629914 }, "prompt_2": { - "accuracy": 0.49185043144774687 + "accuracy": 0.8235858101629914 }, "prompt_3": { - "accuracy": 0.49760306807286675 + "accuracy": 0.8245445829338447 }, "prompt_4": { - "accuracy": 0.5282837967401726 + "accuracy": 0.8216682646212847 }, "prompt_5": { - "accuracy": 0.5215723873441994 + "accuracy": 0.8235858101629914 } }, "qqp": { "prompt_1": { - "accuracy": 0.511 + "accuracy": 0.8415 }, "prompt_2": { - "accuracy": 0.4905 + "accuracy": 0.8435 }, "prompt_3": { - "accuracy": 0.5085 + "accuracy": 0.838 }, "prompt_4": { - "accuracy": 0.502 + "accuracy": 0.849 }, "prompt_5": { - "accuracy": 0.499 + "accuracy": 0.843 } }, "mnli": { "prompt_1": { - "accuracy": 0.3395 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.339 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.326 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.3325 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.324 + "accuracy": 0.9 } }, "qnli": { "prompt_1": { - "accuracy": 0.4955 + "accuracy": 1.0 }, "prompt_2": { - "accuracy": 0.484 + "accuracy": 1.0 }, "prompt_3": { - "accuracy": 0.5185 + "accuracy": 1.0 }, "prompt_4": { - "accuracy": 0.4895 + "accuracy": 1.0 }, "prompt_5": { - "accuracy": 0.5345 + "accuracy": 1.0 } }, "wnli": { "prompt_1": { - "accuracy": 0.5915492957746479 + "accuracy": 0.6 }, "prompt_2": { - "accuracy": 0.4507042253521127 + "accuracy": 0.6 }, "prompt_3": { - "accuracy": 0.5211267605633803 + "accuracy": 0.6 }, "prompt_4": { - "accuracy": 0.5352112676056338 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.49295774647887325 + "accuracy": 0.6 } }, "rte": { "prompt_1": { - "accuracy": 0.48736462093862815 + "accuracy": 1.0 }, "prompt_2": { - "accuracy": 0.45126353790613716 + "accuracy": 1.0 }, "prompt_3": { - "accuracy": 0.4332129963898917 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.4620938628158845 + "accuracy": 1.0 }, "prompt_5": { - "accuracy": 0.4981949458483754 + "accuracy": 1.0 } }, "mrpc": { "prompt_1": { - "accuracy": 0.49754901960784315 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.49754901960784315 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.4583333333333333 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.5024509803921569 + "accuracy": 0.8 }, "prompt_5": { - "accuracy": 0.47058823529411764 + "accuracy": 0.9 } } }, "five_shot": { + "cross_xquad": { + "prompt_1": -1 + }, + "cross_mmlu": { + "prompt_1": -1 + }, + "cross_logiqa": { + "prompt_1": -1 + }, + "sg_eval": { + "prompt_1": -1 + }, + "cn_eval": { + "prompt_1": -1 + }, + "us_eval": { + "prompt_1": -1 + }, + "ph_eval": { + "prompt_1": -1 + }, + "sing2eng": { + "prompt_1": -1 + }, + "indommlu": { + "prompt_1": -1 + }, + "flores_ind2eng": { + "prompt_1": -1 + }, + "flores_vie2eng": { + "prompt_1": -1 + }, + "flores_zho2eng": { + "prompt_1": -1 + }, + "flores_zsm2eng": { + "prompt_1": -1 + }, + "mmlu": { + "prompt_1": -1 + }, + "mmlu_full": { + "prompt_1": -1 + }, + "c_eval": { + "prompt_1": -1 + }, + "c_eval_full": { + "prompt_1": -1 + }, + "cmmlu": { + "prompt_1": -1 + }, + "cmmlu_full": { + "prompt_1": -1 + }, + "zbench": { + "prompt_1": -1 + }, + "ind_emotion": { + "prompt_1": -1 + }, + "ocnli": { + "prompt_1": -1 + }, + "c3": { + "prompt_1": -1 + }, + "dream": { + "prompt_1": -1 + }, + "samsum": { + "prompt_1": -1 + }, + "dialogsum": { + "prompt_1": -1 + }, + "sst2": { + "prompt_1": -1 + }, + "cola": { + "prompt_1": -1 + }, + "qqp": { + "prompt_1": -1 + }, + "mnli": { + "prompt_1": -1 + }, + "qnli": { + "prompt_1": -1 + }, + "wnli": { + "prompt_1": -1 + }, + "rte": { + "prompt_1": -1 + }, + "mrpc": { + "prompt_1": -1 + } + } + }, + "flan_ul2": { + "model_size": "20B", + "model_link": "https://huggingface.co/google/flan-t5-ul2", + "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.2636554621848739, + "overall_acc": 0.7878151260504201, "language_acc": { - "English": 0.2714285714285714, - "Chinese": 0.25210084033613445, - "Spanish": 0.26134453781512607, - "Vietnamese": 0.26974789915966385 + "Spanish": 0.9621848739495799, + "English": 0.9781512605042016, + "Chinese": 0.3663865546218487, + "Vietnamese": 0.8445378151260504 }, - "consistency_score_2": 0.24747899159663866, - "consistency_score_3": 0.06113445378151261, - "consistency_score_4": 0.015966386554621848, + "consistency_score_2": 0.6305322128851542, + "consistency_score_3": 0.46554621848739497, + "consistency_score_4": 0.3302521008403361, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.23865546218487396, - "English,Spanish": 0.2495798319327731, - "English,Vietnamese": 0.246218487394958, - "Chinese,Spanish": 0.25126050420168067, - "Chinese,Vietnamese": 0.2445378151260504, - "Spanish,Vietnamese": 0.2546218487394958 + "Spanish,English": 0.9663865546218487, + "Spanish,Chinese": 0.37058823529411766, + "Spanish,Vietnamese": 0.8394957983193277, + "English,Chinese": 0.3722689075630252, + "English,Vietnamese": 0.8436974789915966, + "Chinese,Vietnamese": 0.3907563025210084 }, "3_combine": { - "English,Chinese,Spanish": 0.06134453781512605, - "English,Chinese,Vietnamese": 0.06134453781512605, - "English,Spanish,Vietnamese": 0.05714285714285714, - "Chinese,Spanish,Vietnamese": 0.06470588235294118 + "Spanish,English,Chinese": 0.3621848739495798, + "Spanish,English,Vietnamese": 0.8285714285714286, + "Spanish,Chinese,Vietnamese": 0.3352941176470588, + "English,Chinese,Vietnamese": 0.33613445378151263 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.015966386554621848 + "Spanish,English,Chinese,Vietnamese": 0.3302521008403361 } }, - "AC3_2": 0.25531124898727287, - "AC3_3": 0.0992545141942765, - "AC3_4": 0.030109414153094483 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.26285714285714284, + "AC3_2": 0.7004529865170198, + "AC3_3": 0.5852491851117807, + "AC3_4": 0.4654060045248004 + }, + "prompt_2": { + "overall_acc": 0.7903361344537815, "language_acc": { - "Spanish": 0.2733333333333333, - "English": 0.24, - "Chinese": 0.26, - "Vietnamese": 0.24666666666666667, - "Filipino": 0.30666666666666664, - "Indonesian": 0.24666666666666667, - "Malay": 0.26666666666666666 + "Spanish": 0.9621848739495799, + "English": 0.9773109243697479, + "Chinese": 0.373109243697479, + "Vietnamese": 0.8487394957983193 }, - "consistency_score_2": 0.24222222222222223, - "consistency_score_3": 0.056380952380952386, - "consistency_score_4": 0.01257142857142857, - "consistency_score_5": 0.0019047619047619048, - "consistency_score_6": 0.0, - "consistency_score_7": 0.0, + "consistency_score_2": 0.634733893557423, + "consistency_score_3": 0.47016806722689086, + "consistency_score_4": 0.33613445378151263, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.22666666666666666, - "Spanish,Chinese": 0.24, - "Spanish,Vietnamese": 0.19333333333333333, - "Spanish,Filipino": 0.24, - "Spanish,Indonesian": 0.2, - "Spanish,Malay": 0.24666666666666667, - "English,Chinese": 0.3, - "English,Vietnamese": 0.2866666666666667, - "English,Filipino": 0.26, - "English,Indonesian": 0.2733333333333333, - "English,Malay": 0.21333333333333335, - "Chinese,Vietnamese": 0.20666666666666667, - "Chinese,Filipino": 0.26666666666666666, - "Chinese,Indonesian": 0.24666666666666667, - "Chinese,Malay": 0.22666666666666666, - "Vietnamese,Filipino": 0.20666666666666667, - "Vietnamese,Indonesian": 0.3, - "Vietnamese,Malay": 0.24666666666666667, - "Filipino,Indonesian": 0.24666666666666667, - "Filipino,Malay": 0.22666666666666666, - "Indonesian,Malay": 0.23333333333333334 + "Spanish,English": 0.9672268907563025, + "Spanish,Chinese": 0.3773109243697479, + "Spanish,Vietnamese": 0.8411764705882353, + "English,Chinese": 0.3773109243697479, + "English,Vietnamese": 0.846218487394958, + "Chinese,Vietnamese": 0.39915966386554624 }, "3_combine": { - "Spanish,English,Chinese": 0.07333333333333333, - "Spanish,English,Vietnamese": 0.03333333333333333, - "Spanish,English,Filipino": 0.05333333333333334, - "Spanish,English,Indonesian": 0.04, - "Spanish,English,Malay": 0.04, - "Spanish,Chinese,Vietnamese": 0.04, - "Spanish,Chinese,Filipino": 0.07333333333333333, - "Spanish,Chinese,Indonesian": 0.02666666666666667, - "Spanish,Chinese,Malay": 0.06666666666666667, - "Spanish,Vietnamese,Filipino": 0.02666666666666667, - "Spanish,Vietnamese,Indonesian": 0.04, - "Spanish,Vietnamese,Malay": 0.04, - "Spanish,Filipino,Indonesian": 0.04, - "Spanish,Filipino,Malay": 0.04666666666666667, - "Spanish,Indonesian,Malay": 0.06, - "English,Chinese,Vietnamese": 0.1, - "English,Chinese,Filipino": 0.09333333333333334, - "English,Chinese,Indonesian": 0.04666666666666667, - "English,Chinese,Malay": 0.04666666666666667, - "English,Vietnamese,Filipino": 0.07333333333333333, - "English,Vietnamese,Indonesian": 0.08666666666666667, - "English,Vietnamese,Malay": 0.08, - "English,Filipino,Indonesian": 0.07333333333333333, - "English,Filipino,Malay": 0.06, - "English,Indonesian,Malay": 0.06666666666666667, - "Chinese,Vietnamese,Filipino": 0.05333333333333334, - "Chinese,Vietnamese,Indonesian": 0.06, - "Chinese,Vietnamese,Malay": 0.04, - "Chinese,Filipino,Indonesian": 0.1, - "Chinese,Filipino,Malay": 0.05333333333333334, - "Chinese,Indonesian,Malay": 0.04, - "Vietnamese,Filipino,Indonesian": 0.04666666666666667, - "Vietnamese,Filipino,Malay": 0.05333333333333334, - "Vietnamese,Indonesian,Malay": 0.06666666666666667, - "Filipino,Indonesian,Malay": 0.03333333333333333 + "Spanish,English,Chinese": 0.36722689075630255, + "Spanish,English,Vietnamese": 0.83109243697479, + "Spanish,Chinese,Vietnamese": 0.3411764705882353, + "English,Chinese,Vietnamese": 0.3411764705882353 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.02, - "Spanish,English,Chinese,Filipino": 0.03333333333333333, - "Spanish,English,Chinese,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Malay": 0.02666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.0, - "Spanish,English,Vietnamese,Indonesian": 0.0, - "Spanish,English,Vietnamese,Malay": 0.0, - "Spanish,English,Filipino,Indonesian": 0.006666666666666667, - "Spanish,English,Filipino,Malay": 0.013333333333333334, - "Spanish,English,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.0, - "Spanish,Chinese,Vietnamese,Indonesian": 0.006666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.006666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.0, - "Spanish,Chinese,Filipino,Malay": 0.02, - "Spanish,Chinese,Indonesian,Malay": 0.006666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.02666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.03333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.02666666666666667, - "English,Chinese,Vietnamese,Malay": 0.02, - "English,Chinese,Filipino,Indonesian": 0.02, - "English,Chinese,Filipino,Malay": 0.02, - "English,Chinese,Indonesian,Malay": 0.0, - "English,Vietnamese,Filipino,Indonesian": 0.02666666666666667, - "English,Vietnamese,Filipino,Malay": 0.02666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "English,Filipino,Indonesian,Malay": 0.006666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.02, - "Chinese,Vietnamese,Filipino,Malay": 0.013333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.013333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.0 + "Spanish,English,Chinese,Vietnamese": 0.33613445378151263 + } + }, + "AC3_2": 0.7040399726964512, + "AC3_3": 0.589590756255747, + "AC3_4": 0.47166647333599176 + }, + "prompt_3": { + "overall_acc": 0.792016806722689, + "language_acc": { + "Spanish": 0.9621848739495799, + "English": 0.9773109243697479, + "Chinese": 0.37563025210084033, + "Vietnamese": 0.8529411764705882 + }, + "consistency_score_2": 0.6372549019607844, + "consistency_score_3": 0.4756302521008404, + "consistency_score_4": 0.3420168067226891, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.965546218487395, + "Spanish,Chinese": 0.3815126050420168, + "Spanish,Vietnamese": 0.8487394957983193, + "English,Chinese": 0.37815126050420167, + "English,Vietnamese": 0.8521008403361344, + "Chinese,Vietnamese": 0.39747899159663863 }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.0, - "Spanish,English,Chinese,Vietnamese,Malay": 0.0, - "Spanish,English,Chinese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Filipino,Malay": 0.013333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "English,Chinese,Filipino,Indonesian,Malay": 0.0, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "3_combine": { + "Spanish,English,Chinese": 0.3697478991596639, + "Spanish,English,Vietnamese": 0.8369747899159664, + "Spanish,Chinese,Vietnamese": 0.3487394957983193, + "English,Chinese,Vietnamese": 0.34705882352941175 }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.3420168067226891 + } + }, + "AC3_2": 0.7062570250536211, + "AC3_3": 0.594340752495431, + "AC3_4": 0.4777337388776721 + }, + "prompt_4": { + "overall_acc": 0.7913865546218487, + "language_acc": { + "Spanish": 0.9630252100840336, + "English": 0.9773109243697479, + "Chinese": 0.3764705882352941, + "Vietnamese": 0.8487394957983193 + }, + "consistency_score_2": 0.6355742296918768, + "consistency_score_3": 0.4726890756302522, + "consistency_score_4": 0.33949579831932775, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.9663865546218487, + "Spanish,Chinese": 0.3815126050420168, + "Spanish,Vietnamese": 0.8436974789915966, + "English,Chinese": 0.37815126050420167, + "English,Vietnamese": 0.8478991596638655, + "Chinese,Vietnamese": 0.3957983193277311 }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 + "3_combine": { + "Spanish,English,Chinese": 0.3697478991596639, + "Spanish,English,Vietnamese": 0.8327731092436975, + "Spanish,Chinese,Vietnamese": 0.3445378151260504, + "English,Chinese,Vietnamese": 0.34369747899159664 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.33949579831932775 } }, - "AC3_2": 0.2521181646263877, - "AC3_3": 0.09284691439300952, - "AC3_4": 0.02399525784546704, - "AC3_5": 0.0037821171619836294, - "AC3_6": 0.0, - "AC3_7": 0.0 + "AC3_2": 0.7049736829999066, + "AC3_3": 0.5918629708353986, + "AC3_4": 0.47515536769675665 + }, + "prompt_5": { + "overall_acc": 0.7922268907563025, + "language_acc": { + "Spanish": 0.9613445378151261, + "English": 0.9773109243697479, + "Chinese": 0.37899159663865545, + "Vietnamese": 0.8512605042016806 + }, + "consistency_score_2": 0.6378151260504202, + "consistency_score_3": 0.47478991596638664, + "consistency_score_4": 0.3403361344537815, + "detailed_consistency_score": { + "2_combine": { + "Spanish,English": 0.9672268907563025, + "Spanish,Chinese": 0.38571428571428573, + "Spanish,Vietnamese": 0.8445378151260504, + "English,Chinese": 0.3831932773109244, + "English,Vietnamese": 0.8487394957983193, + "Chinese,Vietnamese": 0.39747899159663863 + }, + "3_combine": { + "Spanish,English,Chinese": 0.3739495798319328, + "Spanish,English,Vietnamese": 0.8336134453781513, + "Spanish,Chinese,Vietnamese": 0.346218487394958, + "English,Chinese,Vietnamese": 0.3453781512605042 + }, + "4_combine": { + "Spanish,English,Chinese,Vietnamese": 0.3403361344537815 + } + }, + "AC3_2": 0.7066845424321856, + "AC3_3": 0.5937432508598683, + "AC3_4": 0.47612968388495314 } }, - "cross_logiqa": { + "cross_mmlu": { "prompt_1": { - "overall_acc": 0.2597402597402597, + "overall_acc": 0.3914285714285714, "language_acc": { - "English": 0.2784090909090909, - "Filipino": 0.2727272727272727, - "Spanish": 0.2727272727272727, - "Malay": 0.23295454545454544, - "Chinese": 0.26136363636363635, - "Indonesian": 0.2840909090909091, - "Vietnamese": 0.2159090909090909 + "Filipino": 0.36, + "Vietnamese": 0.30666666666666664, + "Chinese": 0.3, + "Spanish": 0.41333333333333333, + "Malay": 0.36, + "Indonesian": 0.4066666666666667, + "English": 0.5933333333333334 }, - "consistency_score_2": 0.25189393939393945, - "consistency_score_3": 0.0625, - "consistency_score_4": 0.01623376623376623, - "consistency_score_5": 0.005140692640692638, - "consistency_score_6": 0.0016233766233766235, - "consistency_score_7": 0.0, + "consistency_score_2": 0.4546031746031745, + "consistency_score_3": 0.2617142857142857, + "consistency_score_4": 0.16952380952380952, + "consistency_score_5": 0.11936507936507937, + "consistency_score_6": 0.09047619047619049, + "consistency_score_7": 0.07333333333333333, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.2897727272727273, - "English,Spanish": 0.25, - "English,Malay": 0.23863636363636365, - "English,Chinese": 0.26704545454545453, - "English,Indonesian": 0.26704545454545453, - "English,Vietnamese": 0.2840909090909091, - "Filipino,Spanish": 0.23863636363636365, - "Filipino,Malay": 0.18181818181818182, - "Filipino,Chinese": 0.2840909090909091, - "Filipino,Indonesian": 0.2556818181818182, - "Filipino,Vietnamese": 0.21022727272727273, - "Spanish,Malay": 0.25, - "Spanish,Chinese": 0.2556818181818182, - "Spanish,Indonesian": 0.25, - "Spanish,Vietnamese": 0.19318181818181818, - "Malay,Chinese": 0.22727272727272727, - "Malay,Indonesian": 0.2556818181818182, - "Malay,Vietnamese": 0.32386363636363635, - "Chinese,Indonesian": 0.32386363636363635, - "Chinese,Vietnamese": 0.21022727272727273, - "Indonesian,Vietnamese": 0.23295454545454544 + "Filipino,Vietnamese": 0.4, + "Filipino,Chinese": 0.36666666666666664, + "Filipino,Spanish": 0.44, + "Filipino,Malay": 0.43333333333333335, + "Filipino,Indonesian": 0.5066666666666667, + "Filipino,English": 0.5, + "Vietnamese,Chinese": 0.4266666666666667, + "Vietnamese,Spanish": 0.3933333333333333, + "Vietnamese,Malay": 0.4, + "Vietnamese,Indonesian": 0.49333333333333335, + "Vietnamese,English": 0.37333333333333335, + "Chinese,Spanish": 0.42, + "Chinese,Malay": 0.36666666666666664, + "Chinese,Indonesian": 0.41333333333333333, + "Chinese,English": 0.3933333333333333, + "Spanish,Malay": 0.4866666666666667, + "Spanish,Indonesian": 0.49333333333333335, + "Spanish,English": 0.6466666666666666, + "Malay,Indonesian": 0.5933333333333334, + "Malay,English": 0.4666666666666667, + "Indonesian,English": 0.5333333333333333 }, "3_combine": { - "English,Filipino,Spanish": 0.09659090909090909, - "English,Filipino,Malay": 0.0625, - "English,Filipino,Chinese": 0.06818181818181818, - "English,Filipino,Indonesian": 0.07386363636363637, - "English,Filipino,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay": 0.056818181818181816, - "English,Spanish,Chinese": 0.05113636363636364, - "English,Spanish,Indonesian": 0.08522727272727272, - "English,Spanish,Vietnamese": 0.03409090909090909, - "English,Malay,Chinese": 0.056818181818181816, - "English,Malay,Indonesian": 0.07386363636363637, - "English,Malay,Vietnamese": 0.0625, - "English,Chinese,Indonesian": 0.08522727272727272, - "English,Chinese,Vietnamese": 0.0625, - "English,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay": 0.03977272727272727, - "Filipino,Spanish,Chinese": 0.07386363636363637, - "Filipino,Spanish,Indonesian": 0.06818181818181818, - "Filipino,Spanish,Vietnamese": 0.017045454545454544, - "Filipino,Malay,Chinese": 0.056818181818181816, - "Filipino,Malay,Indonesian": 0.03409090909090909, - "Filipino,Malay,Vietnamese": 0.05113636363636364, - "Filipino,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Chinese,Vietnamese": 0.03977272727272727, - "Filipino,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese": 0.045454545454545456, - "Spanish,Malay,Indonesian": 0.07386363636363637, - "Spanish,Malay,Vietnamese": 0.08522727272727272, - "Spanish,Chinese,Indonesian": 0.08522727272727272, - "Spanish,Chinese,Vietnamese": 0.056818181818181816, - "Spanish,Indonesian,Vietnamese": 0.05113636363636364, - "Malay,Chinese,Indonesian": 0.07386363636363637, - "Malay,Chinese,Vietnamese": 0.03977272727272727, - "Malay,Indonesian,Vietnamese": 0.0625, - "Chinese,Indonesian,Vietnamese": 0.09090909090909091 + "Filipino,Vietnamese,Chinese": 0.20666666666666667, + "Filipino,Vietnamese,Spanish": 0.21333333333333335, + "Filipino,Vietnamese,Malay": 0.21333333333333335, + "Filipino,Vietnamese,Indonesian": 0.26, + "Filipino,Vietnamese,English": 0.23333333333333334, + "Filipino,Chinese,Spanish": 0.21333333333333335, + "Filipino,Chinese,Malay": 0.16666666666666666, + "Filipino,Chinese,Indonesian": 0.22, + "Filipino,Chinese,English": 0.20666666666666667, + "Filipino,Spanish,Malay": 0.2733333333333333, + "Filipino,Spanish,Indonesian": 0.30666666666666664, + "Filipino,Spanish,English": 0.34, + "Filipino,Malay,Indonesian": 0.3333333333333333, + "Filipino,Malay,English": 0.2866666666666667, + "Filipino,Indonesian,English": 0.3466666666666667, + "Vietnamese,Chinese,Spanish": 0.23333333333333334, + "Vietnamese,Chinese,Malay": 0.20666666666666667, + "Vietnamese,Chinese,Indonesian": 0.26, + "Vietnamese,Chinese,English": 0.2, + "Vietnamese,Spanish,Malay": 0.21333333333333335, + "Vietnamese,Spanish,Indonesian": 0.26, + "Vietnamese,Spanish,English": 0.2733333333333333, + "Vietnamese,Malay,Indonesian": 0.30666666666666664, + "Vietnamese,Malay,English": 0.22, + "Vietnamese,Indonesian,English": 0.26666666666666666, + "Chinese,Spanish,Malay": 0.22666666666666666, + "Chinese,Spanish,Indonesian": 0.23333333333333334, + "Chinese,Spanish,English": 0.29333333333333333, + "Chinese,Malay,Indonesian": 0.24666666666666667, + "Chinese,Malay,English": 0.22666666666666666, + "Chinese,Indonesian,English": 0.23333333333333334, + "Spanish,Malay,Indonesian": 0.35333333333333333, + "Spanish,Malay,English": 0.36, + "Spanish,Indonesian,English": 0.37333333333333335, + "Malay,Indonesian,English": 0.35333333333333333 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.022727272727272728, - "English,Filipino,Spanish,Chinese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese": 0.017045454545454544, - "English,Filipino,Malay,Indonesian": 0.022727272727272728, - "English,Filipino,Malay,Vietnamese": 0.017045454545454544, - "English,Filipino,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese": 0.011363636363636364, - "English,Spanish,Malay,Indonesian": 0.022727272727272728, - "English,Spanish,Malay,Vietnamese": 0.005681818181818182, - "English,Spanish,Chinese,Indonesian": 0.022727272727272728, - "English,Spanish,Chinese,Vietnamese": 0.005681818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.011363636363636364, - "English,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese": 0.011363636363636364, - "Filipino,Spanish,Malay,Indonesian": 0.011363636363636364, - "Filipino,Spanish,Malay,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Spanish,Chinese,Vietnamese": 0.0, - "Filipino,Spanish,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.017045454545454544, - "Filipino,Malay,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "Spanish,Malay,Indonesian,Vietnamese": 0.022727272727272728, - "Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 + "Filipino,Vietnamese,Chinese,Spanish": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.14, + "Filipino,Vietnamese,Chinese,English": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Indonesian": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,English": 0.16666666666666666, + "Filipino,Vietnamese,Malay,Indonesian": 0.17333333333333334, + "Filipino,Vietnamese,Malay,English": 0.15333333333333332, + "Filipino,Vietnamese,Indonesian,English": 0.18, + "Filipino,Chinese,Spanish,Malay": 0.13333333333333333, + "Filipino,Chinese,Spanish,Indonesian": 0.14666666666666667, + "Filipino,Chinese,Spanish,English": 0.16666666666666666, + "Filipino,Chinese,Malay,Indonesian": 0.13333333333333333, + "Filipino,Chinese,Malay,English": 0.12, + "Filipino,Chinese,Indonesian,English": 0.14666666666666667, + "Filipino,Spanish,Malay,Indonesian": 0.24, + "Filipino,Spanish,Malay,English": 0.22, + "Filipino,Spanish,Indonesian,English": 0.25333333333333335, + "Filipino,Malay,Indonesian,English": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.14, + "Vietnamese,Chinese,Spanish,Indonesian": 0.16, + "Vietnamese,Chinese,Spanish,English": 0.16, + "Vietnamese,Chinese,Malay,Indonesian": 0.16666666666666666, + "Vietnamese,Chinese,Malay,English": 0.14, + "Vietnamese,Chinese,Indonesian,English": 0.14666666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, + "Vietnamese,Spanish,Malay,English": 0.16666666666666666, + "Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, + "Vietnamese,Malay,Indonesian,English": 0.18666666666666668, + "Chinese,Spanish,Malay,Indonesian": 0.16, + "Chinese,Spanish,Malay,English": 0.19333333333333333, + "Chinese,Spanish,Indonesian,English": 0.18, + "Chinese,Malay,Indonesian,English": 0.16666666666666666, + "Spanish,Malay,Indonesian,English": 0.2733333333333333 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Malay,English": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.09333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Malay,English": 0.12666666666666668, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.13333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, + "Filipino,Chinese,Spanish,Malay,English": 0.11333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.12, + "Filipino,Chinese,Malay,Indonesian,English": 0.10666666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.2, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.12, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.11333333333333333, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.14, + "Chinese,Spanish,Malay,Indonesian,English": 0.14 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.07333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.11333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 } }, - "AC3_2": 0.25575693471276356, - "AC3_3": 0.10075566747502998, - "AC3_4": 0.03055767760542847, - "AC3_5": 0.010081848688869882, - "AC3_6": 0.0032265870762842345, - "AC3_7": 0.0 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.24271844660194175 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.2761904761904762 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.27102803738317754 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.19, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.0, - "history": 0.06666666666666667, - "literature": 0.3, - "politics": 0.1, - "culture": 0.2, - "film": 0.1, - "law": 0.3, - "geography": 0.3 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.008655093137572354 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.009334732184866329 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.009408623117346356 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.009446622541446521 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.009253570424978683 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.26254375729288215 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.2572041473006793, - "category_acc": { - "high_school_european_history": 0.2804878048780488, - "business_ethics": 0.25252525252525254, - "clinical_knowledge": 0.25, - "medical_genetics": 0.32323232323232326, - "high_school_us_history": 0.2413793103448276, - "high_school_physics": 0.2733333333333333, - "high_school_world_history": 0.2669491525423729, - "virology": 0.19393939393939394, - "high_school_microeconomics": 0.25738396624472576, - "econometrics": 0.3185840707964602, - "college_computer_science": 0.1717171717171717, - "high_school_biology": 0.3268608414239482, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.24555160142348753, - "philosophy": 0.2806451612903226, - "professional_medicine": 0.2767527675276753, - "nutrition": 0.26229508196721313, - "global_facts": 0.21212121212121213, - "machine_learning": 0.24324324324324326, - "security_studies": 0.28688524590163933, - "public_relations": 0.1559633027522936, - "professional_psychology": 0.2684124386252046, - "prehistory": 0.21981424148606812, - "anatomy": 0.2835820895522388, - "human_sexuality": 0.2923076923076923, - "college_medicine": 0.3546511627906977, - "high_school_government_and_politics": 0.2864583333333333, - "college_chemistry": 0.1919191919191919, - "logical_fallacies": 0.2345679012345679, - "high_school_geography": 0.3299492385786802, - "elementary_mathematics": 0.27851458885941643, - "human_aging": 0.19369369369369369, - "college_mathematics": 0.25252525252525254, - "high_school_psychology": 0.2610294117647059, - "formal_logic": 0.176, - "high_school_statistics": 0.2372093023255814, - "international_law": 0.25, - "high_school_mathematics": 0.26765799256505574, - "high_school_computer_science": 0.2222222222222222, - "conceptual_physics": 0.24358974358974358, - "miscellaneous": 0.27365728900255754, - "high_school_chemistry": 0.18811881188118812, - "marketing": 0.2532188841201717, - "professional_law": 0.25244618395303325, - "management": 0.17647058823529413, - "college_physics": 0.25742574257425743, - "jurisprudence": 0.18691588785046728, - "world_religions": 0.2647058823529412, - "sociology": 0.265, - "us_foreign_policy": 0.21212121212121213, - "high_school_macroeconomics": 0.23393316195372751, - "computer_security": 0.21212121212121213, - "moral_scenarios": 0.2785234899328859, - "moral_disputes": 0.2289855072463768, - "electrical_engineering": 0.25, - "astronomy": 0.271523178807947, - "college_biology": 0.25874125874125875 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.2399702823179792 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.27334993773349936, - "category_acc": { - "computer_network": 0.125, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.15384615384615385, - "college_programming": 0.35714285714285715, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.25, - "high_school_chemistry": 0.25, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.11538461538461539, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.23333333333333334, - "business_administration": 0.3157894736842105, - "marxism": 0.16666666666666666, - "mao_zedong_thought": 0.20689655172413793, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.32653061224489793, - "high_school_politics": 0.25, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.25, - "logic": 0.3333333333333333, - "law": 0.4482758620689655, - "chinese_language_and_literature": 0.14285714285714285, - "art_studies": 0.23684210526315788, - "professional_tour_guide": 0.20588235294117646, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.44, - "middle_school_history": 0.14814814814814814, - "civil_servant": 0.3076923076923077, - "sports_science": 0.25, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.125, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.25925925925925924, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.24074074074074073, - "physician": 0.2962962962962963 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.2903225806451613 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.24840269383526162, - "category_acc": { - "agronomy": 0.25443786982248523, - "anatomy": 0.19594594594594594, - "ancient_chinese": 0.24390243902439024, - "arts": 0.1875, - "astronomy": 0.22424242424242424, - "business_ethics": 0.2822966507177033, - "chinese_civil_service_exam": 0.20625, - "chinese_driving_rule": 0.2824427480916031, - "chinese_food_culture": 0.25735294117647056, - "chinese_foreign_policy": 0.2523364485981308, - "chinese_history": 0.2724458204334365, - "chinese_literature": 0.20588235294117646, - "chinese_teacher_qualification": 0.18435754189944134, - "clinical_knowledge": 0.24050632911392406, - "college_actuarial_science": 0.20754716981132076, - "college_education": 0.2616822429906542, - "college_engineering_hydrology": 0.2830188679245283, - "college_law": 0.2962962962962963, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.2641509433962264, - "college_medicine": 0.25274725274725274, - "computer_science": 0.24509803921568626, - "computer_security": 0.2807017543859649, - "conceptual_physics": 0.22448979591836735, - "construction_project_management": 0.23741007194244604, - "economics": 0.25157232704402516, - "education": 0.294478527607362, - "electrical_engineering": 0.2441860465116279, - "elementary_chinese": 0.21428571428571427, - "elementary_commonsense": 0.23232323232323232, - "elementary_information_and_technology": 0.27310924369747897, - "elementary_mathematics": 0.22608695652173913, - "ethnology": 0.31851851851851853, - "food_science": 0.2867132867132867, - "genetics": 0.23863636363636365, - "global_facts": 0.2550335570469799, - "high_school_biology": 0.21301775147928995, - "high_school_chemistry": 0.30303030303030304, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.2682926829268293, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.1888111888111888, - "human_sexuality": 0.23015873015873015, - "international_law": 0.24864864864864866, - "journalism": 0.23255813953488372, - "jurisprudence": 0.24087591240875914, - "legal_and_moral_basis": 0.2850467289719626, - "logical": 0.18699186991869918, - "machine_learning": 0.20491803278688525, - "management": 0.19523809523809524, - "marketing": 0.21666666666666667, - "marxist_theory": 0.2275132275132275, - "modern_chinese": 0.2672413793103448, - "nutrition": 0.22758620689655173, - "philosophy": 0.22857142857142856, - "professional_accounting": 0.28, - "professional_law": 0.3127962085308057, - "professional_medicine": 0.22872340425531915, - "professional_psychology": 0.2801724137931034, - "public_relations": 0.28160919540229884, - "security_study": 0.34814814814814815, - "sociology": 0.2743362831858407, - "sports_science": 0.26666666666666666, - "traditional_chinese_medicine": 0.2756756756756757, - "virology": 0.21893491124260356, - "world_history": 0.2484472049689441, - "world_religions": 0.26875 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.2727272727272727 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.1409090909090909 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3345762711864407 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.27711293941660436 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.34149926506614403 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.09095816345387557, - "rouge2": 0.00156516877981128, - "rougeL": 0.07275089832117783, - "avg_rouge": 0.05509141018495489 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.10544977163926846, - "rouge2": 0.0018442575084971875, - "rougeL": 0.0867857990304355, - "avg_rouge": 0.06469327605940038 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.49655963302752293 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.5311601150527325 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.4935 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.336 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.508 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.4507042253521127 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.516245487364621 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.5098039215686274 - } - } - } - }, - "qwen_1_5_7b": { - "model_size": "7B", - "model_link": "https://huggingface.co/Qwen/Qwen1.5-7B", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.9178571428571428, - "language_acc": { - "English": 0.9092436974789916, - "Chinese": 0.938655462184874, - "Spanish": 0.9218487394957983, - "Vietnamese": 0.9016806722689076 - }, - "consistency_score_2": 0.8908963585434174, - "consistency_score_3": 0.8413865546218487, - "consistency_score_4": 0.8058823529411765, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8966386554621849, - "English,Spanish": 0.8949579831932774, - "English,Vietnamese": 0.8764705882352941, - "Chinese,Spanish": 0.9067226890756303, - "Chinese,Vietnamese": 0.8848739495798319, - "Spanish,Vietnamese": 0.8857142857142857 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8546218487394958, - "English,Chinese,Vietnamese": 0.8336134453781513, - "English,Spanish,Vietnamese": 0.8344537815126051, - "Chinese,Spanish,Vietnamese": 0.8428571428571429 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8058823529411765 - } - }, - "AC3_2": 0.9041758156167865, - "AC3_3": 0.8779598416369483, - "AC3_4": 0.8582327848374748 + "AC3_2": 0.4206571964123532, + "AC3_3": 0.31369078860339067, + "AC3_4": 0.23658501087220715, + "AC3_5": 0.18294237766106827, + "AC3_6": 0.14697910781817347, + "AC3_7": 0.1235245901373564 }, "prompt_2": { - "overall_acc": 0.9252100840336134, + "overall_acc": 0.3885714285714285, "language_acc": { - "English": 0.9294117647058824, - "Chinese": 0.9310924369747899, - "Spanish": 0.9201680672268907, - "Vietnamese": 0.9201680672268907 + "Filipino": 0.36, + "Vietnamese": 0.32, + "Chinese": 0.29333333333333333, + "Spanish": 0.4266666666666667, + "Malay": 0.36, + "Indonesian": 0.38666666666666666, + "English": 0.5733333333333334 }, - "consistency_score_2": 0.9130252100840336, - "consistency_score_3": 0.873109243697479, - "consistency_score_4": 0.846218487394958, + "consistency_score_2": 0.4565079365079365, + "consistency_score_3": 0.26552380952380955, + "consistency_score_4": 0.17542857142857146, + "consistency_score_5": 0.12666666666666668, + "consistency_score_6": 0.09809523809523811, + "consistency_score_7": 0.08, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9218487394957983, - "English,Spanish": 0.9159663865546218, - "English,Vietnamese": 0.9042016806722689, - "Chinese,Spanish": 0.9134453781512605, - "Chinese,Vietnamese": 0.9092436974789916, - "Spanish,Vietnamese": 0.9134453781512605 + "Filipino,Vietnamese": 0.41333333333333333, + "Filipino,Chinese": 0.3933333333333333, + "Filipino,Spanish": 0.4266666666666667, + "Filipino,Malay": 0.46, + "Filipino,Indonesian": 0.4866666666666667, + "Filipino,English": 0.49333333333333335, + "Vietnamese,Chinese": 0.41333333333333333, + "Vietnamese,Spanish": 0.4066666666666667, + "Vietnamese,Malay": 0.4, + "Vietnamese,Indonesian": 0.49333333333333335, + "Vietnamese,English": 0.38, + "Chinese,Spanish": 0.4066666666666667, + "Chinese,Malay": 0.44, + "Chinese,Indonesian": 0.4666666666666667, + "Chinese,English": 0.38, + "Spanish,Malay": 0.4866666666666667, + "Spanish,Indonesian": 0.49333333333333335, + "Spanish,English": 0.6133333333333333, + "Malay,Indonesian": 0.6066666666666667, + "Malay,English": 0.4533333333333333, + "Indonesian,English": 0.47333333333333333 }, "3_combine": { - "English,Chinese,Spanish": 0.8798319327731092, - "English,Chinese,Vietnamese": 0.8714285714285714, - "English,Spanish,Vietnamese": 0.8697478991596639, - "Chinese,Spanish,Vietnamese": 0.8714285714285714 + "Filipino,Vietnamese,Chinese": 0.21333333333333335, + "Filipino,Vietnamese,Spanish": 0.22666666666666666, + "Filipino,Vietnamese,Malay": 0.23333333333333334, + "Filipino,Vietnamese,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,English": 0.22666666666666666, + "Filipino,Chinese,Spanish": 0.20666666666666667, + "Filipino,Chinese,Malay": 0.22, + "Filipino,Chinese,Indonesian": 0.24666666666666667, + "Filipino,Chinese,English": 0.22666666666666666, + "Filipino,Spanish,Malay": 0.26666666666666666, + "Filipino,Spanish,Indonesian": 0.3, + "Filipino,Spanish,English": 0.32666666666666666, + "Filipino,Malay,Indonesian": 0.3333333333333333, + "Filipino,Malay,English": 0.29333333333333333, + "Filipino,Indonesian,English": 0.30666666666666664, + "Vietnamese,Chinese,Spanish": 0.22666666666666666, + "Vietnamese,Chinese,Malay": 0.24666666666666667, + "Vietnamese,Chinese,Indonesian": 0.2733333333333333, + "Vietnamese,Chinese,English": 0.2, + "Vietnamese,Spanish,Malay": 0.22666666666666666, + "Vietnamese,Spanish,Indonesian": 0.25333333333333335, + "Vietnamese,Spanish,English": 0.26, + "Vietnamese,Malay,Indonesian": 0.30666666666666664, + "Vietnamese,Malay,English": 0.22666666666666666, + "Vietnamese,Indonesian,English": 0.24666666666666667, + "Chinese,Spanish,Malay": 0.25333333333333335, + "Chinese,Spanish,Indonesian": 0.24, + "Chinese,Spanish,English": 0.28, + "Chinese,Malay,Indonesian": 0.32666666666666666, + "Chinese,Malay,English": 0.24666666666666667, + "Chinese,Indonesian,English": 0.23333333333333334, + "Spanish,Malay,Indonesian": 0.3466666666666667, + "Spanish,Malay,English": 0.34, + "Spanish,Indonesian,English": 0.34, + "Malay,Indonesian,English": 0.32666666666666666 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.846218487394958 + "Filipino,Vietnamese,Chinese,Spanish": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Malay": 0.16, + "Filipino,Vietnamese,Chinese,Indonesian": 0.16, + "Filipino,Vietnamese,Chinese,English": 0.14, + "Filipino,Vietnamese,Spanish,Malay": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,English": 0.16, + "Filipino,Vietnamese,Malay,Indonesian": 0.18666666666666668, + "Filipino,Vietnamese,Malay,English": 0.16, + "Filipino,Vietnamese,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Malay": 0.14666666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.15333333333333332, + "Filipino,Chinese,Spanish,English": 0.17333333333333334, + "Filipino,Chinese,Malay,Indonesian": 0.18, + "Filipino,Chinese,Malay,English": 0.15333333333333332, + "Filipino,Chinese,Indonesian,English": 0.16, + "Filipino,Spanish,Malay,Indonesian": 0.21333333333333335, + "Filipino,Spanish,Malay,English": 0.22, + "Filipino,Spanish,Indonesian,English": 0.24, + "Filipino,Malay,Indonesian,English": 0.22, + "Vietnamese,Chinese,Spanish,Malay": 0.16666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian": 0.16, + "Vietnamese,Chinese,Spanish,English": 0.16, + "Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, + "Vietnamese,Chinese,Malay,English": 0.16, + "Vietnamese,Chinese,Indonesian,English": 0.14666666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, + "Vietnamese,Spanish,Malay,English": 0.16, + "Vietnamese,Spanish,Indonesian,English": 0.16, + "Vietnamese,Malay,Indonesian,English": 0.17333333333333334, + "Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, + "Chinese,Spanish,Malay,English": 0.18666666666666668, + "Chinese,Spanish,Indonesian,English": 0.17333333333333334, + "Chinese,Malay,Indonesian,English": 0.19333333333333333, + "Spanish,Malay,Indonesian,English": 0.24666666666666667 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.12, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Malay,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.10666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14, + "Filipino,Vietnamese,Spanish,Malay,English": 0.12666666666666668, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12, + "Filipino,Chinese,Spanish,Malay,English": 0.12, + "Filipino,Chinese,Spanish,Indonesian,English": 0.12666666666666668, + "Filipino,Chinese,Malay,Indonesian,English": 0.13333333333333333, + "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12, + "Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.12, + "Chinese,Spanish,Malay,Indonesian,English": 0.14 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.09333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 } }, - "AC3_2": 0.9190772628551642, - "AC3_3": 0.8984049320163265, - "AC3_4": 0.8839530878266162 + "AC3_2": 0.4198089513295224, + "AC3_3": 0.31547383304936494, + "AC3_4": 0.24172528581616537, + "AC3_5": 0.19105360439914856, + "AC3_6": 0.15664523340365272, + "AC3_7": 0.1326829268009518 }, "prompt_3": { - "overall_acc": 0.9302521008403362, + "overall_acc": 0.38095238095238093, "language_acc": { - "English": 0.9252100840336135, - "Chinese": 0.938655462184874, - "Spanish": 0.926890756302521, - "Vietnamese": 0.9302521008403362 + "Filipino": 0.32666666666666666, + "Vietnamese": 0.3, + "Chinese": 0.2733333333333333, + "Spanish": 0.41333333333333333, + "Malay": 0.36, + "Indonesian": 0.3933333333333333, + "English": 0.6 }, - "consistency_score_2": 0.9096638655462185, - "consistency_score_3": 0.869327731092437, - "consistency_score_4": 0.8428571428571429, + "consistency_score_2": 0.45968253968253964, + "consistency_score_3": 0.2700952380952381, + "consistency_score_4": 0.1771428571428571, + "consistency_score_5": 0.12571428571428572, + "consistency_score_6": 0.09619047619047619, + "consistency_score_7": 0.08, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9084033613445378, - "English,Spanish": 0.9184873949579831, - "English,Vietnamese": 0.9025210084033614, - "Chinese,Spanish": 0.9117647058823529, - "Chinese,Vietnamese": 0.907563025210084, - "Spanish,Vietnamese": 0.9092436974789916 + "Filipino,Vietnamese": 0.3933333333333333, + "Filipino,Chinese": 0.3933333333333333, + "Filipino,Spanish": 0.44, + "Filipino,Malay": 0.48, + "Filipino,Indonesian": 0.5466666666666666, + "Filipino,English": 0.4866666666666667, + "Vietnamese,Chinese": 0.3933333333333333, + "Vietnamese,Spanish": 0.38666666666666666, + "Vietnamese,Malay": 0.4066666666666667, + "Vietnamese,Indonesian": 0.5, + "Vietnamese,English": 0.36666666666666664, + "Chinese,Spanish": 0.42, + "Chinese,Malay": 0.38, + "Chinese,Indonesian": 0.44666666666666666, + "Chinese,English": 0.37333333333333335, + "Spanish,Malay": 0.4866666666666667, + "Spanish,Indonesian": 0.5066666666666667, + "Spanish,English": 0.6266666666666667, + "Malay,Indonesian": 0.6266666666666667, + "Malay,English": 0.4666666666666667, + "Indonesian,English": 0.5266666666666666 }, "3_combine": { - "English,Chinese,Spanish": 0.8747899159663866, - "English,Chinese,Vietnamese": 0.8647058823529412, - "English,Spanish,Vietnamese": 0.8689075630252101, - "Chinese,Spanish,Vietnamese": 0.8689075630252101 + "Filipino,Vietnamese,Chinese": 0.20666666666666667, + "Filipino,Vietnamese,Spanish": 0.21333333333333335, + "Filipino,Vietnamese,Malay": 0.24, + "Filipino,Vietnamese,Indonesian": 0.28, + "Filipino,Vietnamese,English": 0.21333333333333335, + "Filipino,Chinese,Spanish": 0.22, + "Filipino,Chinese,Malay": 0.2, + "Filipino,Chinese,Indonesian": 0.25333333333333335, + "Filipino,Chinese,English": 0.21333333333333335, + "Filipino,Spanish,Malay": 0.2866666666666667, + "Filipino,Spanish,Indonesian": 0.3333333333333333, + "Filipino,Spanish,English": 0.32666666666666666, + "Filipino,Malay,Indonesian": 0.38, + "Filipino,Malay,English": 0.3, + "Filipino,Indonesian,English": 0.3466666666666667, + "Vietnamese,Chinese,Spanish": 0.21333333333333335, + "Vietnamese,Chinese,Malay": 0.21333333333333335, + "Vietnamese,Chinese,Indonesian": 0.2733333333333333, + "Vietnamese,Chinese,English": 0.18666666666666668, + "Vietnamese,Spanish,Malay": 0.22666666666666666, + "Vietnamese,Spanish,Indonesian": 0.26, + "Vietnamese,Spanish,English": 0.25333333333333335, + "Vietnamese,Malay,Indonesian": 0.32666666666666666, + "Vietnamese,Malay,English": 0.22666666666666666, + "Vietnamese,Indonesian,English": 0.26666666666666666, + "Chinese,Spanish,Malay": 0.23333333333333334, + "Chinese,Spanish,Indonesian": 0.24666666666666667, + "Chinese,Spanish,English": 0.28, + "Chinese,Malay,Indonesian": 0.2866666666666667, + "Chinese,Malay,English": 0.22, + "Chinese,Indonesian,English": 0.23333333333333334, + "Spanish,Malay,Indonesian": 0.38, + "Spanish,Malay,English": 0.36, + "Spanish,Indonesian,English": 0.38, + "Malay,Indonesian,English": 0.37333333333333335 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8428571428571429 + "Filipino,Vietnamese,Chinese,Spanish": 0.14, + "Filipino,Vietnamese,Chinese,Malay": 0.14, + "Filipino,Vietnamese,Chinese,Indonesian": 0.16, + "Filipino,Vietnamese,Chinese,English": 0.12, + "Filipino,Vietnamese,Spanish,Malay": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Indonesian": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,English": 0.14666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.2, + "Filipino,Vietnamese,Malay,English": 0.16, + "Filipino,Vietnamese,Indonesian,English": 0.18, + "Filipino,Chinese,Spanish,Malay": 0.14666666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.17333333333333334, + "Filipino,Chinese,Spanish,English": 0.17333333333333334, + "Filipino,Chinese,Malay,Indonesian": 0.16, + "Filipino,Chinese,Malay,English": 0.14, + "Filipino,Chinese,Indonesian,English": 0.16666666666666666, + "Filipino,Spanish,Malay,Indonesian": 0.25333333333333335, + "Filipino,Spanish,Malay,English": 0.23333333333333334, + "Filipino,Spanish,Indonesian,English": 0.26666666666666666, + "Filipino,Malay,Indonesian,English": 0.25333333333333335, + "Vietnamese,Chinese,Spanish,Malay": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,Indonesian": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,English": 0.14666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.17333333333333334, + "Vietnamese,Chinese,Malay,English": 0.14, + "Vietnamese,Chinese,Indonesian,English": 0.14, + "Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Vietnamese,Spanish,Malay,English": 0.16, + "Vietnamese,Spanish,Indonesian,English": 0.18, + "Vietnamese,Malay,Indonesian,English": 0.2, + "Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, + "Chinese,Spanish,Malay,English": 0.18, + "Chinese,Spanish,Indonesian,English": 0.18, + "Chinese,Malay,Indonesian,English": 0.18666666666666668, + "Spanish,Malay,Indonesian,English": 0.2866666666666667 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.1, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.1, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.1, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Malay,English": 0.12, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.14, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,English": 0.12666666666666668, + "Filipino,Chinese,Spanish,Indonesian,English": 0.14, + "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Spanish,Malay,Indonesian,English": 0.20666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12, + "Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.12, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.14, + "Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.08666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 } }, - "AC3_2": 0.9198427943411128, - "AC3_3": 0.8987586255109518, - "AC3_4": 0.884400812407806 + "AC3_2": 0.41663070056905455, + "AC3_3": 0.3160857086644258, + "AC3_4": 0.241833252028828, + "AC3_5": 0.18904403863078753, + "AC3_6": 0.15359756673902836, + "AC3_7": 0.1322314049299911 }, "prompt_4": { - "overall_acc": 0.9105042016806723, + "overall_acc": 0.3895238095238095, "language_acc": { - "English": 0.9042016806722689, - "Chinese": 0.926890756302521, - "Spanish": 0.9084033613445378, - "Vietnamese": 0.9025210084033614 + "Filipino": 0.36666666666666664, + "Vietnamese": 0.30666666666666664, + "Chinese": 0.3, + "Spanish": 0.41333333333333333, + "Malay": 0.36666666666666664, + "Indonesian": 0.4066666666666667, + "English": 0.5666666666666667 }, - "consistency_score_2": 0.8817927170868348, - "consistency_score_3": 0.8287815126050421, - "consistency_score_4": 0.7932773109243697, + "consistency_score_2": 0.4571428571428572, + "consistency_score_3": 0.26742857142857146, + "consistency_score_4": 0.17676190476190473, + "consistency_score_5": 0.12666666666666665, + "consistency_score_6": 0.09714285714285716, + "consistency_score_7": 0.08, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.8840336134453781, - "English,Spanish": 0.8756302521008403, - "English,Vietnamese": 0.8781512605042017, - "Chinese,Spanish": 0.8915966386554622, - "Chinese,Vietnamese": 0.8798319327731092, - "Spanish,Vietnamese": 0.8815126050420168 + "Filipino,Vietnamese": 0.4066666666666667, + "Filipino,Chinese": 0.4066666666666667, + "Filipino,Spanish": 0.4533333333333333, + "Filipino,Malay": 0.4666666666666667, + "Filipino,Indonesian": 0.5066666666666667, + "Filipino,English": 0.4866666666666667, + "Vietnamese,Chinese": 0.4066666666666667, + "Vietnamese,Spanish": 0.4066666666666667, + "Vietnamese,Malay": 0.4066666666666667, + "Vietnamese,Indonesian": 0.49333333333333335, + "Vietnamese,English": 0.35333333333333333, + "Chinese,Spanish": 0.4266666666666667, + "Chinese,Malay": 0.42, + "Chinese,Indonesian": 0.44, + "Chinese,English": 0.38, + "Spanish,Malay": 0.4866666666666667, + "Spanish,Indonesian": 0.5, + "Spanish,English": 0.6, + "Malay,Indonesian": 0.6133333333333333, + "Malay,English": 0.44666666666666666, + "Indonesian,English": 0.49333333333333335 }, "3_combine": { - "English,Chinese,Spanish": 0.8336134453781513, - "English,Chinese,Vietnamese": 0.8285714285714286, - "English,Spanish,Vietnamese": 0.8218487394957983, - "Chinese,Spanish,Vietnamese": 0.83109243697479 + "Filipino,Vietnamese,Chinese": 0.2, + "Filipino,Vietnamese,Spanish": 0.23333333333333334, + "Filipino,Vietnamese,Malay": 0.23333333333333334, + "Filipino,Vietnamese,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,English": 0.21333333333333335, + "Filipino,Chinese,Spanish": 0.23333333333333334, + "Filipino,Chinese,Malay": 0.21333333333333335, + "Filipino,Chinese,Indonesian": 0.24, + "Filipino,Chinese,English": 0.22666666666666666, + "Filipino,Spanish,Malay": 0.2866666666666667, + "Filipino,Spanish,Indonesian": 0.31333333333333335, + "Filipino,Spanish,English": 0.32666666666666666, + "Filipino,Malay,Indonesian": 0.3466666666666667, + "Filipino,Malay,English": 0.29333333333333333, + "Filipino,Indonesian,English": 0.32, + "Vietnamese,Chinese,Spanish": 0.24, + "Vietnamese,Chinese,Malay": 0.23333333333333334, + "Vietnamese,Chinese,Indonesian": 0.2733333333333333, + "Vietnamese,Chinese,English": 0.19333333333333333, + "Vietnamese,Spanish,Malay": 0.22666666666666666, + "Vietnamese,Spanish,Indonesian": 0.26, + "Vietnamese,Spanish,English": 0.24666666666666667, + "Vietnamese,Malay,Indonesian": 0.32666666666666666, + "Vietnamese,Malay,English": 0.21333333333333335, + "Vietnamese,Indonesian,English": 0.24666666666666667, + "Chinese,Spanish,Malay": 0.26, + "Chinese,Spanish,Indonesian": 0.24666666666666667, + "Chinese,Spanish,English": 0.28, + "Chinese,Malay,Indonesian": 0.31333333333333335, + "Chinese,Malay,English": 0.24, + "Chinese,Indonesian,English": 0.23333333333333334, + "Spanish,Malay,Indonesian": 0.36, + "Spanish,Malay,English": 0.3333333333333333, + "Spanish,Indonesian,English": 0.3466666666666667, + "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7932773109243697 + "Filipino,Vietnamese,Chinese,Spanish": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Malay": 0.14, + "Filipino,Vietnamese,Chinese,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,English": 0.11333333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.18, + "Filipino,Vietnamese,Spanish,English": 0.14666666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.2, + "Filipino,Vietnamese,Malay,English": 0.15333333333333332, + "Filipino,Vietnamese,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Malay": 0.16666666666666666, + "Filipino,Chinese,Spanish,Indonesian": 0.16666666666666666, + "Filipino,Chinese,Spanish,English": 0.18, + "Filipino,Chinese,Malay,Indonesian": 0.17333333333333334, + "Filipino,Chinese,Malay,English": 0.14666666666666667, + "Filipino,Chinese,Indonesian,English": 0.16, + "Filipino,Spanish,Malay,Indonesian": 0.24666666666666667, + "Filipino,Spanish,Malay,English": 0.22666666666666666, + "Filipino,Spanish,Indonesian,English": 0.24666666666666667, + "Filipino,Malay,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Malay": 0.16, + "Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, + "Vietnamese,Chinese,Spanish,English": 0.15333333333333332, + "Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, + "Vietnamese,Chinese,Malay,English": 0.14, + "Vietnamese,Chinese,Indonesian,English": 0.14666666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Vietnamese,Spanish,Malay,English": 0.15333333333333332, + "Vietnamese,Spanish,Indonesian,English": 0.17333333333333334, + "Vietnamese,Malay,Indonesian,English": 0.18, + "Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Chinese,Spanish,Malay,English": 0.19333333333333333, + "Chinese,Spanish,Indonesian,English": 0.18, + "Chinese,Malay,Indonesian,English": 0.18666666666666668, + "Spanish,Malay,Indonesian,English": 0.25333333333333335 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.1, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Malay,English": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.09333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Malay,English": 0.12, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.13333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.14, + "Filipino,Chinese,Spanish,Malay,English": 0.13333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.13333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Spanish,Malay,Indonesian,English": 0.19333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, + "Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.12, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.13333333333333333, + "Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.08, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.11333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 } }, - "AC3_2": 0.8959184892493037, - "AC3_3": 0.8677229316158314, - "AC3_4": 0.8478579199369737 + "AC3_2": 0.42063313509414024, + "AC3_3": 0.31713043113260764, + "AC3_4": 0.24317396244484493, + "AC3_5": 0.19116851164808235, + "AC3_6": 0.15550461277207275, + "AC3_7": 0.1327383366857251 }, "prompt_5": { - "overall_acc": 0.8966386554621849, + "overall_acc": 0.3866666666666667, "language_acc": { - "English": 0.9042016806722689, - "Chinese": 0.9100840336134454, - "Spanish": 0.8873949579831932, - "Vietnamese": 0.8848739495798319 + "Filipino": 0.34, + "Vietnamese": 0.32666666666666666, + "Chinese": 0.28, + "Spanish": 0.43333333333333335, + "Malay": 0.35333333333333333, + "Indonesian": 0.38666666666666666, + "English": 0.5866666666666667 }, - "consistency_score_2": 0.873669467787115, - "consistency_score_3": 0.8178571428571428, - "consistency_score_4": 0.7789915966386555, + "consistency_score_2": 0.4523809523809524, + "consistency_score_3": 0.26285714285714284, + "consistency_score_4": 0.17314285714285715, + "consistency_score_5": 0.12444444444444444, + "consistency_score_6": 0.09619047619047619, + "consistency_score_7": 0.08, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.8823529411764706, - "English,Spanish": 0.8789915966386554, - "English,Vietnamese": 0.8689075630252101, - "Chinese,Spanish": 0.8689075630252101, - "Chinese,Vietnamese": 0.8689075630252101, - "Spanish,Vietnamese": 0.8739495798319328 + "Filipino,Vietnamese": 0.4066666666666667, + "Filipino,Chinese": 0.36, + "Filipino,Spanish": 0.44666666666666666, + "Filipino,Malay": 0.4533333333333333, + "Filipino,Indonesian": 0.5466666666666666, + "Filipino,English": 0.49333333333333335, + "Vietnamese,Chinese": 0.4, + "Vietnamese,Spanish": 0.4, + "Vietnamese,Malay": 0.42, + "Vietnamese,Indonesian": 0.4866666666666667, + "Vietnamese,English": 0.35333333333333333, + "Chinese,Spanish": 0.41333333333333333, + "Chinese,Malay": 0.4, + "Chinese,Indonesian": 0.4066666666666667, + "Chinese,English": 0.3466666666666667, + "Spanish,Malay": 0.5133333333333333, + "Spanish,Indonesian": 0.4666666666666667, + "Spanish,English": 0.6266666666666667, + "Malay,Indonesian": 0.6266666666666667, + "Malay,English": 0.4533333333333333, + "Indonesian,English": 0.48 }, "3_combine": { - "English,Chinese,Spanish": 0.8235294117647058, - "English,Chinese,Vietnamese": 0.8184873949579832, - "English,Spanish,Vietnamese": 0.8176470588235294, - "Chinese,Spanish,Vietnamese": 0.8117647058823529 + "Filipino,Vietnamese,Chinese": 0.2, + "Filipino,Vietnamese,Spanish": 0.22, + "Filipino,Vietnamese,Malay": 0.23333333333333334, + "Filipino,Vietnamese,Indonesian": 0.28, + "Filipino,Vietnamese,English": 0.21333333333333335, + "Filipino,Chinese,Spanish": 0.2, + "Filipino,Chinese,Malay": 0.18, + "Filipino,Chinese,Indonesian": 0.22666666666666666, + "Filipino,Chinese,English": 0.19333333333333333, + "Filipino,Spanish,Malay": 0.29333333333333333, + "Filipino,Spanish,Indonesian": 0.30666666666666664, + "Filipino,Spanish,English": 0.34, + "Filipino,Malay,Indonesian": 0.36666666666666664, + "Filipino,Malay,English": 0.29333333333333333, + "Filipino,Indonesian,English": 0.32, + "Vietnamese,Chinese,Spanish": 0.23333333333333334, + "Vietnamese,Chinese,Malay": 0.24, + "Vietnamese,Chinese,Indonesian": 0.26666666666666666, + "Vietnamese,Chinese,English": 0.18666666666666668, + "Vietnamese,Spanish,Malay": 0.24666666666666667, + "Vietnamese,Spanish,Indonesian": 0.26, + "Vietnamese,Spanish,English": 0.25333333333333335, + "Vietnamese,Malay,Indonesian": 0.32666666666666666, + "Vietnamese,Malay,English": 0.22666666666666666, + "Vietnamese,Indonesian,English": 0.24666666666666667, + "Chinese,Spanish,Malay": 0.24666666666666667, + "Chinese,Spanish,Indonesian": 0.22, + "Chinese,Spanish,English": 0.26666666666666666, + "Chinese,Malay,Indonesian": 0.29333333333333333, + "Chinese,Malay,English": 0.22, + "Chinese,Indonesian,English": 0.19333333333333333, + "Spanish,Malay,Indonesian": 0.36666666666666664, + "Spanish,Malay,English": 0.36666666666666664, + "Spanish,Indonesian,English": 0.34, + "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7789915966386555 + "Filipino,Vietnamese,Chinese,Spanish": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.16, + "Filipino,Vietnamese,Chinese,English": 0.12, + "Filipino,Vietnamese,Spanish,Malay": 0.16, + "Filipino,Vietnamese,Spanish,Indonesian": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,English": 0.15333333333333332, + "Filipino,Vietnamese,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Malay,English": 0.15333333333333332, + "Filipino,Vietnamese,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Malay": 0.14, + "Filipino,Chinese,Spanish,Indonesian": 0.14666666666666667, + "Filipino,Chinese,Spanish,English": 0.15333333333333332, + "Filipino,Chinese,Malay,Indonesian": 0.16, + "Filipino,Chinese,Malay,English": 0.12666666666666668, + "Filipino,Chinese,Indonesian,English": 0.12666666666666668, + "Filipino,Spanish,Malay,Indonesian": 0.24666666666666667, + "Filipino,Spanish,Malay,English": 0.24, + "Filipino,Spanish,Indonesian,English": 0.24, + "Filipino,Malay,Indonesian,English": 0.24, + "Vietnamese,Chinese,Spanish,Malay": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, + "Vietnamese,Chinese,Spanish,English": 0.15333333333333332, + "Vietnamese,Chinese,Malay,Indonesian": 0.2, + "Vietnamese,Chinese,Malay,English": 0.14666666666666667, + "Vietnamese,Chinese,Indonesian,English": 0.14, + "Vietnamese,Spanish,Malay,Indonesian": 0.2, + "Vietnamese,Spanish,Malay,English": 0.16666666666666666, + "Vietnamese,Spanish,Indonesian,English": 0.17333333333333334, + "Vietnamese,Malay,Indonesian,English": 0.18666666666666668, + "Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, + "Chinese,Spanish,Malay,English": 0.18, + "Chinese,Spanish,Indonesian,English": 0.14666666666666667, + "Chinese,Malay,Indonesian,English": 0.17333333333333334, + "Spanish,Malay,Indonesian,English": 0.2733333333333333 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Malay,English": 0.1, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.09333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14, + "Filipino,Vietnamese,Spanish,Malay,English": 0.12, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12666666666666668, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.13333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,English": 0.11333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.1, + "Filipino,Chinese,Malay,Indonesian,English": 0.11333333333333333, + "Filipino,Spanish,Malay,Indonesian,English": 0.2, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, + "Vietnamese,Chinese,Spanish,Malay,English": 0.12, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.14, + "Chinese,Spanish,Malay,Indonesian,English": 0.14 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.08666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 } }, - "AC3_2": 0.8850050526039814, - "AC3_3": 0.8554378839629191, - "AC3_4": 0.8336850888303321 + "AC3_2": 0.41695043506192236, + "AC3_3": 0.3129618767846613, + "AC3_4": 0.23918339567010166, + "AC3_5": 0.18828985503562445, + "AC3_6": 0.15405654171694425, + "AC3_7": 0.1325714285430204 } }, - "cross_mmlu": { + "cross_logiqa": { "prompt_1": { - "overall_acc": 0.5371428571428571, + "overall_acc": 0.39123376623376627, "language_acc": { - "Spanish": 0.5866666666666667, - "English": 0.66, - "Chinese": 0.6066666666666667, - "Vietnamese": 0.4866666666666667, - "Filipino": 0.42, - "Indonesian": 0.52, - "Malay": 0.48 + "Indonesian": 0.3465909090909091, + "English": 0.6136363636363636, + "Filipino": 0.375, + "Spanish": 0.5, + "Chinese": 0.25, + "Malay": 0.3522727272727273, + "Vietnamese": 0.30113636363636365 }, - "consistency_score_2": 0.557142857142857, - "consistency_score_3": 0.383047619047619, - "consistency_score_4": 0.2857142857142857, - "consistency_score_5": 0.2228571428571429, - "consistency_score_6": 0.1790476190476191, - "consistency_score_7": 0.14666666666666667, + "consistency_score_2": 0.3825757575757576, + "consistency_score_3": 0.18295454545454543, + "consistency_score_4": 0.09756493506493508, + "consistency_score_5": 0.053030303030303025, + "consistency_score_6": 0.02516233766233766, + "consistency_score_7": 0.005681818181818182, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6266666666666667, - "Spanish,Chinese": 0.6466666666666666, - "Spanish,Vietnamese": 0.58, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.5666666666666667, - "Spanish,Malay": 0.5333333333333333, - "English,Chinese": 0.6, - "English,Vietnamese": 0.5733333333333334, - "English,Filipino": 0.48, - "English,Indonesian": 0.6133333333333333, - "English,Malay": 0.5266666666666666, - "Chinese,Vietnamese": 0.5066666666666667, - "Chinese,Filipino": 0.49333333333333335, - "Chinese,Indonesian": 0.58, - "Chinese,Malay": 0.49333333333333335, - "Vietnamese,Filipino": 0.52, - "Vietnamese,Indonesian": 0.5866666666666667, - "Vietnamese,Malay": 0.5866666666666667, - "Filipino,Indonesian": 0.58, - "Filipino,Malay": 0.5466666666666666, - "Indonesian,Malay": 0.62 + "Indonesian,English": 0.4034090909090909, + "Indonesian,Filipino": 0.44886363636363635, + "Indonesian,Spanish": 0.5113636363636364, + "Indonesian,Chinese": 0.24431818181818182, + "Indonesian,Malay": 0.5795454545454546, + "Indonesian,Vietnamese": 0.38636363636363635, + "English,Filipino": 0.3465909090909091, + "English,Spanish": 0.6534090909090909, + "English,Chinese": 0.2556818181818182, + "English,Malay": 0.3977272727272727, + "English,Vietnamese": 0.2784090909090909, + "Filipino,Spanish": 0.4431818181818182, + "Filipino,Chinese": 0.22727272727272727, + "Filipino,Malay": 0.48863636363636365, + "Filipino,Vietnamese": 0.4943181818181818, + "Spanish,Chinese": 0.22727272727272727, + "Spanish,Malay": 0.42045454545454547, + "Spanish,Vietnamese": 0.39204545454545453, + "Chinese,Malay": 0.2215909090909091, + "Chinese,Vietnamese": 0.19886363636363635, + "Malay,Vietnamese": 0.4147727272727273 }, "3_combine": { - "Spanish,English,Chinese": 0.4666666666666667, - "Spanish,English,Vietnamese": 0.41333333333333333, - "Spanish,English,Filipino": 0.3, - "Spanish,English,Indonesian": 0.4266666666666667, - "Spanish,English,Malay": 0.38, - "Spanish,Chinese,Vietnamese": 0.3933333333333333, - "Spanish,Chinese,Filipino": 0.34, - "Spanish,Chinese,Indonesian": 0.44, - "Spanish,Chinese,Malay": 0.37333333333333335, - "Spanish,Vietnamese,Filipino": 0.32, - "Spanish,Vietnamese,Indonesian": 0.4, - "Spanish,Vietnamese,Malay": 0.3933333333333333, - "Spanish,Filipino,Indonesian": 0.3466666666666667, - "Spanish,Filipino,Malay": 0.32, - "Spanish,Indonesian,Malay": 0.42, - "English,Chinese,Vietnamese": 0.37333333333333335, - "English,Chinese,Filipino": 0.35333333333333333, - "English,Chinese,Indonesian": 0.43333333333333335, - "English,Chinese,Malay": 0.35333333333333333, - "English,Vietnamese,Filipino": 0.3466666666666667, - "English,Vietnamese,Indonesian": 0.4266666666666667, - "English,Vietnamese,Malay": 0.38666666666666666, - "English,Filipino,Indonesian": 0.38, - "English,Filipino,Malay": 0.32, - "English,Indonesian,Malay": 0.42, - "Chinese,Vietnamese,Filipino": 0.34, - "Chinese,Vietnamese,Indonesian": 0.38666666666666666, - "Chinese,Vietnamese,Malay": 0.35333333333333333, - "Chinese,Filipino,Indonesian": 0.4, - "Chinese,Filipino,Malay": 0.34, - "Chinese,Indonesian,Malay": 0.4066666666666667, - "Vietnamese,Filipino,Indonesian": 0.3933333333333333, - "Vietnamese,Filipino,Malay": 0.38666666666666666, - "Vietnamese,Indonesian,Malay": 0.44666666666666666, - "Filipino,Indonesian,Malay": 0.4266666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.32, - "Spanish,English,Chinese,Filipino": 0.26666666666666666, - "Spanish,English,Chinese,Indonesian": 0.34, - "Spanish,English,Chinese,Malay": 0.28, - "Spanish,English,Vietnamese,Filipino": 0.24, - "Spanish,English,Vietnamese,Indonesian": 0.32, - "Spanish,English,Vietnamese,Malay": 0.2866666666666667, - "Spanish,English,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Filipino,Malay": 0.22, - "Spanish,English,Indonesian,Malay": 0.32, - "Spanish,Chinese,Vietnamese,Filipino": 0.26, - "Spanish,Chinese,Vietnamese,Indonesian": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Malay": 0.26666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Filipino,Malay": 0.24666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.32, - "Spanish,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino": 0.25333333333333335, - "English,Chinese,Vietnamese,Indonesian": 0.30666666666666664, - "English,Chinese,Vietnamese,Malay": 0.25333333333333335, - "English,Chinese,Filipino,Indonesian": 0.29333333333333333, - "English,Chinese,Filipino,Malay": 0.26, - "English,Chinese,Indonesian,Malay": 0.32, - "English,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "English,Vietnamese,Filipino,Malay": 0.25333333333333335, - "English,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "English,Filipino,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.2733333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.3, - "Vietnamese,Filipino,Indonesian,Malay": 0.32666666666666666 + "Indonesian,English,Filipino": 0.19318181818181818, + "Indonesian,English,Spanish": 0.3352272727272727, + "Indonesian,English,Chinese": 0.09090909090909091, + "Indonesian,English,Malay": 0.2556818181818182, + "Indonesian,English,Vietnamese": 0.16477272727272727, + "Indonesian,Filipino,Spanish": 0.26136363636363635, + "Indonesian,Filipino,Chinese": 0.10795454545454546, + "Indonesian,Filipino,Malay": 0.3068181818181818, + "Indonesian,Filipino,Vietnamese": 0.23295454545454544, + "Indonesian,Spanish,Chinese": 0.125, + "Indonesian,Spanish,Malay": 0.29545454545454547, + "Indonesian,Spanish,Vietnamese": 0.24431818181818182, + "Indonesian,Chinese,Malay": 0.1534090909090909, + "Indonesian,Chinese,Vietnamese": 0.09659090909090909, + "Indonesian,Malay,Vietnamese": 0.26704545454545453, + "English,Filipino,Spanish": 0.2840909090909091, + "English,Filipino,Chinese": 0.06818181818181818, + "English,Filipino,Malay": 0.2215909090909091, + "English,Filipino,Vietnamese": 0.1590909090909091, + "English,Spanish,Chinese": 0.14204545454545456, + "English,Spanish,Malay": 0.29545454545454547, + "English,Spanish,Vietnamese": 0.24431818181818182, + "English,Chinese,Malay": 0.07954545454545454, + "English,Chinese,Vietnamese": 0.056818181818181816, + "English,Malay,Vietnamese": 0.1590909090909091, + "Filipino,Spanish,Chinese": 0.10227272727272728, + "Filipino,Spanish,Malay": 0.2556818181818182, + "Filipino,Spanish,Vietnamese": 0.23863636363636365, + "Filipino,Chinese,Malay": 0.11363636363636363, + "Filipino,Chinese,Vietnamese": 0.11931818181818182, + "Filipino,Malay,Vietnamese": 0.25, + "Spanish,Chinese,Malay": 0.08522727272727272, + "Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Spanish,Malay,Vietnamese": 0.2159090909090909, + "Chinese,Malay,Vietnamese": 0.09659090909090909 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.17045454545454544, + "Indonesian,English,Filipino,Chinese": 0.028409090909090908, + "Indonesian,English,Filipino,Malay": 0.14204545454545456, + "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, + "Indonesian,English,Spanish,Chinese": 0.06818181818181818, + "Indonesian,English,Spanish,Malay": 0.2159090909090909, + "Indonesian,English,Spanish,Vietnamese": 0.1590909090909091, + "Indonesian,English,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Spanish,Chinese": 0.0625, + "Indonesian,Filipino,Spanish,Malay": 0.1875, + "Indonesian,Filipino,Spanish,Vietnamese": 0.1534090909090909, + "Indonesian,Filipino,Chinese,Malay": 0.07954545454545454, + "Indonesian,Filipino,Chinese,Vietnamese": 0.0625, + "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Malay": 0.07954545454545454, + "Indonesian,Spanish,Chinese,Vietnamese": 0.0625, + "Indonesian,Spanish,Malay,Vietnamese": 0.17045454545454544, + "Indonesian,Chinese,Malay,Vietnamese": 0.07386363636363637, + "English,Filipino,Spanish,Chinese": 0.05113636363636364, + "English,Filipino,Spanish,Malay": 0.18181818181818182, + "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "English,Filipino,Chinese,Malay": 0.03977272727272727, + "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, + "English,Filipino,Malay,Vietnamese": 0.11363636363636363, + "English,Spanish,Chinese,Malay": 0.05113636363636364, + "English,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "English,Spanish,Malay,Vietnamese": 0.14772727272727273, + "English,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Filipino,Spanish,Chinese,Malay": 0.05113636363636364, + "Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, + "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, + "Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,English,Chinese,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.22, - "Spanish,English,Chinese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.24, - "English,Chinese,Filipino,Indonesian,Malay": 0.24, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24 + "Indonesian,English,Filipino,Spanish,Chinese": 0.022727272727272728, + "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, + "Indonesian,English,Filipino,Chinese,Malay": 0.022727272727272728, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Spanish,Chinese,Malay": 0.045454545454545456, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.125, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.028409090909090908, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, + "English,Filipino,Spanish,Chinese,Malay": 0.022727272727272728, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "English,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, + "English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, + "English,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 } }, - "AC3_2": 0.5469600894688529, - "AC3_3": 0.4471928319589723, - "AC3_4": 0.3730158729705412, - "AC3_5": 0.3150161116663916, - "AC3_6": 0.2685714285339286, - "AC3_7": 0.23041782726435392 + "AC3_2": 0.3868563254426954, + "AC3_3": 0.24931888858844456, + "AC3_4": 0.1561816628750512, + "AC3_5": 0.09340050933563672, + "AC3_6": 0.04728361306173132, + "AC3_7": 0.011200966719617114 }, "prompt_2": { - "overall_acc": 0.5304761904761904, + "overall_acc": 0.39285714285714285, "language_acc": { - "Spanish": 0.5666666666666667, - "English": 0.64, - "Chinese": 0.6266666666666667, - "Vietnamese": 0.4866666666666667, - "Filipino": 0.3933333333333333, - "Indonesian": 0.54, - "Malay": 0.46 + "Indonesian": 0.36363636363636365, + "English": 0.6022727272727273, + "Filipino": 0.38636363636363635, + "Spanish": 0.48863636363636365, + "Chinese": 0.26136363636363635, + "Malay": 0.3465909090909091, + "Vietnamese": 0.30113636363636365 }, - "consistency_score_2": 0.5057142857142858, - "consistency_score_3": 0.32342857142857157, - "consistency_score_4": 0.2306666666666667, - "consistency_score_5": 0.17682539682539683, - "consistency_score_6": 0.14476190476190479, - "consistency_score_7": 0.12666666666666668, + "consistency_score_2": 0.3831168831168831, + "consistency_score_3": 0.17987012987012987, + "consistency_score_4": 0.09334415584415583, + "consistency_score_5": 0.04978354978354977, + "consistency_score_6": 0.023538961038961033, + "consistency_score_7": 0.005681818181818182, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6533333333333333, - "Spanish,Chinese": 0.58, - "Spanish,Vietnamese": 0.5466666666666666, - "Spanish,Filipino": 0.38666666666666666, - "Spanish,Indonesian": 0.5866666666666667, - "Spanish,Malay": 0.4866666666666667, - "English,Chinese": 0.6066666666666667, - "English,Vietnamese": 0.5733333333333334, - "English,Filipino": 0.4, - "English,Indonesian": 0.5533333333333333, - "English,Malay": 0.46, - "Chinese,Vietnamese": 0.49333333333333335, - "Chinese,Filipino": 0.42, - "Chinese,Indonesian": 0.56, - "Chinese,Malay": 0.49333333333333335, - "Vietnamese,Filipino": 0.3933333333333333, - "Vietnamese,Indonesian": 0.5466666666666666, - "Vietnamese,Malay": 0.5333333333333333, - "Filipino,Indonesian": 0.38, - "Filipino,Malay": 0.4266666666666667, - "Indonesian,Malay": 0.54 + "Indonesian,English": 0.42045454545454547, + "Indonesian,Filipino": 0.48295454545454547, + "Indonesian,Spanish": 0.5, + "Indonesian,Chinese": 0.23295454545454544, + "Indonesian,Malay": 0.5852272727272727, + "Indonesian,Vietnamese": 0.3977272727272727, + "English,Filipino": 0.36363636363636365, + "English,Spanish": 0.6477272727272727, + "English,Chinese": 0.26136363636363635, + "English,Malay": 0.3977272727272727, + "English,Vietnamese": 0.30113636363636365, + "Filipino,Spanish": 0.4431818181818182, + "Filipino,Chinese": 0.2159090909090909, + "Filipino,Malay": 0.45454545454545453, + "Filipino,Vietnamese": 0.48295454545454547, + "Spanish,Chinese": 0.25, + "Spanish,Malay": 0.39204545454545453, + "Spanish,Vietnamese": 0.375, + "Chinese,Malay": 0.24431818181818182, + "Chinese,Vietnamese": 0.18181818181818182, + "Malay,Vietnamese": 0.4147727272727273 }, "3_combine": { - "Spanish,English,Chinese": 0.44666666666666666, - "Spanish,English,Vietnamese": 0.42, - "Spanish,English,Filipino": 0.2733333333333333, - "Spanish,English,Indonesian": 0.4266666666666667, - "Spanish,English,Malay": 0.34, - "Spanish,Chinese,Vietnamese": 0.36, - "Spanish,Chinese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Indonesian": 0.41333333333333333, - "Spanish,Chinese,Malay": 0.3466666666666667, - "Spanish,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,Vietnamese,Indonesian": 0.38666666666666666, - "Spanish,Vietnamese,Malay": 0.35333333333333333, - "Spanish,Filipino,Indonesian": 0.24, - "Spanish,Filipino,Malay": 0.22666666666666666, - "Spanish,Indonesian,Malay": 0.36666666666666664, - "English,Chinese,Vietnamese": 0.38666666666666666, - "English,Chinese,Filipino": 0.3, - "English,Chinese,Indonesian": 0.4066666666666667, - "English,Chinese,Malay": 0.3466666666666667, - "English,Vietnamese,Filipino": 0.2866666666666667, - "English,Vietnamese,Indonesian": 0.38666666666666666, - "English,Vietnamese,Malay": 0.3466666666666667, - "English,Filipino,Indonesian": 0.24, - "English,Filipino,Malay": 0.22666666666666666, - "English,Indonesian,Malay": 0.3466666666666667, - "Chinese,Vietnamese,Filipino": 0.26, - "Chinese,Vietnamese,Indonesian": 0.35333333333333333, - "Chinese,Vietnamese,Malay": 0.32, - "Chinese,Filipino,Indonesian": 0.25333333333333335, - "Chinese,Filipino,Malay": 0.26666666666666666, - "Chinese,Indonesian,Malay": 0.35333333333333333, - "Vietnamese,Filipino,Indonesian": 0.24, - "Vietnamese,Filipino,Malay": 0.25333333333333335, - "Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Filipino,Indonesian,Malay": 0.26 + "Indonesian,English,Filipino": 0.2159090909090909, + "Indonesian,English,Spanish": 0.3352272727272727, + "Indonesian,English,Chinese": 0.09090909090909091, + "Indonesian,English,Malay": 0.2727272727272727, + "Indonesian,English,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Spanish": 0.26704545454545453, + "Indonesian,Filipino,Chinese": 0.10227272727272728, + "Indonesian,Filipino,Malay": 0.29545454545454547, + "Indonesian,Filipino,Vietnamese": 0.23863636363636365, + "Indonesian,Spanish,Chinese": 0.125, + "Indonesian,Spanish,Malay": 0.2897727272727273, + "Indonesian,Spanish,Vietnamese": 0.23295454545454544, + "Indonesian,Chinese,Malay": 0.16477272727272727, + "Indonesian,Chinese,Vietnamese": 0.07954545454545454, + "Indonesian,Malay,Vietnamese": 0.2556818181818182, + "English,Filipino,Spanish": 0.2897727272727273, + "English,Filipino,Chinese": 0.07386363636363637, + "English,Filipino,Malay": 0.21022727272727273, + "English,Filipino,Vietnamese": 0.1590909090909091, + "English,Spanish,Chinese": 0.1534090909090909, + "English,Spanish,Malay": 0.2784090909090909, + "English,Spanish,Vietnamese": 0.23863636363636365, + "English,Chinese,Malay": 0.08522727272727272, + "English,Chinese,Vietnamese": 0.056818181818181816, + "English,Malay,Vietnamese": 0.1534090909090909, + "Filipino,Spanish,Chinese": 0.10227272727272728, + "Filipino,Spanish,Malay": 0.2215909090909091, + "Filipino,Spanish,Vietnamese": 0.22727272727272727, + "Filipino,Chinese,Malay": 0.10795454545454546, + "Filipino,Chinese,Vietnamese": 0.09659090909090909, + "Filipino,Malay,Vietnamese": 0.24431818181818182, + "Spanish,Chinese,Malay": 0.09659090909090909, + "Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Spanish,Malay,Vietnamese": 0.19318181818181818, + "Chinese,Malay,Vietnamese": 0.09659090909090909 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.2866666666666667, - "Spanish,English,Chinese,Filipino": 0.20666666666666667, - "Spanish,English,Chinese,Indonesian": 0.32666666666666666, - "Spanish,English,Chinese,Malay": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,English,Vietnamese,Malay": 0.2733333333333333, - "Spanish,English,Filipino,Indonesian": 0.2, - "Spanish,English,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.26, - "Spanish,Chinese,Filipino,Indonesian": 0.2, - "Spanish,Chinese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Indonesian,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Vietnamese,Filipino,Malay": 0.18, - "Spanish,Vietnamese,Indonesian,Malay": 0.28, - "Spanish,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino": 0.22666666666666666, - "English,Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "English,Chinese,Vietnamese,Malay": 0.25333333333333335, - "English,Chinese,Filipino,Indonesian": 0.20666666666666667, - "English,Chinese,Filipino,Malay": 0.21333333333333335, - "English,Chinese,Indonesian,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "English,Filipino,Indonesian,Malay": 0.17333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Chinese,Vietnamese,Filipino,Malay": 0.2, - "Chinese,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 + "Indonesian,English,Filipino,Spanish": 0.17613636363636365, + "Indonesian,English,Filipino,Chinese": 0.03409090909090909, + "Indonesian,English,Filipino,Malay": 0.14772727272727273, + "Indonesian,English,Filipino,Vietnamese": 0.10795454545454546, + "Indonesian,English,Spanish,Chinese": 0.07386363636363637, + "Indonesian,English,Spanish,Malay": 0.2159090909090909, + "Indonesian,English,Spanish,Vietnamese": 0.1534090909090909, + "Indonesian,English,Chinese,Malay": 0.06818181818181818, + "Indonesian,English,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,English,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Spanish,Chinese": 0.05113636363636364, + "Indonesian,Filipino,Spanish,Malay": 0.17613636363636365, + "Indonesian,Filipino,Spanish,Vietnamese": 0.14772727272727273, + "Indonesian,Filipino,Chinese,Malay": 0.06818181818181818, + "Indonesian,Filipino,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,Spanish,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,Spanish,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,Chinese,Malay,Vietnamese": 0.0625, + "English,Filipino,Spanish,Chinese": 0.0625, + "English,Filipino,Spanish,Malay": 0.1590909090909091, + "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "English,Filipino,Chinese,Malay": 0.03977272727272727, + "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, + "English,Filipino,Malay,Vietnamese": 0.11363636363636363, + "English,Spanish,Chinese,Malay": 0.06818181818181818, + "English,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "English,Spanish,Malay,Vietnamese": 0.13068181818181818, + "English,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Filipino,Spanish,Chinese,Malay": 0.045454545454545456, + "Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, + "Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.16, - "Spanish,English,Chinese,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "English,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.2, - "English,Chinese,Filipino,Indonesian,Malay": 0.16, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 + "Indonesian,English,Filipino,Spanish,Chinese": 0.028409090909090908, + "Indonesian,English,Filipino,Spanish,Malay": 0.125, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, + "Indonesian,English,Filipino,Chinese,Malay": 0.022727272727272728, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,English,Spanish,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.028409090909090908, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, + "English,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, + "English,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, + "English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.017045454545454544, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.005681818181818182 } }, - "AC3_2": 0.5177993696979277, - "AC3_3": 0.4018508013933017, - "AC3_4": 0.3215248581492825, - "AC3_5": 0.2652380952005952, - "AC3_6": 0.2274538249377705, - "AC3_7": 0.20450241542781725 + "AC3_2": 0.38792588159973884, + "AC3_3": 0.24676060896242177, + "AC3_4": 0.15084664914617452, + "AC3_5": 0.08836884385010232, + "AC3_6": 0.04441659703750908, + "AC3_7": 0.011201629325091566 }, "prompt_3": { - "overall_acc": 0.49619047619047624, + "overall_acc": 0.39204545454545453, "language_acc": { - "Spanish": 0.4533333333333333, - "English": 0.7133333333333334, - "Chinese": 0.5733333333333334, - "Vietnamese": 0.5, - "Filipino": 0.37333333333333335, - "Indonesian": 0.47333333333333333, - "Malay": 0.38666666666666666 + "Indonesian": 0.3693181818181818, + "English": 0.6079545454545454, + "Filipino": 0.3806818181818182, + "Spanish": 0.5113636363636364, + "Chinese": 0.25, + "Malay": 0.3352272727272727, + "Vietnamese": 0.2897727272727273 }, - "consistency_score_2": 0.4523809523809524, - "consistency_score_3": 0.2619047619047619, - "consistency_score_4": 0.17257142857142854, - "consistency_score_5": 0.12126984126984126, - "consistency_score_6": 0.08857142857142856, - "consistency_score_7": 0.06666666666666667, + "consistency_score_2": 0.39420995670995673, + "consistency_score_3": 0.1943181818181819, + "consistency_score_4": 0.10649350649350653, + "consistency_score_5": 0.05979437229437229, + "consistency_score_6": 0.030844155844155844, + "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.49333333333333335, - "Spanish,Chinese": 0.4533333333333333, - "Spanish,Vietnamese": 0.41333333333333333, - "Spanish,Filipino": 0.38, - "Spanish,Indonesian": 0.48, - "Spanish,Malay": 0.46, - "English,Chinese": 0.56, - "English,Vietnamese": 0.5133333333333333, - "English,Filipino": 0.42, - "English,Indonesian": 0.5133333333333333, - "English,Malay": 0.38666666666666666, - "Chinese,Vietnamese": 0.4666666666666667, - "Chinese,Filipino": 0.3933333333333333, - "Chinese,Indonesian": 0.48, - "Chinese,Malay": 0.38666666666666666, - "Vietnamese,Filipino": 0.38, - "Vietnamese,Indonesian": 0.5066666666666667, - "Vietnamese,Malay": 0.48, - "Filipino,Indonesian": 0.4066666666666667, - "Filipino,Malay": 0.38666666666666666, - "Indonesian,Malay": 0.54 + "Indonesian,English": 0.42613636363636365, + "Indonesian,Filipino": 0.4659090909090909, + "Indonesian,Spanish": 0.5056818181818182, + "Indonesian,Chinese": 0.26704545454545453, + "Indonesian,Malay": 0.6079545454545454, + "Indonesian,Vietnamese": 0.4318181818181818, + "English,Filipino": 0.3522727272727273, + "English,Spanish": 0.6534090909090909, + "English,Chinese": 0.26704545454545453, + "English,Malay": 0.42613636363636365, + "English,Vietnamese": 0.2784090909090909, + "Filipino,Spanish": 0.42045454545454547, + "Filipino,Chinese": 0.25, + "Filipino,Malay": 0.4659090909090909, + "Filipino,Vietnamese": 0.4715909090909091, + "Spanish,Chinese": 0.2556818181818182, + "Spanish,Malay": 0.4090909090909091, + "Spanish,Vietnamese": 0.3977272727272727, + "Chinese,Malay": 0.26704545454545453, + "Chinese,Vietnamese": 0.23295454545454544, + "Malay,Vietnamese": 0.42613636363636365 }, "3_combine": { - "Spanish,English,Chinese": 0.32, - "Spanish,English,Vietnamese": 0.28, - "Spanish,English,Filipino": 0.23333333333333334, - "Spanish,English,Indonesian": 0.29333333333333333, - "Spanish,English,Malay": 0.24, - "Spanish,Chinese,Vietnamese": 0.26, - "Spanish,Chinese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Malay": 0.24, - "Spanish,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,Vietnamese,Indonesian": 0.28, - "Spanish,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Filipino,Indonesian": 0.24666666666666667, - "Spanish,Filipino,Malay": 0.21333333333333335, - "Spanish,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese": 0.32666666666666666, - "English,Chinese,Filipino": 0.24666666666666667, - "English,Chinese,Indonesian": 0.34, - "English,Chinese,Malay": 0.24666666666666667, - "English,Vietnamese,Filipino": 0.26666666666666666, - "English,Vietnamese,Indonesian": 0.34, - "English,Vietnamese,Malay": 0.28, - "English,Filipino,Indonesian": 0.2866666666666667, - "English,Filipino,Malay": 0.19333333333333333, - "English,Indonesian,Malay": 0.2866666666666667, - "Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian": 0.29333333333333333, - "Chinese,Vietnamese,Malay": 0.24666666666666667, - "Chinese,Filipino,Indonesian": 0.22666666666666666, - "Chinese,Filipino,Malay": 0.18, - "Chinese,Indonesian,Malay": 0.28, - "Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Vietnamese,Filipino,Malay": 0.22, - "Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Filipino,Indonesian,Malay": 0.26 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.19333333333333333, - "Spanish,English,Chinese,Filipino": 0.15333333333333332, - "Spanish,English,Chinese,Indonesian": 0.20666666666666667, - "Spanish,English,Chinese,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Malay": 0.18, - "Spanish,English,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Filipino,Malay": 0.14, - "Spanish,English,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Malay": 0.15333333333333332, - "Spanish,Chinese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "English,Chinese,Vietnamese,Malay": 0.18, - "English,Chinese,Filipino,Indonesian": 0.18, - "English,Chinese,Filipino,Malay": 0.12, - "English,Chinese,Indonesian,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "English,Vietnamese,Filipino,Malay": 0.16, - "English,Vietnamese,Indonesian,Malay": 0.24, - "English,Filipino,Indonesian,Malay": 0.18, - "Chinese,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.2, - "Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 + "Indonesian,English,Filipino": 0.19886363636363635, + "Indonesian,English,Spanish": 0.3465909090909091, + "Indonesian,English,Chinese": 0.11931818181818182, + "Indonesian,English,Malay": 0.2897727272727273, + "Indonesian,English,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Spanish": 0.25, + "Indonesian,Filipino,Chinese": 0.13068181818181818, + "Indonesian,Filipino,Malay": 0.3181818181818182, + "Indonesian,Filipino,Vietnamese": 0.2556818181818182, + "Indonesian,Spanish,Chinese": 0.14204545454545456, + "Indonesian,Spanish,Malay": 0.30113636363636365, + "Indonesian,Spanish,Vietnamese": 0.2556818181818182, + "Indonesian,Chinese,Malay": 0.17045454545454544, + "Indonesian,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,Malay,Vietnamese": 0.29545454545454547, + "English,Filipino,Spanish": 0.2784090909090909, + "English,Filipino,Chinese": 0.08522727272727272, + "English,Filipino,Malay": 0.2215909090909091, + "English,Filipino,Vietnamese": 0.14772727272727273, + "English,Spanish,Chinese": 0.17045454545454544, + "English,Spanish,Malay": 0.3068181818181818, + "English,Spanish,Vietnamese": 0.24431818181818182, + "English,Chinese,Malay": 0.11363636363636363, + "English,Chinese,Vietnamese": 0.07386363636363637, + "English,Malay,Vietnamese": 0.1590909090909091, + "Filipino,Spanish,Chinese": 0.10795454545454546, + "Filipino,Spanish,Malay": 0.23295454545454544, + "Filipino,Spanish,Vietnamese": 0.22727272727272727, + "Filipino,Chinese,Malay": 0.11931818181818182, + "Filipino,Chinese,Vietnamese": 0.125, + "Filipino,Malay,Vietnamese": 0.2556818181818182, + "Spanish,Chinese,Malay": 0.10227272727272728, + "Spanish,Chinese,Vietnamese": 0.09659090909090909, + "Spanish,Malay,Vietnamese": 0.2215909090909091, + "Chinese,Malay,Vietnamese": 0.13636363636363635 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.16477272727272727, + "Indonesian,English,Filipino,Chinese": 0.03977272727272727, + "Indonesian,English,Filipino,Malay": 0.14204545454545456, + "Indonesian,English,Filipino,Vietnamese": 0.10227272727272728, + "Indonesian,English,Spanish,Chinese": 0.09659090909090909, + "Indonesian,English,Spanish,Malay": 0.23863636363636365, + "Indonesian,English,Spanish,Vietnamese": 0.16477272727272727, + "Indonesian,English,Chinese,Malay": 0.08522727272727272, + "Indonesian,English,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,English,Malay,Vietnamese": 0.125, + "Indonesian,Filipino,Spanish,Chinese": 0.06818181818181818, + "Indonesian,Filipino,Spanish,Malay": 0.17613636363636365, + "Indonesian,Filipino,Spanish,Vietnamese": 0.1590909090909091, + "Indonesian,Filipino,Chinese,Malay": 0.07954545454545454, + "Indonesian,Filipino,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,Filipino,Malay,Vietnamese": 0.19318181818181818, + "Indonesian,Spanish,Chinese,Malay": 0.09090909090909091, + "Indonesian,Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,Spanish,Malay,Vietnamese": 0.17045454545454544, + "Indonesian,Chinese,Malay,Vietnamese": 0.09659090909090909, + "English,Filipino,Spanish,Chinese": 0.06818181818181818, + "English,Filipino,Spanish,Malay": 0.17045454545454544, + "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, + "English,Filipino,Chinese,Malay": 0.05113636363636364, + "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, + "English,Filipino,Malay,Vietnamese": 0.10795454545454546, + "English,Spanish,Chinese,Malay": 0.08522727272727272, + "English,Spanish,Chinese,Vietnamese": 0.0625, + "English,Spanish,Malay,Vietnamese": 0.14772727272727273, + "English,Chinese,Malay,Vietnamese": 0.0625, + "Filipino,Spanish,Chinese,Malay": 0.05113636363636364, + "Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, + "Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, + "Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 + "Indonesian,English,Filipino,Spanish,Chinese": 0.03409090909090909, + "Indonesian,English,Filipino,Spanish,Malay": 0.125, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, + "Indonesian,English,Filipino,Chinese,Malay": 0.028409090909090908, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,English,Spanish,Chinese,Malay": 0.07386363636363637, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.0625, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, + "English,Filipino,Spanish,Chinese,Malay": 0.03409090909090909, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, + "English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, + "English,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.08, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.022727272727272728, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, - "AC3_2": 0.47327404852536004, - "AC3_3": 0.3428451782270894, - "AC3_4": 0.25608007482845363, - "AC3_5": 0.1949046394591922, - "AC3_6": 0.15031177289193132, - "AC3_7": 0.11754089112406915 + "AC3_2": 0.39312472622166406, + "AC3_3": 0.25984408029395195, + "AC3_4": 0.16749060119194473, + "AC3_5": 0.10376292866511384, + "AC3_6": 0.057188972242627374, + "AC3_7": 0.022087067856240646 }, "prompt_4": { - "overall_acc": 0.5257142857142857, + "overall_acc": 0.387987012987013, "language_acc": { - "Spanish": 0.6, - "English": 0.6733333333333333, - "Chinese": 0.6066666666666667, - "Vietnamese": 0.46, - "Filipino": 0.4066666666666667, - "Indonesian": 0.48, - "Malay": 0.4533333333333333 + "Indonesian": 0.3693181818181818, + "English": 0.5965909090909091, + "Filipino": 0.3806818181818182, + "Spanish": 0.48863636363636365, + "Chinese": 0.25, + "Malay": 0.32954545454545453, + "Vietnamese": 0.30113636363636365 }, - "consistency_score_2": 0.5326984126984127, - "consistency_score_3": 0.3535238095238095, - "consistency_score_4": 0.25447619047619047, - "consistency_score_5": 0.18888888888888888, - "consistency_score_6": 0.13999999999999999, - "consistency_score_7": 0.1, + "consistency_score_2": 0.3898809523809524, + "consistency_score_3": 0.1876623376623377, + "consistency_score_4": 0.10048701298701297, + "consistency_score_5": 0.056006493506493504, + "consistency_score_6": 0.02922077922077922, + "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6733333333333333, - "Spanish,Chinese": 0.5866666666666667, - "Spanish,Vietnamese": 0.54, - "Spanish,Filipino": 0.43333333333333335, - "Spanish,Indonesian": 0.5333333333333333, - "Spanish,Malay": 0.49333333333333335, - "English,Chinese": 0.66, - "English,Vietnamese": 0.5733333333333334, - "English,Filipino": 0.4533333333333333, - "English,Indonesian": 0.5733333333333334, - "English,Malay": 0.5266666666666666, - "Chinese,Vietnamese": 0.4866666666666667, - "Chinese,Filipino": 0.46, - "Chinese,Indonesian": 0.5266666666666666, - "Chinese,Malay": 0.5333333333333333, - "Vietnamese,Filipino": 0.4066666666666667, - "Vietnamese,Indonesian": 0.56, - "Vietnamese,Malay": 0.56, - "Filipino,Indonesian": 0.47333333333333333, - "Filipino,Malay": 0.52, - "Indonesian,Malay": 0.6133333333333333 + "Indonesian,English": 0.42613636363636365, + "Indonesian,Filipino": 0.4431818181818182, + "Indonesian,Spanish": 0.5113636363636364, + "Indonesian,Chinese": 0.26136363636363635, + "Indonesian,Malay": 0.5909090909090909, + "Indonesian,Vietnamese": 0.4090909090909091, + "English,Filipino": 0.36363636363636365, + "English,Spanish": 0.6477272727272727, + "English,Chinese": 0.2727272727272727, + "English,Malay": 0.42045454545454547, + "English,Vietnamese": 0.2840909090909091, + "Filipino,Spanish": 0.42045454545454547, + "Filipino,Chinese": 0.24431818181818182, + "Filipino,Malay": 0.4659090909090909, + "Filipino,Vietnamese": 0.48863636363636365, + "Spanish,Chinese": 0.2556818181818182, + "Spanish,Malay": 0.39204545454545453, + "Spanish,Vietnamese": 0.4090909090909091, + "Chinese,Malay": 0.25, + "Chinese,Vietnamese": 0.20454545454545456, + "Malay,Vietnamese": 0.42613636363636365 }, "3_combine": { - "Spanish,English,Chinese": 0.49333333333333335, - "Spanish,English,Vietnamese": 0.44, - "Spanish,English,Filipino": 0.34, - "Spanish,English,Indonesian": 0.42, - "Spanish,English,Malay": 0.37333333333333335, - "Spanish,Chinese,Vietnamese": 0.36666666666666664, - "Spanish,Chinese,Filipino": 0.29333333333333333, - "Spanish,Chinese,Indonesian": 0.37333333333333335, - "Spanish,Chinese,Malay": 0.36666666666666664, - "Spanish,Vietnamese,Filipino": 0.26, - "Spanish,Vietnamese,Indonesian": 0.36666666666666664, - "Spanish,Vietnamese,Malay": 0.36, - "Spanish,Filipino,Indonesian": 0.28, - "Spanish,Filipino,Malay": 0.2866666666666667, - "Spanish,Indonesian,Malay": 0.36666666666666664, - "English,Chinese,Vietnamese": 0.4066666666666667, - "English,Chinese,Filipino": 0.3333333333333333, - "English,Chinese,Indonesian": 0.41333333333333333, - "English,Chinese,Malay": 0.4, - "English,Vietnamese,Filipino": 0.2866666666666667, - "English,Vietnamese,Indonesian": 0.4066666666666667, - "English,Vietnamese,Malay": 0.38, - "English,Filipino,Indonesian": 0.32, - "English,Filipino,Malay": 0.31333333333333335, - "English,Indonesian,Malay": 0.4066666666666667, - "Chinese,Vietnamese,Filipino": 0.24666666666666667, - "Chinese,Vietnamese,Indonesian": 0.36, - "Chinese,Vietnamese,Malay": 0.35333333333333333, - "Chinese,Filipino,Indonesian": 0.30666666666666664, - "Chinese,Filipino,Malay": 0.30666666666666664, - "Chinese,Indonesian,Malay": 0.4, - "Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Vietnamese,Filipino,Malay": 0.30666666666666664, - "Vietnamese,Indonesian,Malay": 0.41333333333333333, - "Filipino,Indonesian,Malay": 0.34 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.32666666666666666, - "Spanish,English,Chinese,Filipino": 0.26666666666666666, - "Spanish,English,Chinese,Indonesian": 0.32666666666666666, - "Spanish,English,Chinese,Malay": 0.30666666666666664, - "Spanish,English,Vietnamese,Filipino": 0.22666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.30666666666666664, - "Spanish,English,Vietnamese,Malay": 0.2866666666666667, - "Spanish,English,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Indonesian,Malay": 0.3, - "Spanish,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian": 0.28, - "Spanish,Chinese,Vietnamese,Malay": 0.28, - "Spanish,Chinese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.2, - "Spanish,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.31333333333333335, - "English,Chinese,Vietnamese,Malay": 0.29333333333333333, - "English,Chinese,Filipino,Indonesian": 0.24, - "English,Chinese,Filipino,Malay": 0.23333333333333334, - "English,Chinese,Indonesian,Malay": 0.31333333333333335, - "English,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "English,Vietnamese,Filipino,Malay": 0.23333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.32, - "English,Filipino,Indonesian,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.24, - "Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 + "Indonesian,English,Filipino": 0.21022727272727273, + "Indonesian,English,Spanish": 0.3465909090909091, + "Indonesian,English,Chinese": 0.10795454545454546, + "Indonesian,English,Malay": 0.2897727272727273, + "Indonesian,English,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Spanish": 0.24431818181818182, + "Indonesian,Filipino,Chinese": 0.11931818181818182, + "Indonesian,Filipino,Malay": 0.30113636363636365, + "Indonesian,Filipino,Vietnamese": 0.22727272727272727, + "Indonesian,Spanish,Chinese": 0.13068181818181818, + "Indonesian,Spanish,Malay": 0.29545454545454547, + "Indonesian,Spanish,Vietnamese": 0.25, + "Indonesian,Chinese,Malay": 0.16477272727272727, + "Indonesian,Chinese,Vietnamese": 0.10227272727272728, + "Indonesian,Malay,Vietnamese": 0.2784090909090909, + "English,Filipino,Spanish": 0.2784090909090909, + "English,Filipino,Chinese": 0.08522727272727272, + "English,Filipino,Malay": 0.2215909090909091, + "English,Filipino,Vietnamese": 0.14772727272727273, + "English,Spanish,Chinese": 0.17045454545454544, + "English,Spanish,Malay": 0.29545454545454547, + "English,Spanish,Vietnamese": 0.2556818181818182, + "English,Chinese,Malay": 0.09659090909090909, + "English,Chinese,Vietnamese": 0.056818181818181816, + "English,Malay,Vietnamese": 0.1590909090909091, + "Filipino,Spanish,Chinese": 0.10795454545454546, + "Filipino,Spanish,Malay": 0.22727272727272727, + "Filipino,Spanish,Vietnamese": 0.2215909090909091, + "Filipino,Chinese,Malay": 0.125, + "Filipino,Chinese,Vietnamese": 0.11931818181818182, + "Filipino,Malay,Vietnamese": 0.25, + "Spanish,Chinese,Malay": 0.09659090909090909, + "Spanish,Chinese,Vietnamese": 0.09090909090909091, + "Spanish,Malay,Vietnamese": 0.21022727272727273, + "Chinese,Malay,Vietnamese": 0.11363636363636363 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.17613636363636365, + "Indonesian,English,Filipino,Chinese": 0.045454545454545456, + "Indonesian,English,Filipino,Malay": 0.1534090909090909, + "Indonesian,English,Filipino,Vietnamese": 0.09659090909090909, + "Indonesian,English,Spanish,Chinese": 0.09090909090909091, + "Indonesian,English,Spanish,Malay": 0.23295454545454544, + "Indonesian,English,Spanish,Vietnamese": 0.16477272727272727, + "Indonesian,English,Chinese,Malay": 0.07954545454545454, + "Indonesian,English,Chinese,Vietnamese": 0.03977272727272727, + "Indonesian,English,Malay,Vietnamese": 0.125, + "Indonesian,Filipino,Spanish,Chinese": 0.056818181818181816, + "Indonesian,Filipino,Spanish,Malay": 0.17613636363636365, + "Indonesian,Filipino,Spanish,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Chinese,Malay": 0.07954545454545454, + "Indonesian,Filipino,Chinese,Vietnamese": 0.0625, + "Indonesian,Filipino,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Spanish,Chinese,Malay": 0.07954545454545454, + "Indonesian,Spanish,Chinese,Vietnamese": 0.0625, + "Indonesian,Spanish,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Chinese,Malay,Vietnamese": 0.07386363636363637, + "English,Filipino,Spanish,Chinese": 0.07386363636363637, + "English,Filipino,Spanish,Malay": 0.16477272727272727, + "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, + "English,Filipino,Chinese,Malay": 0.05113636363636364, + "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, + "English,Filipino,Malay,Vietnamese": 0.10227272727272728, + "English,Spanish,Chinese,Malay": 0.07954545454545454, + "English,Spanish,Chinese,Vietnamese": 0.05113636363636364, + "English,Spanish,Malay,Vietnamese": 0.14772727272727273, + "English,Chinese,Malay,Vietnamese": 0.05113636363636364, + "Filipino,Spanish,Chinese,Malay": 0.056818181818181816, + "Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, + "Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, + "Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, + "Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.24, - "Spanish,English,Chinese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Filipino,Malay": 0.18, - "Spanish,English,Chinese,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.24, - "Spanish,English,Filipino,Indonesian,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "English,Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 + "Indonesian,English,Filipino,Spanish,Chinese": 0.03977272727272727, + "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, + "Indonesian,English,Filipino,Chinese,Malay": 0.03409090909090909, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Spanish,Chinese,Malay": 0.06818181818181818, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, + "English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.2, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, - "AC3_2": 0.5291833061459058, - "AC3_3": 0.42275810839460154, - "AC3_4": 0.3429464285274718, - "AC3_5": 0.2779209239948711, - "AC3_6": 0.2211158797951113, - "AC3_7": 0.16803652965351015 + "AC3_2": 0.3889316769686338, + "AC3_3": 0.2529684077467979, + "AC3_4": 0.159630416078295, + "AC3_5": 0.09788337802586573, + "AC3_6": 0.05434837534702182, + "AC3_7": 0.02208056170917894 }, "prompt_5": { - "overall_acc": 0.5038095238095238, + "overall_acc": 0.3928571428571429, "language_acc": { - "Spanish": 0.5066666666666667, - "English": 0.6333333333333333, - "Chinese": 0.5733333333333334, - "Vietnamese": 0.47333333333333333, - "Filipino": 0.38666666666666666, - "Indonesian": 0.47333333333333333, - "Malay": 0.48 + "Indonesian": 0.3693181818181818, + "English": 0.6022727272727273, + "Filipino": 0.36363636363636365, + "Spanish": 0.5056818181818182, + "Chinese": 0.2727272727272727, + "Malay": 0.3352272727272727, + "Vietnamese": 0.30113636363636365 }, - "consistency_score_2": 0.5514285714285714, - "consistency_score_3": 0.3817142857142857, - "consistency_score_4": 0.2866666666666666, - "consistency_score_5": 0.22317460317460316, - "consistency_score_6": 0.1761904761904762, - "consistency_score_7": 0.14, + "consistency_score_2": 0.38988095238095244, + "consistency_score_3": 0.18879870129870135, + "consistency_score_4": 0.10162337662337663, + "consistency_score_5": 0.056547619047619034, + "consistency_score_6": 0.029220779220779224, + "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6466666666666666, - "Spanish,Chinese": 0.54, - "Spanish,Vietnamese": 0.5333333333333333, - "Spanish,Filipino": 0.43333333333333335, - "Spanish,Indonesian": 0.56, - "Spanish,Malay": 0.54, - "English,Chinese": 0.64, - "English,Vietnamese": 0.5466666666666666, - "English,Filipino": 0.49333333333333335, - "English,Indonesian": 0.56, - "English,Malay": 0.5466666666666666, - "Chinese,Vietnamese": 0.52, - "Chinese,Filipino": 0.4533333333333333, - "Chinese,Indonesian": 0.5666666666666667, - "Chinese,Malay": 0.5866666666666667, - "Vietnamese,Filipino": 0.4533333333333333, - "Vietnamese,Indonesian": 0.62, - "Vietnamese,Malay": 0.6133333333333333, - "Filipino,Indonesian": 0.52, - "Filipino,Malay": 0.52, - "Indonesian,Malay": 0.6866666666666666 + "Indonesian,English": 0.42045454545454547, + "Indonesian,Filipino": 0.4659090909090909, + "Indonesian,Spanish": 0.48295454545454547, + "Indonesian,Chinese": 0.23863636363636365, + "Indonesian,Malay": 0.5909090909090909, + "Indonesian,Vietnamese": 0.42613636363636365, + "English,Filipino": 0.3465909090909091, + "English,Spanish": 0.6647727272727273, + "English,Chinese": 0.24431818181818182, + "English,Malay": 0.42045454545454547, + "English,Vietnamese": 0.30113636363636365, + "Filipino,Spanish": 0.42045454545454547, + "Filipino,Chinese": 0.25, + "Filipino,Malay": 0.4602272727272727, + "Filipino,Vietnamese": 0.48295454545454547, + "Spanish,Chinese": 0.23295454545454544, + "Spanish,Malay": 0.42045454545454547, + "Spanish,Vietnamese": 0.3977272727272727, + "Chinese,Malay": 0.26136363636363635, + "Chinese,Vietnamese": 0.23295454545454544, + "Malay,Vietnamese": 0.42613636363636365 }, "3_combine": { - "Spanish,English,Chinese": 0.44666666666666666, - "Spanish,English,Vietnamese": 0.41333333333333333, - "Spanish,English,Filipino": 0.34, - "Spanish,English,Indonesian": 0.42, - "Spanish,English,Malay": 0.3933333333333333, - "Spanish,Chinese,Vietnamese": 0.36666666666666664, - "Spanish,Chinese,Filipino": 0.26, - "Spanish,Chinese,Indonesian": 0.36666666666666664, - "Spanish,Chinese,Malay": 0.37333333333333335, - "Spanish,Vietnamese,Filipino": 0.28, - "Spanish,Vietnamese,Indonesian": 0.41333333333333333, - "Spanish,Vietnamese,Malay": 0.42, - "Spanish,Filipino,Indonesian": 0.32, - "Spanish,Filipino,Malay": 0.3, - "Spanish,Indonesian,Malay": 0.44666666666666666, - "English,Chinese,Vietnamese": 0.41333333333333333, - "English,Chinese,Filipino": 0.3333333333333333, - "English,Chinese,Indonesian": 0.42, - "English,Chinese,Malay": 0.4266666666666667, - "English,Vietnamese,Filipino": 0.32, - "English,Vietnamese,Indonesian": 0.4266666666666667, - "English,Vietnamese,Malay": 0.4266666666666667, - "English,Filipino,Indonesian": 0.35333333333333333, - "English,Filipino,Malay": 0.34, - "English,Indonesian,Malay": 0.4533333333333333, - "Chinese,Vietnamese,Filipino": 0.29333333333333333, - "Chinese,Vietnamese,Indonesian": 0.4, - "Chinese,Vietnamese,Malay": 0.42, - "Chinese,Filipino,Indonesian": 0.3466666666666667, - "Chinese,Filipino,Malay": 0.34, - "Chinese,Indonesian,Malay": 0.47333333333333333, - "Vietnamese,Filipino,Indonesian": 0.36, - "Vietnamese,Filipino,Malay": 0.35333333333333333, - "Vietnamese,Indonesian,Malay": 0.4866666666666667, - "Filipino,Indonesian,Malay": 0.41333333333333333 + "Indonesian,English,Filipino": 0.20454545454545456, + "Indonesian,English,Spanish": 0.3352272727272727, + "Indonesian,English,Chinese": 0.09090909090909091, + "Indonesian,English,Malay": 0.2840909090909091, + "Indonesian,English,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Spanish": 0.24431818181818182, + "Indonesian,Filipino,Chinese": 0.125, + "Indonesian,Filipino,Malay": 0.30113636363636365, + "Indonesian,Filipino,Vietnamese": 0.25, + "Indonesian,Spanish,Chinese": 0.11363636363636363, + "Indonesian,Spanish,Malay": 0.29545454545454547, + "Indonesian,Spanish,Vietnamese": 0.2556818181818182, + "Indonesian,Chinese,Malay": 0.16477272727272727, + "Indonesian,Chinese,Vietnamese": 0.10795454545454546, + "Indonesian,Malay,Vietnamese": 0.2784090909090909, + "English,Filipino,Spanish": 0.2727272727272727, + "English,Filipino,Chinese": 0.06818181818181818, + "English,Filipino,Malay": 0.2159090909090909, + "English,Filipino,Vietnamese": 0.1534090909090909, + "English,Spanish,Chinese": 0.1534090909090909, + "English,Spanish,Malay": 0.3125, + "English,Spanish,Vietnamese": 0.2556818181818182, + "English,Chinese,Malay": 0.10795454545454546, + "English,Chinese,Vietnamese": 0.07386363636363637, + "English,Malay,Vietnamese": 0.1590909090909091, + "Filipino,Spanish,Chinese": 0.09659090909090909, + "Filipino,Spanish,Malay": 0.23295454545454544, + "Filipino,Spanish,Vietnamese": 0.23295454545454544, + "Filipino,Chinese,Malay": 0.13636363636363635, + "Filipino,Chinese,Vietnamese": 0.11931818181818182, + "Filipino,Malay,Vietnamese": 0.2556818181818182, + "Spanish,Chinese,Malay": 0.10795454545454546, + "Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Spanish,Malay,Vietnamese": 0.2215909090909091, + "Chinese,Malay,Vietnamese": 0.125 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.32, - "Spanish,English,Chinese,Filipino": 0.22, - "Spanish,English,Chinese,Indonesian": 0.30666666666666664, - "Spanish,English,Chinese,Malay": 0.30666666666666664, - "Spanish,English,Vietnamese,Filipino": 0.24, - "Spanish,English,Vietnamese,Indonesian": 0.34, - "Spanish,English,Vietnamese,Malay": 0.32666666666666666, - "Spanish,English,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Indonesian,Malay": 0.3466666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.2, - "Spanish,Chinese,Vietnamese,Indonesian": 0.3, - "Spanish,Chinese,Vietnamese,Malay": 0.31333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.32666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,Vietnamese,Filipino,Malay": 0.24, - "Spanish,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino": 0.24, - "English,Chinese,Vietnamese,Indonesian": 0.3333333333333333, - "English,Chinese,Vietnamese,Malay": 0.3466666666666667, - "English,Chinese,Filipino,Indonesian": 0.26, - "English,Chinese,Filipino,Malay": 0.26666666666666666, - "English,Chinese,Indonesian,Malay": 0.36666666666666664, - "English,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "English,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.38, - "English,Filipino,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Indonesian,Malay": 0.36, - "Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333 + "Indonesian,English,Filipino,Spanish": 0.16477272727272727, + "Indonesian,English,Filipino,Chinese": 0.03409090909090909, + "Indonesian,English,Filipino,Malay": 0.14204545454545456, + "Indonesian,English,Filipino,Vietnamese": 0.09659090909090909, + "Indonesian,English,Spanish,Chinese": 0.07386363636363637, + "Indonesian,English,Spanish,Malay": 0.22727272727272727, + "Indonesian,English,Spanish,Vietnamese": 0.16477272727272727, + "Indonesian,English,Chinese,Malay": 0.07386363636363637, + "Indonesian,English,Chinese,Vietnamese": 0.03977272727272727, + "Indonesian,English,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Spanish,Chinese": 0.0625, + "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Vietnamese": 0.1590909090909091, + "Indonesian,Filipino,Chinese,Malay": 0.07954545454545454, + "Indonesian,Filipino,Chinese,Vietnamese": 0.06818181818181818, + "Indonesian,Filipino,Malay,Vietnamese": 0.17613636363636365, + "Indonesian,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,Spanish,Chinese,Vietnamese": 0.0625, + "Indonesian,Spanish,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Chinese,Malay,Vietnamese": 0.08522727272727272, + "English,Filipino,Spanish,Chinese": 0.056818181818181816, + "English,Filipino,Spanish,Malay": 0.17045454545454544, + "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, + "English,Filipino,Chinese,Malay": 0.05113636363636364, + "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, + "English,Filipino,Malay,Vietnamese": 0.10795454545454546, + "English,Spanish,Chinese,Malay": 0.07954545454545454, + "English,Spanish,Chinese,Vietnamese": 0.05113636363636364, + "English,Spanish,Malay,Vietnamese": 0.14772727272727273, + "English,Chinese,Malay,Vietnamese": 0.056818181818181816, + "Filipino,Spanish,Chinese,Malay": 0.0625, + "Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, + "Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, + "Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, + "Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.26, - "Spanish,English,Chinese,Vietnamese,Malay": 0.26666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.2, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Spanish,English,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335 + "Indonesian,English,Filipino,Spanish,Chinese": 0.03409090909090909, + "Indonesian,English,Filipino,Spanish,Malay": 0.11931818181818182, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, + "Indonesian,English,Filipino,Chinese,Malay": 0.022727272727272728, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Spanish,Chinese,Malay": 0.0625, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.05113636363636364, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.05113636363636364, + "English,Filipino,Spanish,Chinese,Malay": 0.03977272727272727, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "English,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, + "English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.022727272727272728, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.011363636363636364, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, - "AC3_2": 0.5265446105741348, - "AC3_3": 0.43434471309970757, - "AC3_4": 0.36541365457224684, - "AC3_5": 0.30932584732696633, - "AC3_6": 0.2610777644007152, - "AC3_7": 0.2191124260014692 + "AC3_2": 0.3913633894122495, + "AC3_3": 0.2550336907978269, + "AC3_4": 0.16147640930980742, + "AC3_5": 0.09886471142549386, + "AC3_6": 0.05439560438271683, + "AC3_7": 0.02208835340819019 } }, - "cross_logiqa": { + "sg_eval": { "prompt_1": { - "overall_acc": 0.4391233766233765, - "language_acc": { - "English": 0.4602272727272727, - "Filipino": 0.3181818181818182, - "Spanish": 0.4943181818181818, - "Malay": 0.39204545454545453, - "Chinese": 0.5454545454545454, - "Indonesian": 0.4318181818181818, - "Vietnamese": 0.4318181818181818 - }, - "consistency_score_2": 0.4586038961038961, - "consistency_score_3": 0.26980519480519477, - "consistency_score_4": 0.18003246753246746, - "consistency_score_5": 0.12851731601731603, - "consistency_score_6": 0.09415584415584417, - "consistency_score_7": 0.06818181818181818, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.35795454545454547, - "English,Spanish": 0.5056818181818182, - "English,Malay": 0.4318181818181818, - "English,Chinese": 0.5, - "English,Indonesian": 0.5340909090909091, - "English,Vietnamese": 0.48863636363636365, - "Filipino,Spanish": 0.42045454545454547, - "Filipino,Malay": 0.3522727272727273, - "Filipino,Chinese": 0.3977272727272727, - "Filipino,Indonesian": 0.38636363636363635, - "Filipino,Vietnamese": 0.3977272727272727, - "Spanish,Malay": 0.4602272727272727, - "Spanish,Chinese": 0.5454545454545454, - "Spanish,Indonesian": 0.5568181818181818, - "Spanish,Vietnamese": 0.48295454545454547, - "Malay,Chinese": 0.4318181818181818, - "Malay,Indonesian": 0.5397727272727273, - "Malay,Vietnamese": 0.4034090909090909, - "Chinese,Indonesian": 0.48863636363636365, - "Chinese,Vietnamese": 0.4375, - "Indonesian,Vietnamese": 0.5113636363636364 - }, - "3_combine": { - "English,Filipino,Spanish": 0.23295454545454544, - "English,Filipino,Malay": 0.17045454545454544, - "English,Filipino,Chinese": 0.2215909090909091, - "English,Filipino,Indonesian": 0.2215909090909091, - "English,Filipino,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay": 0.2784090909090909, - "English,Spanish,Chinese": 0.3352272727272727, - "English,Spanish,Indonesian": 0.35795454545454547, - "English,Spanish,Vietnamese": 0.32954545454545453, - "English,Malay,Chinese": 0.26704545454545453, - "English,Malay,Indonesian": 0.3125, - "English,Malay,Vietnamese": 0.23863636363636365, - "English,Chinese,Indonesian": 0.3181818181818182, - "English,Chinese,Vietnamese": 0.2840909090909091, - "English,Indonesian,Vietnamese": 0.3409090909090909, - "Filipino,Spanish,Malay": 0.19886363636363635, - "Filipino,Spanish,Chinese": 0.2727272727272727, - "Filipino,Spanish,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Vietnamese": 0.24431818181818182, - "Filipino,Malay,Chinese": 0.1875, - "Filipino,Malay,Indonesian": 0.2215909090909091, - "Filipino,Malay,Vietnamese": 0.19318181818181818, - "Filipino,Chinese,Indonesian": 0.23295454545454544, - "Filipino,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Indonesian,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Chinese": 0.2897727272727273, - "Spanish,Malay,Indonesian": 0.3409090909090909, - "Spanish,Malay,Vietnamese": 0.24431818181818182, - "Spanish,Chinese,Indonesian": 0.36363636363636365, - "Spanish,Chinese,Vietnamese": 0.3068181818181818, - "Spanish,Indonesian,Vietnamese": 0.3522727272727273, - "Malay,Chinese,Indonesian": 0.2897727272727273, - "Malay,Chinese,Vietnamese": 0.22727272727272727, - "Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Chinese,Indonesian,Vietnamese": 0.32386363636363635 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.125, - "English,Filipino,Spanish,Chinese": 0.18181818181818182, - "English,Filipino,Spanish,Indonesian": 0.18181818181818182, - "English,Filipino,Spanish,Vietnamese": 0.18181818181818182, - "English,Filipino,Malay,Chinese": 0.10795454545454546, - "English,Filipino,Malay,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Vietnamese": 0.11931818181818182, - "English,Filipino,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay,Chinese": 0.21022727272727273, - "English,Spanish,Malay,Indonesian": 0.22727272727272727, - "English,Spanish,Malay,Vietnamese": 0.19318181818181818, - "English,Spanish,Chinese,Indonesian": 0.26136363636363635, - "English,Spanish,Chinese,Vietnamese": 0.22727272727272727, - "English,Spanish,Indonesian,Vietnamese": 0.26704545454545453, - "English,Malay,Chinese,Indonesian": 0.21022727272727273, - "English,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.14772727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Chinese,Indonesian": 0.21022727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "Malay,Chinese,Indonesian,Vietnamese": 0.1875 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese,Indonesian": 0.17045454545454544, - "English,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - } - }, - "AC3_2": 0.44865227443864514, - "AC3_3": 0.334244585227893, - "AC3_4": 0.25536854986715835, - "AC3_5": 0.1988404231261756, - "AC3_6": 0.15506335366441334, - "AC3_7": 0.11803636361309629 + "accuracy": 0.5631067961165048 }, "prompt_2": { - "overall_acc": 0.43993506493506496, - "language_acc": { - "English": 0.5056818181818182, - "Filipino": 0.3068181818181818, - "Spanish": 0.4715909090909091, - "Malay": 0.42045454545454547, - "Chinese": 0.5227272727272727, - "Indonesian": 0.4602272727272727, - "Vietnamese": 0.39204545454545453 - }, - "consistency_score_2": 0.4826839826839827, - "consistency_score_3": 0.30016233766233774, - "consistency_score_4": 0.21233766233766233, - "consistency_score_5": 0.16260822510822512, - "consistency_score_6": 0.13068181818181818, - "consistency_score_7": 0.10795454545454546, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3693181818181818, - "English,Spanish": 0.5852272727272727, - "English,Malay": 0.48295454545454547, - "English,Chinese": 0.44886363636363635, - "English,Indonesian": 0.5056818181818182, - "English,Vietnamese": 0.5284090909090909, - "Filipino,Spanish": 0.4318181818181818, - "Filipino,Malay": 0.42613636363636365, - "Filipino,Chinese": 0.4090909090909091, - "Filipino,Indonesian": 0.42613636363636365, - "Filipino,Vietnamese": 0.4147727272727273, - "Spanish,Malay": 0.5511363636363636, - "Spanish,Chinese": 0.44886363636363635, - "Spanish,Indonesian": 0.5397727272727273, - "Spanish,Vietnamese": 0.5511363636363636, - "Malay,Chinese": 0.48295454545454547, - "Malay,Indonesian": 0.5568181818181818, - "Malay,Vietnamese": 0.5056818181818182, - "Chinese,Indonesian": 0.48295454545454547, - "Chinese,Vietnamese": 0.42045454545454547, - "Indonesian,Vietnamese": 0.5681818181818182 - }, - "3_combine": { - "English,Filipino,Spanish": 0.26704545454545453, - "English,Filipino,Malay": 0.21022727272727273, - "English,Filipino,Chinese": 0.20454545454545456, - "English,Filipino,Indonesian": 0.23295454545454544, - "English,Filipino,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay": 0.3693181818181818, - "English,Spanish,Chinese": 0.30113636363636365, - "English,Spanish,Indonesian": 0.3806818181818182, - "English,Spanish,Vietnamese": 0.38636363636363635, - "English,Malay,Chinese": 0.2897727272727273, - "English,Malay,Indonesian": 0.3125, - "English,Malay,Vietnamese": 0.32386363636363635, - "English,Chinese,Indonesian": 0.3181818181818182, - "English,Chinese,Vietnamese": 0.2897727272727273, - "English,Indonesian,Vietnamese": 0.36363636363636365, - "Filipino,Spanish,Malay": 0.26136363636363635, - "Filipino,Spanish,Chinese": 0.22727272727272727, - "Filipino,Spanish,Indonesian": 0.2897727272727273, - "Filipino,Spanish,Vietnamese": 0.2784090909090909, - "Filipino,Malay,Chinese": 0.24431818181818182, - "Filipino,Malay,Indonesian": 0.2897727272727273, - "Filipino,Malay,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Indonesian": 0.24431818181818182, - "Filipino,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Chinese": 0.32954545454545453, - "Spanish,Malay,Indonesian": 0.3806818181818182, - "Spanish,Malay,Vietnamese": 0.3693181818181818, - "Spanish,Chinese,Indonesian": 0.32386363636363635, - "Spanish,Chinese,Vietnamese": 0.2897727272727273, - "Spanish,Indonesian,Vietnamese": 0.38636363636363635, - "Malay,Chinese,Indonesian": 0.3409090909090909, - "Malay,Chinese,Vietnamese": 0.3125, - "Malay,Indonesian,Vietnamese": 0.375, - "Chinese,Indonesian,Vietnamese": 0.32386363636363635 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.1590909090909091, - "English,Filipino,Spanish,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Vietnamese": 0.19886363636363635, - "English,Filipino,Malay,Chinese": 0.14204545454545456, - "English,Filipino,Malay,Indonesian": 0.17045454545454544, - "English,Filipino,Malay,Vietnamese": 0.17045454545454544, - "English,Filipino,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay,Chinese": 0.23295454545454544, - "English,Spanish,Malay,Indonesian": 0.2727272727272727, - "English,Spanish,Malay,Vietnamese": 0.26136363636363635, - "English,Spanish,Chinese,Indonesian": 0.25, - "English,Spanish,Chinese,Vietnamese": 0.22727272727272727, - "English,Spanish,Indonesian,Vietnamese": 0.2897727272727273, - "English,Malay,Chinese,Indonesian": 0.23295454545454544, - "English,Malay,Chinese,Vietnamese": 0.23295454545454544, - "English,Malay,Indonesian,Vietnamese": 0.25, - "English,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay,Chinese": 0.17045454545454544, - "Filipino,Spanish,Malay,Indonesian": 0.21022727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese,Indonesian": 0.1875, - "Filipino,Malay,Chinese,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Chinese,Indonesian": 0.2727272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.24431818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.29545454545454547, - "Spanish,Chinese,Indonesian,Vietnamese": 0.25, - "Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "English,Filipino,Malay,Chinese,Indonesian": 0.125, - "English,Filipino,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese,Indonesian": 0.21022727272727273, - "English,Spanish,Malay,Chinese,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2159090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 - } - }, - "AC3_2": 0.46031915293788206, - "AC3_3": 0.3568501579096404, - "AC3_4": 0.2864286037067421, - "AC3_5": 0.2374503583135098, - "AC3_6": 0.20150653041858504, - "AC3_7": 0.1733670033353607 - }, - "prompt_3": { - "overall_acc": 0.4375, - "language_acc": { - "English": 0.42045454545454547, - "Filipino": 0.29545454545454547, - "Spanish": 0.4318181818181818, - "Malay": 0.4431818181818182, - "Chinese": 0.5340909090909091, - "Indonesian": 0.48863636363636365, - "Vietnamese": 0.44886363636363635 - }, - "consistency_score_2": 0.4372294372294373, - "consistency_score_3": 0.24383116883116887, - "consistency_score_4": 0.1566558441558442, - "consistency_score_5": 0.1103896103896104, - "consistency_score_6": 0.0836038961038961, - "consistency_score_7": 0.06818181818181818, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.48863636363636365, - "English,Malay": 0.4147727272727273, - "English,Chinese": 0.48863636363636365, - "English,Indonesian": 0.4431818181818182, - "English,Vietnamese": 0.4318181818181818, - "Filipino,Spanish": 0.3806818181818182, - "Filipino,Malay": 0.39204545454545453, - "Filipino,Chinese": 0.38636363636363635, - "Filipino,Indonesian": 0.35795454545454547, - "Filipino,Vietnamese": 0.3806818181818182, - "Spanish,Malay": 0.48295454545454547, - "Spanish,Chinese": 0.4375, - "Spanish,Indonesian": 0.4602272727272727, - "Spanish,Vietnamese": 0.5056818181818182, - "Malay,Chinese": 0.4375, - "Malay,Indonesian": 0.5454545454545454, - "Malay,Vietnamese": 0.5284090909090909, - "Chinese,Indonesian": 0.42613636363636365, - "Chinese,Vietnamese": 0.38636363636363635, - "Indonesian,Vietnamese": 0.4715909090909091 - }, - "3_combine": { - "English,Filipino,Spanish": 0.20454545454545456, - "English,Filipino,Malay": 0.19318181818181818, - "English,Filipino,Chinese": 0.19318181818181818, - "English,Filipino,Indonesian": 0.18181818181818182, - "English,Filipino,Vietnamese": 0.1875, - "English,Spanish,Malay": 0.2840909090909091, - "English,Spanish,Chinese": 0.2727272727272727, - "English,Spanish,Indonesian": 0.2840909090909091, - "English,Spanish,Vietnamese": 0.2897727272727273, - "English,Malay,Chinese": 0.24431818181818182, - "English,Malay,Indonesian": 0.2840909090909091, - "English,Malay,Vietnamese": 0.2556818181818182, - "English,Chinese,Indonesian": 0.26136363636363635, - "English,Chinese,Vietnamese": 0.24431818181818182, - "English,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay": 0.2215909090909091, - "Filipino,Spanish,Chinese": 0.19318181818181818, - "Filipino,Spanish,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Chinese": 0.23295454545454544, - "Filipino,Malay,Indonesian": 0.23863636363636365, - "Filipino,Malay,Vietnamese": 0.23295454545454544, - "Filipino,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Chinese": 0.2556818181818182, - "Spanish,Malay,Indonesian": 0.3068181818181818, - "Spanish,Malay,Vietnamese": 0.3181818181818182, - "Spanish,Chinese,Indonesian": 0.23295454545454544, - "Spanish,Chinese,Vietnamese": 0.23295454545454544, - "Spanish,Indonesian,Vietnamese": 0.2840909090909091, - "Malay,Chinese,Indonesian": 0.26704545454545453, - "Malay,Chinese,Vietnamese": 0.2556818181818182, - "Malay,Indonesian,Vietnamese": 0.35795454545454547, - "Chinese,Indonesian,Vietnamese": 0.23863636363636365 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.1590909090909091, - "English,Filipino,Spanish,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese": 0.125, - "English,Filipino,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Malay,Vietnamese": 0.125, - "English,Filipino,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese": 0.16477272727272727, - "English,Spanish,Malay,Indonesian": 0.21022727272727273, - "English,Spanish,Malay,Vietnamese": 0.1875, - "English,Spanish,Chinese,Indonesian": 0.1590909090909091, - "English,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "English,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "English,Malay,Chinese,Indonesian": 0.18181818181818182, - "English,Malay,Chinese,Vietnamese": 0.1590909090909091, - "English,Malay,Indonesian,Vietnamese": 0.1875, - "English,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Chinese": 0.14204545454545456, - "Filipino,Spanish,Malay,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Indonesian,Vietnamese": 0.1875, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.18181818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.17045454545454544, - "Spanish,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - } - }, - "AC3_2": 0.43736467672080115, - "AC3_3": 0.3131403382909473, - "AC3_4": 0.23070355187373956, - "AC3_5": 0.17629629626411897, - "AC3_6": 0.14038161991075543, - "AC3_7": 0.11797752806655726 - }, - "prompt_4": { - "overall_acc": 0.44237012987012986, - "language_acc": { - "English": 0.4772727272727273, - "Filipino": 0.3181818181818182, - "Spanish": 0.4318181818181818, - "Malay": 0.3977272727272727, - "Chinese": 0.5625, - "Indonesian": 0.48295454545454547, - "Vietnamese": 0.42613636363636365 - }, - "consistency_score_2": 0.4488636363636364, - "consistency_score_3": 0.2498376623376624, - "consistency_score_4": 0.1571428571428571, - "consistency_score_5": 0.108495670995671, - "consistency_score_6": 0.08035714285714286, - "consistency_score_7": 0.0625, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3409090909090909, - "English,Spanish": 0.4943181818181818, - "English,Malay": 0.4090909090909091, - "English,Chinese": 0.5, - "English,Indonesian": 0.4772727272727273, - "English,Vietnamese": 0.4375, - "Filipino,Spanish": 0.36363636363636365, - "Filipino,Malay": 0.3977272727272727, - "Filipino,Chinese": 0.42045454545454547, - "Filipino,Indonesian": 0.42045454545454547, - "Filipino,Vietnamese": 0.4034090909090909, - "Spanish,Malay": 0.48863636363636365, - "Spanish,Chinese": 0.4659090909090909, - "Spanish,Indonesian": 0.5170454545454546, - "Spanish,Vietnamese": 0.5056818181818182, - "Malay,Chinese": 0.3977272727272727, - "Malay,Indonesian": 0.5397727272727273, - "Malay,Vietnamese": 0.4602272727272727, - "Chinese,Indonesian": 0.45454545454545453, - "Chinese,Vietnamese": 0.42045454545454547, - "Indonesian,Vietnamese": 0.5113636363636364 - }, - "3_combine": { - "English,Filipino,Spanish": 0.1875, - "English,Filipino,Malay": 0.1534090909090909, - "English,Filipino,Chinese": 0.2215909090909091, - "English,Filipino,Indonesian": 0.20454545454545456, - "English,Filipino,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay": 0.26136363636363635, - "English,Spanish,Chinese": 0.2897727272727273, - "English,Spanish,Indonesian": 0.3068181818181818, - "English,Spanish,Vietnamese": 0.2784090909090909, - "English,Malay,Chinese": 0.22727272727272727, - "English,Malay,Indonesian": 0.2556818181818182, - "English,Malay,Vietnamese": 0.23863636363636365, - "English,Chinese,Indonesian": 0.30113636363636365, - "English,Chinese,Vietnamese": 0.26136363636363635, - "English,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Malay": 0.19318181818181818, - "Filipino,Spanish,Chinese": 0.2215909090909091, - "Filipino,Spanish,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese": 0.20454545454545456, - "Filipino,Malay,Indonesian": 0.24431818181818182, - "Filipino,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Chinese,Indonesian": 0.23295454545454544, - "Filipino,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Indonesian,Vietnamese": 0.2556818181818182, - "Spanish,Malay,Chinese": 0.24431818181818182, - "Spanish,Malay,Indonesian": 0.32954545454545453, - "Spanish,Malay,Vietnamese": 0.2897727272727273, - "Spanish,Chinese,Indonesian": 0.2840909090909091, - "Spanish,Chinese,Vietnamese": 0.2727272727272727, - "Spanish,Indonesian,Vietnamese": 0.3352272727272727, - "Malay,Chinese,Indonesian": 0.2556818181818182, - "Malay,Chinese,Vietnamese": 0.22727272727272727, - "Malay,Indonesian,Vietnamese": 0.3352272727272727, - "Chinese,Indonesian,Vietnamese": 0.2784090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.09659090909090909, - "English,Filipino,Spanish,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese": 0.09659090909090909, - "English,Filipino,Malay,Indonesian": 0.10795454545454546, - "English,Filipino,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Chinese,Vietnamese": 0.125, - "English,Filipino,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Chinese": 0.1590909090909091, - "English,Spanish,Malay,Indonesian": 0.18181818181818182, - "English,Spanish,Malay,Vietnamese": 0.17045454545454544, - "English,Spanish,Chinese,Indonesian": 0.21022727272727273, - "English,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.2159090909090909, - "English,Malay,Chinese,Indonesian": 0.16477272727272727, - "English,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Malay,Indonesian,Vietnamese": 0.1875, - "English,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Malay,Chinese": 0.11931818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese,Indonesian": 0.125, - "Filipino,Malay,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.17613636363636365, - "Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Spanish,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0625, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - } - }, - "AC3_2": 0.44559322730552514, - "AC3_3": 0.3193281565533662, - "AC3_4": 0.23190592236277208, - "AC3_5": 0.1742538526971709, - "AC3_6": 0.1360082076048595, - "AC3_7": 0.10952572345097496 - }, - "prompt_5": { - "overall_acc": 0.4228896103896105, - "language_acc": { - "English": 0.4659090909090909, - "Filipino": 0.29545454545454547, - "Spanish": 0.42045454545454547, - "Malay": 0.3977272727272727, - "Chinese": 0.5227272727272727, - "Indonesian": 0.4431818181818182, - "Vietnamese": 0.4147727272727273 - }, - "consistency_score_2": 0.43804112554112556, - "consistency_score_3": 0.24090909090909096, - "consistency_score_4": 0.14983766233766238, - "consistency_score_5": 0.1003787878787879, - "consistency_score_6": 0.07061688311688312, - "consistency_score_7": 0.05113636363636364, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3522727272727273, - "English,Spanish": 0.5170454545454546, - "English,Malay": 0.39204545454545453, - "English,Chinese": 0.5170454545454546, - "English,Indonesian": 0.4431818181818182, - "English,Vietnamese": 0.4943181818181818, - "Filipino,Spanish": 0.3806818181818182, - "Filipino,Malay": 0.3977272727272727, - "Filipino,Chinese": 0.3806818181818182, - "Filipino,Indonesian": 0.4090909090909091, - "Filipino,Vietnamese": 0.3806818181818182, - "Spanish,Malay": 0.4090909090909091, - "Spanish,Chinese": 0.4715909090909091, - "Spanish,Indonesian": 0.5340909090909091, - "Spanish,Vietnamese": 0.4772727272727273, - "Malay,Chinese": 0.4090909090909091, - "Malay,Indonesian": 0.4715909090909091, - "Malay,Vietnamese": 0.4659090909090909, - "Chinese,Indonesian": 0.4147727272727273, - "Chinese,Vietnamese": 0.3977272727272727, - "Indonesian,Vietnamese": 0.48295454545454547 - }, - "3_combine": { - "English,Filipino,Spanish": 0.22727272727272727, - "English,Filipino,Malay": 0.17045454545454544, - "English,Filipino,Chinese": 0.21022727272727273, - "English,Filipino,Indonesian": 0.19318181818181818, - "English,Filipino,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay": 0.23863636363636365, - "English,Spanish,Chinese": 0.30113636363636365, - "English,Spanish,Indonesian": 0.2897727272727273, - "English,Spanish,Vietnamese": 0.3125, - "English,Malay,Chinese": 0.2215909090909091, - "English,Malay,Indonesian": 0.23295454545454544, - "English,Malay,Vietnamese": 0.24431818181818182, - "English,Chinese,Indonesian": 0.23863636363636365, - "English,Chinese,Vietnamese": 0.26136363636363635, - "English,Indonesian,Vietnamese": 0.2840909090909091, - "Filipino,Spanish,Malay": 0.18181818181818182, - "Filipino,Spanish,Chinese": 0.1875, - "Filipino,Spanish,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Vietnamese": 0.23295454545454544, - "Filipino,Malay,Chinese": 0.1875, - "Filipino,Malay,Indonesian": 0.25, - "Filipino,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Indonesian,Vietnamese": 0.24431818181818182, - "Spanish,Malay,Chinese": 0.24431818181818182, - "Spanish,Malay,Indonesian": 0.2727272727272727, - "Spanish,Malay,Vietnamese": 0.2556818181818182, - "Spanish,Chinese,Indonesian": 0.2784090909090909, - "Spanish,Chinese,Vietnamese": 0.2840909090909091, - "Spanish,Indonesian,Vietnamese": 0.32386363636363635, - "Malay,Chinese,Indonesian": 0.23295454545454544, - "Malay,Chinese,Vietnamese": 0.23295454545454544, - "Malay,Indonesian,Vietnamese": 0.2727272727272727, - "Chinese,Indonesian,Vietnamese": 0.24431818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.10795454545454546, - "English,Filipino,Spanish,Chinese": 0.13068181818181818, - "English,Filipino,Spanish,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese": 0.10795454545454546, - "English,Filipino,Malay,Indonesian": 0.125, - "English,Filipino,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese": 0.1534090909090909, - "English,Spanish,Malay,Indonesian": 0.16477272727272727, - "English,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Spanish,Chinese,Indonesian": 0.17613636363636365, - "English,Spanish,Chinese,Vietnamese": 0.1875, - "English,Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "English,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese": 0.09659090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Malay,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.125, - "Filipino,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.17045454545454544, - "Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Indonesian,Vietnamese": 0.1875, - "Spanish,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Indonesian": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 - } - }, - "AC3_2": 0.43033204220066246, - "AC3_3": 0.30695435643122526, - "AC3_4": 0.22127387206431864, - "AC3_5": 0.16224616900161573, - "AC3_6": 0.12102432926460857, - "AC3_7": 0.09123988167440096 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.49514563106796117 - }, - "prompt_2": { - "accuracy": 0.5436893203883495 + "accuracy": 0.5533980582524272 }, "prompt_3": { "accuracy": 0.5533980582524272 }, "prompt_4": { - "accuracy": 0.42718446601941745 + "accuracy": 0.5728155339805825 }, "prompt_5": { - "accuracy": 0.5242718446601942 + "accuracy": 0.5631067961165048 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.7428571428571429 + "accuracy": 0.18095238095238095 }, "prompt_2": { - "accuracy": 0.6571428571428571 + "accuracy": 0.18095238095238095 }, "prompt_3": { - "accuracy": 0.6666666666666666 + "accuracy": 0.20952380952380953 }, "prompt_4": { - "accuracy": 0.6571428571428571 + "accuracy": 0.17142857142857143 }, "prompt_5": { - "accuracy": 0.7333333333333333 + "accuracy": 0.20952380952380953 } }, "us_eval": { "prompt_1": { - "accuracy": 0.6074766355140186 + "accuracy": 0.6542056074766355 }, "prompt_2": { - "accuracy": 0.6635514018691588 + "accuracy": 0.6355140186915887 }, "prompt_3": { - "accuracy": 0.6074766355140186 + "accuracy": 0.6728971962616822 }, "prompt_4": { - "accuracy": 0.4205607476635514 + "accuracy": 0.6822429906542056 }, "prompt_5": { - "accuracy": 0.616822429906542 + "accuracy": 0.6728971962616822 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.36, + "accuracy": 0.32, "category_acc": { - "brand": 0.4, + "brand": 0.2, "demographics": 0.0, - "biology": 0.2, - "history": 0.26666666666666666, + "biology": 0.4, + "history": 0.13333333333333333, "literature": 0.3, - "politics": 0.4, + "politics": 0.7, "culture": 0.4, - "film": 0.2, - "law": 0.7, - "geography": 0.6 + "film": 0.3, + "law": 0.2, + "geography": 0.5 } }, "prompt_2": { - "accuracy": 0.4, + "accuracy": 0.29, "category_acc": { - "brand": 0.5, - "demographics": 0.0, - "biology": 0.3, - "history": 0.3333333333333333, + "brand": 0.2, + "demographics": 0.2, + "biology": 0.4, + "history": 0.06666666666666667, "literature": 0.3, - "politics": 0.3, + "politics": 0.5, "culture": 0.4, "film": 0.3, - "law": 0.7, - "geography": 0.7 + "law": 0.2, + "geography": 0.4 } }, "prompt_3": { - "accuracy": 0.42, + "accuracy": 0.34, "category_acc": { - "brand": 0.5, + "brand": 0.3, "demographics": 0.2, "biology": 0.4, - "history": 0.3333333333333333, + "history": 0.13333333333333333, "literature": 0.3, - "politics": 0.6, - "culture": 0.3, + "politics": 0.7, + "culture": 0.4, "film": 0.3, - "law": 0.5, - "geography": 0.7 + "law": 0.2, + "geography": 0.5 } }, "prompt_4": { "accuracy": 0.35, "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.1, - "history": 0.26666666666666666, - "literature": 0.4, - "politics": 0.4, + "brand": 0.2, + "demographics": 0.2, + "biology": 0.5, + "history": 0.2, + "literature": 0.3, + "politics": 0.7, "culture": 0.4, - "film": 0.4, - "law": 0.4, + "film": 0.3, + "law": 0.2, "geography": 0.5 } }, "prompt_5": { - "accuracy": 0.43, + "accuracy": 0.35, "category_acc": { - "brand": 0.6, - "demographics": 0.0, - "biology": 0.2, - "history": 0.3333333333333333, - "literature": 0.3, - "politics": 0.5, - "culture": 0.3, + "brand": 0.3, + "demographics": 0.2, + "biology": 0.5, + "history": 0.13333333333333333, + "literature": 0.4, + "politics": 0.6, + "culture": 0.4, "film": 0.3, - "law": 0.7, - "geography": 0.9 + "law": 0.2, + "geography": 0.5 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.15204278000075216 + "bleu_score": 0.17143237165650724 }, "prompt_2": { - "bleu_score": 0.22347669110653468 + "bleu_score": 0.17307422379138457 }, "prompt_3": { - "bleu_score": 0.19675288557189102 + "bleu_score": 0.17719935633819425 }, "prompt_4": { - "bleu_score": 0.08752275491508515 + "bleu_score": 0.17445808927858594 }, "prompt_5": { - "bleu_score": 0.09328384966786438 + "bleu_score": 0.17398886376784886 } }, "indommlu": { "prompt_1": { - "accuracy": 0.3468856398958542, + "accuracy": 0.3235863542292543, "category_acc": { - "History": 0.3072289156626506, - "Geography": 0.3163265306122449, - "Lampungic": 0.24489795918367346, - "Social science": 0.4741235392320534, - "Balinese": 0.29723991507430997, - "Makassarese": 0.3225806451612903, - "Banjarese": 0.3680555555555556, - "Chemistry": 0.28905109489051095, - "Biology": 0.27928994082840236, - "Science": 0.4241486068111455, - "Christian religion": 0.417910447761194, - "Art": 0.39767054908485855, - "Islam religion": 0.38264580369843526, - "Hindu religion": 0.3466666666666667, - "Madurese": 0.2711864406779661, + "History": 0.29116465863453816, + "Geography": 0.25918367346938775, + "Lampungic": 0.2585034013605442, + "Social science": 0.41736227045075125, + "Balinese": 0.2951167728237792, + "Makassarese": 0.3010752688172043, + "Banjarese": 0.2569444444444444, + "Chemistry": 0.21897810218978103, + "Biology": 0.31005917159763313, + "Science": 0.3302373581011352, + "Christian religion": 0.3781094527363184, + "Art": 0.3544093178036606, + "Islam religion": 0.3456614509246088, + "Hindu religion": 0.3, + "Madurese": 0.31186440677966104, "Sport": 0.3716216216216216, - "Indonesian language": 0.38387297633872974, - "Physics": 0.32323232323232326, - "Minangkabau culture": 0.2964824120603015, - "Dayak language": 0.3119266055045872, + "Indonesian language": 0.37484433374844334, + "Physics": 0.24646464646464647, + "Minangkabau culture": 0.271356783919598, + "Dayak language": 0.25688073394495414, "Sociology": 0.3165322580645161, - "Economy": 0.2725409836065574, - "Sundanese": 0.3267070008643042, - "Javanese": 0.28225806451612906, - "Civic education": 0.3676680972818312 + "Economy": 0.3073770491803279, + "Sundanese": 0.31892826274848746, + "Javanese": 0.2772177419354839, + "Civic education": 0.34334763948497854 } }, "prompt_2": { - "accuracy": 0.34661859937245476, + "accuracy": 0.3191134254623139, "category_acc": { - "History": 0.2891566265060241, - "Geography": 0.3183673469387755, - "Lampungic": 0.2789115646258503, - "Social science": 0.48414023372287146, - "Balinese": 0.27388535031847133, - "Makassarese": 0.2956989247311828, + "History": 0.3112449799196787, + "Geography": 0.24897959183673468, + "Lampungic": 0.3401360544217687, + "Social science": 0.3989983305509182, + "Balinese": 0.28874734607218683, + "Makassarese": 0.3279569892473118, "Banjarese": 0.2847222222222222, - "Chemistry": 0.26277372262773724, - "Biology": 0.27928994082840236, - "Science": 0.4107327141382869, - "Christian religion": 0.36318407960199006, - "Art": 0.3860232945091514, - "Islam religion": 0.406827880512091, - "Hindu religion": 0.32666666666666666, - "Madurese": 0.28135593220338984, - "Sport": 0.3918918918918919, - "Indonesian language": 0.3972602739726027, - "Physics": 0.3151515151515151, - "Minangkabau culture": 0.3316582914572864, - "Dayak language": 0.25688073394495414, - "Sociology": 0.3125, - "Economy": 0.28688524590163933, - "Sundanese": 0.31547104580812446, - "Javanese": 0.3165322580645161, - "Civic education": 0.3447782546494993 + "Chemistry": 0.23065693430656933, + "Biology": 0.3005917159763314, + "Science": 0.31682146542827655, + "Christian religion": 0.38308457711442784, + "Art": 0.3211314475873544, + "Islam religion": 0.31152204836415365, + "Hindu religion": 0.29333333333333333, + "Madurese": 0.2983050847457627, + "Sport": 0.39864864864864863, + "Indonesian language": 0.3664383561643836, + "Physics": 0.2545454545454545, + "Minangkabau culture": 0.27638190954773867, + "Dayak language": 0.23853211009174313, + "Sociology": 0.3225806451612903, + "Economy": 0.29713114754098363, + "Sundanese": 0.324114088159032, + "Javanese": 0.2842741935483871, + "Civic education": 0.33047210300429186 } }, "prompt_3": { - "accuracy": 0.37826290139528673, - "category_acc": { - "History": 0.3293172690763052, - "Geography": 0.3448979591836735, - "Lampungic": 0.3401360544217687, - "Social science": 0.5559265442404007, - "Balinese": 0.2951167728237792, - "Makassarese": 0.3387096774193548, - "Banjarese": 0.3541666666666667, - "Chemistry": 0.2656934306569343, - "Biology": 0.3230769230769231, - "Science": 0.46542827657378744, - "Christian religion": 0.4577114427860697, - "Art": 0.4242928452579035, - "Islam religion": 0.4366998577524893, - "Hindu religion": 0.3333333333333333, - "Madurese": 0.2847457627118644, - "Sport": 0.40540540540540543, - "Indonesian language": 0.424346201743462, - "Physics": 0.3656565656565657, - "Minangkabau culture": 0.3316582914572864, - "Dayak language": 0.42201834862385323, - "Sociology": 0.3588709677419355, - "Economy": 0.32581967213114754, - "Sundanese": 0.320656871218669, - "Javanese": 0.3125, - "Civic education": 0.38483547925608014 - } - }, - "prompt_4": { - "accuracy": 0.38807664063021563, + "accuracy": 0.31857934441551505, "category_acc": { - "History": 0.36947791164658633, - "Geography": 0.37551020408163266, - "Lampungic": 0.21768707482993196, - "Social science": 0.5659432387312187, - "Balinese": 0.28662420382165604, - "Makassarese": 0.3655913978494624, - "Banjarese": 0.2777777777777778, - "Chemistry": 0.2773722627737226, - "Biology": 0.35384615384615387, - "Science": 0.48194014447884415, - "Christian religion": 0.44776119402985076, - "Art": 0.4209650582362729, - "Islam religion": 0.4310099573257468, - "Hindu religion": 0.36, - "Madurese": 0.29152542372881357, - "Sport": 0.41216216216216217, - "Indonesian language": 0.43773349937733497, - "Physics": 0.34949494949494947, - "Minangkabau culture": 0.34673366834170855, - "Dayak language": 0.3211009174311927, - "Sociology": 0.36693548387096775, - "Economy": 0.3463114754098361, - "Sundanese": 0.3465859982713915, - "Javanese": 0.30443548387096775, - "Civic education": 0.41630901287553645 + "History": 0.2971887550200803, + "Geography": 0.24693877551020407, + "Lampungic": 0.2857142857142857, + "Social science": 0.41402337228714525, + "Balinese": 0.2802547770700637, + "Makassarese": 0.34946236559139787, + "Banjarese": 0.2708333333333333, + "Chemistry": 0.21897810218978103, + "Biology": 0.2982248520710059, + "Science": 0.32920536635706915, + "Christian religion": 0.373134328358209, + "Art": 0.3261231281198003, + "Islam religion": 0.33712660028449504, + "Hindu religion": 0.28, + "Madurese": 0.2677966101694915, + "Sport": 0.39864864864864863, + "Indonesian language": 0.3707970112079701, + "Physics": 0.24848484848484848, + "Minangkabau culture": 0.25125628140703515, + "Dayak language": 0.26605504587155965, + "Sociology": 0.31451612903225806, + "Economy": 0.2766393442622951, + "Sundanese": 0.3292999135695765, + "Javanese": 0.27419354838709675, + "Civic education": 0.33047210300429186 } }, + "prompt_4": -1, "prompt_5": { - "accuracy": 0.3819347085920288, + "accuracy": 0.32592295880899924, "category_acc": { - "History": 0.3393574297188755, - "Geography": 0.35306122448979593, - "Lampungic": 0.25170068027210885, - "Social science": 0.5392320534223706, - "Balinese": 0.3205944798301486, - "Makassarese": 0.34408602150537637, - "Banjarese": 0.3819444444444444, - "Chemistry": 0.2759124087591241, - "Biology": 0.33964497041420116, - "Science": 0.478844169246646, - "Christian religion": 0.4577114427860697, - "Art": 0.43261231281198004, - "Islam religion": 0.44523470839260315, - "Hindu religion": 0.41333333333333333, - "Madurese": 0.3016949152542373, - "Sport": 0.36486486486486486, - "Indonesian language": 0.41594022415940224, - "Physics": 0.3151515151515151, - "Minangkabau culture": 0.36180904522613067, - "Dayak language": 0.3761467889908257, - "Sociology": 0.36088709677419356, - "Economy": 0.33811475409836067, - "Sundanese": 0.331028522039758, - "Javanese": 0.3014112903225806, - "Civic education": 0.4406294706723891 + "History": 0.2991967871485944, + "Geography": 0.2836734693877551, + "Lampungic": 0.29931972789115646, + "Social science": 0.4307178631051753, + "Balinese": 0.28662420382165604, + "Makassarese": 0.3333333333333333, + "Banjarese": 0.24305555555555555, + "Chemistry": 0.2510948905109489, + "Biology": 0.2982248520710059, + "Science": 0.32301341589267285, + "Christian religion": 0.36318407960199006, + "Art": 0.34608985024958405, + "Islam religion": 0.3357041251778094, + "Hindu religion": 0.30666666666666664, + "Madurese": 0.30847457627118646, + "Sport": 0.3918918918918919, + "Indonesian language": 0.37359900373599003, + "Physics": 0.25252525252525254, + "Minangkabau culture": 0.2814070351758794, + "Dayak language": 0.28440366972477066, + "Sociology": 0.3084677419354839, + "Economy": 0.28688524590163933, + "Sundanese": 0.3327571305099395, + "Javanese": 0.2752016129032258, + "Civic education": 0.3547925608011445 } } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.20274880824482147 + "bleu_score": 0.28907320267771625 }, "prompt_2": { - "bleu_score": 0.3236460772194234 + "bleu_score": 0.2893892328430556 }, "prompt_3": { - "bleu_score": 0.2825241768229112 + "bleu_score": 0.2886733513466506 }, "prompt_4": { - "bleu_score": 0.10795479204893771 + "bleu_score": 0.29284370210142285 }, "prompt_5": { - "bleu_score": 0.09273251454339947 + "bleu_score": 0.2747670406496726 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.13979629067247779 + "bleu_score": 0.07158636430680222 }, "prompt_2": { - "bleu_score": 0.2686012591998093 + "bleu_score": 0.07103723230064819 }, "prompt_3": { - "bleu_score": 0.22230468586443336 + "bleu_score": 0.07070585299769382 }, "prompt_4": { - "bleu_score": 0.07007283498846292 + "bleu_score": 0.07161514805550445 }, "prompt_5": { - "bleu_score": 0.06845931958611609 + "bleu_score": 0.06633933982982537 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.0862941907114682 + "bleu_score": 0.018213298322017683 }, "prompt_2": { - "bleu_score": 0.22666388297966877 + "bleu_score": 0.01861580300222175 }, "prompt_3": { - "bleu_score": 0.23328928459329606 + "bleu_score": 0.01940857812030854 }, "prompt_4": { - "bleu_score": 0.08999103645081159 + "bleu_score": 0.015585331792968693 }, "prompt_5": { - "bleu_score": 0.08169806519027373 + "bleu_score": 0.018660857793531816 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.26298590883139666 + "bleu_score": 0.27139437453475074 }, "prompt_2": { - "bleu_score": 0.3124590600264869 + "bleu_score": 0.26884915209700333 }, "prompt_3": { - "bleu_score": 0.32255754147682697 + "bleu_score": 0.26967135172638884 }, "prompt_4": { - "bleu_score": 0.09597465649022448 + "bleu_score": 0.2718225078074264 }, "prompt_5": { - "bleu_score": 0.08148789569809947 + "bleu_score": 0.25925138729269165 } }, "mmlu": { "prompt_1": { - "accuracy": 0.5717619603267211 + "accuracy": 0.5530921820303384 }, "prompt_2": { - "accuracy": 0.411901983663944 + "accuracy": 0.558926487747958 }, "prompt_3": { - "accuracy": 0.5670945157526255 + "accuracy": 0.544924154025671 }, "prompt_4": { - "accuracy": 0.5694282380396732 + "accuracy": 0.5472578763127188 }, "prompt_5": { - "accuracy": 0.5624270711785297 + "accuracy": 0.5577596266044341 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.5254200929567394, + "accuracy": 0.5392205934930282, "category_acc": { - "high_school_european_history": 0.676829268292683, - "business_ethics": 0.6161616161616161, - "clinical_knowledge": 0.6136363636363636, - "medical_genetics": 0.5858585858585859, - "high_school_us_history": 0.7438423645320197, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.75, - "virology": 0.43636363636363634, - "high_school_microeconomics": 0.5189873417721519, - "econometrics": 0.3805309734513274, - "college_computer_science": 0.43434343434343436, - "high_school_biology": 0.6245954692556634, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.41637010676156583, - "philosophy": 0.5935483870967742, - "professional_medicine": 0.5867158671586716, - "nutrition": 0.5737704918032787, - "global_facts": 0.30303030303030304, - "machine_learning": 0.3963963963963964, - "security_studies": 0.569672131147541, - "public_relations": 0.5321100917431193, - "professional_psychology": 0.5253682487725041, - "prehistory": 0.5944272445820433, - "anatomy": 0.4253731343283582, - "human_sexuality": 0.6, - "college_medicine": 0.563953488372093, - "high_school_government_and_politics": 0.6979166666666666, - "college_chemistry": 0.41414141414141414, - "logical_fallacies": 0.6049382716049383, - "high_school_geography": 0.649746192893401, - "elementary_mathematics": 0.5225464190981433, - "human_aging": 0.6261261261261262, - "college_mathematics": 0.40404040404040403, - "high_school_psychology": 0.7003676470588235, - "formal_logic": 0.312, - "high_school_statistics": 0.40930232558139534, - "international_law": 0.6916666666666667, - "high_school_mathematics": 0.3680297397769517, - "high_school_computer_science": 0.5252525252525253, - "conceptual_physics": 0.5085470085470085, - "miscellaneous": 0.6355498721227621, - "high_school_chemistry": 0.504950495049505, - "marketing": 0.7639484978540773, - "professional_law": 0.4096542726679713, - "management": 0.6176470588235294, - "college_physics": 0.3465346534653465, + "high_school_european_history": 0.7378048780487805, + "business_ethics": 0.7070707070707071, + "clinical_knowledge": 0.6060606060606061, + "medical_genetics": 0.5252525252525253, + "high_school_us_history": 0.7684729064039408, + "high_school_physics": 0.2866666666666667, + "high_school_world_history": 0.7711864406779662, + "virology": 0.4909090909090909, + "high_school_microeconomics": 0.5864978902953587, + "econometrics": 0.35398230088495575, + "college_computer_science": 0.4444444444444444, + "high_school_biology": 0.6634304207119741, + "abstract_algebra": 0.20202020202020202, + "professional_accounting": 0.4306049822064057, + "philosophy": 0.5580645161290323, + "professional_medicine": 0.5571955719557196, + "nutrition": 0.6065573770491803, + "global_facts": 0.2828282828282828, + "machine_learning": 0.40540540540540543, + "security_studies": 0.6557377049180327, + "public_relations": 0.5504587155963303, + "professional_psychology": 0.5515548281505729, + "prehistory": 0.5913312693498453, + "anatomy": 0.47761194029850745, + "human_sexuality": 0.6538461538461539, + "college_medicine": 0.48255813953488375, + "high_school_government_and_politics": 0.8333333333333334, + "college_chemistry": 0.3838383838383838, + "logical_fallacies": 0.6666666666666666, + "high_school_geography": 0.7258883248730964, + "elementary_mathematics": 0.3740053050397878, + "human_aging": 0.6036036036036037, + "college_mathematics": 0.30303030303030304, + "high_school_psychology": 0.7794117647058824, + "formal_logic": 0.288, + "high_school_statistics": 0.4, + "international_law": 0.7, + "high_school_mathematics": 0.2379182156133829, + "high_school_computer_science": 0.4444444444444444, + "conceptual_physics": 0.452991452991453, + "miscellaneous": 0.7186700767263428, + "high_school_chemistry": 0.40594059405940597, + "marketing": 0.8369098712446352, + "professional_law": 0.4253098499673842, + "management": 0.7941176470588235, + "college_physics": 0.42574257425742573, "jurisprudence": 0.616822429906542, - "world_religions": 0.5882352941176471, - "sociology": 0.73, - "us_foreign_policy": 0.7272727272727273, - "high_school_macroeconomics": 0.4704370179948586, - "computer_security": 0.6666666666666666, - "moral_scenarios": 0.24384787472035793, - "moral_disputes": 0.5797101449275363, - "electrical_engineering": 0.4583333333333333, - "astronomy": 0.5165562913907285, - "college_biology": 0.6363636363636364 + "world_religions": 0.7, + "sociology": 0.68, + "us_foreign_policy": 0.7474747474747475, + "high_school_macroeconomics": 0.5295629820051414, + "computer_security": 0.6363636363636364, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.6405797101449275, + "electrical_engineering": 0.5277777777777778, + "astronomy": 0.5629139072847682, + "college_biology": 0.5104895104895105 } }, "prompt_2": { - "accuracy": 0.43139077583124774, + "accuracy": 0.5386485520200215, "category_acc": { - "high_school_european_history": 0.6890243902439024, - "business_ethics": 0.42424242424242425, - "clinical_knowledge": 0.5681818181818182, - "medical_genetics": 0.45454545454545453, - "high_school_us_history": 0.7635467980295566, - "high_school_physics": 0.34, - "high_school_world_history": 0.7330508474576272, - "virology": 0.3393939393939394, - "high_school_microeconomics": 0.4767932489451477, - "econometrics": 0.3274336283185841, - "college_computer_science": 0.41414141414141414, - "high_school_biology": 0.4401294498381877, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.4092526690391459, - "philosophy": 0.49032258064516127, - "professional_medicine": 0.5756457564575646, - "nutrition": 0.4852459016393443, + "high_school_european_history": 0.7439024390243902, + "business_ethics": 0.7272727272727273, + "clinical_knowledge": 0.6022727272727273, + "medical_genetics": 0.5252525252525253, + "high_school_us_history": 0.7684729064039408, + "high_school_physics": 0.26, + "high_school_world_history": 0.7627118644067796, + "virology": 0.47878787878787876, + "high_school_microeconomics": 0.5738396624472574, + "econometrics": 0.34513274336283184, + "college_computer_science": 0.45454545454545453, + "high_school_biology": 0.6601941747572816, + "abstract_algebra": 0.1919191919191919, + "professional_accounting": 0.4234875444839858, + "philosophy": 0.5709677419354838, + "professional_medicine": 0.5682656826568265, + "nutrition": 0.6032786885245902, "global_facts": 0.2727272727272727, - "machine_learning": 0.3063063063063063, - "security_studies": 0.6065573770491803, - "public_relations": 0.22018348623853212, - "professional_psychology": 0.3944353518821604, - "prehistory": 0.5015479876160991, - "anatomy": 0.291044776119403, - "human_sexuality": 0.4153846153846154, + "machine_learning": 0.40540540540540543, + "security_studies": 0.6639344262295082, + "public_relations": 0.5412844036697247, + "professional_psychology": 0.5482815057283142, + "prehistory": 0.5913312693498453, + "anatomy": 0.4925373134328358, + "human_sexuality": 0.6384615384615384, "college_medicine": 0.48255813953488375, - "high_school_government_and_politics": 0.4270833333333333, - "college_chemistry": 0.32323232323232326, - "logical_fallacies": 0.4691358024691358, - "high_school_geography": 0.467005076142132, - "elementary_mathematics": 0.5278514588859416, - "human_aging": 0.32432432432432434, - "college_mathematics": 0.37373737373737376, - "high_school_psychology": 0.38419117647058826, - "formal_logic": 0.344, - "high_school_statistics": 0.4325581395348837, - "international_law": 0.575, - "high_school_mathematics": 0.3345724907063197, - "high_school_computer_science": 0.5151515151515151, - "conceptual_physics": 0.38461538461538464, - "miscellaneous": 0.3312020460358056, - "high_school_chemistry": 0.36633663366336633, - "marketing": 0.6738197424892703, - "professional_law": 0.41487279843444225, - "management": 0.37254901960784315, - "college_physics": 0.3465346534653465, - "jurisprudence": 0.5607476635514018, - "world_religions": 0.37058823529411766, - "sociology": 0.39, - "us_foreign_policy": 0.6363636363636364, - "high_school_macroeconomics": 0.4241645244215938, - "computer_security": 0.3838383838383838, - "moral_scenarios": 0.2483221476510067, - "moral_disputes": 0.5159420289855072, - "electrical_engineering": 0.4930555555555556, - "astronomy": 0.4900662251655629, - "college_biology": 0.4405594405594406 + "high_school_government_and_politics": 0.8333333333333334, + "college_chemistry": 0.3838383838383838, + "logical_fallacies": 0.6604938271604939, + "high_school_geography": 0.7055837563451777, + "elementary_mathematics": 0.3687002652519894, + "human_aging": 0.5900900900900901, + "college_mathematics": 0.31313131313131315, + "high_school_psychology": 0.7738970588235294, + "formal_logic": 0.312, + "high_school_statistics": 0.386046511627907, + "international_law": 0.7, + "high_school_mathematics": 0.24907063197026022, + "high_school_computer_science": 0.43434343434343436, + "conceptual_physics": 0.452991452991453, + "miscellaneous": 0.7199488491048593, + "high_school_chemistry": 0.39603960396039606, + "marketing": 0.8326180257510729, + "professional_law": 0.4298760600130463, + "management": 0.803921568627451, + "college_physics": 0.40594059405940597, + "jurisprudence": 0.6448598130841121, + "world_religions": 0.6882352941176471, + "sociology": 0.69, + "us_foreign_policy": 0.7474747474747475, + "high_school_macroeconomics": 0.5398457583547558, + "computer_security": 0.6464646464646465, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.6521739130434783, + "electrical_engineering": 0.4861111111111111, + "astronomy": 0.5695364238410596, + "college_biology": 0.5104895104895105 } }, "prompt_3": { - "accuracy": 0.5528780836610654, + "accuracy": 0.5380765105470147, "category_acc": { - "high_school_european_history": 0.6951219512195121, - "business_ethics": 0.6262626262626263, - "clinical_knowledge": 0.6590909090909091, - "medical_genetics": 0.6060606060606061, - "high_school_us_history": 0.729064039408867, - "high_school_physics": 0.34, - "high_school_world_history": 0.7203389830508474, - "virology": 0.4303030303030303, - "high_school_microeconomics": 0.5738396624472574, - "econometrics": 0.3893805309734513, - "college_computer_science": 0.5151515151515151, - "high_school_biology": 0.6472491909385113, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.42704626334519574, - "philosophy": 0.5774193548387097, - "professional_medicine": 0.6088560885608856, - "nutrition": 0.6327868852459017, - "global_facts": 0.41414141414141414, - "machine_learning": 0.4594594594594595, - "security_studies": 0.610655737704918, - "public_relations": 0.5412844036697247, - "professional_psychology": 0.5564648117839607, - "prehistory": 0.6408668730650154, - "anatomy": 0.5223880597014925, - "human_sexuality": 0.6692307692307692, - "college_medicine": 0.5813953488372093, - "high_school_government_and_politics": 0.7552083333333334, - "college_chemistry": 0.41414141414141414, - "logical_fallacies": 0.6358024691358025, - "high_school_geography": 0.6649746192893401, - "elementary_mathematics": 0.5119363395225465, - "human_aging": 0.5945945945945946, - "college_mathematics": 0.37373737373737376, - "high_school_psychology": 0.7279411764705882, - "formal_logic": 0.4, - "high_school_statistics": 0.4325581395348837, - "international_law": 0.7416666666666667, - "high_school_mathematics": 0.3420074349442379, - "high_school_computer_science": 0.6262626262626263, - "conceptual_physics": 0.5341880341880342, - "miscellaneous": 0.7455242966751918, - "high_school_chemistry": 0.47029702970297027, - "marketing": 0.7854077253218884, - "professional_law": 0.4044357469015003, - "management": 0.7156862745098039, - "college_physics": 0.37623762376237624, - "jurisprudence": 0.7102803738317757, - "world_religions": 0.7176470588235294, - "sociology": 0.775, - "us_foreign_policy": 0.797979797979798, - "high_school_macroeconomics": 0.5578406169665809, + "high_school_european_history": 0.75, + "business_ethics": 0.7070707070707071, + "clinical_knowledge": 0.6022727272727273, + "medical_genetics": 0.5151515151515151, + "high_school_us_history": 0.7733990147783252, + "high_school_physics": 0.28, + "high_school_world_history": 0.7669491525423728, + "virology": 0.48484848484848486, + "high_school_microeconomics": 0.569620253164557, + "econometrics": 0.34513274336283184, + "college_computer_science": 0.4444444444444444, + "high_school_biology": 0.6731391585760518, + "abstract_algebra": 0.21212121212121213, + "professional_accounting": 0.4234875444839858, + "philosophy": 0.5741935483870968, + "professional_medicine": 0.5682656826568265, + "nutrition": 0.6131147540983607, + "global_facts": 0.2727272727272727, + "machine_learning": 0.3783783783783784, + "security_studies": 0.6557377049180327, + "public_relations": 0.5779816513761468, + "professional_psychology": 0.5450081833060556, + "prehistory": 0.5789473684210527, + "anatomy": 0.4701492537313433, + "human_sexuality": 0.6384615384615384, + "college_medicine": 0.47093023255813954, + "high_school_government_and_politics": 0.84375, + "college_chemistry": 0.3939393939393939, + "logical_fallacies": 0.6604938271604939, + "high_school_geography": 0.7208121827411168, + "elementary_mathematics": 0.36074270557029176, + "human_aging": 0.5765765765765766, + "college_mathematics": 0.31313131313131315, + "high_school_psychology": 0.78125, + "formal_logic": 0.328, + "high_school_statistics": 0.386046511627907, + "international_law": 0.7, + "high_school_mathematics": 0.2379182156133829, + "high_school_computer_science": 0.4444444444444444, + "conceptual_physics": 0.43162393162393164, + "miscellaneous": 0.7071611253196931, + "high_school_chemistry": 0.40594059405940597, + "marketing": 0.8326180257510729, + "professional_law": 0.43118069145466403, + "management": 0.7745098039215687, + "college_physics": 0.42574257425742573, + "jurisprudence": 0.6261682242990654, + "world_religions": 0.6882352941176471, + "sociology": 0.69, + "us_foreign_policy": 0.7575757575757576, + "high_school_macroeconomics": 0.5372750642673522, "computer_security": 0.6363636363636364, - "moral_scenarios": 0.24272930648769575, - "moral_disputes": 0.5971014492753624, - "electrical_engineering": 0.4652777777777778, - "astronomy": 0.5695364238410596, - "college_biology": 0.6223776223776224 + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.6521739130434783, + "electrical_engineering": 0.5069444444444444, + "astronomy": 0.5827814569536424, + "college_biology": 0.4965034965034965 } }, "prompt_4": { - "accuracy": 0.5422238112263139, + "accuracy": 0.5392920986771541, "category_acc": { - "high_school_european_history": 0.6524390243902439, - "business_ethics": 0.6363636363636364, - "clinical_knowledge": 0.6325757575757576, - "medical_genetics": 0.6161616161616161, - "high_school_us_history": 0.729064039408867, - "high_school_physics": 0.38666666666666666, - "high_school_world_history": 0.7203389830508474, - "virology": 0.46060606060606063, - "high_school_microeconomics": 0.5907172995780591, - "econometrics": 0.39823008849557523, - "college_computer_science": 0.5555555555555556, - "high_school_biology": 0.6504854368932039, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.398576512455516, - "philosophy": 0.6, - "professional_medicine": 0.5830258302583026, - "nutrition": 0.5868852459016394, - "global_facts": 0.3838383838383838, - "machine_learning": 0.44144144144144143, - "security_studies": 0.6311475409836066, + "high_school_european_history": 0.75, + "business_ethics": 0.7272727272727273, + "clinical_knowledge": 0.6060606060606061, + "medical_genetics": 0.5151515151515151, + "high_school_us_history": 0.7684729064039408, + "high_school_physics": 0.2733333333333333, + "high_school_world_history": 0.7627118644067796, + "virology": 0.48484848484848486, + "high_school_microeconomics": 0.5780590717299579, + "econometrics": 0.336283185840708, + "college_computer_science": 0.46464646464646464, + "high_school_biology": 0.6666666666666666, + "abstract_algebra": 0.18181818181818182, + "professional_accounting": 0.4377224199288256, + "philosophy": 0.5741935483870968, + "professional_medicine": 0.5535055350553506, + "nutrition": 0.6131147540983607, + "global_facts": 0.2727272727272727, + "machine_learning": 0.38738738738738737, + "security_studies": 0.6639344262295082, "public_relations": 0.5504587155963303, - "professional_psychology": 0.530278232405892, - "prehistory": 0.6068111455108359, + "professional_psychology": 0.5499181669394435, + "prehistory": 0.5913312693498453, "anatomy": 0.4701492537313433, - "human_sexuality": 0.6923076923076923, - "college_medicine": 0.563953488372093, - "high_school_government_and_politics": 0.71875, - "college_chemistry": 0.42424242424242425, - "logical_fallacies": 0.6049382716049383, - "high_school_geography": 0.6852791878172588, - "elementary_mathematics": 0.5623342175066313, - "human_aging": 0.6261261261261262, - "college_mathematics": 0.36363636363636365, - "high_school_psychology": 0.71875, - "formal_logic": 0.336, - "high_school_statistics": 0.5023255813953489, - "international_law": 0.7, - "high_school_mathematics": 0.36059479553903345, - "high_school_computer_science": 0.5454545454545454, - "conceptual_physics": 0.5256410256410257, - "miscellaneous": 0.6943734015345269, - "high_school_chemistry": 0.4801980198019802, - "marketing": 0.7725321888412017, - "professional_law": 0.3744292237442922, - "management": 0.7156862745098039, - "college_physics": 0.44554455445544555, - "jurisprudence": 0.616822429906542, - "world_religions": 0.7176470588235294, - "sociology": 0.755, - "us_foreign_policy": 0.7474747474747475, - "high_school_macroeconomics": 0.5141388174807198, - "computer_security": 0.6363636363636364, - "moral_scenarios": 0.2483221476510067, - "moral_disputes": 0.5884057971014492, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.5827814569536424, - "college_biology": 0.6013986013986014 + "human_sexuality": 0.6384615384615384, + "college_medicine": 0.47674418604651164, + "high_school_government_and_politics": 0.828125, + "college_chemistry": 0.3939393939393939, + "logical_fallacies": 0.6728395061728395, + "high_school_geography": 0.7309644670050761, + "elementary_mathematics": 0.3687002652519894, + "human_aging": 0.5630630630630631, + "college_mathematics": 0.32323232323232326, + "high_school_psychology": 0.78125, + "formal_logic": 0.32, + "high_school_statistics": 0.4046511627906977, + "international_law": 0.7166666666666667, + "high_school_mathematics": 0.25650557620817843, + "high_school_computer_science": 0.4444444444444444, + "conceptual_physics": 0.4230769230769231, + "miscellaneous": 0.7071611253196931, + "high_school_chemistry": 0.4158415841584158, + "marketing": 0.8240343347639485, + "professional_law": 0.4292237442922374, + "management": 0.7843137254901961, + "college_physics": 0.39603960396039606, + "jurisprudence": 0.6448598130841121, + "world_religions": 0.711764705882353, + "sociology": 0.695, + "us_foreign_policy": 0.7373737373737373, + "high_school_macroeconomics": 0.5449871465295629, + "computer_security": 0.6464646464646465, + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.6405797101449275, + "electrical_engineering": 0.5138888888888888, + "astronomy": 0.5761589403973509, + "college_biology": 0.5034965034965035 } }, "prompt_5": { - "accuracy": 0.5415087593850554, + "accuracy": 0.5432248838040757, "category_acc": { - "high_school_european_history": 0.6585365853658537, - "business_ethics": 0.5959595959595959, - "clinical_knowledge": 0.5984848484848485, - "medical_genetics": 0.6464646464646465, - "high_school_us_history": 0.7536945812807881, - "high_school_physics": 0.38666666666666666, - "high_school_world_history": 0.7203389830508474, - "virology": 0.4727272727272727, - "high_school_microeconomics": 0.569620253164557, - "econometrics": 0.3805309734513274, - "college_computer_science": 0.46464646464646464, - "high_school_biology": 0.6310679611650486, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.39501779359430605, - "philosophy": 0.5903225806451613, - "professional_medicine": 0.5867158671586716, - "nutrition": 0.5967213114754099, - "global_facts": 0.35353535353535354, - "machine_learning": 0.43243243243243246, - "security_studies": 0.610655737704918, - "public_relations": 0.5596330275229358, - "professional_psychology": 0.5237315875613748, - "prehistory": 0.6253869969040248, - "anatomy": 0.5, + "high_school_european_history": 0.75, + "business_ethics": 0.7373737373737373, + "clinical_knowledge": 0.6022727272727273, + "medical_genetics": 0.5252525252525253, + "high_school_us_history": 0.7832512315270936, + "high_school_physics": 0.26, + "high_school_world_history": 0.7627118644067796, + "virology": 0.4666666666666667, + "high_school_microeconomics": 0.5738396624472574, + "econometrics": 0.36283185840707965, + "college_computer_science": 0.45454545454545453, + "high_school_biology": 0.6634304207119741, + "abstract_algebra": 0.1919191919191919, + "professional_accounting": 0.4234875444839858, + "philosophy": 0.5709677419354838, + "professional_medicine": 0.5535055350553506, + "nutrition": 0.6295081967213115, + "global_facts": 0.31313131313131315, + "machine_learning": 0.38738738738738737, + "security_studies": 0.6762295081967213, + "public_relations": 0.5321100917431193, + "professional_psychology": 0.55810147299509, + "prehistory": 0.6068111455108359, + "anatomy": 0.47761194029850745, "human_sexuality": 0.6538461538461539, - "college_medicine": 0.5406976744186046, - "high_school_government_and_politics": 0.703125, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.6419753086419753, - "high_school_geography": 0.700507614213198, - "elementary_mathematics": 0.47745358090185674, - "human_aging": 0.6531531531531531, - "college_mathematics": 0.35353535353535354, - "high_school_psychology": 0.7297794117647058, - "formal_logic": 0.304, - "high_school_statistics": 0.4046511627906977, - "international_law": 0.6833333333333333, - "high_school_mathematics": 0.31226765799256506, - "high_school_computer_science": 0.5757575757575758, - "conceptual_physics": 0.49572649572649574, - "miscellaneous": 0.7404092071611253, - "high_school_chemistry": 0.4801980198019802, - "marketing": 0.7939914163090128, - "professional_law": 0.3933463796477495, - "management": 0.6666666666666666, - "college_physics": 0.38613861386138615, - "jurisprudence": 0.6635514018691588, - "world_religions": 0.7352941176470589, - "sociology": 0.755, - "us_foreign_policy": 0.7474747474747475, - "high_school_macroeconomics": 0.4781491002570694, + "college_medicine": 0.47674418604651164, + "high_school_government_and_politics": 0.8333333333333334, + "college_chemistry": 0.3838383838383838, + "logical_fallacies": 0.6728395061728395, + "high_school_geography": 0.7360406091370558, + "elementary_mathematics": 0.3740053050397878, + "human_aging": 0.5900900900900901, + "college_mathematics": 0.32323232323232326, + "high_school_psychology": 0.7886029411764706, + "formal_logic": 0.32, + "high_school_statistics": 0.4, + "international_law": 0.7083333333333334, + "high_school_mathematics": 0.25650557620817843, + "high_school_computer_science": 0.46464646464646464, + "conceptual_physics": 0.4230769230769231, + "miscellaneous": 0.7212276214833759, + "high_school_chemistry": 0.40594059405940597, + "marketing": 0.8283261802575107, + "professional_law": 0.43313763861709065, + "management": 0.7941176470588235, + "college_physics": 0.40594059405940597, + "jurisprudence": 0.6822429906542056, + "world_religions": 0.7235294117647059, + "sociology": 0.69, + "us_foreign_policy": 0.7575757575757576, + "high_school_macroeconomics": 0.5295629820051414, "computer_security": 0.6666666666666666, - "moral_scenarios": 0.27181208053691275, - "moral_disputes": 0.6115942028985507, - "electrical_engineering": 0.5138888888888888, - "astronomy": 0.5695364238410596, - "college_biology": 0.6013986013986014 + "moral_scenarios": 0.23825503355704697, + "moral_disputes": 0.6608695652173913, + "electrical_engineering": 0.5, + "astronomy": 0.5761589403973509, + "college_biology": 0.48951048951048953 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.6693907875185735 + "accuracy": 0.2473997028231798 }, "prompt_2": { - "accuracy": 0.6203566121842496 + "accuracy": 0.24294205052005943 }, "prompt_3": { - "accuracy": 0.6017830609212481 + "accuracy": 0.2526002971768202 }, "prompt_4": { - "accuracy": 0.6352154531946508 + "accuracy": 0.24814264487369986 }, "prompt_5": { - "accuracy": 0.5832095096582467 + "accuracy": 0.24962852897473997 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.6811955168119551, + "accuracy": 0.2559153175591532, "category_acc": { - "computer_network": 0.5833333333333334, - "operating_system": 0.375, - "computer_architecture": 0.5769230769230769, - "college_programming": 0.6190476190476191, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.5862068965517241, + "computer_network": 0.2916666666666667, + "operating_system": 0.20833333333333334, + "computer_architecture": 0.19230769230769232, + "college_programming": 0.3333333333333333, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.27586206896551724, "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.40476190476190477, - "metrology_engineer": 0.6896551724137931, - "high_school_mathematics": 0.4782608695652174, - "high_school_physics": 0.7916666666666666, - "high_school_chemistry": 0.7083333333333334, - "high_school_biology": 0.8333333333333334, - "middle_school_mathematics": 0.8333333333333334, - "middle_school_biology": 0.9615384615384616, - "middle_school_physics": 0.9166666666666666, - "middle_school_chemistry": 1.0, - "veterinary_medicine": 0.7142857142857143, - "college_economics": 0.5666666666666667, - "business_administration": 0.6052631578947368, - "marxism": 0.875, - "mao_zedong_thought": 0.8275862068965517, - "education_science": 0.8235294117647058, - "teacher_qualification": 0.7755102040816326, - "high_school_politics": 0.9583333333333334, - "high_school_geography": 0.7916666666666666, - "middle_school_politics": 0.9615384615384616, - "middle_school_geography": 0.9411764705882353, - "modern_chinese_history": 0.8571428571428571, - "ideological_and_moral_cultivation": 0.9583333333333334, - "logic": 0.4444444444444444, - "law": 0.6896551724137931, - "chinese_language_and_literature": 0.6785714285714286, - "art_studies": 0.6578947368421053, - "professional_tour_guide": 0.7647058823529411, - "legal_professional": 0.6428571428571429, - "high_school_chinese": 0.7916666666666666, - "high_school_history": 0.88, - "middle_school_history": 0.9629629629629629, - "civil_servant": 0.5769230769230769, - "sports_science": 0.75, - "plant_protection": 0.8148148148148148, - "basic_medicine": 0.75, - "clinical_medicine": 0.5925925925925926, - "urban_and_rural_planner": 0.6078431372549019, - "accountant": 0.6666666666666666, - "fire_engineer": 0.6111111111111112, - "environmental_impact_assessment_engineer": 0.6666666666666666, - "tax_accountant": 0.5925925925925926, - "physician": 0.6481481481481481 + "probability_and_statistics": 0.34782608695652173, + "discrete_mathematics": 0.3333333333333333, + "electrical_engineer": 0.2857142857142857, + "metrology_engineer": 0.13793103448275862, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.20833333333333334, + "high_school_biology": 0.20833333333333334, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.2692307692307692, + "middle_school_physics": 0.20833333333333334, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.3, + "business_administration": 0.18421052631578946, + "marxism": 0.25, + "mao_zedong_thought": 0.3103448275862069, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.2857142857142857, + "high_school_politics": 0.3333333333333333, + "high_school_geography": 0.20833333333333334, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.058823529411764705, + "modern_chinese_history": 0.21428571428571427, + "ideological_and_moral_cultivation": 0.2916666666666667, + "logic": 0.4074074074074074, + "law": 0.20689655172413793, + "chinese_language_and_literature": 0.25, + "art_studies": 0.23684210526315788, + "professional_tour_guide": 0.3235294117647059, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.4166666666666667, + "high_school_history": 0.28, + "middle_school_history": 0.18518518518518517, + "civil_servant": 0.19230769230769232, + "sports_science": 0.125, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.20833333333333334, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.35294117647058826, + "accountant": 0.2777777777777778, + "fire_engineer": 0.1388888888888889, + "environmental_impact_assessment_engineer": 0.19444444444444445, + "tax_accountant": 0.2037037037037037, + "physician": 0.25925925925925924 } }, "prompt_2": { - "accuracy": 0.6189290161892902, + "accuracy": 0.26089663760896636, "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.5384615384615384, - "college_programming": 0.6190476190476191, - "college_physics": 0.25, - "college_chemistry": 0.4482758620689655, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.4523809523809524, - "metrology_engineer": 0.5862068965517241, - "high_school_mathematics": 0.43478260869565216, - "high_school_physics": 0.7083333333333334, - "high_school_chemistry": 0.5416666666666666, - "high_school_biology": 0.8333333333333334, - "middle_school_mathematics": 0.6666666666666666, - "middle_school_biology": 0.9615384615384616, - "middle_school_physics": 0.7916666666666666, - "middle_school_chemistry": 0.72, - "veterinary_medicine": 0.7142857142857143, - "college_economics": 0.6, - "business_administration": 0.6052631578947368, - "marxism": 0.75, - "mao_zedong_thought": 0.8620689655172413, - "education_science": 0.7352941176470589, - "teacher_qualification": 0.7142857142857143, - "high_school_politics": 0.9583333333333334, - "high_school_geography": 0.75, - "middle_school_politics": 0.9230769230769231, - "middle_school_geography": 0.9411764705882353, - "modern_chinese_history": 0.6785714285714286, - "ideological_and_moral_cultivation": 0.8333333333333334, - "logic": 0.4444444444444444, - "law": 0.4827586206896552, - "chinese_language_and_literature": 0.5714285714285714, - "art_studies": 0.6578947368421053, - "professional_tour_guide": 0.7647058823529411, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.7916666666666666, - "high_school_history": 0.92, - "middle_school_history": 0.9629629629629629, - "civil_servant": 0.5576923076923077, - "sports_science": 0.5833333333333334, - "plant_protection": 0.5925925925925926, - "basic_medicine": 0.75, - "clinical_medicine": 0.5185185185185185, - "urban_and_rural_planner": 0.5686274509803921, - "accountant": 0.5185185185185185, - "fire_engineer": 0.5, - "environmental_impact_assessment_engineer": 0.4722222222222222, - "tax_accountant": 0.42592592592592593, - "physician": 0.7037037037037037 + "computer_network": 0.2916666666666667, + "operating_system": 0.16666666666666666, + "computer_architecture": 0.19230769230769232, + "college_programming": 0.2857142857142857, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.41379310344827586, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.43478260869565216, + "discrete_mathematics": 0.47619047619047616, + "electrical_engineer": 0.21428571428571427, + "metrology_engineer": 0.1724137931034483, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.16666666666666666, + "high_school_biology": 0.25, + "middle_school_mathematics": 0.20833333333333334, + "middle_school_biology": 0.2692307692307692, + "middle_school_physics": 0.20833333333333334, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.32142857142857145, + "college_economics": 0.26666666666666666, + "business_administration": 0.21052631578947367, + "marxism": 0.25, + "mao_zedong_thought": 0.3103448275862069, + "education_science": 0.17647058823529413, + "teacher_qualification": 0.30612244897959184, + "high_school_politics": 0.3333333333333333, + "high_school_geography": 0.20833333333333334, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.058823529411764705, + "modern_chinese_history": 0.21428571428571427, + "ideological_and_moral_cultivation": 0.3333333333333333, + "logic": 0.4074074074074074, + "law": 0.20689655172413793, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.2631578947368421, + "professional_tour_guide": 0.3235294117647059, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.4166666666666667, + "high_school_history": 0.28, + "middle_school_history": 0.18518518518518517, + "civil_servant": 0.19230769230769232, + "sports_science": 0.20833333333333334, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.16666666666666666, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.37254901960784315, + "accountant": 0.2962962962962963, + "fire_engineer": 0.1111111111111111, + "environmental_impact_assessment_engineer": 0.16666666666666666, + "tax_accountant": 0.24074074074074073, + "physician": 0.2962962962962963 } }, "prompt_3": { - "accuracy": 0.5853051058530511, + "accuracy": 0.26089663760896636, "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.5384615384615384, - "college_programming": 0.6904761904761905, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.41379310344827586, + "computer_network": 0.3333333333333333, + "operating_system": 0.16666666666666666, + "computer_architecture": 0.19230769230769232, + "college_programming": 0.2619047619047619, + "college_physics": 0.25, + "college_chemistry": 0.3103448275862069, "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.40476190476190477, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.6666666666666666, - "high_school_chemistry": 0.625, - "high_school_biology": 0.5416666666666666, - "middle_school_mathematics": 0.5833333333333334, - "middle_school_biology": 0.9230769230769231, - "middle_school_physics": 0.75, - "middle_school_chemistry": 0.64, - "veterinary_medicine": 0.6428571428571429, - "college_economics": 0.6, - "business_administration": 0.47368421052631576, - "marxism": 0.75, - "mao_zedong_thought": 0.7931034482758621, - "education_science": 0.7647058823529411, - "teacher_qualification": 0.7346938775510204, - "high_school_politics": 0.8333333333333334, - "high_school_geography": 0.7916666666666666, - "middle_school_politics": 0.9230769230769231, - "middle_school_geography": 0.8235294117647058, - "modern_chinese_history": 0.6785714285714286, - "ideological_and_moral_cultivation": 0.875, - "logic": 0.37037037037037035, - "law": 0.4827586206896552, - "chinese_language_and_literature": 0.5357142857142857, - "art_studies": 0.6052631578947368, - "professional_tour_guide": 0.7058823529411765, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.5416666666666666, - "high_school_history": 0.84, - "middle_school_history": 0.7407407407407407, - "civil_servant": 0.5192307692307693, - "sports_science": 0.5833333333333334, - "plant_protection": 0.6296296296296297, - "basic_medicine": 0.75, - "clinical_medicine": 0.5555555555555556, - "urban_and_rural_planner": 0.5686274509803921, - "accountant": 0.46296296296296297, - "fire_engineer": 0.5, - "environmental_impact_assessment_engineer": 0.5555555555555556, - "tax_accountant": 0.4074074074074074, - "physician": 0.5925925925925926 + "probability_and_statistics": 0.34782608695652173, + "discrete_mathematics": 0.3333333333333333, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.13793103448275862, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.20833333333333334, + "high_school_biology": 0.25, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.3076923076923077, + "middle_school_physics": 0.25, + "middle_school_chemistry": 0.28, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.26666666666666666, + "business_administration": 0.18421052631578946, + "marxism": 0.25, + "mao_zedong_thought": 0.3103448275862069, + "education_science": 0.2647058823529412, + "teacher_qualification": 0.2653061224489796, + "high_school_politics": 0.3333333333333333, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.058823529411764705, + "modern_chinese_history": 0.25, + "ideological_and_moral_cultivation": 0.3333333333333333, + "logic": 0.4074074074074074, + "law": 0.20689655172413793, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.2894736842105263, + "professional_tour_guide": 0.35294117647058826, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.4166666666666667, + "high_school_history": 0.32, + "middle_school_history": 0.18518518518518517, + "civil_servant": 0.21153846153846154, + "sports_science": 0.16666666666666666, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.25, + "clinical_medicine": 0.2962962962962963, + "urban_and_rural_planner": 0.37254901960784315, + "accountant": 0.2777777777777778, + "fire_engineer": 0.1388888888888889, + "environmental_impact_assessment_engineer": 0.19444444444444445, + "tax_accountant": 0.2222222222222222, + "physician": 0.24074074074074073 } }, "prompt_4": { - "accuracy": 0.6382316313823163, + "accuracy": 0.2671232876712329, "category_acc": { - "computer_network": 0.5416666666666666, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.5769230769230769, - "college_programming": 0.5952380952380952, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.5172413793103449, - "advanced_mathematics": 0.4166666666666667, + "computer_network": 0.20833333333333334, + "operating_system": 0.16666666666666666, + "computer_architecture": 0.19230769230769232, + "college_programming": 0.2619047619047619, + "college_physics": 0.2916666666666667, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.3333333333333333, "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.4523809523809524, - "metrology_engineer": 0.5862068965517241, - "high_school_mathematics": 0.4782608695652174, - "high_school_physics": 0.6666666666666666, - "high_school_chemistry": 0.625, - "high_school_biology": 0.6666666666666666, - "middle_school_mathematics": 0.75, - "middle_school_biology": 0.9230769230769231, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.8, - "veterinary_medicine": 0.6428571428571429, - "college_economics": 0.5333333333333333, - "business_administration": 0.631578947368421, - "marxism": 0.7083333333333334, - "mao_zedong_thought": 0.8275862068965517, - "education_science": 0.8235294117647058, - "teacher_qualification": 0.7551020408163265, - "high_school_politics": 0.9583333333333334, - "high_school_geography": 0.75, - "middle_school_politics": 0.9230769230769231, - "middle_school_geography": 0.8823529411764706, - "modern_chinese_history": 0.6785714285714286, - "ideological_and_moral_cultivation": 0.8333333333333334, - "logic": 0.48148148148148145, - "law": 0.6206896551724138, - "chinese_language_and_literature": 0.6428571428571429, - "art_studies": 0.6052631578947368, - "professional_tour_guide": 0.7647058823529411, - "legal_professional": 0.5357142857142857, - "high_school_chinese": 0.7083333333333334, - "high_school_history": 0.84, - "middle_school_history": 0.7777777777777778, - "civil_servant": 0.5961538461538461, - "sports_science": 0.7083333333333334, - "plant_protection": 0.7407407407407407, - "basic_medicine": 0.7916666666666666, - "clinical_medicine": 0.6666666666666666, - "urban_and_rural_planner": 0.6078431372549019, - "accountant": 0.5925925925925926, - "fire_engineer": 0.5833333333333334, - "environmental_impact_assessment_engineer": 0.5555555555555556, - "tax_accountant": 0.5555555555555556, - "physician": 0.6666666666666666 + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.2857142857142857, + "metrology_engineer": 0.1724137931034483, + "high_school_mathematics": 0.391304347826087, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.2916666666666667, + "high_school_biology": 0.2916666666666667, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.15384615384615385, + "middle_school_physics": 0.125, + "middle_school_chemistry": 0.36, + "veterinary_medicine": 0.35714285714285715, + "college_economics": 0.25, + "business_administration": 0.21052631578947367, + "marxism": 0.3333333333333333, + "mao_zedong_thought": 0.27586206896551724, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.2857142857142857, + "high_school_politics": 0.3333333333333333, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.11764705882352941, + "modern_chinese_history": 0.21428571428571427, + "ideological_and_moral_cultivation": 0.3333333333333333, + "logic": 0.37037037037037035, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.4473684210526316, + "professional_tour_guide": 0.38235294117647056, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.4166666666666667, + "high_school_history": 0.36, + "middle_school_history": 0.2222222222222222, + "civil_servant": 0.2692307692307692, + "sports_science": 0.16666666666666666, + "plant_protection": 0.2222222222222222, + "basic_medicine": 0.16666666666666666, + "clinical_medicine": 0.2222222222222222, + "urban_and_rural_planner": 0.29411764705882354, + "accountant": 0.2777777777777778, + "fire_engineer": 0.16666666666666666, + "environmental_impact_assessment_engineer": 0.2222222222222222, + "tax_accountant": 0.25925925925925924, + "physician": 0.24074074074074073 } }, "prompt_5": { - "accuracy": 0.5884184308841843, + "accuracy": 0.2577833125778331, "category_acc": { - "computer_network": 0.5, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.5384615384615384, - "college_programming": 0.5, + "computer_network": 0.3333333333333333, + "operating_system": 0.16666666666666666, + "computer_architecture": 0.11538461538461539, + "college_programming": 0.2619047619047619, "college_physics": 0.20833333333333334, - "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.2857142857142857, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.34782608695652173, + "discrete_mathematics": 0.42857142857142855, "electrical_engineer": 0.21428571428571427, - "metrology_engineer": 0.5172413793103449, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.7083333333333334, - "high_school_chemistry": 0.7083333333333334, - "high_school_biology": 0.875, - "middle_school_mathematics": 0.625, - "middle_school_biology": 0.9230769230769231, - "middle_school_physics": 0.8333333333333334, - "middle_school_chemistry": 0.88, - "veterinary_medicine": 0.6785714285714286, - "college_economics": 0.4666666666666667, - "business_administration": 0.5526315789473685, - "marxism": 0.8333333333333334, - "mao_zedong_thought": 0.7586206896551724, - "education_science": 0.7058823529411765, - "teacher_qualification": 0.6530612244897959, - "high_school_politics": 0.9166666666666666, - "high_school_geography": 0.75, - "middle_school_politics": 0.8461538461538461, - "middle_school_geography": 0.9411764705882353, - "modern_chinese_history": 0.75, - "ideological_and_moral_cultivation": 0.7083333333333334, - "logic": 0.37037037037037035, - "law": 0.5172413793103449, - "chinese_language_and_literature": 0.5, - "art_studies": 0.631578947368421, - "professional_tour_guide": 0.7352941176470589, - "legal_professional": 0.5357142857142857, - "high_school_chinese": 0.7083333333333334, - "high_school_history": 0.84, - "middle_school_history": 0.9259259259259259, - "civil_servant": 0.5576923076923077, - "sports_science": 0.5833333333333334, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.625, - "clinical_medicine": 0.5185185185185185, - "urban_and_rural_planner": 0.6078431372549019, - "accountant": 0.5925925925925926, - "fire_engineer": 0.3611111111111111, - "environmental_impact_assessment_engineer": 0.4722222222222222, - "tax_accountant": 0.5185185185185185, - "physician": 0.5370370370370371 + "metrology_engineer": 0.1724137931034483, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.20833333333333334, + "high_school_biology": 0.20833333333333334, + "middle_school_mathematics": 0.20833333333333334, + "middle_school_biology": 0.2692307692307692, + "middle_school_physics": 0.20833333333333334, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.2857142857142857, + "college_economics": 0.3333333333333333, + "business_administration": 0.18421052631578946, + "marxism": 0.25, + "mao_zedong_thought": 0.3103448275862069, + "education_science": 0.2647058823529412, + "teacher_qualification": 0.2857142857142857, + "high_school_politics": 0.3333333333333333, + "high_school_geography": 0.16666666666666666, + "middle_school_politics": 0.2692307692307692, + "middle_school_geography": 0.058823529411764705, + "modern_chinese_history": 0.21428571428571427, + "ideological_and_moral_cultivation": 0.375, + "logic": 0.4074074074074074, + "law": 0.20689655172413793, + "chinese_language_and_literature": 0.21428571428571427, + "art_studies": 0.2631578947368421, + "professional_tour_guide": 0.3235294117647059, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.4166666666666667, + "high_school_history": 0.32, + "middle_school_history": 0.18518518518518517, + "civil_servant": 0.21153846153846154, + "sports_science": 0.16666666666666666, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.2916666666666667, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.37254901960784315, + "accountant": 0.2777777777777778, + "fire_engineer": 0.1111111111111111, + "environmental_impact_assessment_engineer": 0.16666666666666666, + "tax_accountant": 0.2037037037037037, + "physician": 0.2962962962962963 } } }, "cmmlu": { "prompt_1": { - "accuracy": 0.7168458781362007 + "accuracy": 0.2724014336917563 }, "prompt_2": { - "accuracy": 0.6810035842293907 + "accuracy": 0.30824372759856633 }, "prompt_3": { - "accuracy": 0.6200716845878136 + "accuracy": 0.27956989247311825 }, "prompt_4": { - "accuracy": 0.6810035842293907 + "accuracy": 0.2939068100358423 }, "prompt_5": { - "accuracy": 0.6738351254480287 + "accuracy": 0.2831541218637993 } }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.6888274909342083, - "category_acc": { - "agronomy": 0.6094674556213018, - "anatomy": 0.6891891891891891, - "ancient_chinese": 0.3719512195121951, - "arts": 0.875, - "astronomy": 0.42424242424242425, - "business_ethics": 0.645933014354067, - "chinese_civil_service_exam": 0.64375, - "chinese_driving_rule": 0.9007633587786259, - "chinese_food_culture": 0.6544117647058824, - "chinese_foreign_policy": 0.794392523364486, - "chinese_history": 0.9040247678018576, - "chinese_literature": 0.5735294117647058, - "chinese_teacher_qualification": 0.8491620111731844, - "clinical_knowledge": 0.6540084388185654, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.8037383177570093, - "college_engineering_hydrology": 0.5849056603773585, - "college_law": 0.6666666666666666, - "college_mathematics": 0.29523809523809524, - "college_medical_statistics": 0.5943396226415094, - "college_medicine": 0.73992673992674, - "computer_science": 0.7009803921568627, - "computer_security": 0.8128654970760234, - "conceptual_physics": 0.8299319727891157, - "construction_project_management": 0.5467625899280576, - "economics": 0.6415094339622641, - "education": 0.7177914110429447, - "electrical_engineering": 0.7093023255813954, - "elementary_chinese": 0.7063492063492064, - "elementary_commonsense": 0.6868686868686869, - "elementary_information_and_technology": 0.8529411764705882, - "elementary_mathematics": 0.5043478260869565, - "ethnology": 0.6814814814814815, - "food_science": 0.5804195804195804, - "genetics": 0.5397727272727273, - "global_facts": 0.6912751677852349, - "high_school_biology": 0.8698224852071006, - "high_school_chemistry": 0.7727272727272727, - "high_school_geography": 0.7457627118644068, - "high_school_mathematics": 0.4878048780487805, - "high_school_physics": 0.6909090909090909, - "high_school_politics": 0.7482517482517482, - "human_sexuality": 0.626984126984127, - "international_law": 0.5621621621621622, - "journalism": 0.5988372093023255, - "jurisprudence": 0.6982968369829684, - "legal_and_moral_basis": 0.9485981308411215, - "logical": 0.5609756097560976, - "machine_learning": 0.5409836065573771, - "management": 0.7619047619047619, - "marketing": 0.7222222222222222, - "marxist_theory": 0.8888888888888888, - "modern_chinese": 0.5172413793103449, - "nutrition": 0.6482758620689655, - "philosophy": 0.7142857142857143, - "professional_accounting": 0.8057142857142857, - "professional_law": 0.6113744075829384, - "professional_medicine": 0.6196808510638298, - "professional_psychology": 0.7887931034482759, - "public_relations": 0.6666666666666666, - "security_study": 0.7851851851851852, - "sociology": 0.6769911504424779, - "sports_science": 0.6727272727272727, - "traditional_chinese_medicine": 0.745945945945946, - "virology": 0.7159763313609467, - "world_history": 0.8757763975155279, - "world_religions": 0.70625 - } - }, - "prompt_2": { - "accuracy": 0.6354688309445692, - "category_acc": { - "agronomy": 0.5384615384615384, - "anatomy": 0.6418918918918919, - "ancient_chinese": 0.3048780487804878, - "arts": 0.86875, - "astronomy": 0.4, - "business_ethics": 0.6172248803827751, - "chinese_civil_service_exam": 0.5625, - "chinese_driving_rule": 0.8778625954198473, - "chinese_food_culture": 0.6029411764705882, - "chinese_foreign_policy": 0.8037383177570093, - "chinese_history": 0.8668730650154799, - "chinese_literature": 0.553921568627451, - "chinese_teacher_qualification": 0.8100558659217877, - "clinical_knowledge": 0.4936708860759494, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.7850467289719626, - "college_engineering_hydrology": 0.5943396226415094, - "college_law": 0.5740740740740741, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.5471698113207547, - "college_medicine": 0.6446886446886447, - "computer_science": 0.6421568627450981, - "computer_security": 0.7602339181286549, - "conceptual_physics": 0.7210884353741497, - "construction_project_management": 0.4892086330935252, - "economics": 0.660377358490566, - "education": 0.7116564417177914, - "electrical_engineering": 0.6802325581395349, - "elementary_chinese": 0.6309523809523809, - "elementary_commonsense": 0.6616161616161617, - "elementary_information_and_technology": 0.8109243697478992, - "elementary_mathematics": 0.4434782608695652, - "ethnology": 0.6370370370370371, - "food_science": 0.5734265734265734, - "genetics": 0.5170454545454546, - "global_facts": 0.6778523489932886, - "high_school_biology": 0.7100591715976331, - "high_school_chemistry": 0.5227272727272727, - "high_school_geography": 0.6864406779661016, - "high_school_mathematics": 0.40853658536585363, - "high_school_physics": 0.5545454545454546, - "high_school_politics": 0.7412587412587412, - "human_sexuality": 0.6111111111111112, - "international_law": 0.518918918918919, - "journalism": 0.5872093023255814, - "jurisprudence": 0.6423357664233577, - "legal_and_moral_basis": 0.9299065420560748, - "logical": 0.4634146341463415, - "machine_learning": 0.5163934426229508, - "management": 0.7761904761904762, - "marketing": 0.6888888888888889, - "marxist_theory": 0.8571428571428571, - "modern_chinese": 0.46551724137931033, - "nutrition": 0.5724137931034483, - "philosophy": 0.6857142857142857, - "professional_accounting": 0.68, - "professional_law": 0.5260663507109005, - "professional_medicine": 0.48936170212765956, - "professional_psychology": 0.7758620689655172, - "public_relations": 0.6264367816091954, - "security_study": 0.7185185185185186, - "sociology": 0.6415929203539823, - "sports_science": 0.6363636363636364, - "traditional_chinese_medicine": 0.6432432432432432, - "virology": 0.6390532544378699, - "world_history": 0.8633540372670807, - "world_religions": 0.70625 - } - }, - "prompt_3": { - "accuracy": 0.6204455189086514, - "category_acc": { - "agronomy": 0.5207100591715976, - "anatomy": 0.5945945945945946, - "ancient_chinese": 0.3353658536585366, - "arts": 0.84375, - "astronomy": 0.40606060606060607, - "business_ethics": 0.583732057416268, - "chinese_civil_service_exam": 0.4875, - "chinese_driving_rule": 0.9007633587786259, - "chinese_food_culture": 0.6102941176470589, - "chinese_foreign_policy": 0.7663551401869159, - "chinese_history": 0.826625386996904, - "chinese_literature": 0.4803921568627451, - "chinese_teacher_qualification": 0.8156424581005587, - "clinical_knowledge": 0.6160337552742616, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.7850467289719626, - "college_engineering_hydrology": 0.5283018867924528, - "college_law": 0.5925925925925926, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.5094339622641509, - "college_medicine": 0.6776556776556777, - "computer_science": 0.6764705882352942, - "computer_security": 0.7192982456140351, - "conceptual_physics": 0.7142857142857143, - "construction_project_management": 0.45323741007194246, - "economics": 0.5974842767295597, - "education": 0.6625766871165644, - "electrical_engineering": 0.6511627906976745, - "elementary_chinese": 0.5912698412698413, - "elementary_commonsense": 0.6212121212121212, - "elementary_information_and_technology": 0.7899159663865546, - "elementary_mathematics": 0.4956521739130435, - "ethnology": 0.5925925925925926, - "food_science": 0.5664335664335665, - "genetics": 0.5284090909090909, - "global_facts": 0.6510067114093959, - "high_school_biology": 0.6449704142011834, - "high_school_chemistry": 0.5227272727272727, - "high_school_geography": 0.6864406779661016, - "high_school_mathematics": 0.3902439024390244, - "high_school_physics": 0.5181818181818182, - "high_school_politics": 0.7062937062937062, - "human_sexuality": 0.5793650793650794, - "international_law": 0.5243243243243243, - "journalism": 0.5581395348837209, - "jurisprudence": 0.5912408759124088, - "legal_and_moral_basis": 0.897196261682243, - "logical": 0.43902439024390244, - "machine_learning": 0.47540983606557374, - "management": 0.7619047619047619, - "marketing": 0.65, - "marxist_theory": 0.8042328042328042, - "modern_chinese": 0.4051724137931034, - "nutrition": 0.6068965517241379, - "philosophy": 0.6571428571428571, - "professional_accounting": 0.6857142857142857, - "professional_law": 0.5118483412322274, - "professional_medicine": 0.5531914893617021, - "professional_psychology": 0.7672413793103449, - "public_relations": 0.6149425287356322, - "security_study": 0.7185185185185186, - "sociology": 0.6150442477876106, - "sports_science": 0.6121212121212121, - "traditional_chinese_medicine": 0.6432432432432432, - "virology": 0.6686390532544378, - "world_history": 0.7888198757763976, - "world_religions": 0.65 - } - }, + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, "prompt_4": { - "accuracy": 0.647815575893628, + "accuracy": 0.25392850975651876, "category_acc": { - "agronomy": 0.5266272189349113, - "anatomy": 0.6081081081081081, - "ancient_chinese": 0.35365853658536583, - "arts": 0.83125, - "astronomy": 0.42424242424242425, - "business_ethics": 0.6220095693779905, - "chinese_civil_service_exam": 0.51875, - "chinese_driving_rule": 0.8702290076335878, - "chinese_food_culture": 0.6176470588235294, - "chinese_foreign_policy": 0.7663551401869159, - "chinese_history": 0.848297213622291, - "chinese_literature": 0.4852941176470588, - "chinese_teacher_qualification": 0.8379888268156425, - "clinical_knowledge": 0.6286919831223629, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.7850467289719626, - "college_engineering_hydrology": 0.5471698113207547, - "college_law": 0.6574074074074074, - "college_mathematics": 0.3142857142857143, - "college_medical_statistics": 0.5283018867924528, - "college_medicine": 0.7106227106227107, - "computer_science": 0.6372549019607843, - "computer_security": 0.7719298245614035, - "conceptual_physics": 0.7414965986394558, - "construction_project_management": 0.5251798561151079, - "economics": 0.5849056603773585, - "education": 0.6748466257668712, - "electrical_engineering": 0.6453488372093024, - "elementary_chinese": 0.6587301587301587, - "elementary_commonsense": 0.6666666666666666, - "elementary_information_and_technology": 0.8235294117647058, - "elementary_mathematics": 0.4608695652173913, - "ethnology": 0.6444444444444445, - "food_science": 0.5944055944055944, - "genetics": 0.5170454545454546, - "global_facts": 0.6644295302013423, - "high_school_biology": 0.7751479289940828, - "high_school_chemistry": 0.7045454545454546, - "high_school_geography": 0.7033898305084746, - "high_school_mathematics": 0.43902439024390244, - "high_school_physics": 0.6636363636363637, - "high_school_politics": 0.7132867132867133, - "human_sexuality": 0.5634920634920635, - "international_law": 0.518918918918919, - "journalism": 0.5581395348837209, - "jurisprudence": 0.6642335766423357, - "legal_and_moral_basis": 0.9158878504672897, - "logical": 0.5203252032520326, - "machine_learning": 0.5245901639344263, - "management": 0.7619047619047619, - "marketing": 0.6722222222222223, - "marxist_theory": 0.8359788359788359, - "modern_chinese": 0.5086206896551724, - "nutrition": 0.6344827586206897, - "philosophy": 0.6666666666666666, - "professional_accounting": 0.7942857142857143, - "professional_law": 0.5592417061611374, - "professional_medicine": 0.5851063829787234, - "professional_psychology": 0.7629310344827587, - "public_relations": 0.6436781609195402, - "security_study": 0.7111111111111111, - "sociology": 0.6106194690265486, - "sports_science": 0.6363636363636364, - "traditional_chinese_medicine": 0.6756756756756757, - "virology": 0.6804733727810651, - "world_history": 0.8198757763975155, - "world_religions": 0.64375 + "agronomy": 0.25443786982248523, + "anatomy": 0.25675675675675674, + "ancient_chinese": 0.2682926829268293, + "arts": 0.24375, + "astronomy": 0.19393939393939394, + "business_ethics": 0.2535885167464115, + "chinese_civil_service_exam": 0.24375, + "chinese_driving_rule": 0.3053435114503817, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.27102803738317754, + "chinese_history": 0.25077399380804954, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.24022346368715083, + "clinical_knowledge": 0.2489451476793249, + "college_actuarial_science": 0.27358490566037735, + "college_education": 0.2897196261682243, + "college_engineering_hydrology": 0.3018867924528302, + "college_law": 0.2222222222222222, + "college_mathematics": 0.20952380952380953, + "college_medical_statistics": 0.24528301886792453, + "college_medicine": 0.2490842490842491, + "computer_science": 0.30392156862745096, + "computer_security": 0.2573099415204678, + "conceptual_physics": 0.2857142857142857, + "construction_project_management": 0.26618705035971224, + "economics": 0.2389937106918239, + "education": 0.25153374233128833, + "electrical_engineering": 0.2441860465116279, + "elementary_chinese": 0.28174603174603174, + "elementary_commonsense": 0.2474747474747475, + "elementary_information_and_technology": 0.2857142857142857, + "elementary_mathematics": 0.26956521739130435, + "ethnology": 0.23703703703703705, + "food_science": 0.3006993006993007, + "genetics": 0.2159090909090909, + "global_facts": 0.2751677852348993, + "high_school_biology": 0.23076923076923078, + "high_school_chemistry": 0.25757575757575757, + "high_school_geography": 0.2457627118644068, + "high_school_mathematics": 0.1951219512195122, + "high_school_physics": 0.2727272727272727, + "high_school_politics": 0.26573426573426573, + "human_sexuality": 0.24603174603174602, + "international_law": 0.2648648648648649, + "journalism": 0.2441860465116279, + "jurisprudence": 0.26277372262773724, + "legal_and_moral_basis": 0.2570093457943925, + "logical": 0.25203252032520324, + "machine_learning": 0.22950819672131148, + "management": 0.24285714285714285, + "marketing": 0.25555555555555554, + "marxist_theory": 0.25925925925925924, + "modern_chinese": 0.27586206896551724, + "nutrition": 0.2, + "philosophy": 0.2761904761904762, + "professional_accounting": 0.2571428571428571, + "professional_law": 0.23696682464454977, + "professional_medicine": 0.24202127659574468, + "professional_psychology": 0.24568965517241378, + "public_relations": 0.2413793103448276, + "security_study": 0.2740740740740741, + "sociology": 0.2610619469026549, + "sports_science": 0.23636363636363636, + "traditional_chinese_medicine": 0.2594594594594595, + "virology": 0.25443786982248523, + "world_history": 0.2360248447204969, + "world_religions": 0.275 } }, - "prompt_5": { - "accuracy": 0.605076843377655, - "category_acc": { - "agronomy": 0.5443786982248521, - "anatomy": 0.4594594594594595, - "ancient_chinese": 0.36585365853658536, - "arts": 0.8, - "astronomy": 0.3515151515151515, - "business_ethics": 0.5263157894736842, - "chinese_civil_service_exam": 0.56875, - "chinese_driving_rule": 0.7480916030534351, - "chinese_food_culture": 0.5661764705882353, - "chinese_foreign_policy": 0.7663551401869159, - "chinese_history": 0.8173374613003096, - "chinese_literature": 0.4803921568627451, - "chinese_teacher_qualification": 0.7821229050279329, - "clinical_knowledge": 0.5738396624472574, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.719626168224299, - "college_engineering_hydrology": 0.4528301886792453, - "college_law": 0.4444444444444444, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.5660377358490566, - "college_medicine": 0.5457875457875457, - "computer_science": 0.5931372549019608, - "computer_security": 0.7777777777777778, - "conceptual_physics": 0.8503401360544217, - "construction_project_management": 0.4748201438848921, - "economics": 0.6163522012578616, - "education": 0.6319018404907976, - "electrical_engineering": 0.6046511627906976, - "elementary_chinese": 0.6904761904761905, - "elementary_commonsense": 0.6111111111111112, - "elementary_information_and_technology": 0.7857142857142857, - "elementary_mathematics": 0.4782608695652174, - "ethnology": 0.6074074074074074, - "food_science": 0.46153846153846156, - "genetics": 0.5284090909090909, - "global_facts": 0.6375838926174496, - "high_school_biology": 0.757396449704142, - "high_school_chemistry": 0.6515151515151515, - "high_school_geography": 0.652542372881356, - "high_school_mathematics": 0.3780487804878049, - "high_school_physics": 0.5636363636363636, - "high_school_politics": 0.7342657342657343, - "human_sexuality": 0.5793650793650794, - "international_law": 0.4648648648648649, - "journalism": 0.4941860465116279, - "jurisprudence": 0.6301703163017032, - "legal_and_moral_basis": 0.9018691588785047, - "logical": 0.5284552845528455, - "machine_learning": 0.5327868852459017, - "management": 0.719047619047619, - "marketing": 0.65, - "marxist_theory": 0.8201058201058201, - "modern_chinese": 0.5775862068965517, - "nutrition": 0.6137931034482759, - "philosophy": 0.638095238095238, - "professional_accounting": 0.6685714285714286, - "professional_law": 0.5213270142180095, - "professional_medicine": 0.4734042553191489, - "professional_psychology": 0.6896551724137931, - "public_relations": 0.5229885057471264, - "security_study": 0.6666666666666666, - "sociology": 0.588495575221239, - "sports_science": 0.5333333333333333, - "traditional_chinese_medicine": 0.6270270270270271, - "virology": 0.5798816568047337, - "world_history": 0.7329192546583851, - "world_religions": 0.61875 - } - } + "prompt_5": -1 }, "zbench": { "prompt_1": { - "accuracy": 0.3333333333333333 + "accuracy": 0.36363636363636365 }, "prompt_2": { - "accuracy": 0.36363636363636365 + "accuracy": 0.2727272727272727 }, "prompt_3": { - "accuracy": 0.36363636363636365 + "accuracy": 0.2727272727272727 }, "prompt_4": { - "accuracy": 0.30303030303030304 + "accuracy": 0.2727272727272727 }, "prompt_5": { - "accuracy": 0.24242424242424243 + "accuracy": 0.30303030303030304 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.4159090909090909 + "accuracy": 0.4340909090909091 }, "prompt_2": { - "accuracy": 0.43863636363636366 + "accuracy": 0.42727272727272725 }, "prompt_3": { - "accuracy": 0.40454545454545454 + "accuracy": 0.42272727272727273 }, "prompt_4": { - "accuracy": 0.325 + "accuracy": 0.4590909090909091 }, "prompt_5": { - "accuracy": 0.3340909090909091 + "accuracy": 0.46136363636363636 } }, "ocnli": { "prompt_1": { - "accuracy": 0.49796610169491523 + "accuracy": 0.3454237288135593 }, "prompt_2": { - "accuracy": 0.49322033898305084 + "accuracy": 0.3454237288135593 }, "prompt_3": { - "accuracy": 0.5128813559322034 + "accuracy": 0.34508474576271186 }, "prompt_4": { - "accuracy": 0.5010169491525424 + "accuracy": 0.348135593220339 }, "prompt_5": { - "accuracy": 0.4894915254237288 + "accuracy": 0.3461016949152542 } }, "c3": { "prompt_1": { - "accuracy": 0.8908002991772626 + "accuracy": 0.2715033657442034 }, "prompt_2": { - "accuracy": 0.8777112939416605 + "accuracy": 0.2696335078534031 }, "prompt_3": { - "accuracy": 0.8878085265519821 + "accuracy": 0.2696335078534031 }, "prompt_4": { - "accuracy": 0.881077038145101 + "accuracy": 0.2726252804786836 }, "prompt_5": { - "accuracy": 0.8403141361256544 + "accuracy": 0.2707554225878833 } }, "dream": { "prompt_1": { - "accuracy": 0.8535031847133758 + "accuracy": 0.9299363057324841 }, "prompt_2": { - "accuracy": 0.8735913767760901 + "accuracy": 0.9294463498285155 }, "prompt_3": { - "accuracy": 0.8878000979911808 + "accuracy": 0.9294463498285155 }, "prompt_4": { - "accuracy": 0.8652621264086232 + "accuracy": 0.9284664380205782 }, "prompt_5": { - "accuracy": 0.8530132288094071 + "accuracy": 0.9279764821166095 } }, "samsum": { "prompt_1": { - "rouge1": 0.317109537534875, - "rouge2": 0.12301652641451623, - "rougeL": 0.23944069685998487, - "avg_rouge": 0.22652225360312536 + "rouge1": 0.5431764893139259, + "rouge2": 0.29470708142921725, + "rougeL": 0.4528347926542214, + "avg_rouge": 0.43023945446578815 }, "prompt_2": { - "rouge1": 0.3737595119698359, - "rouge2": 0.1485860570070288, - "rougeL": 0.2897299796770693, - "avg_rouge": 0.2706918495513113 + "rouge1": 0.542488939942541, + "rouge2": 0.2933689952477829, + "rougeL": 0.45139235454365445, + "avg_rouge": 0.42908342991132614 }, "prompt_3": { - "rouge1": 0.3365359548611154, - "rouge2": 0.12992216132759968, - "rougeL": 0.25767319370683117, - "avg_rouge": 0.24137710329851544 + "rouge1": 0.5373293668982885, + "rouge2": 0.2908203324216751, + "rougeL": 0.44950047209810673, + "avg_rouge": 0.42588339047269014 }, "prompt_4": { - "rouge1": 0.3604801882697693, - "rouge2": 0.1418836115841394, - "rougeL": 0.2787780695556606, - "avg_rouge": 0.2603806231365231 + "rouge1": 0.5401317921433638, + "rouge2": 0.28908707215882196, + "rougeL": 0.44795067378763737, + "avg_rouge": 0.42572317936327436 }, "prompt_5": { - "rouge1": 0.3654942720016746, - "rouge2": 0.14243458890960486, - "rougeL": 0.28105658752469315, - "avg_rouge": 0.26299514947865754 + "rouge1": 0.5430637681899185, + "rouge2": 0.292544342841456, + "rougeL": 0.4526876943035778, + "avg_rouge": 0.42943193511165073 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.2035379224548374, - "rouge2": 0.059397887117575474, - "rougeL": 0.15185952067665914, - "avg_rouge": 0.138265110083024 + "rouge1": 0.2923360623216347, + "rouge2": 0.10186037739881519, + "rougeL": 0.22620543198594276, + "avg_rouge": 0.20680062390213091 }, "prompt_2": { - "rouge1": 0.20867076426895081, - "rouge2": 0.06080638399409097, - "rougeL": 0.15459241113736594, - "avg_rouge": 0.14135651980013592 + "rouge1": 0.3058212189179107, + "rouge2": 0.11173668321767392, + "rougeL": 0.2398256417068377, + "avg_rouge": 0.2191278479474741 }, "prompt_3": { - "rouge1": 0.21637061196740395, - "rouge2": 0.06219209114251397, - "rougeL": 0.15968062007097034, - "avg_rouge": 0.14608110772696276 + "rouge1": 0.30430152353756323, + "rouge2": 0.11015725862414502, + "rougeL": 0.2378957718944913, + "avg_rouge": 0.2174515180187332 }, "prompt_4": { - "rouge1": 0.20748031686111118, - "rouge2": 0.05962748449769534, - "rougeL": 0.15270369819002264, - "avg_rouge": 0.13993716651627638 + "rouge1": 0.2568407137049432, + "rouge2": 0.08122523878099604, + "rougeL": 0.19502090677295342, + "avg_rouge": 0.17769561975296422 }, "prompt_5": { - "rouge1": 0.2009320427907052, - "rouge2": 0.0549186731977667, - "rougeL": 0.15128322730046576, - "avg_rouge": 0.1357113144296459 + "rouge1": 0.3030613159708064, + "rouge2": 0.10838718211190466, + "rougeL": 0.23768183221016057, + "avg_rouge": 0.21637677676429054 } }, "sst2": { "prompt_1": { - "accuracy": 0.7637614678899083 + "accuracy": 0.9621559633027523 }, "prompt_2": { - "accuracy": 0.9105504587155964 + "accuracy": 0.9610091743119266 }, "prompt_3": { - "accuracy": 0.9139908256880734 + "accuracy": 0.9610091743119266 }, "prompt_4": { - "accuracy": 0.7178899082568807 + "accuracy": 0.9621559633027523 }, "prompt_5": { - "accuracy": 0.9311926605504587 + "accuracy": 0.9621559633027523 } }, "cola": { "prompt_1": { - "accuracy": 0.7833173537871524 + "accuracy": 0.8465963566634708 }, "prompt_2": { - "accuracy": 0.7593480345158198 + "accuracy": 0.8475551294343241 }, "prompt_3": { - "accuracy": 0.7794822627037392 + "accuracy": 0.850431447746884 }, "prompt_4": { - "accuracy": 0.7833173537871524 + "accuracy": 0.8437200383509108 }, "prompt_5": { - "accuracy": 0.7976989453499521 + "accuracy": 0.8465963566634708 } }, "qqp": { "prompt_1": { - "accuracy": 0.6965 + "accuracy": 0.841 }, "prompt_2": { - "accuracy": 0.7595 + "accuracy": 0.8655 }, "prompt_3": { - "accuracy": 0.7445 + "accuracy": 0.832 }, "prompt_4": { - "accuracy": 0.76 + "accuracy": 0.866 }, "prompt_5": { - "accuracy": 0.7075 + "accuracy": 0.8695 } }, "mnli": { "prompt_1": { - "accuracy": 0.6295 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.6475 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.6325 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.6375 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.6175 + "accuracy": 0.9 } }, "qnli": { "prompt_1": { - "accuracy": 0.5835 + "accuracy": 1.0 }, "prompt_2": { - "accuracy": 0.6255 + "accuracy": 1.0 }, "prompt_3": { - "accuracy": 0.628 + "accuracy": 1.0 }, "prompt_4": { - "accuracy": 0.7025 + "accuracy": 1.0 }, "prompt_5": { - "accuracy": 0.553 + "accuracy": 1.0 } }, "wnli": { "prompt_1": { - "accuracy": 0.5492957746478874 + "accuracy": 0.8 }, "prompt_2": { - "accuracy": 0.5915492957746479 + "accuracy": 0.8 }, "prompt_3": { - "accuracy": 0.5211267605633803 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.4788732394366197 + "accuracy": 0.8 }, "prompt_5": { - "accuracy": 0.5774647887323944 + "accuracy": 0.8 } }, "rte": { "prompt_1": { - "accuracy": 0.7509025270758123 + "accuracy": 1.0 }, "prompt_2": { - "accuracy": 0.8303249097472925 + "accuracy": 1.0 }, "prompt_3": { - "accuracy": 0.7653429602888087 + "accuracy": 1.0 }, "prompt_4": { - "accuracy": 0.6534296028880866 + "accuracy": 1.0 }, "prompt_5": { - "accuracy": 0.6823104693140795 + "accuracy": 1.0 } }, "mrpc": { "prompt_1": { - "accuracy": 0.7205882352941176 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.7524509803921569 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.7107843137254902 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.6887254901960784 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.7156862745098039 + "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.9195378151260504, - "language_acc": { - "English": 0.9134453781512605, - "Chinese": 0.9302521008403362, - "Spanish": 0.9193277310924369, - "Vietnamese": 0.915126050420168 - }, - "consistency_score_2": 0.896218487394958, - "consistency_score_3": 0.8491596638655462, - "consistency_score_4": 0.8142857142857143, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8941176470588236, - "English,Spanish": 0.8949579831932774, - "English,Vietnamese": 0.8781512605042017, - "Chinese,Spanish": 0.9109243697478991, - "Chinese,Vietnamese": 0.8957983193277311, - "Spanish,Vietnamese": 0.9033613445378151 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8571428571428571, - "English,Chinese,Vietnamese": 0.838655462184874, - "English,Spanish,Vietnamese": 0.8428571428571429, - "Chinese,Spanish,Vietnamese": 0.8579831932773109 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8142857142857143 - } - }, - "AC3_2": 0.9077284089116491, - "AC3_3": 0.8829485327305746, - "AC3_4": 0.8637170898395751 - } + "prompt_1": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.3742857142857143, - "language_acc": { - "Spanish": 0.37333333333333335, - "English": 0.42, - "Chinese": 0.35333333333333333, - "Vietnamese": 0.38, - "Filipino": 0.32, - "Indonesian": 0.44666666666666666, - "Malay": 0.32666666666666666 - }, - "consistency_score_2": 0.45936507936507925, - "consistency_score_3": 0.25619047619047625, - "consistency_score_4": 0.15733333333333333, - "consistency_score_5": 0.10317460317460318, - "consistency_score_6": 0.06952380952380952, - "consistency_score_7": 0.04666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.4866666666666667, - "Spanish,Chinese": 0.4533333333333333, - "Spanish,Vietnamese": 0.5333333333333333, - "Spanish,Filipino": 0.46, - "Spanish,Indonesian": 0.4666666666666667, - "Spanish,Malay": 0.43333333333333335, - "English,Chinese": 0.5066666666666667, - "English,Vietnamese": 0.47333333333333333, - "English,Filipino": 0.38666666666666666, - "English,Indonesian": 0.44666666666666666, - "English,Malay": 0.4266666666666667, - "Chinese,Vietnamese": 0.52, - "Chinese,Filipino": 0.43333333333333335, - "Chinese,Indonesian": 0.4533333333333333, - "Chinese,Malay": 0.4866666666666667, - "Vietnamese,Filipino": 0.4066666666666667, - "Vietnamese,Indonesian": 0.47333333333333333, - "Vietnamese,Malay": 0.4533333333333333, - "Filipino,Indonesian": 0.4, - "Filipino,Malay": 0.47333333333333333, - "Indonesian,Malay": 0.47333333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.2866666666666667, - "Spanish,English,Vietnamese": 0.29333333333333333, - "Spanish,English,Filipino": 0.22666666666666666, - "Spanish,English,Indonesian": 0.25333333333333335, - "Spanish,English,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese": 0.31333333333333335, - "Spanish,Chinese,Filipino": 0.2733333333333333, - "Spanish,Chinese,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Filipino": 0.28, - "Spanish,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,Vietnamese,Malay": 0.2733333333333333, - "Spanish,Filipino,Indonesian": 0.24666666666666667, - "Spanish,Filipino,Malay": 0.24666666666666667, - "Spanish,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese": 0.30666666666666664, - "English,Chinese,Filipino": 0.22666666666666666, - "English,Chinese,Indonesian": 0.2733333333333333, - "English,Chinese,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino": 0.22, - "English,Vietnamese,Indonesian": 0.24666666666666667, - "English,Vietnamese,Malay": 0.24, - "English,Filipino,Indonesian": 0.2, - "English,Filipino,Malay": 0.20666666666666667, - "English,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino": 0.25333333333333335, - "Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Malay": 0.2866666666666667, - "Chinese,Filipino,Indonesian": 0.24, - "Chinese,Filipino,Malay": 0.2733333333333333, - "Chinese,Indonesian,Malay": 0.26, - "Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Vietnamese,Filipino,Malay": 0.22, - "Vietnamese,Indonesian,Malay": 0.26, - "Filipino,Indonesian,Malay": 0.24666666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.2, - "Spanish,English,Chinese,Filipino": 0.15333333333333332, - "Spanish,English,Chinese,Indonesian": 0.16, - "Spanish,English,Chinese,Malay": 0.16, - "Spanish,English,Vietnamese,Filipino": 0.14, - "Spanish,English,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,English,Vietnamese,Malay": 0.16, - "Spanish,English,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.16666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.15333333333333332, - "Spanish,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese,Filipino": 0.16, - "English,Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "English,Chinese,Vietnamese,Malay": 0.17333333333333334, - "English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Filipino,Malay": 0.13333333333333333, - "English,Chinese,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian": 0.14, - "English,Vietnamese,Filipino,Malay": 0.12666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "English,Filipino,Indonesian,Malay": 0.10666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.14 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,English,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.08, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.08, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.08, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.06, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.06, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04666666666666667 - } - }, - "AC3_2": 0.41248395164239315, - "AC3_3": 0.3041778160933167, - "AC3_4": 0.2215406663861308, - "AC3_5": 0.16175911850715435, - "AC3_6": 0.11726548127339358, - "AC3_7": 0.08298642531965256 - } + "prompt_1": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.4967532467532468, - "language_acc": { - "English": 0.5568181818181818, - "Filipino": 0.3352272727272727, - "Spanish": 0.4943181818181818, - "Malay": 0.4659090909090909, - "Chinese": 0.6136363636363636, - "Indonesian": 0.5397727272727273, - "Vietnamese": 0.4715909090909091 - }, - "consistency_score_2": 0.49945887445887444, - "consistency_score_3": 0.3224025974025974, - "consistency_score_4": 0.2349025974025974, - "consistency_score_5": 0.18452380952380956, - "consistency_score_6": 0.1525974025974026, - "consistency_score_7": 0.13068181818181818, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3977272727272727, - "English,Spanish": 0.5454545454545454, - "English,Malay": 0.45454545454545453, - "English,Chinese": 0.6079545454545454, - "English,Indonesian": 0.5568181818181818, - "English,Vietnamese": 0.5227272727272727, - "Filipino,Spanish": 0.4375, - "Filipino,Malay": 0.39204545454545453, - "Filipino,Chinese": 0.42613636363636365, - "Filipino,Indonesian": 0.4602272727272727, - "Filipino,Vietnamese": 0.4147727272727273, - "Spanish,Malay": 0.5170454545454546, - "Spanish,Chinese": 0.5227272727272727, - "Spanish,Indonesian": 0.5340909090909091, - "Spanish,Vietnamese": 0.5227272727272727, - "Malay,Chinese": 0.4715909090909091, - "Malay,Indonesian": 0.5568181818181818, - "Malay,Vietnamese": 0.5170454545454546, - "Chinese,Indonesian": 0.5568181818181818, - "Chinese,Vietnamese": 0.5284090909090909, - "Indonesian,Vietnamese": 0.5454545454545454 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2727272727272727, - "English,Filipino,Malay": 0.20454545454545456, - "English,Filipino,Chinese": 0.2784090909090909, - "English,Filipino,Indonesian": 0.26136363636363635, - "English,Filipino,Vietnamese": 0.2556818181818182, - "English,Spanish,Malay": 0.3352272727272727, - "English,Spanish,Chinese": 0.4090909090909091, - "English,Spanish,Indonesian": 0.3806818181818182, - "English,Spanish,Vietnamese": 0.3693181818181818, - "English,Malay,Chinese": 0.3352272727272727, - "English,Malay,Indonesian": 0.35795454545454547, - "English,Malay,Vietnamese": 0.3181818181818182, - "English,Chinese,Indonesian": 0.4147727272727273, - "English,Chinese,Vietnamese": 0.39204545454545453, - "English,Indonesian,Vietnamese": 0.3806818181818182, - "Filipino,Spanish,Malay": 0.2556818181818182, - "Filipino,Spanish,Chinese": 0.2784090909090909, - "Filipino,Spanish,Indonesian": 0.30113636363636365, - "Filipino,Spanish,Vietnamese": 0.2784090909090909, - "Filipino,Malay,Chinese": 0.25, - "Filipino,Malay,Indonesian": 0.29545454545454547, - "Filipino,Malay,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian": 0.30113636363636365, - "Filipino,Chinese,Vietnamese": 0.2727272727272727, - "Filipino,Indonesian,Vietnamese": 0.29545454545454547, - "Spanish,Malay,Chinese": 0.32386363636363635, - "Spanish,Malay,Indonesian": 0.3693181818181818, - "Spanish,Malay,Vietnamese": 0.3409090909090909, - "Spanish,Chinese,Indonesian": 0.375, - "Spanish,Chinese,Vietnamese": 0.3409090909090909, - "Spanish,Indonesian,Vietnamese": 0.35795454545454547, - "Malay,Chinese,Indonesian": 0.3522727272727273, - "Malay,Chinese,Vietnamese": 0.32386363636363635, - "Malay,Indonesian,Vietnamese": 0.3806818181818182, - "Chinese,Indonesian,Vietnamese": 0.375 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17045454545454544, - "English,Filipino,Spanish,Chinese": 0.2159090909090909, - "English,Filipino,Spanish,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Vietnamese": 0.2159090909090909, - "English,Filipino,Malay,Chinese": 0.17045454545454544, - "English,Filipino,Malay,Indonesian": 0.1875, - "English,Filipino,Malay,Vietnamese": 0.17045454545454544, - "English,Filipino,Chinese,Indonesian": 0.2215909090909091, - "English,Filipino,Chinese,Vietnamese": 0.21022727272727273, - "English,Filipino,Indonesian,Vietnamese": 0.20454545454545456, - "English,Spanish,Malay,Chinese": 0.2727272727272727, - "English,Spanish,Malay,Indonesian": 0.2727272727272727, - "English,Spanish,Malay,Vietnamese": 0.24431818181818182, - "English,Spanish,Chinese,Indonesian": 0.32954545454545453, - "English,Spanish,Chinese,Vietnamese": 0.30113636363636365, - "English,Spanish,Indonesian,Vietnamese": 0.2897727272727273, - "English,Malay,Chinese,Indonesian": 0.2897727272727273, - "English,Malay,Chinese,Vietnamese": 0.25, - "English,Malay,Indonesian,Vietnamese": 0.2727272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.3125, - "Filipino,Spanish,Malay,Chinese": 0.18181818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.2159090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Chinese,Indonesian": 0.2556818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.23863636363636365, - "Spanish,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.2784090909090909 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Indonesian": 0.1875, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.23863636363636365, - "English,Spanish,Malay,Chinese,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - } - }, - "AC3_2": 0.4981023864282695, - "AC3_3": 0.3910233642217649, - "AC3_4": 0.31897135476090743, - "AC3_5": 0.2690911153582099, - "AC3_6": 0.23347402593807098, - "AC3_7": 0.20692696691986037 - } + "prompt_1": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.30097087378640774 - } + "prompt_1": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.5523809523809524 - } + "prompt_1": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.4205607476635514 - } + "prompt_1": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.33, - "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.3, - "history": 0.26666666666666666, - "literature": 0.4, - "politics": 0.6, - "culture": 0.2, - "film": 0.4, - "law": 0.2, - "geography": 0.3 - } - } + "prompt_1": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.07553564389712619 - } + "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.08874817365716853 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.0648988290647751 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.08852443675439182 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.08892355148135121 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.4072345390898483 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.40457633178405433, - "category_acc": { - "high_school_european_history": 0.676829268292683, - "business_ethics": 0.4444444444444444, - "clinical_knowledge": 0.4166666666666667, - "medical_genetics": 0.494949494949495, - "high_school_us_history": 0.5812807881773399, - "high_school_physics": 0.3466666666666667, - "high_school_world_history": 0.6228813559322034, - "virology": 0.30303030303030304, - "high_school_microeconomics": 0.41350210970464135, - "econometrics": 0.3008849557522124, - "college_computer_science": 0.3939393939393939, - "high_school_biology": 0.4627831715210356, - "abstract_algebra": 0.31313131313131315, - "professional_accounting": 0.34519572953736655, - "philosophy": 0.4096774193548387, - "professional_medicine": 0.46494464944649444, - "nutrition": 0.36721311475409835, - "global_facts": 0.20202020202020202, - "machine_learning": 0.34234234234234234, - "security_studies": 0.4918032786885246, - "public_relations": 0.3394495412844037, - "professional_psychology": 0.4026186579378069, - "prehistory": 0.39628482972136225, - "anatomy": 0.3283582089552239, - "human_sexuality": 0.4307692307692308, - "college_medicine": 0.3953488372093023, - "high_school_government_and_politics": 0.4583333333333333, - "college_chemistry": 0.2727272727272727, - "logical_fallacies": 0.41975308641975306, - "high_school_geography": 0.4467005076142132, - "elementary_mathematics": 0.3395225464190981, - "human_aging": 0.46846846846846846, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.4834558823529412, - "formal_logic": 0.312, - "high_school_statistics": 0.35348837209302325, - "international_law": 0.4166666666666667, - "high_school_mathematics": 0.27137546468401486, - "high_school_computer_science": 0.40404040404040403, - "conceptual_physics": 0.3504273504273504, - "miscellaneous": 0.45907928388746805, - "high_school_chemistry": 0.4207920792079208, - "marketing": 0.48497854077253216, - "professional_law": 0.3776908023483366, - "management": 0.49019607843137253, - "college_physics": 0.36633663366336633, - "jurisprudence": 0.45794392523364486, - "world_religions": 0.47058823529411764, - "sociology": 0.42, - "us_foreign_policy": 0.494949494949495, - "high_school_macroeconomics": 0.38817480719794345, - "computer_security": 0.41414141414141414, - "moral_scenarios": 0.2751677852348993, - "moral_disputes": 0.43478260869565216, - "electrical_engineering": 0.3680555555555556, - "astronomy": 0.44370860927152317, - "college_biology": 0.46153846153846156 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.4576523031203566 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.47073474470734744, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.375, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.47619047619047616, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.5217391304347826, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.4166666666666667, - "high_school_biology": 0.7083333333333334, - "middle_school_mathematics": 0.5833333333333334, - "middle_school_biology": 0.6153846153846154, - "middle_school_physics": 0.7083333333333334, - "middle_school_chemistry": 0.56, - "veterinary_medicine": 0.5, - "college_economics": 0.43333333333333335, - "business_administration": 0.34210526315789475, - "marxism": 0.5416666666666666, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.5714285714285714, - "high_school_politics": 0.6666666666666666, - "high_school_geography": 0.7083333333333334, - "middle_school_politics": 0.5384615384615384, - "middle_school_geography": 0.7058823529411765, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.75, - "logic": 0.2962962962962963, - "law": 0.5862068965517241, - "chinese_language_and_literature": 0.4642857142857143, - "art_studies": 0.5263157894736842, - "professional_tour_guide": 0.5, - "legal_professional": 0.25, - "high_school_chinese": 0.4583333333333333, - "high_school_history": 0.6, - "middle_school_history": 0.6666666666666666, - "civil_servant": 0.38461538461538464, - "sports_science": 0.3333333333333333, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.6666666666666666, - "clinical_medicine": 0.48148148148148145, - "urban_and_rural_planner": 0.45098039215686275, - "accountant": 0.42592592592592593, - "fire_engineer": 0.5, - "environmental_impact_assessment_engineer": 0.4722222222222222, - "tax_accountant": 0.35185185185185186, - "physician": 0.5925925925925926 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.6666666666666666 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.6284752201692281, - "category_acc": { - "agronomy": 0.6035502958579881, - "anatomy": 0.6486486486486487, - "ancient_chinese": 0.34146341463414637, - "arts": 0.79375, - "astronomy": 0.43636363636363634, - "business_ethics": 0.6267942583732058, - "chinese_civil_service_exam": 0.51875, - "chinese_driving_rule": 0.7862595419847328, - "chinese_food_culture": 0.6397058823529411, - "chinese_foreign_policy": 0.7663551401869159, - "chinese_history": 0.7832817337461301, - "chinese_literature": 0.5098039215686274, - "chinese_teacher_qualification": 0.7150837988826816, - "clinical_knowledge": 0.6329113924050633, - "college_actuarial_science": 0.33962264150943394, - "college_education": 0.6915887850467289, - "college_engineering_hydrology": 0.5188679245283019, - "college_law": 0.5833333333333334, - "college_mathematics": 0.34285714285714286, - "college_medical_statistics": 0.5849056603773585, - "college_medicine": 0.7032967032967034, - "computer_science": 0.6176470588235294, - "computer_security": 0.7076023391812866, - "conceptual_physics": 0.5578231292517006, - "construction_project_management": 0.5827338129496403, - "economics": 0.610062893081761, - "education": 0.7423312883435583, - "electrical_engineering": 0.6627906976744186, - "elementary_chinese": 0.6388888888888888, - "elementary_commonsense": 0.6363636363636364, - "elementary_information_and_technology": 0.6680672268907563, - "elementary_mathematics": 0.46956521739130436, - "ethnology": 0.6296296296296297, - "food_science": 0.6503496503496503, - "genetics": 0.5625, - "global_facts": 0.6912751677852349, - "high_school_biology": 0.6331360946745562, - "high_school_chemistry": 0.4318181818181818, - "high_school_geography": 0.6271186440677966, - "high_school_mathematics": 0.4878048780487805, - "high_school_physics": 0.5545454545454546, - "high_school_politics": 0.6993006993006993, - "human_sexuality": 0.5555555555555556, - "international_law": 0.5513513513513514, - "journalism": 0.5348837209302325, - "jurisprudence": 0.5620437956204379, - "legal_and_moral_basis": 0.8598130841121495, - "logical": 0.5609756097560976, - "machine_learning": 0.5573770491803278, - "management": 0.7523809523809524, - "marketing": 0.7111111111111111, - "marxist_theory": 0.783068783068783, - "modern_chinese": 0.46551724137931033, - "nutrition": 0.6620689655172414, - "philosophy": 0.5714285714285714, - "professional_accounting": 0.6914285714285714, - "professional_law": 0.4881516587677725, - "professional_medicine": 0.6037234042553191, - "professional_psychology": 0.8060344827586207, - "public_relations": 0.6206896551724138, - "security_study": 0.7037037037037037, - "sociology": 0.672566371681416, - "sports_science": 0.6303030303030303, - "traditional_chinese_medicine": 0.6864864864864865, - "virology": 0.6982248520710059, - "world_history": 0.8136645962732919, - "world_religions": 0.61875 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.3333333333333333 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.5931818181818181 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.4752542372881356 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.40089753178758414 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.8000979911807937 - } + "prompt_1": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.2206914957168043, - "rouge2": 0.0875176528869922, - "rougeL": 0.16523011446196761, - "avg_rouge": 0.15781308768858804 - } + "prompt_1": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.21076971169141254, - "rouge2": 0.07677428579445292, - "rougeL": 0.16026281357813363, - "avg_rouge": 0.14926893702133304 - } + "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.49655963302752293 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.40172579098753597 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.6105 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.576 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.6035 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.5492957746478874 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.6570397111913358 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.5294117647058824 - } + "prompt_1": -1 } } }, - "seallm_7b_v2": { - "model_size": "7B", - "model_link": "https://huggingface.co/SeaLLMs/SeaLLM-7B-v2", + "flan_t5_small": { + "model_size": "0.06B", + "model_link": "https://huggingface.co/google/flan-t5-small", "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.9235294117647058, + "overall_acc": 0.4852941176470588, "language_acc": { - "English": 0.9453781512605042, - "Chinese": 0.9067226890756303, - "Spanish": 0.9285714285714286, - "Vietnamese": 0.9134453781512605 + "Spanish": 0.5714285714285714, + "English": 0.6621848739495798, + "Chinese": 0.31596638655462184, + "Vietnamese": 0.3915966386554622 }, - "consistency_score_2": 0.9127450980392157, - "consistency_score_3": 0.8724789915966387, - "consistency_score_4": 0.8436974789915966, + "consistency_score_2": 0.4644257703081232, + "consistency_score_3": 0.2743697478991597, + "consistency_score_4": 0.16974789915966387, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9117647058823529, - "English,Spanish": 0.9445378151260504, - "English,Vietnamese": 0.9159663865546218, - "Chinese,Spanish": 0.9, - "Chinese,Vietnamese": 0.8915966386554622, - "Spanish,Vietnamese": 0.9126050420168067 + "Spanish,English": 0.6714285714285714, + "Spanish,Chinese": 0.3773109243697479, + "Spanish,Vietnamese": 0.5310924369747899, + "English,Chinese": 0.3815126050420168, + "English,Vietnamese": 0.5, + "Chinese,Vietnamese": 0.32521008403361346 }, "3_combine": { - "English,Chinese,Spanish": 0.880672268907563, - "English,Chinese,Vietnamese": 0.8638655462184874, - "English,Spanish,Vietnamese": 0.8890756302521008, - "Chinese,Spanish,Vietnamese": 0.8563025210084033 + "Spanish,English,Chinese": 0.2781512605042017, + "Spanish,English,Vietnamese": 0.39663865546218485, + "Spanish,Chinese,Vietnamese": 0.21008403361344538, + "English,Chinese,Vietnamese": 0.21260504201680672 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8436974789915966 + "Spanish,English,Chinese,Vietnamese": 0.16974789915966387 } }, - "AC3_2": 0.9181055870860841, - "AC3_3": 0.8972786634332136, - "AC3_4": 0.881810298962717 + "AC3_2": 0.47463067215696914, + "AC3_3": 0.3505498437851925, + "AC3_4": 0.2515186959582823 }, "prompt_2": { - "overall_acc": 0.9157563025210084, + "overall_acc": 0.486764705882353, "language_acc": { - "English": 0.9378151260504202, - "Chinese": 0.9042016806722689, - "Spanish": 0.9235294117647059, - "Vietnamese": 0.8974789915966387 + "Spanish": 0.5739495798319327, + "English": 0.6495798319327731, + "Chinese": 0.3243697478991597, + "Vietnamese": 0.39915966386554624 }, - "consistency_score_2": 0.8952380952380953, - "consistency_score_3": 0.8470588235294119, - "consistency_score_4": 0.8084033613445378, + "consistency_score_2": 0.4558823529411764, + "consistency_score_3": 0.26701680672268907, + "consistency_score_4": 0.16554621848739495, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9100840336134454, - "English,Spanish": 0.9319327731092437, - "English,Vietnamese": 0.8949579831932774, - "Chinese,Spanish": 0.8899159663865546, - "Chinese,Vietnamese": 0.8638655462184874, - "Spanish,Vietnamese": 0.880672268907563 + "Spanish,English": 0.6974789915966386, + "Spanish,Chinese": 0.38907563025210085, + "Spanish,Vietnamese": 0.5092436974789916, + "English,Chinese": 0.37142857142857144, + "English,Vietnamese": 0.4865546218487395, + "Chinese,Vietnamese": 0.2815126050420168 }, "3_combine": { - "English,Chinese,Spanish": 0.8705882352941177, - "English,Chinese,Vietnamese": 0.8394957983193277, - "English,Spanish,Vietnamese": 0.8554621848739495, - "Chinese,Spanish,Vietnamese": 0.8226890756302521 + "Spanish,English,Chinese": 0.2857142857142857, + "Spanish,English,Vietnamese": 0.39327731092436974, + "Spanish,Chinese,Vietnamese": 0.1957983193277311, + "English,Chinese,Vietnamese": 0.19327731092436976 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8084033613445378 + "Spanish,English,Chinese,Vietnamese": 0.16554621848739495 } }, - "AC3_2": 0.9053809652203805, - "AC3_3": 0.880068981421821, - "AC3_4": 0.8587377243563935 + "AC3_2": 0.4708176561857148, + "AC3_3": 0.34485949893778933, + "AC3_4": 0.24706640140192743 }, "prompt_3": { - "overall_acc": 0.9065126050420168, + "overall_acc": 0.475, "language_acc": { - "English": 0.938655462184874, - "Chinese": 0.9025210084033614, - "Spanish": 0.9117647058823529, - "Vietnamese": 0.873109243697479 + "Spanish": 0.5563025210084034, + "English": 0.6386554621848739, + "Chinese": 0.32605042016806723, + "Vietnamese": 0.37899159663865545 }, - "consistency_score_2": 0.884453781512605, - "consistency_score_3": 0.8338235294117649, - "consistency_score_4": 0.7949579831932773, + "consistency_score_2": 0.4414565826330532, + "consistency_score_3": 0.25168067226890756, + "consistency_score_4": 0.15378151260504203, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.907563025210084, - "English,Spanish": 0.9193277310924369, - "English,Vietnamese": 0.8815126050420168, - "Chinese,Spanish": 0.8831932773109243, - "Chinese,Vietnamese": 0.8563025210084033, - "Spanish,Vietnamese": 0.8588235294117647 + "Spanish,English": 0.66890756302521, + "Spanish,Chinese": 0.3420168067226891, + "Spanish,Vietnamese": 0.5100840336134453, + "English,Chinese": 0.33949579831932775, + "English,Vietnamese": 0.48823529411764705, + "Chinese,Vietnamese": 0.3 }, "3_combine": { - "English,Chinese,Spanish": 0.8630252100840337, - "English,Chinese,Vietnamese": 0.8285714285714286, - "English,Spanish,Vietnamese": 0.8344537815126051, - "Chinese,Spanish,Vietnamese": 0.8092436974789916 + "Spanish,English,Chinese": 0.2504201680672269, + "Spanish,English,Vietnamese": 0.38823529411764707, + "Spanish,Chinese,Vietnamese": 0.18319327731092436, + "English,Chinese,Vietnamese": 0.18487394957983194 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7949579831932773 + "Spanish,English,Chinese,Vietnamese": 0.15378151260504203 } }, - "AC3_2": 0.8953473471000113, - "AC3_3": 0.8686500553368002, - "AC3_4": 0.8470783299857728 + "AC3_2": 0.45761442648021133, + "AC3_3": 0.32902572993448503, + "AC3_4": 0.2323421316035433 }, "prompt_4": { - "overall_acc": 0.9163865546218487, + "overall_acc": 0.4859243697478992, "language_acc": { - "English": 0.9361344537815126, - "Chinese": 0.9058823529411765, - "Spanish": 0.9260504201680673, - "Vietnamese": 0.8974789915966387 + "Spanish": 0.5756302521008403, + "English": 0.6495798319327731, + "Chinese": 0.31932773109243695, + "Vietnamese": 0.39915966386554624 }, - "consistency_score_2": 0.9008403361344538, - "consistency_score_3": 0.8556722689075631, - "consistency_score_4": 0.8252100840336134, + "consistency_score_2": 0.4598039215686274, + "consistency_score_3": 0.2710084033613445, + "consistency_score_4": 0.16554621848739495, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9050420168067227, - "English,Spanish": 0.9310924369747899, - "English,Vietnamese": 0.9016806722689076, - "Chinese,Spanish": 0.8974789915966387, - "Chinese,Vietnamese": 0.8789915966386554, - "Spanish,Vietnamese": 0.8907563025210085 + "Spanish,English": 0.6957983193277311, + "Spanish,Chinese": 0.37478991596638656, + "Spanish,Vietnamese": 0.5218487394957984, + "English,Chinese": 0.37815126050420167, + "English,Vietnamese": 0.5042016806722689, + "Chinese,Vietnamese": 0.28403361344537814 }, "3_combine": { - "English,Chinese,Spanish": 0.8714285714285714, - "English,Chinese,Vietnamese": 0.8487394957983193, - "English,Spanish,Vietnamese": 0.8638655462184874, - "Chinese,Spanish,Vietnamese": 0.838655462184874 + "Spanish,English,Chinese": 0.2823529411764706, + "Spanish,English,Vietnamese": 0.40588235294117647, + "Spanish,Chinese,Vietnamese": 0.1957983193277311, + "English,Chinese,Vietnamese": 0.2 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8252100840336134 + "Spanish,English,Chinese,Vietnamese": 0.16554621848739495 } }, - "AC3_2": 0.908546947052544, - "AC3_3": 0.8849893151782248, - "AC3_4": 0.8684116734253171 + "AC3_2": 0.47250343005217904, + "AC3_3": 0.3479558350774224, + "AC3_4": 0.24695801567402034 }, "prompt_5": { - "overall_acc": 0.9210084033613445, + "overall_acc": 0.4915966386554622, "language_acc": { - "English": 0.9411764705882353, - "Chinese": 0.915126050420168, - "Spanish": 0.9252100840336135, - "Vietnamese": 0.9025210084033614 + "Spanish": 0.565546218487395, + "English": 0.6823529411764706, + "Chinese": 0.3134453781512605, + "Vietnamese": 0.4050420168067227 }, - "consistency_score_2": 0.9067226890756303, - "consistency_score_3": 0.8649159663865547, - "consistency_score_4": 0.8352941176470589, + "consistency_score_2": 0.457703081232493, + "consistency_score_3": 0.2619747899159664, + "consistency_score_4": 0.17058823529411765, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.9193277310924369, - "English,Spanish": 0.9336134453781513, - "English,Vietnamese": 0.9033613445378151, - "Chinese,Spanish": 0.9058823529411765, - "Chinese,Vietnamese": 0.8857142857142857, - "Spanish,Vietnamese": 0.892436974789916 + "Spanish,English": 0.673109243697479, + "Spanish,Chinese": 0.3420168067226891, + "Spanish,Vietnamese": 0.5092436974789916, + "English,Chinese": 0.3596638655462185, + "English,Vietnamese": 0.48403361344537815, + "Chinese,Vietnamese": 0.37815126050420167 }, "3_combine": { - "English,Chinese,Spanish": 0.8840336134453781, - "English,Chinese,Vietnamese": 0.8579831932773109, - "English,Spanish,Vietnamese": 0.8680672268907563, - "Chinese,Spanish,Vietnamese": 0.8495798319327731 + "Spanish,English,Chinese": 0.2529411764705882, + "Spanish,English,Vietnamese": 0.37899159663865545, + "Spanish,Chinese,Vietnamese": 0.20756302521008405, + "English,Chinese,Vietnamese": 0.20840336134453782 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8352941176470589 + "Spanish,English,Chinese,Vietnamese": 0.17058823529411765 } }, - "AC3_2": 0.9138097169402474, - "AC3_3": 0.89208130724064, - "AC3_4": 0.8760596678364123 + "AC3_2": 0.4740447963873758, + "AC3_3": 0.34180151007639403, + "AC3_4": 0.253284562517737 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.6085714285714285, + "overall_acc": 0.26952380952380955, "language_acc": { - "Spanish": 0.6266666666666667, - "English": 0.7133333333333334, - "Chinese": 0.5866666666666667, - "Vietnamese": 0.5666666666666667, - "Filipino": 0.5733333333333334, - "Indonesian": 0.66, - "Malay": 0.5333333333333333 - }, - "consistency_score_2": 0.6358730158730159, - "consistency_score_3": 0.4965714285714287, - "consistency_score_4": 0.41695238095238096, - "consistency_score_5": 0.36253968253968255, - "consistency_score_6": 0.32095238095238093, - "consistency_score_7": 0.2866666666666667, + "Filipino": 0.24, + "Vietnamese": 0.3, + "Chinese": 0.29333333333333333, + "Spanish": 0.29333333333333333, + "Malay": 0.23333333333333334, + "Indonesian": 0.26, + "English": 0.26666666666666666 + }, + "consistency_score_2": 0.45269841269841266, + "consistency_score_3": 0.2617142857142857, + "consistency_score_4": 0.17485714285714288, + "consistency_score_5": 0.1292063492063492, + "consistency_score_6": 0.10285714285714287, + "consistency_score_7": 0.08666666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.7666666666666667, - "Spanish,Chinese": 0.6733333333333333, - "Spanish,Vietnamese": 0.64, - "Spanish,Filipino": 0.6133333333333333, - "Spanish,Indonesian": 0.7066666666666667, - "Spanish,Malay": 0.6666666666666666, - "English,Chinese": 0.64, - "English,Vietnamese": 0.6933333333333334, - "English,Filipino": 0.6266666666666667, - "English,Indonesian": 0.7, - "English,Malay": 0.6066666666666667, - "Chinese,Vietnamese": 0.6666666666666666, - "Chinese,Filipino": 0.5533333333333333, - "Chinese,Indonesian": 0.68, - "Chinese,Malay": 0.5866666666666667, - "Vietnamese,Filipino": 0.5666666666666667, - "Vietnamese,Indonesian": 0.62, - "Vietnamese,Malay": 0.5733333333333334, - "Filipino,Indonesian": 0.6066666666666667, - "Filipino,Malay": 0.56, - "Indonesian,Malay": 0.6066666666666667 + "Filipino,Vietnamese": 0.41333333333333333, + "Filipino,Chinese": 0.38, + "Filipino,Spanish": 0.43333333333333335, + "Filipino,Malay": 0.47333333333333333, + "Filipino,Indonesian": 0.43333333333333335, + "Filipino,English": 0.4666666666666667, + "Vietnamese,Chinese": 0.5533333333333333, + "Vietnamese,Spanish": 0.43333333333333335, + "Vietnamese,Malay": 0.5, + "Vietnamese,Indonesian": 0.5066666666666667, + "Vietnamese,English": 0.38666666666666666, + "Chinese,Spanish": 0.4266666666666667, + "Chinese,Malay": 0.4266666666666667, + "Chinese,Indonesian": 0.4266666666666667, + "Chinese,English": 0.43333333333333335, + "Spanish,Malay": 0.46, + "Spanish,Indonesian": 0.3933333333333333, + "Spanish,English": 0.48, + "Malay,Indonesian": 0.6333333333333333, + "Malay,English": 0.4266666666666667, + "Indonesian,English": 0.42 }, "3_combine": { - "Spanish,English,Chinese": 0.5666666666666667, - "Spanish,English,Vietnamese": 0.5866666666666667, - "Spanish,English,Filipino": 0.5333333333333333, - "Spanish,English,Indonesian": 0.6133333333333333, - "Spanish,English,Malay": 0.5666666666666667, - "Spanish,Chinese,Vietnamese": 0.5266666666666666, - "Spanish,Chinese,Filipino": 0.4666666666666667, - "Spanish,Chinese,Indonesian": 0.56, - "Spanish,Chinese,Malay": 0.5066666666666667, - "Spanish,Vietnamese,Filipino": 0.4533333333333333, - "Spanish,Vietnamese,Indonesian": 0.52, - "Spanish,Vietnamese,Malay": 0.48, - "Spanish,Filipino,Indonesian": 0.5066666666666667, - "Spanish,Filipino,Malay": 0.4666666666666667, - "Spanish,Indonesian,Malay": 0.5266666666666666, - "English,Chinese,Vietnamese": 0.5466666666666666, - "English,Chinese,Filipino": 0.4533333333333333, - "English,Chinese,Indonesian": 0.54, - "English,Chinese,Malay": 0.46, - "English,Vietnamese,Filipino": 0.48, - "English,Vietnamese,Indonesian": 0.5466666666666666, - "English,Vietnamese,Malay": 0.4666666666666667, - "English,Filipino,Indonesian": 0.5133333333333333, - "English,Filipino,Malay": 0.44, - "English,Indonesian,Malay": 0.4866666666666667, - "Chinese,Vietnamese,Filipino": 0.4533333333333333, - "Chinese,Vietnamese,Indonesian": 0.5266666666666666, - "Chinese,Vietnamese,Malay": 0.4533333333333333, - "Chinese,Filipino,Indonesian": 0.4666666666666667, - "Chinese,Filipino,Malay": 0.42, - "Chinese,Indonesian,Malay": 0.48, - "Vietnamese,Filipino,Indonesian": 0.46, - "Vietnamese,Filipino,Malay": 0.42, - "Vietnamese,Indonesian,Malay": 0.44666666666666666, - "Filipino,Indonesian,Malay": 0.44 + "Filipino,Vietnamese,Chinese": 0.26666666666666666, + "Filipino,Vietnamese,Spanish": 0.23333333333333334, + "Filipino,Vietnamese,Malay": 0.28, + "Filipino,Vietnamese,Indonesian": 0.28, + "Filipino,Vietnamese,English": 0.24, + "Filipino,Chinese,Spanish": 0.21333333333333335, + "Filipino,Chinese,Malay": 0.24, + "Filipino,Chinese,Indonesian": 0.22666666666666666, + "Filipino,Chinese,English": 0.24, + "Filipino,Spanish,Malay": 0.25333333333333335, + "Filipino,Spanish,Indonesian": 0.22666666666666666, + "Filipino,Spanish,English": 0.2733333333333333, + "Filipino,Malay,Indonesian": 0.34, + "Filipino,Malay,English": 0.26666666666666666, + "Filipino,Indonesian,English": 0.25333333333333335, + "Vietnamese,Chinese,Spanish": 0.2866666666666667, + "Vietnamese,Chinese,Malay": 0.30666666666666664, + "Vietnamese,Chinese,Indonesian": 0.3, + "Vietnamese,Chinese,English": 0.25333333333333335, + "Vietnamese,Spanish,Malay": 0.28, + "Vietnamese,Spanish,Indonesian": 0.24666666666666667, + "Vietnamese,Spanish,English": 0.25333333333333335, + "Vietnamese,Malay,Indonesian": 0.37333333333333335, + "Vietnamese,Malay,English": 0.22, + "Vietnamese,Indonesian,English": 0.22, + "Chinese,Spanish,Malay": 0.23333333333333334, + "Chinese,Spanish,Indonesian": 0.22666666666666666, + "Chinese,Spanish,English": 0.25333333333333335, + "Chinese,Malay,Indonesian": 0.31333333333333335, + "Chinese,Malay,English": 0.21333333333333335, + "Chinese,Indonesian,English": 0.22666666666666666, + "Spanish,Malay,Indonesian": 0.31333333333333335, + "Spanish,Malay,English": 0.25333333333333335, + "Spanish,Indonesian,English": 0.24666666666666667, + "Malay,Indonesian,English": 0.30666666666666664 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4866666666666667, - "Spanish,English,Chinese,Filipino": 0.41333333333333333, - "Spanish,English,Chinese,Indonesian": 0.5, - "Spanish,English,Chinese,Malay": 0.44666666666666666, - "Spanish,English,Vietnamese,Filipino": 0.4266666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.48, - "Spanish,English,Vietnamese,Malay": 0.44666666666666666, - "Spanish,English,Filipino,Indonesian": 0.4666666666666667, - "Spanish,English,Filipino,Malay": 0.4266666666666667, - "Spanish,English,Indonesian,Malay": 0.48, - "Spanish,Chinese,Vietnamese,Filipino": 0.38666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian": 0.4666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.41333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.41333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.38666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.44, - "Spanish,Vietnamese,Filipino,Indonesian": 0.4066666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.37333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.42, - "Spanish,Filipino,Indonesian,Malay": 0.41333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.4066666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.47333333333333333, - "English,Chinese,Vietnamese,Malay": 0.4, - "English,Chinese,Filipino,Indonesian": 0.42, - "English,Chinese,Filipino,Malay": 0.36, - "English,Chinese,Indonesian,Malay": 0.41333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.4266666666666667, - "English,Vietnamese,Filipino,Malay": 0.35333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.3933333333333333, - "English,Filipino,Indonesian,Malay": 0.38666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.4, - "Chinese,Vietnamese,Filipino,Malay": 0.35333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.3933333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian,Malay": 0.35333333333333333 + "Filipino,Vietnamese,Chinese,Spanish": 0.16666666666666666, + "Filipino,Vietnamese,Chinese,Malay": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Chinese,English": 0.17333333333333334, + "Filipino,Vietnamese,Spanish,Malay": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,English": 0.17333333333333334, + "Filipino,Vietnamese,Malay,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Malay,English": 0.16, + "Filipino,Vietnamese,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Malay": 0.14666666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.14, + "Filipino,Chinese,Spanish,English": 0.16, + "Filipino,Chinese,Malay,Indonesian": 0.19333333333333333, + "Filipino,Chinese,Malay,English": 0.16, + "Filipino,Chinese,Indonesian,English": 0.15333333333333332, + "Filipino,Spanish,Malay,Indonesian": 0.19333333333333333, + "Filipino,Spanish,Malay,English": 0.18, + "Filipino,Spanish,Indonesian,English": 0.17333333333333334, + "Filipino,Malay,Indonesian,English": 0.2, + "Vietnamese,Chinese,Spanish,Malay": 0.18666666666666668, + "Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, + "Vietnamese,Chinese,Spanish,English": 0.18, + "Vietnamese,Chinese,Malay,Indonesian": 0.23333333333333334, + "Vietnamese,Chinese,Malay,English": 0.15333333333333332, + "Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, + "Vietnamese,Spanish,Malay,Indonesian": 0.21333333333333335, + "Vietnamese,Spanish,Malay,English": 0.16, + "Vietnamese,Spanish,Indonesian,English": 0.15333333333333332, + "Vietnamese,Malay,Indonesian,English": 0.16666666666666666, + "Chinese,Spanish,Malay,Indonesian": 0.18, + "Chinese,Spanish,Malay,English": 0.15333333333333332, + "Chinese,Spanish,Indonesian,English": 0.15333333333333332, + "Chinese,Malay,Indonesian,English": 0.16666666666666666, + "Spanish,Malay,Indonesian,English": 0.20666666666666667 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.36666666666666664, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.43333333333333335, - "Spanish,English,Chinese,Vietnamese,Malay": 0.38666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian": 0.38666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.35333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.4066666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.38, - "Spanish,English,Vietnamese,Filipino,Malay": 0.3466666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.38666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.36, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.38, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.35333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.37333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.36, - "English,Chinese,Filipino,Indonesian,Malay": 0.3333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.32, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.31333333333333335 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16, + "Filipino,Vietnamese,Chinese,Malay,English": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.14, + "Filipino,Vietnamese,Spanish,Malay,English": 0.12, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.12, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.13333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,English": 0.12, + "Filipino,Chinese,Spanish,Indonesian,English": 0.12, + "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Spanish,Malay,Indonesian,English": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, + "Vietnamese,Chinese,Spanish,Malay,English": 0.12, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.13333333333333333, + "Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.34, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.30666666666666664, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.3333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.32, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.1, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.10666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.10666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667 } }, - "AC3_2": 0.6219227404748053, - "AC3_3": 0.5468961441372473, - "AC3_4": 0.4948599023079333, - "AC3_5": 0.4543893895763355, - "AC3_6": 0.42026346599694175, - "AC3_7": 0.38974468080752855 + "AC3_2": 0.33788215589251513, + "AC3_3": 0.2655616451889597, + "AC3_4": 0.21210703565124653, + "AC3_5": 0.17467546249789664, + "AC3_6": 0.14889294844375728, + "AC3_7": 0.1311586452394697 }, "prompt_2": { - "overall_acc": 0.5942857142857143, + "overall_acc": 0.259047619047619, "language_acc": { - "Spanish": 0.62, - "English": 0.72, - "Chinese": 0.5666666666666667, - "Vietnamese": 0.5666666666666667, - "Filipino": 0.5133333333333333, - "Indonesian": 0.6266666666666667, - "Malay": 0.5466666666666666 + "Filipino": 0.24, + "Vietnamese": 0.26, + "Chinese": 0.25333333333333335, + "Spanish": 0.2866666666666667, + "Malay": 0.22, + "Indonesian": 0.24666666666666667, + "English": 0.30666666666666664 }, - "consistency_score_2": 0.5904761904761904, - "consistency_score_3": 0.44419047619047614, - "consistency_score_4": 0.3687619047619047, - "consistency_score_5": 0.320952380952381, - "consistency_score_6": 0.2866666666666667, - "consistency_score_7": 0.26, + "consistency_score_2": 0.43777777777777777, + "consistency_score_3": 0.2558095238095238, + "consistency_score_4": 0.1794285714285714, + "consistency_score_5": 0.14063492063492064, + "consistency_score_6": 0.11904761904761904, + "consistency_score_7": 0.10666666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.7333333333333333, - "Spanish,Chinese": 0.6066666666666667, - "Spanish,Vietnamese": 0.5866666666666667, - "Spanish,Filipino": 0.5466666666666666, - "Spanish,Indonesian": 0.66, - "Spanish,Malay": 0.6666666666666666, - "English,Chinese": 0.5866666666666667, - "English,Vietnamese": 0.6066666666666667, - "English,Filipino": 0.6, - "English,Indonesian": 0.62, - "English,Malay": 0.5866666666666667, - "Chinese,Vietnamese": 0.6133333333333333, - "Chinese,Filipino": 0.48, - "Chinese,Indonesian": 0.62, - "Chinese,Malay": 0.5733333333333334, - "Vietnamese,Filipino": 0.47333333333333333, - "Vietnamese,Indonesian": 0.6266666666666667, - "Vietnamese,Malay": 0.58, - "Filipino,Indonesian": 0.52, - "Filipino,Malay": 0.4866666666666667, - "Indonesian,Malay": 0.6266666666666667 + "Filipino,Vietnamese": 0.5133333333333333, + "Filipino,Chinese": 0.26666666666666666, + "Filipino,Spanish": 0.5066666666666667, + "Filipino,Malay": 0.47333333333333333, + "Filipino,Indonesian": 0.5, + "Filipino,English": 0.4666666666666667, + "Vietnamese,Chinese": 0.3333333333333333, + "Vietnamese,Spanish": 0.44666666666666666, + "Vietnamese,Malay": 0.4866666666666667, + "Vietnamese,Indonesian": 0.5, + "Vietnamese,English": 0.38, + "Chinese,Spanish": 0.24, + "Chinese,Malay": 0.3466666666666667, + "Chinese,Indonesian": 0.38, + "Chinese,English": 0.3466666666666667, + "Spanish,Malay": 0.47333333333333333, + "Spanish,Indonesian": 0.4866666666666667, + "Spanish,English": 0.5333333333333333, + "Malay,Indonesian": 0.6733333333333333, + "Malay,English": 0.44, + "Indonesian,English": 0.4 }, "3_combine": { - "Spanish,English,Chinese": 0.5, - "Spanish,English,Vietnamese": 0.5, - "Spanish,English,Filipino": 0.4666666666666667, - "Spanish,English,Indonesian": 0.5466666666666666, - "Spanish,English,Malay": 0.5466666666666666, - "Spanish,Chinese,Vietnamese": 0.4666666666666667, - "Spanish,Chinese,Filipino": 0.38666666666666666, - "Spanish,Chinese,Indonesian": 0.4866666666666667, - "Spanish,Chinese,Malay": 0.46, - "Spanish,Vietnamese,Filipino": 0.38, - "Spanish,Vietnamese,Indonesian": 0.48, - "Spanish,Vietnamese,Malay": 0.46, - "Spanish,Filipino,Indonesian": 0.43333333333333335, - "Spanish,Filipino,Malay": 0.42, - "Spanish,Indonesian,Malay": 0.5266666666666666, - "English,Chinese,Vietnamese": 0.46, - "English,Chinese,Filipino": 0.4066666666666667, - "English,Chinese,Indonesian": 0.46, - "English,Chinese,Malay": 0.4266666666666667, - "English,Vietnamese,Filipino": 0.4, - "English,Vietnamese,Indonesian": 0.4866666666666667, - "English,Vietnamese,Malay": 0.44666666666666666, - "English,Filipino,Indonesian": 0.4266666666666667, - "English,Filipino,Malay": 0.3933333333333333, - "English,Indonesian,Malay": 0.4666666666666667, - "Chinese,Vietnamese,Filipino": 0.36, - "Chinese,Vietnamese,Indonesian": 0.4866666666666667, - "Chinese,Vietnamese,Malay": 0.44666666666666666, - "Chinese,Filipino,Indonesian": 0.38666666666666666, - "Chinese,Filipino,Malay": 0.35333333333333333, - "Chinese,Indonesian,Malay": 0.47333333333333333, - "Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "Vietnamese,Filipino,Malay": 0.36, - "Vietnamese,Indonesian,Malay": 0.46, - "Filipino,Indonesian,Malay": 0.4 + "Filipino,Vietnamese,Chinese": 0.18666666666666668, + "Filipino,Vietnamese,Spanish": 0.31333333333333335, + "Filipino,Vietnamese,Malay": 0.30666666666666664, + "Filipino,Vietnamese,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,English": 0.29333333333333333, + "Filipino,Chinese,Spanish": 0.16666666666666666, + "Filipino,Chinese,Malay": 0.16, + "Filipino,Chinese,Indonesian": 0.18, + "Filipino,Chinese,English": 0.17333333333333334, + "Filipino,Spanish,Malay": 0.30666666666666664, + "Filipino,Spanish,Indonesian": 0.31333333333333335, + "Filipino,Spanish,English": 0.3333333333333333, + "Filipino,Malay,Indonesian": 0.36666666666666664, + "Filipino,Malay,English": 0.28, + "Filipino,Indonesian,English": 0.2733333333333333, + "Vietnamese,Chinese,Spanish": 0.16, + "Vietnamese,Chinese,Malay": 0.20666666666666667, + "Vietnamese,Chinese,Indonesian": 0.21333333333333335, + "Vietnamese,Chinese,English": 0.17333333333333334, + "Vietnamese,Spanish,Malay": 0.29333333333333333, + "Vietnamese,Spanish,Indonesian": 0.3, + "Vietnamese,Spanish,English": 0.28, + "Vietnamese,Malay,Indonesian": 0.38, + "Vietnamese,Malay,English": 0.24, + "Vietnamese,Indonesian,English": 0.24, + "Chinese,Spanish,Malay": 0.16666666666666666, + "Chinese,Spanish,Indonesian": 0.16666666666666666, + "Chinese,Spanish,English": 0.18666666666666668, + "Chinese,Malay,Indonesian": 0.26666666666666666, + "Chinese,Malay,English": 0.18666666666666668, + "Chinese,Indonesian,English": 0.18666666666666668, + "Spanish,Malay,Indonesian": 0.36666666666666664, + "Spanish,Malay,English": 0.30666666666666664, + "Spanish,Indonesian,English": 0.29333333333333333, + "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4066666666666667, - "Spanish,English,Chinese,Filipino": 0.36, - "Spanish,English,Chinese,Indonesian": 0.4266666666666667, - "Spanish,English,Chinese,Malay": 0.4, - "Spanish,English,Vietnamese,Filipino": 0.35333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.43333333333333335, - "Spanish,English,Vietnamese,Malay": 0.42, - "Spanish,English,Filipino,Indonesian": 0.38666666666666666, - "Spanish,English,Filipino,Malay": 0.38666666666666666, - "Spanish,English,Indonesian,Malay": 0.46, - "Spanish,Chinese,Vietnamese,Filipino": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian": 0.4066666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.3933333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.3466666666666667, - "Spanish,Chinese,Filipino,Malay": 0.32666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.42, - "Spanish,Vietnamese,Filipino,Indonesian": 0.34, - "Spanish,Vietnamese,Filipino,Malay": 0.32666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.38, - "English,Chinese,Vietnamese,Filipino": 0.3333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.4066666666666667, - "English,Chinese,Vietnamese,Malay": 0.37333333333333335, - "English,Chinese,Filipino,Indonesian": 0.34, - "English,Chinese,Filipino,Malay": 0.31333333333333335, - "English,Chinese,Indonesian,Malay": 0.38, - "English,Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "English,Vietnamese,Filipino,Malay": 0.32, - "English,Vietnamese,Indonesian,Malay": 0.38666666666666666, - "English,Filipino,Indonesian,Malay": 0.35333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.3, - "Chinese,Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.32666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.32 + "Filipino,Vietnamese,Chinese,Spanish": 0.14, + "Filipino,Vietnamese,Chinese,Malay": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.14666666666666667, + "Filipino,Vietnamese,Chinese,English": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,English": 0.24, + "Filipino,Vietnamese,Malay,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Malay,English": 0.20666666666666667, + "Filipino,Vietnamese,Indonesian,English": 0.20666666666666667, + "Filipino,Chinese,Spanish,Malay": 0.12666666666666668, + "Filipino,Chinese,Spanish,Indonesian": 0.14, + "Filipino,Chinese,Spanish,English": 0.14666666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.16, + "Filipino,Chinese,Malay,English": 0.12666666666666668, + "Filipino,Chinese,Indonesian,English": 0.13333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.25333333333333335, + "Filipino,Spanish,Malay,English": 0.23333333333333334, + "Filipino,Spanish,Indonesian,English": 0.22666666666666666, + "Filipino,Malay,Indonesian,English": 0.23333333333333334, + "Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.14, + "Vietnamese,Chinese,Spanish,English": 0.12666666666666668, + "Vietnamese,Chinese,Malay,Indonesian": 0.18, + "Vietnamese,Chinese,Malay,English": 0.13333333333333333, + "Vietnamese,Chinese,Indonesian,English": 0.12666666666666668, + "Vietnamese,Spanish,Malay,Indonesian": 0.25333333333333335, + "Vietnamese,Spanish,Malay,English": 0.2, + "Vietnamese,Spanish,Indonesian,English": 0.2, + "Vietnamese,Malay,Indonesian,English": 0.21333333333333335, + "Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, + "Chinese,Spanish,Malay,English": 0.14, + "Chinese,Spanish,Indonesian,English": 0.14, + "Chinese,Malay,Indonesian,English": 0.16666666666666666, + "Spanish,Malay,Indonesian,English": 0.25333333333333335 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.3, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.37333333333333335, - "Spanish,English,Chinese,Vietnamese,Malay": 0.35333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.32, - "Spanish,English,Chinese,Filipino,Malay": 0.30666666666666664, - "Spanish,English,Chinese,Indonesian,Malay": 0.37333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.32, - "Spanish,English,Vietnamese,Filipino,Malay": 0.31333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.38, - "Spanish,English,Filipino,Indonesian,Malay": 0.35333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.36, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.3, - "English,Chinese,Vietnamese,Filipino,Malay": 0.28, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.34, - "English,Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.28 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.12, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.10666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.2, + "Filipino,Vietnamese,Spanish,Malay,English": 0.18666666666666668, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.18666666666666668, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12666666666666668, + "Filipino,Chinese,Spanish,Malay,English": 0.12, + "Filipino,Chinese,Spanish,Indonesian,English": 0.12666666666666668, + "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Spanish,Malay,Indonesian,English": 0.2, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, + "Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.18, + "Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.28, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.10666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.16666666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 } }, - "AC3_2": 0.5923748276946031, - "AC3_3": 0.5083911539181338, - "AC3_4": 0.45511754063990284, - "AC3_5": 0.41680392443788455, - "AC3_6": 0.38676756752366426, - "AC3_7": 0.36173913039243855 + "AC3_2": 0.32549126798669864, + "AC3_3": 0.2574183886891454, + "AC3_4": 0.2120094327430101, + "AC3_5": 0.18230038953039385, + "AC3_6": 0.16312822353847933, + "AC3_7": 0.15111111106979166 }, "prompt_3": { - "overall_acc": 0.5780952380952381, + "overall_acc": 0.2733333333333333, "language_acc": { - "Spanish": 0.6266666666666667, - "English": 0.7133333333333334, - "Chinese": 0.56, - "Vietnamese": 0.54, - "Filipino": 0.5, - "Indonesian": 0.5733333333333334, - "Malay": 0.5333333333333333 + "Filipino": 0.28, + "Vietnamese": 0.28, + "Chinese": 0.24666666666666667, + "Spanish": 0.3, + "Malay": 0.26666666666666666, + "Indonesian": 0.25333333333333335, + "English": 0.2866666666666667 }, - "consistency_score_2": 0.5587301587301587, - "consistency_score_3": 0.40514285714285714, - "consistency_score_4": 0.3270476190476191, - "consistency_score_5": 0.2806349206349206, - "consistency_score_6": 0.24952380952380954, - "consistency_score_7": 0.22666666666666666, + "consistency_score_2": 0.3946031746031746, + "consistency_score_3": 0.2085714285714286, + "consistency_score_4": 0.13657142857142857, + "consistency_score_5": 0.10126984126984126, + "consistency_score_6": 0.08095238095238096, + "consistency_score_7": 0.06666666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.7133333333333334, - "Spanish,Chinese": 0.56, - "Spanish,Vietnamese": 0.56, - "Spanish,Filipino": 0.5133333333333333, - "Spanish,Indonesian": 0.5866666666666667, - "Spanish,Malay": 0.5466666666666666, - "English,Chinese": 0.5466666666666666, - "English,Vietnamese": 0.5666666666666667, - "English,Filipino": 0.56, - "English,Indonesian": 0.7, - "English,Malay": 0.5466666666666666, - "Chinese,Vietnamese": 0.58, - "Chinese,Filipino": 0.4866666666666667, - "Chinese,Indonesian": 0.52, - "Chinese,Malay": 0.4866666666666667, - "Vietnamese,Filipino": 0.52, - "Vietnamese,Indonesian": 0.58, - "Vietnamese,Malay": 0.56, - "Filipino,Indonesian": 0.5066666666666667, - "Filipino,Malay": 0.54, - "Indonesian,Malay": 0.5533333333333333 + "Filipino,Vietnamese": 0.3333333333333333, + "Filipino,Chinese": 0.30666666666666664, + "Filipino,Spanish": 0.4666666666666667, + "Filipino,Malay": 0.47333333333333333, + "Filipino,Indonesian": 0.54, + "Filipino,English": 0.4066666666666667, + "Vietnamese,Chinese": 0.30666666666666664, + "Vietnamese,Spanish": 0.35333333333333333, + "Vietnamese,Malay": 0.37333333333333335, + "Vietnamese,Indonesian": 0.35333333333333333, + "Vietnamese,English": 0.2866666666666667, + "Chinese,Spanish": 0.29333333333333333, + "Chinese,Malay": 0.31333333333333335, + "Chinese,Indonesian": 0.34, + "Chinese,English": 0.30666666666666664, + "Spanish,Malay": 0.44666666666666666, + "Spanish,Indonesian": 0.46, + "Spanish,English": 0.47333333333333333, + "Malay,Indonesian": 0.6533333333333333, + "Malay,English": 0.38, + "Indonesian,English": 0.42 }, "3_combine": { - "Spanish,English,Chinese": 0.46, - "Spanish,English,Vietnamese": 0.47333333333333333, - "Spanish,English,Filipino": 0.4266666666666667, - "Spanish,English,Indonesian": 0.5266666666666666, - "Spanish,English,Malay": 0.46, - "Spanish,Chinese,Vietnamese": 0.42, - "Spanish,Chinese,Filipino": 0.36, - "Spanish,Chinese,Indonesian": 0.3933333333333333, - "Spanish,Chinese,Malay": 0.38, - "Spanish,Vietnamese,Filipino": 0.38, - "Spanish,Vietnamese,Indonesian": 0.42, - "Spanish,Vietnamese,Malay": 0.42, - "Spanish,Filipino,Indonesian": 0.38, - "Spanish,Filipino,Malay": 0.38666666666666666, - "Spanish,Indonesian,Malay": 0.42, - "English,Chinese,Vietnamese": 0.42, - "English,Chinese,Filipino": 0.36, - "English,Chinese,Indonesian": 0.4266666666666667, - "English,Chinese,Malay": 0.36, - "English,Vietnamese,Filipino": 0.38, - "English,Vietnamese,Indonesian": 0.47333333333333333, - "English,Vietnamese,Malay": 0.3933333333333333, - "English,Filipino,Indonesian": 0.4266666666666667, - "English,Filipino,Malay": 0.38666666666666666, - "English,Indonesian,Malay": 0.46, - "Chinese,Vietnamese,Filipino": 0.37333333333333335, - "Chinese,Vietnamese,Indonesian": 0.4066666666666667, - "Chinese,Vietnamese,Malay": 0.38, - "Chinese,Filipino,Indonesian": 0.34, - "Chinese,Filipino,Malay": 0.36, - "Chinese,Indonesian,Malay": 0.35333333333333333, - "Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "Vietnamese,Filipino,Malay": 0.4066666666666667, - "Vietnamese,Indonesian,Malay": 0.4, - "Filipino,Indonesian,Malay": 0.38 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.37333333333333335, - "Spanish,English,Chinese,Filipino": 0.31333333333333335, - "Spanish,English,Chinese,Indonesian": 0.37333333333333335, - "Spanish,English,Chinese,Malay": 0.34, - "Spanish,English,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.4, - "Spanish,English,Vietnamese,Malay": 0.36666666666666664, - "Spanish,English,Filipino,Indonesian": 0.3466666666666667, - "Spanish,English,Filipino,Malay": 0.34, - "Spanish,English,Indonesian,Malay": 0.4, - "Spanish,Chinese,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.3466666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.32666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Filipino,Malay": 0.2866666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,Vietnamese,Filipino,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.34, - "Spanish,Filipino,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese,Filipino": 0.3, - "English,Chinese,Vietnamese,Indonesian": 0.36666666666666664, - "English,Chinese,Vietnamese,Malay": 0.31333333333333335, - "English,Chinese,Filipino,Indonesian": 0.3, - "English,Chinese,Filipino,Malay": 0.2866666666666667, - "English,Chinese,Indonesian,Malay": 0.32, - "English,Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "English,Vietnamese,Filipino,Malay": 0.3, - "English,Vietnamese,Indonesian,Malay": 0.3466666666666667, - "English,Filipino,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.2866666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.30666666666666664, - "Spanish,English,Chinese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.30666666666666664, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.26, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 - } - }, - "AC3_2": 0.568247762612745, - "AC3_3": 0.4764078153055073, - "AC3_4": 0.4177565335437133, - "AC3_5": 0.3778456121380257, - "AC3_6": 0.34858677183579184, - "AC3_7": 0.32564891514691147 + "Filipino,Vietnamese,Chinese": 0.14, + "Filipino,Vietnamese,Spanish": 0.18, + "Filipino,Vietnamese,Malay": 0.20666666666666667, + "Filipino,Vietnamese,Indonesian": 0.21333333333333335, + "Filipino,Vietnamese,English": 0.16, + "Filipino,Chinese,Spanish": 0.18666666666666668, + "Filipino,Chinese,Malay": 0.18666666666666668, + "Filipino,Chinese,Indonesian": 0.20666666666666667, + "Filipino,Chinese,English": 0.16, + "Filipino,Spanish,Malay": 0.26666666666666666, + "Filipino,Spanish,Indonesian": 0.32, + "Filipino,Spanish,English": 0.24666666666666667, + "Filipino,Malay,Indonesian": 0.38, + "Filipino,Malay,English": 0.22, + "Filipino,Indonesian,English": 0.25333333333333335, + "Vietnamese,Chinese,Spanish": 0.13333333333333333, + "Vietnamese,Chinese,Malay": 0.14, + "Vietnamese,Chinese,Indonesian": 0.13333333333333333, + "Vietnamese,Chinese,English": 0.12, + "Vietnamese,Spanish,Malay": 0.20666666666666667, + "Vietnamese,Spanish,Indonesian": 0.20666666666666667, + "Vietnamese,Spanish,English": 0.17333333333333334, + "Vietnamese,Malay,Indonesian": 0.26666666666666666, + "Vietnamese,Malay,English": 0.15333333333333332, + "Vietnamese,Indonesian,English": 0.16, + "Chinese,Spanish,Malay": 0.19333333333333333, + "Chinese,Spanish,Indonesian": 0.18666666666666668, + "Chinese,Spanish,English": 0.18, + "Chinese,Malay,Indonesian": 0.23333333333333334, + "Chinese,Malay,English": 0.16, + "Chinese,Indonesian,English": 0.18, + "Spanish,Malay,Indonesian": 0.34, + "Spanish,Malay,English": 0.26, + "Spanish,Indonesian,English": 0.26, + "Malay,Indonesian,English": 0.2866666666666667 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Malay": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.1, + "Filipino,Vietnamese,Chinese,English": 0.08666666666666667, + "Filipino,Vietnamese,Spanish,Malay": 0.14, + "Filipino,Vietnamese,Spanish,Indonesian": 0.14666666666666667, + "Filipino,Vietnamese,Spanish,English": 0.12666666666666668, + "Filipino,Vietnamese,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Malay,English": 0.12, + "Filipino,Vietnamese,Indonesian,English": 0.12, + "Filipino,Chinese,Spanish,Malay": 0.15333333333333332, + "Filipino,Chinese,Spanish,Indonesian": 0.16666666666666666, + "Filipino,Chinese,Spanish,English": 0.12666666666666668, + "Filipino,Chinese,Malay,Indonesian": 0.16666666666666666, + "Filipino,Chinese,Malay,English": 0.11333333333333333, + "Filipino,Chinese,Indonesian,English": 0.12666666666666668, + "Filipino,Spanish,Malay,Indonesian": 0.23333333333333334, + "Filipino,Spanish,Malay,English": 0.16666666666666666, + "Filipino,Spanish,Indonesian,English": 0.18666666666666668, + "Filipino,Malay,Indonesian,English": 0.18, + "Vietnamese,Chinese,Spanish,Malay": 0.11333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.1, + "Vietnamese,Chinese,Spanish,English": 0.08666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.12, + "Vietnamese,Chinese,Malay,English": 0.09333333333333334, + "Vietnamese,Chinese,Indonesian,English": 0.08666666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.18, + "Vietnamese,Spanish,Malay,English": 0.12666666666666668, + "Vietnamese,Spanish,Indonesian,English": 0.12666666666666668, + "Vietnamese,Malay,Indonesian,English": 0.14, + "Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, + "Chinese,Spanish,Malay,English": 0.13333333333333333, + "Chinese,Spanish,Indonesian,English": 0.12666666666666668, + "Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Spanish,Malay,Indonesian,English": 0.21333333333333335 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.1, + "Filipino,Vietnamese,Chinese,Malay,English": 0.08, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.07333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Malay,English": 0.10666666666666667, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.10666666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.11333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, + "Filipino,Chinese,Spanish,Malay,English": 0.10666666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.1, + "Filipino,Spanish,Malay,Indonesian,English": 0.14666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.1, + "Vietnamese,Chinese,Spanish,Malay,English": 0.08, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.07333333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.08, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.12, + "Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.06666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.07333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.1, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.09333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.06666666666666667 + } + }, + "AC3_2": 0.3229594422837141, + "AC3_3": 0.23660079046473698, + "AC3_4": 0.18213754642396737, + "AC3_5": 0.1477853106950121, + "AC3_6": 0.12491039422997602, + "AC3_7": 0.10718954245213379 }, "prompt_4": { - "overall_acc": 0.5980952380952381, + "overall_acc": 0.26571428571428574, "language_acc": { - "Spanish": 0.6066666666666667, - "English": 0.7333333333333333, - "Chinese": 0.56, - "Vietnamese": 0.5533333333333333, - "Filipino": 0.5333333333333333, - "Indonesian": 0.62, - "Malay": 0.58 + "Filipino": 0.26666666666666666, + "Vietnamese": 0.29333333333333333, + "Chinese": 0.23333333333333334, + "Spanish": 0.2866666666666667, + "Malay": 0.21333333333333335, + "Indonesian": 0.26, + "English": 0.30666666666666664 }, - "consistency_score_2": 0.608888888888889, - "consistency_score_3": 0.46171428571428574, - "consistency_score_4": 0.37866666666666676, - "consistency_score_5": 0.32285714285714284, - "consistency_score_6": 0.28095238095238095, - "consistency_score_7": 0.24666666666666667, + "consistency_score_2": 0.42920634920634915, + "consistency_score_3": 0.24285714285714285, + "consistency_score_4": 0.1660952380952381, + "consistency_score_5": 0.12888888888888891, + "consistency_score_6": 0.10952380952380951, + "consistency_score_7": 0.1, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.7066666666666667, - "Spanish,Chinese": 0.6066666666666667, - "Spanish,Vietnamese": 0.62, - "Spanish,Filipino": 0.6266666666666667, - "Spanish,Indonesian": 0.6466666666666666, - "Spanish,Malay": 0.64, - "English,Chinese": 0.6266666666666667, - "English,Vietnamese": 0.6266666666666667, - "English,Filipino": 0.6066666666666667, - "English,Indonesian": 0.6866666666666666, - "English,Malay": 0.5866666666666667, - "Chinese,Vietnamese": 0.6066666666666667, - "Chinese,Filipino": 0.5, - "Chinese,Indonesian": 0.6866666666666666, - "Chinese,Malay": 0.5933333333333334, - "Vietnamese,Filipino": 0.5266666666666666, - "Vietnamese,Indonesian": 0.6133333333333333, - "Vietnamese,Malay": 0.58, - "Filipino,Indonesian": 0.56, - "Filipino,Malay": 0.5133333333333333, - "Indonesian,Malay": 0.6266666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5133333333333333, - "Spanish,English,Vietnamese": 0.5133333333333333, - "Spanish,English,Filipino": 0.49333333333333335, - "Spanish,English,Indonesian": 0.5533333333333333, - "Spanish,English,Malay": 0.52, - "Spanish,Chinese,Vietnamese": 0.48, - "Spanish,Chinese,Filipino": 0.41333333333333333, - "Spanish,Chinese,Indonesian": 0.5266666666666666, - "Spanish,Chinese,Malay": 0.47333333333333333, - "Spanish,Vietnamese,Filipino": 0.42, - "Spanish,Vietnamese,Indonesian": 0.5, - "Spanish,Vietnamese,Malay": 0.46, - "Spanish,Filipino,Indonesian": 0.46, - "Spanish,Filipino,Malay": 0.42, - "Spanish,Indonesian,Malay": 0.5133333333333333, - "English,Chinese,Vietnamese": 0.48, - "English,Chinese,Filipino": 0.4, - "English,Chinese,Indonesian": 0.5333333333333333, - "English,Chinese,Malay": 0.44, - "English,Vietnamese,Filipino": 0.4266666666666667, - "English,Vietnamese,Indonesian": 0.5, - "English,Vietnamese,Malay": 0.44, - "English,Filipino,Indonesian": 0.4666666666666667, - "English,Filipino,Malay": 0.3933333333333333, - "English,Indonesian,Malay": 0.5, - "Chinese,Vietnamese,Filipino": 0.38, - "Chinese,Vietnamese,Indonesian": 0.5133333333333333, - "Chinese,Vietnamese,Malay": 0.44666666666666666, - "Chinese,Filipino,Indonesian": 0.44666666666666666, - "Chinese,Filipino,Malay": 0.36, - "Chinese,Indonesian,Malay": 0.5066666666666667, - "Vietnamese,Filipino,Indonesian": 0.41333333333333333, - "Vietnamese,Filipino,Malay": 0.36666666666666664, - "Vietnamese,Indonesian,Malay": 0.47333333333333333, - "Filipino,Indonesian,Malay": 0.41333333333333333 + "Filipino,Vietnamese": 0.47333333333333333, + "Filipino,Chinese": 0.2866666666666667, + "Filipino,Spanish": 0.4533333333333333, + "Filipino,Malay": 0.44, + "Filipino,Indonesian": 0.4866666666666667, + "Filipino,English": 0.5266666666666666, + "Vietnamese,Chinese": 0.35333333333333333, + "Vietnamese,Spanish": 0.44666666666666666, + "Vietnamese,Malay": 0.4866666666666667, + "Vietnamese,Indonesian": 0.49333333333333335, + "Vietnamese,English": 0.38, + "Chinese,Spanish": 0.24, + "Chinese,Malay": 0.36666666666666664, + "Chinese,Indonesian": 0.38666666666666666, + "Chinese,English": 0.3333333333333333, + "Spanish,Malay": 0.41333333333333333, + "Spanish,Indonesian": 0.44666666666666666, + "Spanish,English": 0.5, + "Malay,Indonesian": 0.66, + "Malay,English": 0.43333333333333335, + "Indonesian,English": 0.4066666666666667 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.16, + "Filipino,Vietnamese,Spanish": 0.2733333333333333, + "Filipino,Vietnamese,Malay": 0.28, + "Filipino,Vietnamese,Indonesian": 0.32666666666666666, + "Filipino,Vietnamese,English": 0.3, + "Filipino,Chinese,Spanish": 0.15333333333333332, + "Filipino,Chinese,Malay": 0.15333333333333332, + "Filipino,Chinese,Indonesian": 0.18666666666666668, + "Filipino,Chinese,English": 0.18, + "Filipino,Spanish,Malay": 0.23333333333333334, + "Filipino,Spanish,Indonesian": 0.26, + "Filipino,Spanish,English": 0.3333333333333333, + "Filipino,Malay,Indonesian": 0.36, + "Filipino,Malay,English": 0.3, + "Filipino,Indonesian,English": 0.3, + "Vietnamese,Chinese,Spanish": 0.16, + "Vietnamese,Chinese,Malay": 0.20666666666666667, + "Vietnamese,Chinese,Indonesian": 0.22, + "Vietnamese,Chinese,English": 0.16, + "Vietnamese,Spanish,Malay": 0.2733333333333333, + "Vietnamese,Spanish,Indonesian": 0.2866666666666667, + "Vietnamese,Spanish,English": 0.26666666666666666, + "Vietnamese,Malay,Indonesian": 0.37333333333333335, + "Vietnamese,Malay,English": 0.24666666666666667, + "Vietnamese,Indonesian,English": 0.24666666666666667, + "Chinese,Spanish,Malay": 0.14666666666666667, + "Chinese,Spanish,Indonesian": 0.16, + "Chinese,Spanish,English": 0.16, + "Chinese,Malay,Indonesian": 0.26666666666666666, + "Chinese,Malay,English": 0.19333333333333333, + "Chinese,Indonesian,English": 0.18, + "Spanish,Malay,Indonesian": 0.31333333333333335, + "Spanish,Malay,English": 0.25333333333333335, + "Spanish,Indonesian,English": 0.26, + "Malay,Indonesian,English": 0.32666666666666666 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4266666666666667, - "Spanish,English,Chinese,Filipino": 0.36, - "Spanish,English,Chinese,Indonesian": 0.46, - "Spanish,English,Chinese,Malay": 0.3933333333333333, - "Spanish,English,Vietnamese,Filipino": 0.36, - "Spanish,English,Vietnamese,Indonesian": 0.44666666666666666, - "Spanish,English,Vietnamese,Malay": 0.4, - "Spanish,English,Filipino,Indonesian": 0.3933333333333333, - "Spanish,English,Filipino,Malay": 0.36, - "Spanish,English,Indonesian,Malay": 0.44666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.34, - "Spanish,Chinese,Vietnamese,Indonesian": 0.43333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.38666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.38666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.3333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.43333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.36666666666666664, - "Spanish,Vietnamese,Filipino,Malay": 0.32, - "Spanish,Vietnamese,Indonesian,Malay": 0.41333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.38, - "English,Chinese,Vietnamese,Filipino": 0.3333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.43333333333333335, - "English,Chinese,Vietnamese,Malay": 0.36666666666666664, - "English,Chinese,Filipino,Indonesian": 0.37333333333333335, - "English,Chinese,Filipino,Malay": 0.3, - "English,Chinese,Indonesian,Malay": 0.42, - "English,Vietnamese,Filipino,Indonesian": 0.36666666666666664, - "English,Vietnamese,Filipino,Malay": 0.31333333333333335, - "English,Vietnamese,Indonesian,Malay": 0.4, - "English,Filipino,Indonesian,Malay": 0.3466666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Indonesian,Malay": 0.41333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.35333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.3333333333333333 + "Filipino,Vietnamese,Chinese,Spanish": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Malay": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,English": 0.12, + "Filipino,Vietnamese,Spanish,Malay": 0.18, + "Filipino,Vietnamese,Spanish,Indonesian": 0.2, + "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, + "Filipino,Vietnamese,Malay,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Malay,English": 0.21333333333333335, + "Filipino,Vietnamese,Indonesian,English": 0.22, + "Filipino,Chinese,Spanish,Malay": 0.10666666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.12666666666666668, + "Filipino,Chinese,Spanish,English": 0.12666666666666668, + "Filipino,Chinese,Malay,Indonesian": 0.15333333333333332, + "Filipino,Chinese,Malay,English": 0.12666666666666668, + "Filipino,Chinese,Indonesian,English": 0.14, + "Filipino,Spanish,Malay,Indonesian": 0.19333333333333333, + "Filipino,Spanish,Malay,English": 0.2, + "Filipino,Spanish,Indonesian,English": 0.21333333333333335, + "Filipino,Malay,Indonesian,English": 0.25333333333333335, + "Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.14666666666666667, + "Vietnamese,Chinese,Spanish,English": 0.12666666666666668, + "Vietnamese,Chinese,Malay,Indonesian": 0.18666666666666668, + "Vietnamese,Chinese,Malay,English": 0.12666666666666668, + "Vietnamese,Chinese,Indonesian,English": 0.12666666666666668, + "Vietnamese,Spanish,Malay,Indonesian": 0.22, + "Vietnamese,Spanish,Malay,English": 0.17333333333333334, + "Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, + "Vietnamese,Malay,Indonesian,English": 0.21333333333333335, + "Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, + "Chinese,Spanish,Malay,English": 0.12, + "Chinese,Spanish,Indonesian,English": 0.12666666666666668, + "Chinese,Malay,Indonesian,English": 0.16, + "Spanish,Malay,Indonesian,English": 0.20666666666666667 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.3933333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.34, - "Spanish,English,Chinese,Filipino,Indonesian": 0.3333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.28, - "Spanish,English,Chinese,Indonesian,Malay": 0.37333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.2866666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.32, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.32666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.36, - "English,Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.1, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.10666666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.15333333333333332, + "Filipino,Vietnamese,Spanish,Malay,English": 0.16, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.19333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.1, + "Filipino,Chinese,Spanish,Indonesian,English": 0.11333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.12666666666666668, + "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, + "Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.12, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.12, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, + "Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.28, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.1, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.1, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.1, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 } }, - "AC3_2": 0.6034438014579824, - "AC3_3": 0.5211297113609134, - "AC3_4": 0.46373374930249744, - "AC3_5": 0.41934702314843847, - "AC3_6": 0.38231439917231574, - "AC3_7": 0.3492822246861992 + "AC3_2": 0.32822815370855946, + "AC3_3": 0.25377207057610424, + "AC3_4": 0.20441363660273362, + "AC3_5": 0.17358004822632528, + "AC3_6": 0.1551124002487282, + "AC3_7": 0.1453124999602661 }, "prompt_5": { - "overall_acc": 0.5790476190476191, + "overall_acc": 0.2733333333333333, "language_acc": { - "Spanish": 0.5466666666666666, - "English": 0.72, - "Chinese": 0.5666666666666667, - "Vietnamese": 0.5733333333333334, - "Filipino": 0.56, - "Indonesian": 0.5933333333333334, - "Malay": 0.49333333333333335 + "Filipino": 0.28, + "Vietnamese": 0.2866666666666667, + "Chinese": 0.2866666666666667, + "Spanish": 0.26, + "Malay": 0.28, + "Indonesian": 0.23333333333333334, + "English": 0.2866666666666667 }, - "consistency_score_2": 0.5926984126984128, - "consistency_score_3": 0.44266666666666654, - "consistency_score_4": 0.36, - "consistency_score_5": 0.30571428571428566, - "consistency_score_6": 0.2676190476190476, - "consistency_score_7": 0.24, + "consistency_score_2": 0.45269841269841266, + "consistency_score_3": 0.25866666666666666, + "consistency_score_4": 0.17314285714285718, + "consistency_score_5": 0.1298412698412698, + "consistency_score_6": 0.10380952380952381, + "consistency_score_7": 0.08666666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.66, - "Spanish,Chinese": 0.6066666666666667, - "Spanish,Vietnamese": 0.64, - "Spanish,Filipino": 0.5933333333333334, - "Spanish,Indonesian": 0.6333333333333333, - "Spanish,Malay": 0.6, - "English,Chinese": 0.5866666666666667, - "English,Vietnamese": 0.6466666666666666, - "English,Filipino": 0.6266666666666667, - "English,Indonesian": 0.68, - "English,Malay": 0.5533333333333333, - "Chinese,Vietnamese": 0.6333333333333333, - "Chinese,Filipino": 0.5333333333333333, - "Chinese,Indonesian": 0.5933333333333334, - "Chinese,Malay": 0.49333333333333335, - "Vietnamese,Filipino": 0.58, - "Vietnamese,Indonesian": 0.6133333333333333, - "Vietnamese,Malay": 0.56, - "Filipino,Indonesian": 0.58, + "Filipino,Vietnamese": 0.4533333333333333, + "Filipino,Chinese": 0.30666666666666664, + "Filipino,Spanish": 0.49333333333333335, "Filipino,Malay": 0.49333333333333335, - "Indonesian,Malay": 0.54 + "Filipino,Indonesian": 0.5133333333333333, + "Filipino,English": 0.4666666666666667, + "Vietnamese,Chinese": 0.5133333333333333, + "Vietnamese,Spanish": 0.43333333333333335, + "Vietnamese,Malay": 0.5066666666666667, + "Vietnamese,Indonesian": 0.44666666666666666, + "Vietnamese,English": 0.4, + "Chinese,Spanish": 0.37333333333333335, + "Chinese,Malay": 0.3333333333333333, + "Chinese,Indonesian": 0.36, + "Chinese,English": 0.34, + "Spanish,Malay": 0.5066666666666667, + "Spanish,Indonesian": 0.49333333333333335, + "Spanish,English": 0.5066666666666667, + "Malay,Indonesian": 0.6933333333333334, + "Malay,English": 0.4266666666666667, + "Indonesian,English": 0.44666666666666666 }, "3_combine": { - "Spanish,English,Chinese": 0.47333333333333333, - "Spanish,English,Vietnamese": 0.52, - "Spanish,English,Filipino": 0.48, - "Spanish,English,Indonesian": 0.5066666666666667, - "Spanish,English,Malay": 0.4666666666666667, - "Spanish,Chinese,Vietnamese": 0.47333333333333333, - "Spanish,Chinese,Filipino": 0.43333333333333335, - "Spanish,Chinese,Indonesian": 0.47333333333333333, - "Spanish,Chinese,Malay": 0.4, - "Spanish,Vietnamese,Filipino": 0.44666666666666666, - "Spanish,Vietnamese,Indonesian": 0.48, - "Spanish,Vietnamese,Malay": 0.4533333333333333, - "Spanish,Filipino,Indonesian": 0.4533333333333333, - "Spanish,Filipino,Malay": 0.4066666666666667, - "Spanish,Indonesian,Malay": 0.4533333333333333, - "English,Chinese,Vietnamese": 0.4866666666666667, - "English,Chinese,Filipino": 0.44, - "English,Chinese,Indonesian": 0.48, - "English,Chinese,Malay": 0.38666666666666666, - "English,Vietnamese,Filipino": 0.47333333333333333, - "English,Vietnamese,Indonesian": 0.5133333333333333, - "English,Vietnamese,Malay": 0.4266666666666667, - "English,Filipino,Indonesian": 0.48, - "English,Filipino,Malay": 0.3933333333333333, - "English,Indonesian,Malay": 0.44, - "Chinese,Vietnamese,Filipino": 0.4266666666666667, - "Chinese,Vietnamese,Indonesian": 0.4666666666666667, - "Chinese,Vietnamese,Malay": 0.38666666666666666, - "Chinese,Filipino,Indonesian": 0.42, - "Chinese,Filipino,Malay": 0.35333333333333333, - "Chinese,Indonesian,Malay": 0.37333333333333335, - "Vietnamese,Filipino,Indonesian": 0.44, - "Vietnamese,Filipino,Malay": 0.38, - "Vietnamese,Indonesian,Malay": 0.41333333333333333, - "Filipino,Indonesian,Malay": 0.3933333333333333 + "Filipino,Vietnamese,Chinese": 0.21333333333333335, + "Filipino,Vietnamese,Spanish": 0.2866666666666667, + "Filipino,Vietnamese,Malay": 0.2866666666666667, + "Filipino,Vietnamese,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,English": 0.25333333333333335, + "Filipino,Chinese,Spanish": 0.19333333333333333, + "Filipino,Chinese,Malay": 0.15333333333333332, + "Filipino,Chinese,Indonesian": 0.16, + "Filipino,Chinese,English": 0.18, + "Filipino,Spanish,Malay": 0.3, + "Filipino,Spanish,Indonesian": 0.31333333333333335, + "Filipino,Spanish,English": 0.32, + "Filipino,Malay,Indonesian": 0.38666666666666666, + "Filipino,Malay,English": 0.2866666666666667, + "Filipino,Indonesian,English": 0.3, + "Vietnamese,Chinese,Spanish": 0.24666666666666667, + "Vietnamese,Chinese,Malay": 0.24, + "Vietnamese,Chinese,Indonesian": 0.23333333333333334, + "Vietnamese,Chinese,English": 0.20666666666666667, + "Vietnamese,Spanish,Malay": 0.29333333333333333, + "Vietnamese,Spanish,Indonesian": 0.2866666666666667, + "Vietnamese,Spanish,English": 0.26666666666666666, + "Vietnamese,Malay,Indonesian": 0.36666666666666664, + "Vietnamese,Malay,English": 0.26, + "Vietnamese,Indonesian,English": 0.23333333333333334, + "Chinese,Spanish,Malay": 0.2, + "Chinese,Spanish,Indonesian": 0.20666666666666667, + "Chinese,Spanish,English": 0.19333333333333333, + "Chinese,Malay,Indonesian": 0.25333333333333335, + "Chinese,Malay,English": 0.16, + "Chinese,Indonesian,English": 0.17333333333333334, + "Spanish,Malay,Indonesian": 0.3933333333333333, + "Spanish,Malay,English": 0.29333333333333333, + "Spanish,Indonesian,English": 0.32666666666666666, + "Malay,Indonesian,English": 0.3333333333333333 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4066666666666667, - "Spanish,English,Chinese,Filipino": 0.38, - "Spanish,English,Chinese,Indonesian": 0.4, - "Spanish,English,Chinese,Malay": 0.34, - "Spanish,English,Vietnamese,Filipino": 0.3933333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.41333333333333333, - "Spanish,English,Vietnamese,Malay": 0.38666666666666666, - "Spanish,English,Filipino,Indonesian": 0.3933333333333333, - "Spanish,English,Filipino,Malay": 0.3466666666666667, - "Spanish,English,Indonesian,Malay": 0.38666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.36, - "Spanish,Chinese,Vietnamese,Indonesian": 0.3933333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.3333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.37333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.31333333333333335, - "Spanish,Chinese,Indonesian,Malay": 0.3466666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.36666666666666664, - "Spanish,Vietnamese,Filipino,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.36, - "English,Chinese,Vietnamese,Filipino": 0.38666666666666666, - "English,Chinese,Vietnamese,Indonesian": 0.41333333333333333, - "English,Chinese,Vietnamese,Malay": 0.32, - "English,Chinese,Filipino,Indonesian": 0.38666666666666666, - "English,Chinese,Filipino,Malay": 0.30666666666666664, - "English,Chinese,Indonesian,Malay": 0.3333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.4, - "English,Vietnamese,Filipino,Malay": 0.3333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.36, - "English,Filipino,Indonesian,Malay": 0.34, - "Chinese,Vietnamese,Filipino,Indonesian": 0.36666666666666664, - "Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.30666666666666664, - "Vietnamese,Filipino,Indonesian,Malay": 0.32666666666666666 + "Filipino,Vietnamese,Chinese,Spanish": 0.15333333333333332, + "Filipino,Vietnamese,Chinese,Malay": 0.12666666666666668, + "Filipino,Vietnamese,Chinese,Indonesian": 0.13333333333333333, + "Filipino,Vietnamese,Chinese,English": 0.13333333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.22666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.20666666666666667, + "Filipino,Vietnamese,Spanish,English": 0.2, + "Filipino,Vietnamese,Malay,Indonesian": 0.22666666666666666, + "Filipino,Vietnamese,Malay,English": 0.19333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.18666666666666668, + "Filipino,Chinese,Spanish,Malay": 0.13333333333333333, + "Filipino,Chinese,Spanish,Indonesian": 0.13333333333333333, + "Filipino,Chinese,Spanish,English": 0.12666666666666668, + "Filipino,Chinese,Malay,Indonesian": 0.13333333333333333, + "Filipino,Chinese,Malay,English": 0.10666666666666667, + "Filipino,Chinese,Indonesian,English": 0.11333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.26, + "Filipino,Spanish,Malay,English": 0.22666666666666666, + "Filipino,Spanish,Indonesian,English": 0.24666666666666667, + "Filipino,Malay,Indonesian,English": 0.24666666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.15333333333333332, + "Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, + "Vietnamese,Chinese,Spanish,English": 0.13333333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, + "Vietnamese,Chinese,Malay,English": 0.12666666666666668, + "Vietnamese,Chinese,Indonesian,English": 0.11333333333333333, + "Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, + "Vietnamese,Spanish,Malay,English": 0.2, + "Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, + "Vietnamese,Malay,Indonesian,English": 0.20666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.16, + "Chinese,Spanish,Malay,English": 0.11333333333333333, + "Chinese,Spanish,Indonesian,English": 0.13333333333333333, + "Chinese,Malay,Indonesian,English": 0.13333333333333333, + "Spanish,Malay,Indonesian,English": 0.26 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.3466666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.29333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.34, - "Spanish,English,Chinese,Filipino,Malay": 0.28, - "Spanish,English,Chinese,Indonesian,Malay": 0.30666666666666664, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.3, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.32, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.3, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.28, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.3, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.10666666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.12, + "Filipino,Vietnamese,Chinese,Malay,English": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.1, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, + "Filipino,Vietnamese,Spanish,Malay,English": 0.16666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16666666666666666, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.17333333333333334, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.12, + "Filipino,Chinese,Spanish,Malay,English": 0.09333333333333334, + "Filipino,Chinese,Spanish,Indonesian,English": 0.1, + "Filipino,Chinese,Malay,Indonesian,English": 0.1, + "Filipino,Spanish,Malay,Indonesian,English": 0.20666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, + "Vietnamese,Chinese,Spanish,Malay,English": 0.1, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.10666666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.18666666666666668, + "Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.09333333333333334, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.09333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08666666666666667 } }, - "AC3_2": 0.585793500227387, - "AC3_3": 0.5017549092225785, - "AC3_4": 0.4439756591819296, - "AC3_5": 0.40015992614268525, - "AC3_6": 0.3660570999621214, - "AC3_7": 0.3393488371678702 + "AC3_2": 0.34085993290743144, + "AC3_3": 0.26579782785312905, + "AC3_4": 0.21199658698323456, + "AC3_5": 0.1760524933946536, + "AC3_6": 0.15047138043148275, + "AC3_7": 0.131604938235048 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.4926948051948052, + "overall_acc": 0.27435064935064934, "language_acc": { - "English": 0.5397727272727273, - "Filipino": 0.45454545454545453, - "Spanish": 0.48863636363636365, - "Malay": 0.4943181818181818, - "Chinese": 0.48295454545454547, - "Indonesian": 0.4772727272727273, - "Vietnamese": 0.5113636363636364 + "Indonesian": 0.26704545454545453, + "English": 0.2897727272727273, + "Filipino": 0.2727272727272727, + "Spanish": 0.2897727272727273, + "Chinese": 0.23863636363636365, + "Malay": 0.2727272727272727, + "Vietnamese": 0.2897727272727273 }, - "consistency_score_2": 0.6144480519480519, - "consistency_score_3": 0.46136363636363636, - "consistency_score_4": 0.3758116883116882, - "consistency_score_5": 0.3192640692640693, - "consistency_score_6": 0.2775974025974026, - "consistency_score_7": 0.24431818181818182, + "consistency_score_2": 0.3890692640692641, + "consistency_score_3": 0.1931818181818182, + "consistency_score_4": 0.1099025974025974, + "consistency_score_5": 0.06737012987012989, + "consistency_score_6": 0.04301948051948052, + "consistency_score_7": 0.028409090909090908, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.6022727272727273, - "English,Spanish": 0.7329545454545454, - "English,Malay": 0.6875, - "English,Chinese": 0.5738636363636364, - "English,Indonesian": 0.6590909090909091, - "English,Vietnamese": 0.6477272727272727, - "Filipino,Spanish": 0.6022727272727273, - "Filipino,Malay": 0.5568181818181818, - "Filipino,Chinese": 0.5284090909090909, - "Filipino,Indonesian": 0.6420454545454546, - "Filipino,Vietnamese": 0.5681818181818182, - "Spanish,Malay": 0.6306818181818182, - "Spanish,Chinese": 0.4943181818181818, - "Spanish,Indonesian": 0.6363636363636364, - "Spanish,Vietnamese": 0.6875, - "Malay,Chinese": 0.5340909090909091, - "Malay,Indonesian": 0.7272727272727273, - "Malay,Vietnamese": 0.6136363636363636, - "Chinese,Indonesian": 0.5681818181818182, - "Chinese,Vietnamese": 0.5284090909090909, - "Indonesian,Vietnamese": 0.6818181818181818 + "Indonesian,English": 0.39204545454545453, + "Indonesian,Filipino": 0.4772727272727273, + "Indonesian,Spanish": 0.3977272727272727, + "Indonesian,Chinese": 0.26136363636363635, + "Indonesian,Malay": 0.5568181818181818, + "Indonesian,Vietnamese": 0.32954545454545453, + "English,Filipino": 0.4318181818181818, + "English,Spanish": 0.5397727272727273, + "English,Chinese": 0.26136363636363635, + "English,Malay": 0.4318181818181818, + "English,Vietnamese": 0.3522727272727273, + "Filipino,Spanish": 0.4772727272727273, + "Filipino,Chinese": 0.32954545454545453, + "Filipino,Malay": 0.48295454545454547, + "Filipino,Vietnamese": 0.38636363636363635, + "Spanish,Chinese": 0.30113636363636365, + "Spanish,Malay": 0.42045454545454547, + "Spanish,Vietnamese": 0.32954545454545453, + "Chinese,Malay": 0.3125, + "Chinese,Vietnamese": 0.35795454545454547, + "Malay,Vietnamese": 0.3409090909090909 }, "3_combine": { - "English,Filipino,Spanish": 0.4943181818181818, - "English,Filipino,Malay": 0.4659090909090909, - "English,Filipino,Chinese": 0.4147727272727273, - "English,Filipino,Indonesian": 0.4943181818181818, - "English,Filipino,Vietnamese": 0.4431818181818182, - "English,Spanish,Malay": 0.5568181818181818, - "English,Spanish,Chinese": 0.4318181818181818, - "English,Spanish,Indonesian": 0.5454545454545454, - "English,Spanish,Vietnamese": 0.5681818181818182, - "English,Malay,Chinese": 0.4375, - "English,Malay,Indonesian": 0.5681818181818182, - "English,Malay,Vietnamese": 0.5056818181818182, - "English,Chinese,Indonesian": 0.4431818181818182, - "English,Chinese,Vietnamese": 0.3977272727272727, - "English,Indonesian,Vietnamese": 0.5284090909090909, - "Filipino,Spanish,Malay": 0.4431818181818182, - "Filipino,Spanish,Chinese": 0.3693181818181818, - "Filipino,Spanish,Indonesian": 0.4715909090909091, - "Filipino,Spanish,Vietnamese": 0.4659090909090909, - "Filipino,Malay,Chinese": 0.375, - "Filipino,Malay,Indonesian": 0.5056818181818182, - "Filipino,Malay,Vietnamese": 0.4090909090909091, - "Filipino,Chinese,Indonesian": 0.42613636363636365, - "Filipino,Chinese,Vietnamese": 0.375, - "Filipino,Indonesian,Vietnamese": 0.4659090909090909, - "Spanish,Malay,Chinese": 0.38636363636363635, - "Spanish,Malay,Indonesian": 0.5340909090909091, - "Spanish,Malay,Vietnamese": 0.4943181818181818, - "Spanish,Chinese,Indonesian": 0.3977272727272727, - "Spanish,Chinese,Vietnamese": 0.3977272727272727, - "Spanish,Indonesian,Vietnamese": 0.5284090909090909, - "Malay,Chinese,Indonesian": 0.44886363636363635, - "Malay,Chinese,Vietnamese": 0.3806818181818182, - "Malay,Indonesian,Vietnamese": 0.5454545454545454, - "Chinese,Indonesian,Vietnamese": 0.4318181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.4090909090909091, - "English,Filipino,Spanish,Chinese": 0.3352272727272727, - "English,Filipino,Spanish,Indonesian": 0.42045454545454547, - "English,Filipino,Spanish,Vietnamese": 0.4034090909090909, - "English,Filipino,Malay,Chinese": 0.3409090909090909, - "English,Filipino,Malay,Indonesian": 0.42613636363636365, - "English,Filipino,Malay,Vietnamese": 0.3806818181818182, - "English,Filipino,Chinese,Indonesian": 0.3693181818181818, - "English,Filipino,Chinese,Vietnamese": 0.30113636363636365, - "English,Filipino,Indonesian,Vietnamese": 0.39204545454545453, - "English,Spanish,Malay,Chinese": 0.36363636363636365, - "English,Spanish,Malay,Indonesian": 0.48863636363636365, - "English,Spanish,Malay,Vietnamese": 0.45454545454545453, - "English,Spanish,Chinese,Indonesian": 0.36363636363636365, - "English,Spanish,Chinese,Vietnamese": 0.35795454545454547, - "English,Spanish,Indonesian,Vietnamese": 0.4659090909090909, - "English,Malay,Chinese,Indonesian": 0.38636363636363635, - "English,Malay,Chinese,Vietnamese": 0.32954545454545453, - "English,Malay,Indonesian,Vietnamese": 0.4772727272727273, - "English,Chinese,Indonesian,Vietnamese": 0.3522727272727273, - "Filipino,Spanish,Malay,Chinese": 0.32386363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.4090909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.3693181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.32954545454545453, - "Filipino,Spanish,Chinese,Vietnamese": 0.3068181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.39204545454545453, - "Filipino,Malay,Chinese,Indonesian": 0.35795454545454547, - "Filipino,Malay,Chinese,Vietnamese": 0.30113636363636365, - "Filipino,Malay,Indonesian,Vietnamese": 0.38636363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.32954545454545453, - "Spanish,Malay,Chinese,Indonesian": 0.3522727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.3181818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.45454545454545453, - "Spanish,Chinese,Indonesian,Vietnamese": 0.3465909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.35795454545454547 + "Indonesian,English,Filipino": 0.25, + "Indonesian,English,Spanish": 0.23863636363636365, + "Indonesian,English,Chinese": 0.11363636363636363, + "Indonesian,English,Malay": 0.2727272727272727, + "Indonesian,English,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Spanish": 0.25, + "Indonesian,Filipino,Chinese": 0.1534090909090909, + "Indonesian,Filipino,Malay": 0.32954545454545453, + "Indonesian,Filipino,Vietnamese": 0.17613636363636365, + "Indonesian,Spanish,Chinese": 0.125, + "Indonesian,Spanish,Malay": 0.2897727272727273, + "Indonesian,Spanish,Vietnamese": 0.13636363636363635, + "Indonesian,Chinese,Malay": 0.17613636363636365, + "Indonesian,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,Malay,Vietnamese": 0.21022727272727273, + "English,Filipino,Spanish": 0.29545454545454547, + "English,Filipino,Chinese": 0.14772727272727273, + "English,Filipino,Malay": 0.2727272727272727, + "English,Filipino,Vietnamese": 0.18181818181818182, + "English,Spanish,Chinese": 0.16477272727272727, + "English,Spanish,Malay": 0.26704545454545453, + "English,Spanish,Vietnamese": 0.19886363636363635, + "English,Chinese,Malay": 0.13636363636363635, + "English,Chinese,Vietnamese": 0.1534090909090909, + "English,Malay,Vietnamese": 0.17613636363636365, + "Filipino,Spanish,Chinese": 0.1534090909090909, + "Filipino,Spanish,Malay": 0.2840909090909091, + "Filipino,Spanish,Vietnamese": 0.18181818181818182, + "Filipino,Chinese,Malay": 0.17045454545454544, + "Filipino,Chinese,Vietnamese": 0.18181818181818182, + "Filipino,Malay,Vietnamese": 0.19886363636363635, + "Spanish,Chinese,Malay": 0.1534090909090909, + "Spanish,Chinese,Vietnamese": 0.14772727272727273, + "Spanish,Malay,Vietnamese": 0.1590909090909091, + "Chinese,Malay,Vietnamese": 0.14772727272727273 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.1590909090909091, + "Indonesian,English,Filipino,Chinese": 0.09090909090909091, + "Indonesian,English,Filipino,Malay": 0.19886363636363635, + "Indonesian,English,Filipino,Vietnamese": 0.09090909090909091, + "Indonesian,English,Spanish,Chinese": 0.08522727272727272, + "Indonesian,English,Spanish,Malay": 0.18181818181818182, + "Indonesian,English,Spanish,Vietnamese": 0.08522727272727272, + "Indonesian,English,Chinese,Malay": 0.07954545454545454, + "Indonesian,English,Chinese,Vietnamese": 0.0625, + "Indonesian,English,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,Filipino,Spanish,Chinese": 0.08522727272727272, + "Indonesian,Filipino,Spanish,Malay": 0.19886363636363635, + "Indonesian,Filipino,Spanish,Vietnamese": 0.09659090909090909, + "Indonesian,Filipino,Chinese,Malay": 0.10227272727272728, + "Indonesian,Filipino,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,Filipino,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,Spanish,Chinese,Malay": 0.10227272727272728, + "Indonesian,Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Indonesian,Spanish,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,Chinese,Malay,Vietnamese": 0.09090909090909091, + "English,Filipino,Spanish,Chinese": 0.10795454545454546, + "English,Filipino,Spanish,Malay": 0.1875, + "English,Filipino,Spanish,Vietnamese": 0.125, + "English,Filipino,Chinese,Malay": 0.09090909090909091, + "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, + "English,Filipino,Malay,Vietnamese": 0.11363636363636363, + "English,Spanish,Chinese,Malay": 0.10227272727272728, + "English,Spanish,Chinese,Vietnamese": 0.10795454545454546, + "English,Spanish,Malay,Vietnamese": 0.10795454545454546, + "English,Chinese,Malay,Vietnamese": 0.07954545454545454, + "Filipino,Spanish,Chinese,Malay": 0.10795454545454546, + "Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, + "Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, + "Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.30113636363636365, - "English,Filipino,Spanish,Malay,Indonesian": 0.375, - "English,Filipino,Spanish,Malay,Vietnamese": 0.3522727272727273, - "English,Filipino,Spanish,Chinese,Indonesian": 0.3068181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.2784090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.35795454545454547, - "English,Filipino,Malay,Chinese,Indonesian": 0.32386363636363635, - "English,Filipino,Malay,Chinese,Vietnamese": 0.2727272727272727, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.35795454545454547, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.32954545454545453, - "English,Spanish,Malay,Chinese,Vietnamese": 0.30113636363636365, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.42613636363636365, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.3181818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.3181818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.3068181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.3465909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.3068181818181818 + "Indonesian,English,Filipino,Spanish,Chinese": 0.0625, + "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.0625, + "Indonesian,English,Filipino,Chinese,Malay": 0.0625, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Spanish,Chinese,Malay": 0.06818181818181818, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.07386363636363637, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.056818181818181816, + "English,Filipino,Spanish,Chinese,Malay": 0.07386363636363637, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, + "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, + "English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, + "English,Spanish,Chinese,Malay,Vietnamese": 0.0625, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.2840909090909091, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.2556818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.32954545454545453, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.05113636363636364, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 } }, - "AC3_2": 0.5468767851888269, - "AC3_3": 0.4765147647496579, - "AC3_4": 0.4263882145400546, - "AC3_5": 0.387457427562991, - "AC3_6": 0.35511406397207235, - "AC3_7": 0.32665448533813757 + "AC3_2": 0.3217913814900218, + "AC3_3": 0.22672032823433533, + "AC3_4": 0.15693738028056856, + "AC3_5": 0.10817626550802964, + "AC3_6": 0.07437639084911933, + "AC3_7": 0.05148671701930909 }, "prompt_2": { - "overall_acc": 0.47159090909090906, + "overall_acc": 0.2792207792207792, "language_acc": { - "English": 0.5454545454545454, - "Filipino": 0.4431818181818182, - "Spanish": 0.48863636363636365, - "Malay": 0.45454545454545453, - "Chinese": 0.4431818181818182, - "Indonesian": 0.4602272727272727, - "Vietnamese": 0.4659090909090909 + "Indonesian": 0.2897727272727273, + "English": 0.29545454545454547, + "Filipino": 0.25, + "Spanish": 0.2784090909090909, + "Chinese": 0.2897727272727273, + "Malay": 0.2840909090909091, + "Vietnamese": 0.26704545454545453 }, - "consistency_score_2": 0.5817099567099566, - "consistency_score_3": 0.41850649350649355, - "consistency_score_4": 0.33165584415584415, - "consistency_score_5": 0.2778679653679653, - "consistency_score_6": 0.24350649350649348, - "consistency_score_7": 0.2215909090909091, + "consistency_score_2": 0.38771645021645024, + "consistency_score_3": 0.18490259740259737, + "consistency_score_4": 0.09724025974025972, + "consistency_score_5": 0.054653679653679677, + "consistency_score_6": 0.03327922077922078, + "consistency_score_7": 0.022727272727272728, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.5625, - "English,Spanish": 0.7443181818181818, - "English,Malay": 0.6420454545454546, - "English,Chinese": 0.6022727272727273, - "English,Indonesian": 0.6193181818181818, - "English,Vietnamese": 0.5852272727272727, - "Filipino,Spanish": 0.5340909090909091, + "Indonesian,English": 0.38636363636363635, + "Indonesian,Filipino": 0.5454545454545454, + "Indonesian,Spanish": 0.4147727272727273, + "Indonesian,Chinese": 0.2784090909090909, + "Indonesian,Malay": 0.6136363636363636, + "Indonesian,Vietnamese": 0.375, + "English,Filipino": 0.44886363636363635, + "English,Spanish": 0.5, + "English,Chinese": 0.2784090909090909, + "English,Malay": 0.4318181818181818, + "English,Vietnamese": 0.36363636363636365, + "Filipino,Spanish": 0.45454545454545453, + "Filipino,Chinese": 0.19318181818181818, "Filipino,Malay": 0.5511363636363636, - "Filipino,Chinese": 0.5227272727272727, - "Filipino,Indonesian": 0.5625, - "Filipino,Vietnamese": 0.48863636363636365, - "Spanish,Malay": 0.5965909090909091, - "Spanish,Chinese": 0.5625, - "Spanish,Indonesian": 0.6306818181818182, - "Spanish,Vietnamese": 0.6193181818181818, - "Malay,Chinese": 0.5056818181818182, - "Malay,Indonesian": 0.6875, - "Malay,Vietnamese": 0.5681818181818182, - "Chinese,Indonesian": 0.5454545454545454, - "Chinese,Vietnamese": 0.48863636363636365, - "Indonesian,Vietnamese": 0.5965909090909091 + "Filipino,Vietnamese": 0.38636363636363635, + "Spanish,Chinese": 0.24431818181818182, + "Spanish,Malay": 0.4772727272727273, + "Spanish,Vietnamese": 0.32386363636363635, + "Chinese,Malay": 0.3068181818181818, + "Chinese,Vietnamese": 0.22727272727272727, + "Malay,Vietnamese": 0.3409090909090909 }, "3_combine": { - "English,Filipino,Spanish": 0.45454545454545453, - "English,Filipino,Malay": 0.42045454545454547, - "English,Filipino,Chinese": 0.4090909090909091, - "English,Filipino,Indonesian": 0.42613636363636365, - "English,Filipino,Vietnamese": 0.36363636363636365, - "English,Spanish,Malay": 0.5340909090909091, - "English,Spanish,Chinese": 0.48863636363636365, - "English,Spanish,Indonesian": 0.5170454545454546, - "English,Spanish,Vietnamese": 0.5113636363636364, - "English,Malay,Chinese": 0.4147727272727273, - "English,Malay,Indonesian": 0.5056818181818182, - "English,Malay,Vietnamese": 0.4318181818181818, - "English,Chinese,Indonesian": 0.4375, - "English,Chinese,Vietnamese": 0.375, - "English,Indonesian,Vietnamese": 0.44886363636363635, - "Filipino,Spanish,Malay": 0.4090909090909091, - "Filipino,Spanish,Chinese": 0.375, - "Filipino,Spanish,Indonesian": 0.3977272727272727, - "Filipino,Spanish,Vietnamese": 0.3693181818181818, - "Filipino,Malay,Chinese": 0.3522727272727273, - "Filipino,Malay,Indonesian": 0.4375, - "Filipino,Malay,Vietnamese": 0.35795454545454547, - "Filipino,Chinese,Indonesian": 0.3693181818181818, - "Filipino,Chinese,Vietnamese": 0.3181818181818182, - "Filipino,Indonesian,Vietnamese": 0.3806818181818182, - "Spanish,Malay,Chinese": 0.38636363636363635, - "Spanish,Malay,Indonesian": 0.4943181818181818, - "Spanish,Malay,Vietnamese": 0.4375, - "Spanish,Chinese,Indonesian": 0.4034090909090909, - "Spanish,Chinese,Vietnamese": 0.38636363636363635, - "Spanish,Indonesian,Vietnamese": 0.4602272727272727, - "Malay,Chinese,Indonesian": 0.4090909090909091, - "Malay,Chinese,Vietnamese": 0.3409090909090909, - "Malay,Indonesian,Vietnamese": 0.4602272727272727, - "Chinese,Indonesian,Vietnamese": 0.36363636363636365 + "Indonesian,English,Filipino": 0.26704545454545453, + "Indonesian,English,Spanish": 0.23295454545454544, + "Indonesian,English,Chinese": 0.11363636363636363, + "Indonesian,English,Malay": 0.2727272727272727, + "Indonesian,English,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Spanish": 0.2784090909090909, + "Indonesian,Filipino,Chinese": 0.125, + "Indonesian,Filipino,Malay": 0.39204545454545453, + "Indonesian,Filipino,Vietnamese": 0.22727272727272727, + "Indonesian,Spanish,Chinese": 0.10227272727272728, + "Indonesian,Spanish,Malay": 0.3181818181818182, + "Indonesian,Spanish,Vietnamese": 0.16477272727272727, + "Indonesian,Chinese,Malay": 0.1875, + "Indonesian,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,Malay,Vietnamese": 0.2556818181818182, + "English,Filipino,Spanish": 0.2556818181818182, + "English,Filipino,Chinese": 0.11363636363636363, + "English,Filipino,Malay": 0.26704545454545453, + "English,Filipino,Vietnamese": 0.19886363636363635, + "English,Spanish,Chinese": 0.125, + "English,Spanish,Malay": 0.26704545454545453, + "English,Spanish,Vietnamese": 0.18181818181818182, + "English,Chinese,Malay": 0.1534090909090909, + "English,Chinese,Vietnamese": 0.09090909090909091, + "English,Malay,Vietnamese": 0.17045454545454544, + "Filipino,Spanish,Chinese": 0.09090909090909091, + "Filipino,Spanish,Malay": 0.30113636363636365, + "Filipino,Spanish,Vietnamese": 0.17045454545454544, + "Filipino,Chinese,Malay": 0.125, + "Filipino,Chinese,Vietnamese": 0.07386363636363637, + "Filipino,Malay,Vietnamese": 0.20454545454545456, + "Spanish,Chinese,Malay": 0.125, + "Spanish,Chinese,Vietnamese": 0.07954545454545454, + "Spanish,Malay,Vietnamese": 0.1875, + "Chinese,Malay,Vietnamese": 0.08522727272727272 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.375, - "English,Filipino,Spanish,Chinese": 0.3522727272727273, - "English,Filipino,Spanish,Indonesian": 0.3522727272727273, - "English,Filipino,Spanish,Vietnamese": 0.32386363636363635, - "English,Filipino,Malay,Chinese": 0.3125, - "English,Filipino,Malay,Indonesian": 0.3522727272727273, - "English,Filipino,Malay,Vietnamese": 0.29545454545454547, - "English,Filipino,Chinese,Indonesian": 0.3352272727272727, - "English,Filipino,Chinese,Vietnamese": 0.2727272727272727, - "English,Filipino,Indonesian,Vietnamese": 0.3181818181818182, - "English,Spanish,Malay,Chinese": 0.3693181818181818, - "English,Spanish,Malay,Indonesian": 0.4431818181818182, - "English,Spanish,Malay,Vietnamese": 0.4034090909090909, - "English,Spanish,Chinese,Indonesian": 0.375, - "English,Spanish,Chinese,Vietnamese": 0.3409090909090909, - "English,Spanish,Indonesian,Vietnamese": 0.39204545454545453, - "English,Malay,Chinese,Indonesian": 0.36363636363636365, - "English,Malay,Chinese,Vietnamese": 0.2897727272727273, - "English,Malay,Indonesian,Vietnamese": 0.3806818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.3181818181818182, - "Filipino,Spanish,Malay,Chinese": 0.3068181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.3522727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.3068181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.3068181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.3125, - "Filipino,Malay,Chinese,Indonesian": 0.3125, - "Filipino,Malay,Chinese,Vietnamese": 0.2556818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.3181818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.3465909090909091, - "Spanish,Malay,Chinese,Vietnamese": 0.2897727272727273, - "Spanish,Malay,Indonesian,Vietnamese": 0.38636363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.3068181818181818, - "Malay,Chinese,Indonesian,Vietnamese": 0.30113636363636365 + "Indonesian,English,Filipino,Spanish": 0.16477272727272727, + "Indonesian,English,Filipino,Chinese": 0.07386363636363637, + "Indonesian,English,Filipino,Malay": 0.19318181818181818, + "Indonesian,English,Filipino,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese": 0.056818181818181816, + "Indonesian,English,Spanish,Malay": 0.17045454545454544, + "Indonesian,English,Spanish,Vietnamese": 0.10227272727272728, + "Indonesian,English,Chinese,Malay": 0.09659090909090909, + "Indonesian,English,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,English,Malay,Vietnamese": 0.125, + "Indonesian,Filipino,Spanish,Chinese": 0.07386363636363637, + "Indonesian,Filipino,Spanish,Malay": 0.20454545454545456, + "Indonesian,Filipino,Spanish,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.10227272727272728, + "Indonesian,Filipino,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Malay,Vietnamese": 0.17045454545454544, + "Indonesian,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,Spanish,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,Chinese,Malay,Vietnamese": 0.06818181818181818, + "English,Filipino,Spanish,Chinese": 0.056818181818181816, + "English,Filipino,Spanish,Malay": 0.17045454545454544, + "English,Filipino,Spanish,Vietnamese": 0.10227272727272728, + "English,Filipino,Chinese,Malay": 0.07954545454545454, + "English,Filipino,Chinese,Vietnamese": 0.05113636363636364, + "English,Filipino,Malay,Vietnamese": 0.10795454545454546, + "English,Spanish,Chinese,Malay": 0.09090909090909091, + "English,Spanish,Chinese,Vietnamese": 0.05113636363636364, + "English,Spanish,Malay,Vietnamese": 0.11363636363636363, + "English,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Filipino,Spanish,Chinese,Malay": 0.07386363636363637, + "Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "Filipino,Spanish,Malay,Vietnamese": 0.125, + "Filipino,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Spanish,Chinese,Malay,Vietnamese": 0.045454545454545456 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.29545454545454547, - "English,Filipino,Spanish,Malay,Indonesian": 0.3181818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.2840909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.29545454545454547, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.25, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.2784090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.2840909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.22727272727272727, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.2727272727272727, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.32954545454545453, - "English,Spanish,Malay,Chinese,Vietnamese": 0.2784090909090909, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.3522727272727273, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2727272727272727 + "Indonesian,English,Filipino,Spanish,Chinese": 0.045454545454545456, + "Indonesian,English,Filipino,Spanish,Malay": 0.11363636363636363, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Chinese,Malay": 0.06818181818181818, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,English,Spanish,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0625, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, + "English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, + "English,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, + "English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.2727272727272727, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.22727272727272727, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.26136363636363635, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 } }, - "AC3_2": 0.5208941456199296, - "AC3_3": 0.443465753601961, - "AC3_4": 0.3894342066098551, - "AC3_5": 0.34969232026840463, - "AC3_6": 0.32117428537454795, - "AC3_7": 0.30150894183429794 + "AC3_2": 0.3246437132360101, + "AC3_3": 0.2224781164490691, + "AC3_4": 0.14424600837328067, + "AC3_5": 0.09141425832366948, + "AC3_6": 0.059470399711109456, + "AC3_7": 0.04203323556770199 }, "prompt_3": { - "overall_acc": 0.45454545454545453, + "overall_acc": 0.2857142857142857, "language_acc": { - "English": 0.5170454545454546, - "Filipino": 0.38636363636363635, - "Spanish": 0.4715909090909091, - "Malay": 0.4715909090909091, - "Chinese": 0.4375, - "Indonesian": 0.44886363636363635, - "Vietnamese": 0.44886363636363635 + "Indonesian": 0.3068181818181818, + "English": 0.3068181818181818, + "Filipino": 0.25, + "Spanish": 0.2840909090909091, + "Chinese": 0.29545454545454547, + "Malay": 0.2840909090909091, + "Vietnamese": 0.2727272727272727 }, - "consistency_score_2": 0.5300324675324675, - "consistency_score_3": 0.35844155844155845, - "consistency_score_4": 0.2724025974025975, - "consistency_score_5": 0.2215909090909091, - "consistency_score_6": 0.1883116883116883, - "consistency_score_7": 0.16477272727272727, + "consistency_score_2": 0.49269480519480524, + "consistency_score_3": 0.3073051948051948, + "consistency_score_4": 0.20990259740259737, + "consistency_score_5": 0.15070346320346323, + "consistency_score_6": 0.11201298701298701, + "consistency_score_7": 0.08522727272727272, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.48863636363636365, - "English,Spanish": 0.6363636363636364, - "English,Malay": 0.5909090909090909, - "English,Chinese": 0.5, - "English,Indonesian": 0.5511363636363636, - "English,Vietnamese": 0.5227272727272727, - "Filipino,Spanish": 0.4659090909090909, - "Filipino,Malay": 0.5397727272727273, - "Filipino,Chinese": 0.4772727272727273, - "Filipino,Indonesian": 0.5454545454545454, - "Filipino,Vietnamese": 0.5, - "Spanish,Malay": 0.5568181818181818, - "Spanish,Chinese": 0.4772727272727273, - "Spanish,Indonesian": 0.5511363636363636, - "Spanish,Vietnamese": 0.5795454545454546, - "Malay,Chinese": 0.4772727272727273, - "Malay,Indonesian": 0.6590909090909091, - "Malay,Vietnamese": 0.5113636363636364, - "Chinese,Indonesian": 0.5056818181818182, - "Chinese,Vietnamese": 0.4715909090909091, - "Indonesian,Vietnamese": 0.5227272727272727 + "Indonesian,English": 0.4602272727272727, + "Indonesian,Filipino": 0.6704545454545454, + "Indonesian,Spanish": 0.5454545454545454, + "Indonesian,Chinese": 0.4090909090909091, + "Indonesian,Malay": 0.7159090909090909, + "Indonesian,Vietnamese": 0.5, + "English,Filipino": 0.4659090909090909, + "English,Spanish": 0.5454545454545454, + "English,Chinese": 0.32386363636363635, + "English,Malay": 0.4659090909090909, + "English,Vietnamese": 0.4034090909090909, + "Filipino,Spanish": 0.5909090909090909, + "Filipino,Chinese": 0.3352272727272727, + "Filipino,Malay": 0.6988636363636364, + "Filipino,Vietnamese": 0.48295454545454547, + "Spanish,Chinese": 0.3125, + "Spanish,Malay": 0.6136363636363636, + "Spanish,Vietnamese": 0.4431818181818182, + "Chinese,Malay": 0.42045454545454547, + "Chinese,Vietnamese": 0.42045454545454547, + "Malay,Vietnamese": 0.5227272727272727 }, "3_combine": { - "English,Filipino,Spanish": 0.35795454545454547, - "English,Filipino,Malay": 0.35795454545454547, - "English,Filipino,Chinese": 0.3125, - "English,Filipino,Indonesian": 0.36363636363636365, - "English,Filipino,Vietnamese": 0.32386363636363635, - "English,Spanish,Malay": 0.44886363636363635, - "English,Spanish,Chinese": 0.375, - "English,Spanish,Indonesian": 0.42613636363636365, - "English,Spanish,Vietnamese": 0.4147727272727273, - "English,Malay,Chinese": 0.3522727272727273, - "English,Malay,Indonesian": 0.4431818181818182, - "English,Malay,Vietnamese": 0.375, - "English,Chinese,Indonesian": 0.36363636363636365, - "English,Chinese,Vietnamese": 0.3181818181818182, - "English,Indonesian,Vietnamese": 0.36363636363636365, - "Filipino,Spanish,Malay": 0.3352272727272727, - "Filipino,Spanish,Chinese": 0.2840909090909091, - "Filipino,Spanish,Indonesian": 0.3409090909090909, - "Filipino,Spanish,Vietnamese": 0.3522727272727273, - "Filipino,Malay,Chinese": 0.32386363636363635, - "Filipino,Malay,Indonesian": 0.4318181818181818, - "Filipino,Malay,Vietnamese": 0.32954545454545453, - "Filipino,Chinese,Indonesian": 0.3465909090909091, - "Filipino,Chinese,Vietnamese": 0.2897727272727273, - "Filipino,Indonesian,Vietnamese": 0.3465909090909091, - "Spanish,Malay,Chinese": 0.3352272727272727, - "Spanish,Malay,Indonesian": 0.42045454545454547, - "Spanish,Malay,Vietnamese": 0.3806818181818182, - "Spanish,Chinese,Indonesian": 0.3352272727272727, - "Spanish,Chinese,Vietnamese": 0.3409090909090909, - "Spanish,Indonesian,Vietnamese": 0.39204545454545453, - "Malay,Chinese,Indonesian": 0.375, - "Malay,Chinese,Vietnamese": 0.2897727272727273, - "Malay,Indonesian,Vietnamese": 0.3806818181818182, - "Chinese,Indonesian,Vietnamese": 0.3181818181818182 + "Indonesian,English,Filipino": 0.3522727272727273, + "Indonesian,English,Spanish": 0.3409090909090909, + "Indonesian,English,Chinese": 0.20454545454545456, + "Indonesian,English,Malay": 0.38636363636363635, + "Indonesian,English,Vietnamese": 0.26136363636363635, + "Indonesian,Filipino,Spanish": 0.42613636363636365, + "Indonesian,Filipino,Chinese": 0.2784090909090909, + "Indonesian,Filipino,Malay": 0.5568181818181818, + "Indonesian,Filipino,Vietnamese": 0.35795454545454547, + "Indonesian,Spanish,Chinese": 0.22727272727272727, + "Indonesian,Spanish,Malay": 0.4715909090909091, + "Indonesian,Spanish,Vietnamese": 0.3068181818181818, + "Indonesian,Chinese,Malay": 0.32386363636363635, + "Indonesian,Chinese,Vietnamese": 0.24431818181818182, + "Indonesian,Malay,Vietnamese": 0.3977272727272727, + "English,Filipino,Spanish": 0.36363636363636365, + "English,Filipino,Chinese": 0.17613636363636365, + "English,Filipino,Malay": 0.375, + "English,Filipino,Vietnamese": 0.2727272727272727, + "English,Spanish,Chinese": 0.18181818181818182, + "English,Spanish,Malay": 0.3806818181818182, + "English,Spanish,Vietnamese": 0.26704545454545453, + "English,Chinese,Malay": 0.19886363636363635, + "English,Chinese,Vietnamese": 0.17613636363636365, + "English,Malay,Vietnamese": 0.2556818181818182, + "Filipino,Spanish,Chinese": 0.21022727272727273, + "Filipino,Spanish,Malay": 0.48863636363636365, + "Filipino,Spanish,Vietnamese": 0.32386363636363635, + "Filipino,Chinese,Malay": 0.2840909090909091, + "Filipino,Chinese,Vietnamese": 0.2159090909090909, + "Filipino,Malay,Vietnamese": 0.3977272727272727, + "Spanish,Chinese,Malay": 0.2556818181818182, + "Spanish,Chinese,Vietnamese": 0.19318181818181818, + "Spanish,Malay,Vietnamese": 0.3409090909090909, + "Chinese,Malay,Vietnamese": 0.26136363636363635 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.2840909090909091, - "English,Filipino,Spanish,Chinese": 0.2556818181818182, - "English,Filipino,Spanish,Indonesian": 0.2897727272727273, - "English,Filipino,Spanish,Vietnamese": 0.2784090909090909, - "English,Filipino,Malay,Chinese": 0.24431818181818182, - "English,Filipino,Malay,Indonesian": 0.3125, - "English,Filipino,Malay,Vietnamese": 0.26704545454545453, - "English,Filipino,Chinese,Indonesian": 0.26136363636363635, - "English,Filipino,Chinese,Vietnamese": 0.2215909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.26704545454545453, - "English,Spanish,Malay,Chinese": 0.29545454545454547, - "English,Spanish,Malay,Indonesian": 0.3522727272727273, - "English,Spanish,Malay,Vietnamese": 0.3181818181818182, - "English,Spanish,Chinese,Indonesian": 0.2840909090909091, - "English,Spanish,Chinese,Vietnamese": 0.2784090909090909, - "English,Spanish,Indonesian,Vietnamese": 0.3181818181818182, - "English,Malay,Chinese,Indonesian": 0.30113636363636365, - "English,Malay,Chinese,Vietnamese": 0.23863636363636365, - "English,Malay,Indonesian,Vietnamese": 0.3125, - "English,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Spanish,Malay,Chinese": 0.23295454545454544, - "Filipino,Spanish,Malay,Indonesian": 0.29545454545454547, - "Filipino,Spanish,Malay,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Chinese,Indonesian": 0.23295454545454544, - "Filipino,Spanish,Chinese,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2840909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.2897727272727273, - "Filipino,Malay,Chinese,Vietnamese": 0.21022727272727273, - "Filipino,Malay,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Chinese,Indonesian": 0.2727272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.24431818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.3181818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182 + "Indonesian,English,Filipino,Spanish": 0.2897727272727273, + "Indonesian,English,Filipino,Chinese": 0.1534090909090909, + "Indonesian,English,Filipino,Malay": 0.32386363636363635, + "Indonesian,English,Filipino,Vietnamese": 0.2159090909090909, + "Indonesian,English,Spanish,Chinese": 0.14772727272727273, + "Indonesian,English,Spanish,Malay": 0.3125, + "Indonesian,English,Spanish,Vietnamese": 0.19318181818181818, + "Indonesian,English,Chinese,Malay": 0.17613636363636365, + "Indonesian,English,Chinese,Vietnamese": 0.13068181818181818, + "Indonesian,English,Malay,Vietnamese": 0.2215909090909091, + "Indonesian,Filipino,Spanish,Chinese": 0.17045454545454544, + "Indonesian,Filipino,Spanish,Malay": 0.39204545454545453, + "Indonesian,Filipino,Spanish,Vietnamese": 0.24431818181818182, + "Indonesian,Filipino,Chinese,Malay": 0.23863636363636365, + "Indonesian,Filipino,Chinese,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Malay,Vietnamese": 0.3181818181818182, + "Indonesian,Spanish,Chinese,Malay": 0.20454545454545456, + "Indonesian,Spanish,Chinese,Vietnamese": 0.14772727272727273, + "Indonesian,Spanish,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,Chinese,Malay,Vietnamese": 0.21022727272727273, + "English,Filipino,Spanish,Chinese": 0.13636363636363635, + "English,Filipino,Spanish,Malay": 0.32954545454545453, + "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, + "English,Filipino,Chinese,Malay": 0.1534090909090909, + "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, + "English,Filipino,Malay,Vietnamese": 0.22727272727272727, + "English,Spanish,Chinese,Malay": 0.1590909090909091, + "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, + "English,Spanish,Malay,Vietnamese": 0.21022727272727273, + "English,Chinese,Malay,Vietnamese": 0.13636363636363635, + "Filipino,Spanish,Chinese,Malay": 0.19318181818181818, + "Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, + "Filipino,Spanish,Malay,Vietnamese": 0.2840909090909091, + "Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, + "Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.21022727272727273, - "English,Filipino,Spanish,Malay,Indonesian": 0.2556818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, - "English,Filipino,Spanish,Chinese,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.23863636363636365, - "English,Filipino,Malay,Chinese,Indonesian": 0.22727272727272727, - "English,Filipino,Malay,Chinese,Vietnamese": 0.18181818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1875, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909 + "Indonesian,English,Filipino,Spanish,Chinese": 0.11931818181818182, + "Indonesian,English,Filipino,Spanish,Malay": 0.2840909090909091, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.17045454545454544, + "Indonesian,English,Filipino,Chinese,Malay": 0.13636363636363635, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.11363636363636363, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.19886363636363635, + "Indonesian,English,Spanish,Chinese,Malay": 0.13636363636363635, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.09659090909090909, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.18181818181818182, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, + "English,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, + "English,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, + "English,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, + "English,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.18181818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 } }, - "AC3_2": 0.48939518843863977, - "AC3_3": 0.40081324421442216, - "AC3_4": 0.3406553249764855, - "AC3_5": 0.2979373567168215, - "AC3_6": 0.2662993571670238, - "AC3_7": 0.24186822348054574 + "AC3_2": 0.36168628030509187, + "AC3_3": 0.2961166946432734, + "AC3_4": 0.2420102006904895, + "AC3_5": 0.19732530329372777, + "AC3_6": 0.16093294456595084, + "AC3_7": 0.1312910284109955 }, "prompt_4": { - "overall_acc": 0.4756493506493507, + "overall_acc": 0.28490259740259744, "language_acc": { - "English": 0.5511363636363636, - "Filipino": 0.42613636363636365, - "Spanish": 0.44886363636363635, - "Malay": 0.5056818181818182, - "Chinese": 0.4659090909090909, - "Indonesian": 0.4715909090909091, - "Vietnamese": 0.4602272727272727 + "Indonesian": 0.2727272727272727, + "English": 0.32386363636363635, + "Filipino": 0.2556818181818182, + "Spanish": 0.32954545454545453, + "Chinese": 0.2727272727272727, + "Malay": 0.2897727272727273, + "Vietnamese": 0.25 }, - "consistency_score_2": 0.5784632034632035, - "consistency_score_3": 0.4180194805194804, - "consistency_score_4": 0.33116883116883117, - "consistency_score_5": 0.2759740259740259, - "consistency_score_6": 0.23782467532467533, - "consistency_score_7": 0.21022727272727273, + "consistency_score_2": 0.39772727272727276, + "consistency_score_3": 0.19155844155844154, + "consistency_score_4": 0.09805194805194804, + "consistency_score_5": 0.05032467532467533, + "consistency_score_6": 0.024350649350649345, + "consistency_score_7": 0.011363636363636364, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.5113636363636364, - "English,Spanish": 0.6988636363636364, - "English,Malay": 0.6420454545454546, - "English,Chinese": 0.5681818181818182, - "English,Indonesian": 0.6420454545454546, - "English,Vietnamese": 0.6193181818181818, - "Filipino,Spanish": 0.5454545454545454, - "Filipino,Malay": 0.5454545454545454, - "Filipino,Chinese": 0.5284090909090909, - "Filipino,Indonesian": 0.5795454545454546, - "Filipino,Vietnamese": 0.5227272727272727, - "Spanish,Malay": 0.5965909090909091, - "Spanish,Chinese": 0.4943181818181818, - "Spanish,Indonesian": 0.5965909090909091, - "Spanish,Vietnamese": 0.6704545454545454, - "Malay,Chinese": 0.5, - "Malay,Indonesian": 0.7045454545454546, - "Malay,Vietnamese": 0.5852272727272727, - "Chinese,Indonesian": 0.4943181818181818, - "Chinese,Vietnamese": 0.5113636363636364, - "Indonesian,Vietnamese": 0.5909090909090909 + "Indonesian,English": 0.3693181818181818, + "Indonesian,Filipino": 0.5738636363636364, + "Indonesian,Spanish": 0.4034090909090909, + "Indonesian,Chinese": 0.26136363636363635, + "Indonesian,Malay": 0.6875, + "Indonesian,Vietnamese": 0.4318181818181818, + "English,Filipino": 0.4147727272727273, + "English,Spanish": 0.5, + "English,Chinese": 0.30113636363636365, + "English,Malay": 0.4318181818181818, + "English,Vietnamese": 0.32386363636363635, + "Filipino,Spanish": 0.5, + "Filipino,Chinese": 0.17045454545454544, + "Filipino,Malay": 0.625, + "Filipino,Vietnamese": 0.4090909090909091, + "Spanish,Chinese": 0.23295454545454544, + "Spanish,Malay": 0.4602272727272727, + "Spanish,Vietnamese": 0.3465909090909091, + "Chinese,Malay": 0.24431818181818182, + "Chinese,Vietnamese": 0.2215909090909091, + "Malay,Vietnamese": 0.4431818181818182 }, "3_combine": { - "English,Filipino,Spanish": 0.42045454545454547, - "English,Filipino,Malay": 0.4034090909090909, - "English,Filipino,Chinese": 0.3806818181818182, - "English,Filipino,Indonesian": 0.4147727272727273, - "English,Filipino,Vietnamese": 0.38636363636363635, - "English,Spanish,Malay": 0.5113636363636364, - "English,Spanish,Chinese": 0.42613636363636365, - "English,Spanish,Indonesian": 0.5056818181818182, - "English,Spanish,Vietnamese": 0.5397727272727273, - "English,Malay,Chinese": 0.4090909090909091, - "English,Malay,Indonesian": 0.5284090909090909, - "English,Malay,Vietnamese": 0.4602272727272727, - "English,Chinese,Indonesian": 0.4034090909090909, - "English,Chinese,Vietnamese": 0.3977272727272727, - "English,Indonesian,Vietnamese": 0.4715909090909091, - "Filipino,Spanish,Malay": 0.39204545454545453, - "Filipino,Spanish,Chinese": 0.35795454545454547, - "Filipino,Spanish,Indonesian": 0.4034090909090909, - "Filipino,Spanish,Vietnamese": 0.4147727272727273, - "Filipino,Malay,Chinese": 0.3522727272727273, - "Filipino,Malay,Indonesian": 0.45454545454545453, - "Filipino,Malay,Vietnamese": 0.375, - "Filipino,Chinese,Indonesian": 0.375, - "Filipino,Chinese,Vietnamese": 0.3522727272727273, - "Filipino,Indonesian,Vietnamese": 0.39204545454545453, - "Spanish,Malay,Chinese": 0.3522727272727273, - "Spanish,Malay,Indonesian": 0.48863636363636365, - "Spanish,Malay,Vietnamese": 0.4659090909090909, - "Spanish,Chinese,Indonesian": 0.3465909090909091, - "Spanish,Chinese,Vietnamese": 0.38636363636363635, - "Spanish,Indonesian,Vietnamese": 0.4715909090909091, - "Malay,Chinese,Indonesian": 0.3977272727272727, - "Malay,Chinese,Vietnamese": 0.3522727272727273, - "Malay,Indonesian,Vietnamese": 0.48863636363636365, - "Chinese,Indonesian,Vietnamese": 0.3522727272727273 + "Indonesian,English,Filipino": 0.23863636363636365, + "Indonesian,English,Spanish": 0.2159090909090909, + "Indonesian,English,Chinese": 0.10795454545454546, + "Indonesian,English,Malay": 0.29545454545454547, + "Indonesian,English,Vietnamese": 0.17045454545454544, + "Indonesian,Filipino,Spanish": 0.2897727272727273, + "Indonesian,Filipino,Chinese": 0.09659090909090909, + "Indonesian,Filipino,Malay": 0.4659090909090909, + "Indonesian,Filipino,Vietnamese": 0.2840909090909091, + "Indonesian,Spanish,Chinese": 0.10795454545454546, + "Indonesian,Spanish,Malay": 0.32386363636363635, + "Indonesian,Spanish,Vietnamese": 0.18181818181818182, + "Indonesian,Chinese,Malay": 0.16477272727272727, + "Indonesian,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,Malay,Vietnamese": 0.3352272727272727, + "English,Filipino,Spanish": 0.26704545454545453, + "English,Filipino,Chinese": 0.09090909090909091, + "English,Filipino,Malay": 0.2784090909090909, + "English,Filipino,Vietnamese": 0.18181818181818182, + "English,Spanish,Chinese": 0.125, + "English,Spanish,Malay": 0.26136363636363635, + "English,Spanish,Vietnamese": 0.1875, + "English,Chinese,Malay": 0.125, + "English,Chinese,Vietnamese": 0.07386363636363637, + "English,Malay,Vietnamese": 0.18181818181818182, + "Filipino,Spanish,Chinese": 0.08522727272727272, + "Filipino,Spanish,Malay": 0.3465909090909091, + "Filipino,Spanish,Vietnamese": 0.21022727272727273, + "Filipino,Chinese,Malay": 0.09659090909090909, + "Filipino,Chinese,Vietnamese": 0.05113636363636364, + "Filipino,Malay,Vietnamese": 0.29545454545454547, + "Spanish,Chinese,Malay": 0.10795454545454546, + "Spanish,Chinese,Vietnamese": 0.07386363636363637, + "Spanish,Malay,Vietnamese": 0.21022727272727273, + "Chinese,Malay,Vietnamese": 0.08522727272727272 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.3465909090909091, - "English,Filipino,Spanish,Chinese": 0.32954545454545453, - "English,Filipino,Spanish,Indonesian": 0.3465909090909091, - "English,Filipino,Spanish,Vietnamese": 0.3522727272727273, - "English,Filipino,Malay,Chinese": 0.3068181818181818, - "English,Filipino,Malay,Indonesian": 0.3522727272727273, - "English,Filipino,Malay,Vietnamese": 0.32386363636363635, - "English,Filipino,Chinese,Indonesian": 0.3068181818181818, - "English,Filipino,Chinese,Vietnamese": 0.29545454545454547, - "English,Filipino,Indonesian,Vietnamese": 0.3181818181818182, - "English,Spanish,Malay,Chinese": 0.3409090909090909, - "English,Spanish,Malay,Indonesian": 0.4375, - "English,Spanish,Malay,Vietnamese": 0.42045454545454547, - "English,Spanish,Chinese,Indonesian": 0.32954545454545453, - "English,Spanish,Chinese,Vietnamese": 0.35795454545454547, - "English,Spanish,Indonesian,Vietnamese": 0.42613636363636365, - "English,Malay,Chinese,Indonesian": 0.3409090909090909, - "English,Malay,Chinese,Vietnamese": 0.3068181818181818, - "English,Malay,Indonesian,Vietnamese": 0.4034090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.3125, - "Filipino,Spanish,Malay,Chinese": 0.2897727272727273, - "Filipino,Spanish,Malay,Indonesian": 0.3465909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.3181818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Indonesian,Vietnamese": 0.32954545454545453, - "Filipino,Malay,Chinese,Indonesian": 0.3068181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.2727272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.3352272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.30113636363636365, - "Spanish,Malay,Chinese,Vietnamese": 0.29545454545454547, - "Spanish,Malay,Indonesian,Vietnamese": 0.3977272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.29545454545454547, - "Malay,Chinese,Indonesian,Vietnamese": 0.29545454545454547 + "Indonesian,English,Filipino,Spanish": 0.1534090909090909, + "Indonesian,English,Filipino,Chinese": 0.05113636363636364, + "Indonesian,English,Filipino,Malay": 0.20454545454545456, + "Indonesian,English,Filipino,Vietnamese": 0.13068181818181818, + "Indonesian,English,Spanish,Chinese": 0.0625, + "Indonesian,English,Spanish,Malay": 0.18181818181818182, + "Indonesian,English,Spanish,Vietnamese": 0.09659090909090909, + "Indonesian,English,Chinese,Malay": 0.09090909090909091, + "Indonesian,English,Chinese,Vietnamese": 0.045454545454545456, + "Indonesian,English,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Spanish,Chinese": 0.056818181818181816, + "Indonesian,Filipino,Spanish,Malay": 0.24431818181818182, + "Indonesian,Filipino,Spanish,Vietnamese": 0.1534090909090909, + "Indonesian,Filipino,Chinese,Malay": 0.07386363636363637, + "Indonesian,Filipino,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,Filipino,Malay,Vietnamese": 0.24431818181818182, + "Indonesian,Spanish,Chinese,Malay": 0.08522727272727272, + "Indonesian,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,Spanish,Malay,Vietnamese": 0.1590909090909091, + "Indonesian,Chinese,Malay,Vietnamese": 0.0625, + "English,Filipino,Spanish,Chinese": 0.045454545454545456, + "English,Filipino,Spanish,Malay": 0.19318181818181818, + "English,Filipino,Spanish,Vietnamese": 0.11363636363636363, + "English,Filipino,Chinese,Malay": 0.045454545454545456, + "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, + "English,Filipino,Malay,Vietnamese": 0.13636363636363635, + "English,Spanish,Chinese,Malay": 0.06818181818181818, + "English,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "English,Spanish,Malay,Vietnamese": 0.10795454545454546, + "English,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Filipino,Spanish,Chinese,Malay": 0.056818181818181816, + "Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, + "Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, + "Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.2784090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.3068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.29545454545454547, - "English,Filipino,Spanish,Chinese,Indonesian": 0.26704545454545453, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.2784090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.29545454545454547, - "English,Filipino,Malay,Chinese,Indonesian": 0.26704545454545453, - "English,Filipino,Malay,Chinese,Vietnamese": 0.25, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.2897727272727273, - "English,Spanish,Malay,Chinese,Vietnamese": 0.2897727272727273, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.3693181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.2556818181818182, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182 + "Indonesian,English,Filipino,Spanish,Chinese": 0.03409090909090909, + "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.08522727272727272, + "Indonesian,English,Filipino,Chinese,Malay": 0.03977272727272727, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,English,Spanish,Chinese,Malay": 0.056818181818181816, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, + "English,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, + "English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, + "English,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.23863636363636365, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.26136363636363635, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.25, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.017045454545454544, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.011363636363636364 } }, - "AC3_2": 0.5220422544105946, - "AC3_3": 0.44497623171118084, - "AC3_4": 0.390472706319682, - "AC3_5": 0.34928893995521165, - "AC3_6": 0.3170995670551227, - "AC3_7": 0.29158149538513706 + "AC3_2": 0.3319911360447612, + "AC3_3": 0.22908692665383723, + "AC3_4": 0.14589331817400056, + "AC3_5": 0.08553976287502078, + "AC3_6": 0.044866550757554545, + "AC3_7": 0.021855541711178424 }, "prompt_5": { - "overall_acc": 0.47564935064935066, + "overall_acc": 0.2930194805194805, "language_acc": { - "English": 0.5625, - "Filipino": 0.42613636363636365, - "Spanish": 0.4318181818181818, - "Malay": 0.48863636363636365, - "Chinese": 0.5113636363636364, - "Indonesian": 0.4431818181818182, - "Vietnamese": 0.4659090909090909 + "Indonesian": 0.3181818181818182, + "English": 0.35795454545454547, + "Filipino": 0.25, + "Spanish": 0.2897727272727273, + "Chinese": 0.2727272727272727, + "Malay": 0.26704545454545453, + "Vietnamese": 0.29545454545454547 }, - "consistency_score_2": 0.5903679653679654, - "consistency_score_3": 0.43165584415584407, - "consistency_score_4": 0.3459415584415585, - "consistency_score_5": 0.29166666666666663, - "consistency_score_6": 0.25243506493506496, - "consistency_score_7": 0.2215909090909091, + "consistency_score_2": 0.35633116883116883, + "consistency_score_3": 0.16201298701298697, + "consistency_score_4": 0.08798701298701296, + "consistency_score_5": 0.05330086580086579, + "consistency_score_6": 0.034090909090909095, + "consistency_score_7": 0.022727272727272728, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.5340909090909091, - "English,Spanish": 0.6988636363636364, - "English,Malay": 0.6306818181818182, - "English,Chinese": 0.6136363636363636, - "English,Indonesian": 0.6420454545454546, - "English,Vietnamese": 0.6420454545454546, - "Filipino,Spanish": 0.5454545454545454, - "Filipino,Malay": 0.5113636363636364, - "Filipino,Chinese": 0.5397727272727273, - "Filipino,Indonesian": 0.5852272727272727, - "Filipino,Vietnamese": 0.5511363636363636, - "Spanish,Malay": 0.6079545454545454, - "Spanish,Chinese": 0.5625, - "Spanish,Indonesian": 0.6306818181818182, - "Spanish,Vietnamese": 0.6704545454545454, - "Malay,Chinese": 0.5056818181818182, - "Malay,Indonesian": 0.6704545454545454, - "Malay,Vietnamese": 0.5852272727272727, - "Chinese,Indonesian": 0.5170454545454546, - "Chinese,Vietnamese": 0.5284090909090909, - "Indonesian,Vietnamese": 0.625 + "Indonesian,English": 0.3977272727272727, + "Indonesian,Filipino": 0.42613636363636365, + "Indonesian,Spanish": 0.4090909090909091, + "Indonesian,Chinese": 0.19318181818181818, + "Indonesian,Malay": 0.5568181818181818, + "Indonesian,Vietnamese": 0.32386363636363635, + "English,Filipino": 0.45454545454545453, + "English,Spanish": 0.5511363636363636, + "English,Chinese": 0.2840909090909091, + "English,Malay": 0.375, + "English,Vietnamese": 0.36363636363636365, + "Filipino,Spanish": 0.4375, + "Filipino,Chinese": 0.125, + "Filipino,Malay": 0.4659090909090909, + "Filipino,Vietnamese": 0.3125, + "Spanish,Chinese": 0.1590909090909091, + "Spanish,Malay": 0.4318181818181818, + "Spanish,Vietnamese": 0.3522727272727273, + "Chinese,Malay": 0.14204545454545456, + "Chinese,Vietnamese": 0.4147727272727273, + "Malay,Vietnamese": 0.3068181818181818 }, "3_combine": { - "English,Filipino,Spanish": 0.42613636363636365, - "English,Filipino,Malay": 0.3977272727272727, - "English,Filipino,Chinese": 0.3977272727272727, - "English,Filipino,Indonesian": 0.4375, - "English,Filipino,Vietnamese": 0.42045454545454547, - "English,Spanish,Malay": 0.5170454545454546, - "English,Spanish,Chinese": 0.4715909090909091, - "English,Spanish,Indonesian": 0.5284090909090909, - "English,Spanish,Vietnamese": 0.5340909090909091, - "English,Malay,Chinese": 0.4147727272727273, - "English,Malay,Indonesian": 0.5056818181818182, - "English,Malay,Vietnamese": 0.4659090909090909, - "English,Chinese,Indonesian": 0.4318181818181818, - "English,Chinese,Vietnamese": 0.42613636363636365, - "English,Indonesian,Vietnamese": 0.48295454545454547, - "Filipino,Spanish,Malay": 0.38636363636363635, - "Filipino,Spanish,Chinese": 0.3806818181818182, - "Filipino,Spanish,Indonesian": 0.42045454545454547, - "Filipino,Spanish,Vietnamese": 0.4318181818181818, - "Filipino,Malay,Chinese": 0.35795454545454547, - "Filipino,Malay,Indonesian": 0.42613636363636365, - "Filipino,Malay,Vietnamese": 0.375, - "Filipino,Chinese,Indonesian": 0.3693181818181818, - "Filipino,Chinese,Vietnamese": 0.375, - "Filipino,Indonesian,Vietnamese": 0.42045454545454547, - "Spanish,Malay,Chinese": 0.39204545454545453, - "Spanish,Malay,Indonesian": 0.48863636363636365, - "Spanish,Malay,Vietnamese": 0.48295454545454547, - "Spanish,Chinese,Indonesian": 0.4090909090909091, - "Spanish,Chinese,Vietnamese": 0.4147727272727273, - "Spanish,Indonesian,Vietnamese": 0.4943181818181818, - "Malay,Chinese,Indonesian": 0.3977272727272727, - "Malay,Chinese,Vietnamese": 0.35795454545454547, - "Malay,Indonesian,Vietnamese": 0.48295454545454547, - "Chinese,Indonesian,Vietnamese": 0.38636363636363635 + "Indonesian,English,Filipino": 0.22727272727272727, + "Indonesian,English,Spanish": 0.26704545454545453, + "Indonesian,English,Chinese": 0.09659090909090909, + "Indonesian,English,Malay": 0.25, + "Indonesian,English,Vietnamese": 0.17613636363636365, + "Indonesian,Filipino,Spanish": 0.23863636363636365, + "Indonesian,Filipino,Chinese": 0.056818181818181816, + "Indonesian,Filipino,Malay": 0.2897727272727273, + "Indonesian,Filipino,Vietnamese": 0.18181818181818182, + "Indonesian,Spanish,Chinese": 0.0625, + "Indonesian,Spanish,Malay": 0.29545454545454547, + "Indonesian,Spanish,Vietnamese": 0.17613636363636365, + "Indonesian,Chinese,Malay": 0.09659090909090909, + "Indonesian,Chinese,Vietnamese": 0.09090909090909091, + "Indonesian,Malay,Vietnamese": 0.21022727272727273, + "English,Filipino,Spanish": 0.2897727272727273, + "English,Filipino,Chinese": 0.08522727272727272, + "English,Filipino,Malay": 0.22727272727272727, + "English,Filipino,Vietnamese": 0.1875, + "English,Spanish,Chinese": 0.11931818181818182, + "English,Spanish,Malay": 0.25, + "English,Spanish,Vietnamese": 0.21022727272727273, + "English,Chinese,Malay": 0.08522727272727272, + "English,Chinese,Vietnamese": 0.14772727272727273, + "English,Malay,Vietnamese": 0.1590909090909091, + "Filipino,Spanish,Chinese": 0.056818181818181816, + "Filipino,Spanish,Malay": 0.24431818181818182, + "Filipino,Spanish,Vietnamese": 0.19318181818181818, + "Filipino,Chinese,Malay": 0.056818181818181816, + "Filipino,Chinese,Vietnamese": 0.0625, + "Filipino,Malay,Vietnamese": 0.17613636363636365, + "Spanish,Chinese,Malay": 0.056818181818181816, + "Spanish,Chinese,Vietnamese": 0.08522727272727272, + "Spanish,Malay,Vietnamese": 0.17613636363636365, + "Chinese,Malay,Vietnamese": 0.08522727272727272 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.3522727272727273, - "English,Filipino,Spanish,Chinese": 0.32954545454545453, - "English,Filipino,Spanish,Indonesian": 0.36363636363636365, - "English,Filipino,Spanish,Vietnamese": 0.3693181818181818, - "English,Filipino,Malay,Chinese": 0.30113636363636365, - "English,Filipino,Malay,Indonesian": 0.3522727272727273, - "English,Filipino,Malay,Vietnamese": 0.3409090909090909, - "English,Filipino,Chinese,Indonesian": 0.32386363636363635, - "English,Filipino,Chinese,Vietnamese": 0.3125, - "English,Filipino,Indonesian,Vietnamese": 0.3522727272727273, - "English,Spanish,Malay,Chinese": 0.35795454545454547, - "English,Spanish,Malay,Indonesian": 0.4318181818181818, - "English,Spanish,Malay,Vietnamese": 0.42613636363636365, - "English,Spanish,Chinese,Indonesian": 0.3693181818181818, - "English,Spanish,Chinese,Vietnamese": 0.375, - "English,Spanish,Indonesian,Vietnamese": 0.4318181818181818, - "English,Malay,Chinese,Indonesian": 0.3465909090909091, - "English,Malay,Chinese,Vietnamese": 0.32386363636363635, - "English,Malay,Indonesian,Vietnamese": 0.4034090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.3465909090909091, - "Filipino,Spanish,Malay,Chinese": 0.30113636363636365, - "Filipino,Spanish,Malay,Indonesian": 0.3409090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.3409090909090909, - "Filipino,Spanish,Chinese,Indonesian": 0.3068181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.3181818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.36363636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.30113636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.2784090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.32954545454545453, - "Filipino,Chinese,Indonesian,Vietnamese": 0.29545454545454547, - "Spanish,Malay,Chinese,Indonesian": 0.3352272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.32386363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.3977272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.3522727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.3125 + "Indonesian,English,Filipino,Spanish": 0.17613636363636365, + "Indonesian,English,Filipino,Chinese": 0.05113636363636364, + "Indonesian,English,Filipino,Malay": 0.1590909090909091, + "Indonesian,English,Filipino,Vietnamese": 0.125, + "Indonesian,English,Spanish,Chinese": 0.05113636363636364, + "Indonesian,English,Spanish,Malay": 0.19318181818181818, + "Indonesian,English,Spanish,Vietnamese": 0.125, + "Indonesian,English,Chinese,Malay": 0.06818181818181818, + "Indonesian,English,Chinese,Vietnamese": 0.05113636363636364, + "Indonesian,English,Malay,Vietnamese": 0.13068181818181818, + "Indonesian,Filipino,Spanish,Chinese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Malay": 0.1875, + "Indonesian,Filipino,Spanish,Vietnamese": 0.13636363636363635, + "Indonesian,Filipino,Chinese,Malay": 0.045454545454545456, + "Indonesian,Filipino,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Malay,Vietnamese": 0.14204545454545456, + "Indonesian,Spanish,Chinese,Malay": 0.045454545454545456, + "Indonesian,Spanish,Chinese,Vietnamese": 0.03977272727272727, + "Indonesian,Spanish,Malay,Vietnamese": 0.13636363636363635, + "Indonesian,Chinese,Malay,Vietnamese": 0.0625, + "English,Filipino,Spanish,Chinese": 0.05113636363636364, + "English,Filipino,Spanish,Malay": 0.16477272727272727, + "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, + "English,Filipino,Chinese,Malay": 0.045454545454545456, + "English,Filipino,Chinese,Vietnamese": 0.03977272727272727, + "English,Filipino,Malay,Vietnamese": 0.10795454545454546, + "English,Spanish,Chinese,Malay": 0.056818181818181816, + "English,Spanish,Chinese,Vietnamese": 0.06818181818181818, + "English,Spanish,Malay,Vietnamese": 0.10227272727272728, + "English,Chinese,Malay,Vietnamese": 0.05113636363636364, + "Filipino,Spanish,Chinese,Malay": 0.03409090909090909, + "Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, + "Filipino,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.2784090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.3068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.3181818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.2784090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.2897727272727273, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.32386363636363635, - "English,Filipino,Malay,Chinese,Indonesian": 0.26704545454545453, - "English,Filipino,Malay,Chinese,Vietnamese": 0.2556818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "English,Spanish,Malay,Chinese,Indonesian": 0.3068181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.3068181818181818, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.36363636363636365, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.32386363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2897727272727273 + "Indonesian,English,Filipino,Spanish,Chinese": 0.03409090909090909, + "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10227272727272728, + "Indonesian,English,Filipino,Chinese,Malay": 0.03977272727272727, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.028409090909090908, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,English,Spanish,Chinese,Malay": 0.045454545454545456, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.03409090909090909, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.09659090909090909, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.045454545454545456, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, + "English,Filipino,Spanish,Chinese,Malay": 0.03409090909090909, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, + "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, + "English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, + "English,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.25, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.028409090909090908, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.028409090909090908, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.022727272727272728 } }, - "AC3_2": 0.5268359812281848, - "AC3_3": 0.452586016545796, - "AC3_4": 0.4005567145269294, - "AC3_5": 0.3616008462152159, - "AC3_6": 0.3298259710850185, - "AC3_7": 0.3023335802296833 + "AC3_2": 0.3215888798206052, + "AC3_3": 0.20865746806576882, + "AC3_4": 0.13533579752865357, + "AC3_5": 0.09019505882135875, + "AC3_6": 0.061076020734768734, + "AC3_7": 0.042182752966308475 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.6116504854368932 + "accuracy": 0.3592233009708738 }, "prompt_2": { - "accuracy": 0.6116504854368932 + "accuracy": 0.36893203883495146 }, "prompt_3": { - "accuracy": 0.6504854368932039 + "accuracy": 0.33980582524271846 }, "prompt_4": { - "accuracy": 0.6796116504854369 + "accuracy": 0.3300970873786408 }, "prompt_5": { - "accuracy": 0.6310679611650486 + "accuracy": 0.34951456310679613 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.3523809523809524 + "accuracy": 0.2571428571428571 }, "prompt_2": { - "accuracy": 0.2761904761904762 + "accuracy": 0.2571428571428571 }, "prompt_3": { - "accuracy": 0.34285714285714286 + "accuracy": 0.2571428571428571 }, "prompt_4": { - "accuracy": 0.38095238095238093 + "accuracy": 0.2571428571428571 }, "prompt_5": { - "accuracy": 0.34285714285714286 + "accuracy": 0.2571428571428571 } }, "us_eval": { "prompt_1": { - "accuracy": 0.719626168224299 + "accuracy": 0.27102803738317754 }, "prompt_2": { - "accuracy": 0.7009345794392523 + "accuracy": 0.29906542056074764 }, "prompt_3": { - "accuracy": 0.7476635514018691 + "accuracy": 0.2616822429906542 }, "prompt_4": { - "accuracy": 0.7476635514018691 + "accuracy": 0.27102803738317754 }, "prompt_5": { - "accuracy": 0.7009345794392523 + "accuracy": 0.2616822429906542 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.48, + "accuracy": 0.28, "category_acc": { - "brand": 0.5, - "demographics": 0.4, - "biology": 0.6, - "history": 0.4, + "brand": 0.4, + "demographics": 0.8, + "biology": 0.1, + "history": 0.4666666666666667, "literature": 0.2, - "politics": 0.6, - "culture": 0.5, - "film": 0.5, - "law": 0.3, - "geography": 0.8 + "politics": 0.2, + "culture": 0.2, + "film": 0.2, + "law": 0.2, + "geography": 0.2 } }, "prompt_2": { - "accuracy": 0.44, + "accuracy": 0.28, "category_acc": { "brand": 0.4, - "demographics": 0.2, - "biology": 0.5, - "history": 0.26666666666666666, + "demographics": 0.6, + "biology": 0.1, + "history": 0.4666666666666667, "literature": 0.2, - "politics": 0.9, - "culture": 0.5, - "film": 0.5, - "law": 0.3, - "geography": 0.6 + "politics": 0.3, + "culture": 0.2, + "film": 0.2, + "law": 0.2, + "geography": 0.2 } }, "prompt_3": { - "accuracy": 0.44, + "accuracy": 0.23, "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.5, - "history": 0.2, - "literature": 0.3, - "politics": 0.7, - "culture": 0.6, - "film": 0.4, - "law": 0.5, - "geography": 0.6 + "brand": 0.3, + "demographics": 0.4, + "biology": 0.2, + "history": 0.3333333333333333, + "literature": 0.2, + "politics": 0.2, + "culture": 0.2, + "film": 0.1, + "law": 0.1, + "geography": 0.3 } }, "prompt_4": { - "accuracy": 0.46, + "accuracy": 0.31, "category_acc": { - "brand": 0.6, - "demographics": 0.2, - "biology": 0.3, - "history": 0.3333333333333333, + "brand": 0.4, + "demographics": 0.8, + "biology": 0.2, + "history": 0.4666666666666667, "literature": 0.3, - "politics": 0.8, - "culture": 0.4, - "film": 0.5, - "law": 0.4, - "geography": 0.7 + "politics": 0.2, + "culture": 0.2, + "film": 0.2, + "law": 0.3, + "geography": 0.2 } }, "prompt_5": { - "accuracy": 0.45, + "accuracy": 0.28, "category_acc": { - "brand": 0.4, - "demographics": 0.4, - "biology": 0.4, + "brand": 0.3, + "demographics": 0.8, + "biology": 0.3, "history": 0.3333333333333333, "literature": 0.3, - "politics": 0.7, - "culture": 0.5, - "film": 0.4, - "law": 0.4, - "geography": 0.7 + "politics": 0.2, + "culture": 0.2, + "film": 0.2, + "law": 0.2, + "geography": 0.2 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.25097405111508586 + "bleu_score": 0.1778634553271136 }, "prompt_2": { - "bleu_score": 0.23362449725214837 + "bleu_score": 0.17587938617494803 }, "prompt_3": { - "bleu_score": 0.22930123896916854 + "bleu_score": 0.18016948741331085 }, "prompt_4": { - "bleu_score": 0.2154515974928525 + "bleu_score": 0.1755807406460382 }, "prompt_5": { - "bleu_score": 0.21095311759318322 + "bleu_score": 0.15871208497539763 } }, "indommlu": { "prompt_1": { - "accuracy": 0.4166499766339542, + "accuracy": 0.22377995860871888, "category_acc": { - "History": 0.3654618473895582, - "Geography": 0.33877551020408164, - "Lampungic": 0.3741496598639456, - "Social science": 0.656093489148581, - "Balinese": 0.30148619957537154, - "Makassarese": 0.34408602150537637, - "Banjarese": 0.3125, - "Chemistry": 0.24671532846715327, - "Biology": 0.3668639053254438, - "Science": 0.5799793601651186, - "Christian religion": 0.46766169154228854, - "Art": 0.5108153078202995, - "Islam religion": 0.4850640113798009, - "Hindu religion": 0.4, - "Madurese": 0.3423728813559322, - "Sport": 0.3716216216216216, - "Indonesian language": 0.4735367372353674, - "Physics": 0.3939393939393939, - "Minangkabau culture": 0.36683417085427134, - "Dayak language": 0.27522935779816515, - "Sociology": 0.3709677419354839, - "Economy": 0.35655737704918034, - "Sundanese": 0.337942955920484, - "Javanese": 0.3034274193548387, - "Civic education": 0.4663805436337625 + "History": 0.15863453815261044, + "Geography": 0.20612244897959184, + "Lampungic": 0.25170068027210885, + "Social science": 0.2621035058430718, + "Balinese": 0.25902335456475584, + "Makassarese": 0.23655913978494625, + "Banjarese": 0.1736111111111111, + "Chemistry": 0.1635036496350365, + "Biology": 0.17159763313609466, + "Science": 0.24045407636738905, + "Christian religion": 0.22885572139303484, + "Art": 0.22129783693843594, + "Islam religion": 0.22048364153627312, + "Hindu religion": 0.21333333333333335, + "Madurese": 0.22033898305084745, + "Sport": 0.19594594594594594, + "Indonesian language": 0.24097135740971357, + "Physics": 0.20202020202020202, + "Minangkabau culture": 0.24623115577889448, + "Dayak language": 0.28440366972477066, + "Sociology": 0.1774193548387097, + "Economy": 0.19672131147540983, + "Sundanese": 0.24546240276577355, + "Javanese": 0.2560483870967742, + "Civic education": 0.23032904148783978 } }, "prompt_2": { - "accuracy": 0.4250617531210361, + "accuracy": 0.22077575272047534, "category_acc": { - "History": 0.3714859437751004, - "Geography": 0.3816326530612245, - "Lampungic": 0.3877551020408163, - "Social science": 0.67779632721202, - "Balinese": 0.29723991507430997, - "Makassarese": 0.34946236559139787, - "Banjarese": 0.3819444444444444, - "Chemistry": 0.25985401459854013, - "Biology": 0.34911242603550297, - "Science": 0.5593395252837977, - "Christian religion": 0.4975124378109453, - "Art": 0.4908485856905158, - "Islam religion": 0.49359886201991465, - "Hindu religion": 0.46, - "Madurese": 0.29152542372881357, - "Sport": 0.38513513513513514, - "Indonesian language": 0.488480697384807, - "Physics": 0.40606060606060607, - "Minangkabau culture": 0.32160804020100503, - "Dayak language": 0.27522935779816515, - "Sociology": 0.3951612903225806, - "Economy": 0.36885245901639346, - "Sundanese": 0.34572169403630076, - "Javanese": 0.32056451612903225, - "Civic education": 0.49356223175965663 + "History": 0.15060240963855423, + "Geography": 0.2, + "Lampungic": 0.2789115646258503, + "Social science": 0.2637729549248748, + "Balinese": 0.2611464968152866, + "Makassarese": 0.21505376344086022, + "Banjarese": 0.1736111111111111, + "Chemistry": 0.1562043795620438, + "Biology": 0.17159763313609466, + "Science": 0.23839009287925697, + "Christian religion": 0.22885572139303484, + "Art": 0.22462562396006655, + "Islam religion": 0.21906116642958748, + "Hindu religion": 0.2, + "Madurese": 0.2305084745762712, + "Sport": 0.1891891891891892, + "Indonesian language": 0.2387920298879203, + "Physics": 0.19797979797979798, + "Minangkabau culture": 0.23618090452261306, + "Dayak language": 0.24770642201834864, + "Sociology": 0.18346774193548387, + "Economy": 0.17827868852459017, + "Sundanese": 0.23768366464995677, + "Javanese": 0.2560483870967742, + "Civic education": 0.2246065808297568 } }, "prompt_3": { - "accuracy": 0.41778489885840175, + "accuracy": 0.22104279324387477, "category_acc": { - "History": 0.3654618473895582, - "Geography": 0.35714285714285715, - "Lampungic": 0.3741496598639456, - "Social science": 0.679465776293823, - "Balinese": 0.27388535031847133, - "Makassarese": 0.3655913978494624, - "Banjarese": 0.3194444444444444, - "Chemistry": 0.27153284671532846, - "Biology": 0.35384615384615387, - "Science": 0.5851393188854489, - "Christian religion": 0.48756218905472637, - "Art": 0.46921797004991683, - "Islam religion": 0.5021337126600285, - "Hindu religion": 0.38, - "Madurese": 0.2576271186440678, - "Sport": 0.41216216216216217, - "Indonesian language": 0.476027397260274, - "Physics": 0.40606060606060607, - "Minangkabau culture": 0.33668341708542715, - "Dayak language": 0.29357798165137616, - "Sociology": 0.3588709677419355, - "Economy": 0.35450819672131145, - "Sundanese": 0.36041486603284356, - "Javanese": 0.2963709677419355, - "Civic education": 0.4663805436337625 + "History": 0.15461847389558234, + "Geography": 0.19795918367346937, + "Lampungic": 0.24489795918367346, + "Social science": 0.2587646076794658, + "Balinese": 0.2611464968152866, + "Makassarese": 0.23118279569892472, + "Banjarese": 0.16666666666666666, + "Chemistry": 0.1635036496350365, + "Biology": 0.17514792899408285, + "Science": 0.22910216718266255, + "Christian religion": 0.23880597014925373, + "Art": 0.2113144758735441, + "Islam religion": 0.22617354196301565, + "Hindu religion": 0.2, + "Madurese": 0.20677966101694914, + "Sport": 0.20945945945945946, + "Indonesian language": 0.23692403486924035, + "Physics": 0.19595959595959597, + "Minangkabau culture": 0.23618090452261306, + "Dayak language": 0.24770642201834864, + "Sociology": 0.18346774193548387, + "Economy": 0.19467213114754098, + "Sundanese": 0.2428694900605013, + "Javanese": 0.2620967741935484, + "Civic education": 0.22746781115879827 } }, "prompt_4": { - "accuracy": 0.4221243073636424, + "accuracy": 0.22077575272047534, "category_acc": { - "History": 0.35542168674698793, - "Geography": 0.3816326530612245, - "Lampungic": 0.3197278911564626, - "Social science": 0.6978297161936561, - "Balinese": 0.28874734607218683, - "Makassarese": 0.3333333333333333, - "Banjarese": 0.3472222222222222, - "Chemistry": 0.25985401459854013, - "Biology": 0.357396449704142, - "Science": 0.5789473684210527, - "Christian religion": 0.48258706467661694, - "Art": 0.47920133111480867, - "Islam religion": 0.5092460881934566, - "Hindu religion": 0.4266666666666667, - "Madurese": 0.3016949152542373, - "Sport": 0.42567567567567566, - "Indonesian language": 0.4828767123287671, - "Physics": 0.4202020202020202, - "Minangkabau culture": 0.3165829145728643, - "Dayak language": 0.30275229357798167, - "Sociology": 0.3649193548387097, - "Economy": 0.3668032786885246, - "Sundanese": 0.334485738980121, - "Javanese": 0.31048387096774194, - "Civic education": 0.48068669527896996 + "History": 0.15060240963855423, + "Geography": 0.18775510204081633, + "Lampungic": 0.23129251700680273, + "Social science": 0.2587646076794658, + "Balinese": 0.25690021231422505, + "Makassarese": 0.22580645161290322, + "Banjarese": 0.1736111111111111, + "Chemistry": 0.1635036496350365, + "Biology": 0.17514792899408285, + "Science": 0.2301341589267286, + "Christian religion": 0.23383084577114427, + "Art": 0.2146422628951747, + "Islam religion": 0.2233285917496444, + "Hindu religion": 0.20666666666666667, + "Madurese": 0.2135593220338983, + "Sport": 0.1891891891891892, + "Indonesian language": 0.2425280199252802, + "Physics": 0.20202020202020202, + "Minangkabau culture": 0.22613065326633167, + "Dayak language": 0.24770642201834864, + "Sociology": 0.18346774193548387, + "Economy": 0.19262295081967212, + "Sundanese": 0.24546240276577355, + "Javanese": 0.2550403225806452, + "Civic education": 0.21745350500715308 } }, "prompt_5": { - "accuracy": 0.4134454903531611, + "accuracy": 0.22731824554376126, "category_acc": { - "History": 0.35542168674698793, - "Geography": 0.3510204081632653, - "Lampungic": 0.35374149659863946, - "Social science": 0.67779632721202, - "Balinese": 0.27176220806794055, - "Makassarese": 0.3172043010752688, - "Banjarese": 0.3541666666666667, - "Chemistry": 0.25255474452554744, - "Biology": 0.3349112426035503, - "Science": 0.5593395252837977, - "Christian religion": 0.47761194029850745, - "Art": 0.4875207986688852, - "Islam religion": 0.4950213371266003, - "Hindu religion": 0.44666666666666666, - "Madurese": 0.31186440677966104, - "Sport": 0.3716216216216216, - "Indonesian language": 0.4735367372353674, - "Physics": 0.4101010101010101, - "Minangkabau culture": 0.32160804020100503, - "Dayak language": 0.3119266055045872, - "Sociology": 0.3528225806451613, - "Economy": 0.35860655737704916, - "Sundanese": 0.3353500432152118, - "Javanese": 0.3094758064516129, - "Civic education": 0.474964234620887 + "History": 0.16265060240963855, + "Geography": 0.19387755102040816, + "Lampungic": 0.2789115646258503, + "Social science": 0.2570951585976628, + "Balinese": 0.2760084925690021, + "Makassarese": 0.22580645161290322, + "Banjarese": 0.18055555555555555, + "Chemistry": 0.15766423357664233, + "Biology": 0.178698224852071, + "Science": 0.23529411764705882, + "Christian religion": 0.23880597014925373, + "Art": 0.22129783693843594, + "Islam religion": 0.2233285917496444, + "Hindu religion": 0.17333333333333334, + "Madurese": 0.2576271186440678, + "Sport": 0.21621621621621623, + "Indonesian language": 0.24937733499377335, + "Physics": 0.20202020202020202, + "Minangkabau culture": 0.2613065326633166, + "Dayak language": 0.29357798165137616, + "Sociology": 0.19153225806451613, + "Economy": 0.19057377049180327, + "Sundanese": 0.2428694900605013, + "Javanese": 0.2550403225806452, + "Civic education": 0.24320457796852646 } } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.33408705348595125 + "bleu_score": 0.03374471826935659 }, "prompt_2": { - "bleu_score": 0.3501301067031636 + "bleu_score": 0.03354584784772205 }, "prompt_3": { - "bleu_score": 0.3462657581574131 + "bleu_score": 0.030709309536872887 }, "prompt_4": { - "bleu_score": 0.35113264082672785 + "bleu_score": 0.033406005624105214 }, "prompt_5": { - "bleu_score": 0.3390807182637613 + "bleu_score": 0.03408654681827874 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.28059782107454206 + "bleu_score": 0.023547771145113035 }, "prompt_2": { - "bleu_score": 0.29249653516843094 + "bleu_score": 0.023013474143517277 }, "prompt_3": { - "bleu_score": 0.2902003571177129 + "bleu_score": 0.020874466049663108 }, "prompt_4": { - "bleu_score": 0.2935942632940225 + "bleu_score": 0.02411759361984998 }, "prompt_5": { - "bleu_score": 0.2810281660242982 + "bleu_score": 0.02376678283625278 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.2090525103556877 + "bleu_score": 0.0061225167958979445 }, "prompt_2": { - "bleu_score": 0.21264359157323018 + "bleu_score": 0.008007806300952039 }, "prompt_3": { - "bleu_score": 0.20987595870104309 + "bleu_score": 0.008798554334606034 }, "prompt_4": { - "bleu_score": 0.2152698819273367 + "bleu_score": 0.008702923563946885 }, "prompt_5": { - "bleu_score": 0.20769235239001288 + "bleu_score": 0.00887049961254504 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.3550885327726007 + "bleu_score": 0.028766122560955086 }, "prompt_2": { - "bleu_score": 0.36067476426396256 + "bleu_score": 0.03126042463241794 }, "prompt_3": { - "bleu_score": 0.3595949002023275 + "bleu_score": 0.030200207989540616 }, "prompt_4": { - "bleu_score": 0.3591822199501052 + "bleu_score": 0.03010036911944672 }, "prompt_5": { - "bleu_score": 0.35117255717346696 + "bleu_score": 0.03042639392728816 } }, "mmlu": { "prompt_1": { - "accuracy": 0.6184364060676779 + "accuracy": 0.29638273045507585 }, "prompt_2": { - "accuracy": 0.6184364060676779 + "accuracy": 0.2928821470245041 }, "prompt_3": { - "accuracy": 0.617269544924154 + "accuracy": 0.2870478413068845 }, "prompt_4": { - "accuracy": 0.6219369894982497 + "accuracy": 0.294049008168028 }, "prompt_5": { - "accuracy": 0.632438739789965 + "accuracy": 0.2847141190198366 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.601430103682517, + "accuracy": 0.2889524490525563, "category_acc": { - "high_school_european_history": 0.7378048780487805, - "business_ethics": 0.5757575757575758, - "clinical_knowledge": 0.6742424242424242, - "medical_genetics": 0.6464646464646465, - "high_school_us_history": 0.8078817733990148, - "high_school_physics": 0.34, - "high_school_world_history": 0.7966101694915254, - "virology": 0.4666666666666667, - "high_school_microeconomics": 0.6497890295358649, - "econometrics": 0.415929203539823, - "college_computer_science": 0.5151515151515151, - "high_school_biology": 0.7508090614886731, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.4377224199288256, - "philosophy": 0.6419354838709678, - "professional_medicine": 0.6457564575645757, - "nutrition": 0.6655737704918033, - "global_facts": 0.30303030303030304, - "machine_learning": 0.36936936936936937, - "security_studies": 0.7008196721311475, - "public_relations": 0.6146788990825688, - "professional_psychology": 0.6235679214402619, - "prehistory": 0.6811145510835913, - "anatomy": 0.5671641791044776, - "human_sexuality": 0.6538461538461539, - "college_medicine": 0.6337209302325582, - "high_school_government_and_politics": 0.8645833333333334, - "college_chemistry": 0.46464646464646464, - "logical_fallacies": 0.7160493827160493, - "high_school_geography": 0.7969543147208121, - "elementary_mathematics": 0.6153846153846154, - "human_aging": 0.6531531531531531, - "college_mathematics": 0.32323232323232326, - "high_school_psychology": 0.8125, - "formal_logic": 0.408, - "high_school_statistics": 0.4604651162790698, - "international_law": 0.7833333333333333, + "high_school_european_history": 0.3231707317073171, + "business_ethics": 0.32323232323232326, + "clinical_knowledge": 0.30303030303030304, + "medical_genetics": 0.1919191919191919, + "high_school_us_history": 0.3399014778325123, + "high_school_physics": 0.29333333333333333, + "high_school_world_history": 0.3220338983050847, + "virology": 0.26666666666666666, + "high_school_microeconomics": 0.3291139240506329, + "econometrics": 0.24778761061946902, + "college_computer_science": 0.25252525252525254, + "high_school_biology": 0.2977346278317152, + "abstract_algebra": 0.2828282828282828, + "professional_accounting": 0.26334519572953735, + "philosophy": 0.3, + "professional_medicine": 0.36531365313653136, + "nutrition": 0.3442622950819672, + "global_facts": 0.21212121212121213, + "machine_learning": 0.17117117117117117, + "security_studies": 0.22950819672131148, + "public_relations": 0.3302752293577982, + "professional_psychology": 0.2684124386252046, + "prehistory": 0.28792569659442724, + "anatomy": 0.2462686567164179, + "human_sexuality": 0.3384615384615385, + "college_medicine": 0.26744186046511625, + "high_school_government_and_politics": 0.328125, + "college_chemistry": 0.29292929292929293, + "logical_fallacies": 0.345679012345679, + "high_school_geography": 0.27411167512690354, + "elementary_mathematics": 0.26790450928381965, + "human_aging": 0.25225225225225223, + "college_mathematics": 0.23232323232323232, + "high_school_psychology": 0.3180147058823529, + "formal_logic": 0.32, + "high_school_statistics": 0.25116279069767444, + "international_law": 0.2916666666666667, "high_school_mathematics": 0.2527881040892193, - "high_school_computer_science": 0.7171717171717171, - "conceptual_physics": 0.5213675213675214, - "miscellaneous": 0.8120204603580563, - "high_school_chemistry": 0.4158415841584158, - "marketing": 0.8369098712446352, - "professional_law": 0.4429223744292237, - "management": 0.7745098039215687, - "college_physics": 0.4158415841584158, - "jurisprudence": 0.7383177570093458, - "world_religions": 0.8352941176470589, - "sociology": 0.815, - "us_foreign_policy": 0.8383838383838383, - "high_school_macroeconomics": 0.6118251928020566, - "computer_security": 0.6565656565656566, - "moral_scenarios": 0.3187919463087248, - "moral_disputes": 0.6695652173913044, - "electrical_engineering": 0.5347222222222222, - "astronomy": 0.6423841059602649, - "college_biology": 0.7552447552447552 + "high_school_computer_science": 0.26262626262626265, + "conceptual_physics": 0.29914529914529914, + "miscellaneous": 0.27365728900255754, + "high_school_chemistry": 0.23267326732673269, + "marketing": 0.41630901287553645, + "professional_law": 0.30071754729288974, + "management": 0.3627450980392157, + "college_physics": 0.19801980198019803, + "jurisprudence": 0.29906542056074764, + "world_religions": 0.23529411764705882, + "sociology": 0.325, + "us_foreign_policy": 0.35353535353535354, + "high_school_macroeconomics": 0.3033419023136247, + "computer_security": 0.21212121212121213, + "moral_scenarios": 0.24608501118568232, + "moral_disputes": 0.2898550724637681, + "electrical_engineering": 0.2986111111111111, + "astronomy": 0.3443708609271523, + "college_biology": 0.27972027972027974 } }, "prompt_2": { - "accuracy": 0.6, + "accuracy": 0.2925277082588488, "category_acc": { - "high_school_european_history": 0.7378048780487805, - "business_ethics": 0.5858585858585859, - "clinical_knowledge": 0.678030303030303, - "medical_genetics": 0.6262626262626263, - "high_school_us_history": 0.8226600985221675, - "high_school_physics": 0.3, - "high_school_world_history": 0.809322033898305, - "virology": 0.4666666666666667, - "high_school_microeconomics": 0.6497890295358649, - "econometrics": 0.40707964601769914, - "college_computer_science": 0.47474747474747475, - "high_school_biology": 0.7605177993527508, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.4092526690391459, - "philosophy": 0.6548387096774193, - "professional_medicine": 0.5977859778597786, - "nutrition": 0.6459016393442623, - "global_facts": 0.30303030303030304, - "machine_learning": 0.4144144144144144, - "security_studies": 0.680327868852459, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.6121112929623568, - "prehistory": 0.6934984520123839, - "anatomy": 0.5597014925373134, - "human_sexuality": 0.6615384615384615, - "college_medicine": 0.622093023255814, - "high_school_government_and_politics": 0.8541666666666666, - "college_chemistry": 0.5050505050505051, - "logical_fallacies": 0.7345679012345679, - "high_school_geography": 0.7918781725888325, - "elementary_mathematics": 0.5755968169761273, - "human_aging": 0.6531531531531531, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.8180147058823529, - "formal_logic": 0.448, - "high_school_statistics": 0.4372093023255814, - "international_law": 0.7916666666666666, - "high_school_mathematics": 0.30111524163568776, - "high_school_computer_science": 0.6666666666666666, - "conceptual_physics": 0.5384615384615384, - "miscellaneous": 0.8056265984654731, - "high_school_chemistry": 0.400990099009901, - "marketing": 0.8454935622317596, - "professional_law": 0.43966079582517936, - "management": 0.7941176470588235, - "college_physics": 0.37623762376237624, - "jurisprudence": 0.7383177570093458, - "world_religions": 0.7941176470588235, - "sociology": 0.815, - "us_foreign_policy": 0.8181818181818182, - "high_school_macroeconomics": 0.6401028277634961, - "computer_security": 0.696969696969697, - "moral_scenarios": 0.3434004474272931, - "moral_disputes": 0.6579710144927536, - "electrical_engineering": 0.5625, - "astronomy": 0.6357615894039735, - "college_biology": 0.7132867132867133 + "high_school_european_history": 0.3353658536585366, + "business_ethics": 0.31313131313131315, + "clinical_knowledge": 0.32196969696969696, + "medical_genetics": 0.1919191919191919, + "high_school_us_history": 0.3399014778325123, + "high_school_physics": 0.2866666666666667, + "high_school_world_history": 0.3305084745762712, + "virology": 0.2606060606060606, + "high_school_microeconomics": 0.2742616033755274, + "econometrics": 0.22123893805309736, + "college_computer_science": 0.2828282828282828, + "high_school_biology": 0.2783171521035599, + "abstract_algebra": 0.2222222222222222, + "professional_accounting": 0.25622775800711745, + "philosophy": 0.3225806451612903, + "professional_medicine": 0.33579335793357934, + "nutrition": 0.32131147540983607, + "global_facts": 0.29292929292929293, + "machine_learning": 0.27927927927927926, + "security_studies": 0.22950819672131148, + "public_relations": 0.3119266055045872, + "professional_psychology": 0.2765957446808511, + "prehistory": 0.30340557275541796, + "anatomy": 0.22388059701492538, + "human_sexuality": 0.3076923076923077, + "college_medicine": 0.3023255813953488, + "high_school_government_and_politics": 0.3072916666666667, + "college_chemistry": 0.24242424242424243, + "logical_fallacies": 0.3333333333333333, + "high_school_geography": 0.28426395939086296, + "elementary_mathematics": 0.23342175066312998, + "human_aging": 0.3108108108108108, + "college_mathematics": 0.29292929292929293, + "high_school_psychology": 0.3235294117647059, + "formal_logic": 0.288, + "high_school_statistics": 0.24651162790697675, + "international_law": 0.3333333333333333, + "high_school_mathematics": 0.2788104089219331, + "high_school_computer_science": 0.32323232323232326, + "conceptual_physics": 0.2948717948717949, + "miscellaneous": 0.30818414322250637, + "high_school_chemistry": 0.22772277227722773, + "marketing": 0.44206008583690987, + "professional_law": 0.29549902152641877, + "management": 0.3235294117647059, + "college_physics": 0.2079207920792079, + "jurisprudence": 0.32710280373831774, + "world_religions": 0.2529411764705882, + "sociology": 0.355, + "us_foreign_policy": 0.31313131313131315, + "high_school_macroeconomics": 0.29562982005141386, + "computer_security": 0.31313131313131315, + "moral_scenarios": 0.24720357941834453, + "moral_disputes": 0.3217391304347826, + "electrical_engineering": 0.3194444444444444, + "astronomy": 0.2847682119205298, + "college_biology": 0.26573426573426573 } }, "prompt_3": { - "accuracy": 0.6035037540221666, + "accuracy": 0.285305684662138, "category_acc": { - "high_school_european_history": 0.7134146341463414, - "business_ethics": 0.5555555555555556, - "clinical_knowledge": 0.6818181818181818, - "medical_genetics": 0.6363636363636364, - "high_school_us_history": 0.7980295566502463, - "high_school_physics": 0.3933333333333333, - "high_school_world_history": 0.8050847457627118, - "virology": 0.48484848484848486, - "high_school_microeconomics": 0.6540084388185654, - "econometrics": 0.415929203539823, - "college_computer_science": 0.46464646464646464, - "high_school_biology": 0.7508090614886731, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.4412811387900356, - "philosophy": 0.6709677419354839, - "professional_medicine": 0.6273062730627307, - "nutrition": 0.6721311475409836, - "global_facts": 0.3939393939393939, - "machine_learning": 0.43243243243243246, - "security_studies": 0.7254098360655737, - "public_relations": 0.6055045871559633, - "professional_psychology": 0.6202945990180033, - "prehistory": 0.6842105263157895, - "anatomy": 0.5373134328358209, - "human_sexuality": 0.6615384615384615, - "college_medicine": 0.6162790697674418, - "high_school_government_and_politics": 0.8541666666666666, - "college_chemistry": 0.5151515151515151, - "logical_fallacies": 0.7160493827160493, - "high_school_geography": 0.7817258883248731, - "elementary_mathematics": 0.5862068965517241, - "human_aging": 0.6441441441441441, - "college_mathematics": 0.3838383838383838, - "high_school_psychology": 0.8161764705882353, - "formal_logic": 0.408, - "high_school_statistics": 0.4558139534883721, - "international_law": 0.7916666666666666, - "high_school_mathematics": 0.24907063197026022, - "high_school_computer_science": 0.6767676767676768, - "conceptual_physics": 0.5256410256410257, - "miscellaneous": 0.8158567774936062, - "high_school_chemistry": 0.4405940594059406, - "marketing": 0.8369098712446352, - "professional_law": 0.4390084801043705, - "management": 0.7941176470588235, - "college_physics": 0.3564356435643564, - "jurisprudence": 0.7757009345794392, - "world_religions": 0.8176470588235294, - "sociology": 0.805, - "us_foreign_policy": 0.8484848484848485, - "high_school_macroeconomics": 0.6323907455012854, - "computer_security": 0.6666666666666666, - "moral_scenarios": 0.3277404921700224, - "moral_disputes": 0.663768115942029, - "electrical_engineering": 0.5347222222222222, - "astronomy": 0.6291390728476821, - "college_biology": 0.7272727272727273 + "high_school_european_history": 0.3231707317073171, + "business_ethics": 0.2727272727272727, + "clinical_knowledge": 0.29924242424242425, + "medical_genetics": 0.16161616161616163, + "high_school_us_history": 0.33497536945812806, + "high_school_physics": 0.29333333333333333, + "high_school_world_history": 0.3347457627118644, + "virology": 0.3090909090909091, + "high_school_microeconomics": 0.3037974683544304, + "econometrics": 0.23008849557522124, + "college_computer_science": 0.24242424242424243, + "high_school_biology": 0.3042071197411003, + "abstract_algebra": 0.21212121212121213, + "professional_accounting": 0.23487544483985764, + "philosophy": 0.27741935483870966, + "professional_medicine": 0.35793357933579334, + "nutrition": 0.3016393442622951, + "global_facts": 0.2222222222222222, + "machine_learning": 0.27927927927927926, + "security_studies": 0.2540983606557377, + "public_relations": 0.3119266055045872, + "professional_psychology": 0.2602291325695581, + "prehistory": 0.2848297213622291, + "anatomy": 0.21641791044776118, + "human_sexuality": 0.27692307692307694, + "college_medicine": 0.3023255813953488, + "high_school_government_and_politics": 0.3541666666666667, + "college_chemistry": 0.26262626262626265, + "logical_fallacies": 0.3333333333333333, + "high_school_geography": 0.28426395939086296, + "elementary_mathematics": 0.22015915119363394, + "human_aging": 0.26576576576576577, + "college_mathematics": 0.2727272727272727, + "high_school_psychology": 0.3382352941176471, + "formal_logic": 0.272, + "high_school_statistics": 0.20465116279069767, + "international_law": 0.25, + "high_school_mathematics": 0.22304832713754646, + "high_school_computer_science": 0.2828282828282828, + "conceptual_physics": 0.29914529914529914, + "miscellaneous": 0.2749360613810742, + "high_school_chemistry": 0.21287128712871287, + "marketing": 0.43776824034334766, + "professional_law": 0.28701891715590344, + "management": 0.35294117647058826, + "college_physics": 0.19801980198019803, + "jurisprudence": 0.32710280373831774, + "world_religions": 0.2411764705882353, + "sociology": 0.355, + "us_foreign_policy": 0.30303030303030304, + "high_school_macroeconomics": 0.30848329048843187, + "computer_security": 0.24242424242424243, + "moral_scenarios": 0.2505592841163311, + "moral_disputes": 0.32463768115942027, + "electrical_engineering": 0.3263888888888889, + "astronomy": 0.36423841059602646, + "college_biology": 0.27972027972027974 } }, "prompt_4": { - "accuracy": 0.6063639613872006, - "category_acc": { - "high_school_european_history": 0.7317073170731707, - "business_ethics": 0.5858585858585859, - "clinical_knowledge": 0.678030303030303, - "medical_genetics": 0.6060606060606061, - "high_school_us_history": 0.7980295566502463, - "high_school_physics": 0.38666666666666666, - "high_school_world_history": 0.7923728813559322, - "virology": 0.4666666666666667, - "high_school_microeconomics": 0.6666666666666666, - "econometrics": 0.40707964601769914, - "college_computer_science": 0.5454545454545454, - "high_school_biology": 0.7605177993527508, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.4306049822064057, - "philosophy": 0.6645161290322581, - "professional_medicine": 0.6162361623616236, - "nutrition": 0.6655737704918033, - "global_facts": 0.37373737373737376, - "machine_learning": 0.4144144144144144, - "security_studies": 0.7090163934426229, - "public_relations": 0.6238532110091743, - "professional_psychology": 0.6252045826513911, - "prehistory": 0.6996904024767802, - "anatomy": 0.5671641791044776, - "human_sexuality": 0.7, - "college_medicine": 0.622093023255814, - "high_school_government_and_politics": 0.8541666666666666, - "college_chemistry": 0.47474747474747475, - "logical_fallacies": 0.7098765432098766, - "high_school_geography": 0.7868020304568528, - "elementary_mathematics": 0.5888594164456233, - "human_aging": 0.6531531531531531, - "college_mathematics": 0.35353535353535354, - "high_school_psychology": 0.8198529411764706, - "formal_logic": 0.376, - "high_school_statistics": 0.4697674418604651, - "international_law": 0.7833333333333333, - "high_school_mathematics": 0.30111524163568776, - "high_school_computer_science": 0.696969696969697, - "conceptual_physics": 0.5256410256410257, - "miscellaneous": 0.8145780051150895, - "high_school_chemistry": 0.4306930693069307, - "marketing": 0.8540772532188842, - "professional_law": 0.4455316373124592, - "management": 0.7745098039215687, - "college_physics": 0.3069306930693069, - "jurisprudence": 0.7289719626168224, - "world_religions": 0.8058823529411765, - "sociology": 0.82, - "us_foreign_policy": 0.8181818181818182, - "high_school_macroeconomics": 0.6580976863753213, - "computer_security": 0.6767676767676768, - "moral_scenarios": 0.3277404921700224, - "moral_disputes": 0.6869565217391305, - "electrical_engineering": 0.5416666666666666, - "astronomy": 0.6423841059602649, - "college_biology": 0.7412587412587412 - } - }, - "prompt_5": { - "accuracy": 0.5982123703968538, + "accuracy": 0.2910260993922059, "category_acc": { - "high_school_european_history": 0.6890243902439024, - "business_ethics": 0.5555555555555556, - "clinical_knowledge": 0.6363636363636364, - "medical_genetics": 0.6161616161616161, - "high_school_us_history": 0.7733990147783252, - "high_school_physics": 0.36666666666666664, - "high_school_world_history": 0.8050847457627118, - "virology": 0.4909090909090909, - "high_school_microeconomics": 0.6624472573839663, - "econometrics": 0.37168141592920356, - "college_computer_science": 0.5353535353535354, - "high_school_biology": 0.7572815533980582, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.41637010676156583, - "philosophy": 0.6516129032258065, - "professional_medicine": 0.5830258302583026, - "nutrition": 0.6557377049180327, - "global_facts": 0.43434343434343436, - "machine_learning": 0.38738738738738737, - "security_studies": 0.7172131147540983, - "public_relations": 0.6055045871559633, - "professional_psychology": 0.6284779050736498, - "prehistory": 0.6934984520123839, - "anatomy": 0.5298507462686567, - "human_sexuality": 0.6692307692307692, - "college_medicine": 0.6104651162790697, - "high_school_government_and_politics": 0.8541666666666666, - "college_chemistry": 0.4444444444444444, - "logical_fallacies": 0.7283950617283951, - "high_school_geography": 0.7868020304568528, - "elementary_mathematics": 0.5676392572944297, - "human_aging": 0.6576576576576577, - "college_mathematics": 0.35353535353535354, - "high_school_psychology": 0.8180147058823529, - "formal_logic": 0.36, - "high_school_statistics": 0.44651162790697674, - "international_law": 0.7833333333333333, - "high_school_mathematics": 0.24535315985130113, - "high_school_computer_science": 0.6464646464646465, - "conceptual_physics": 0.5341880341880342, - "miscellaneous": 0.8158567774936062, - "high_school_chemistry": 0.47029702970297027, - "marketing": 0.8369098712446352, - "professional_law": 0.4383561643835616, - "management": 0.7745098039215687, - "college_physics": 0.36633663366336633, - "jurisprudence": 0.7102803738317757, - "world_religions": 0.8294117647058824, - "sociology": 0.82, - "us_foreign_policy": 0.8383838383838383, - "high_school_macroeconomics": 0.6298200514138818, - "computer_security": 0.6767676767676768, - "moral_scenarios": 0.3232662192393736, - "moral_disputes": 0.6521739130434783, - "electrical_engineering": 0.5555555555555556, - "astronomy": 0.6622516556291391, - "college_biology": 0.6923076923076923 + "high_school_european_history": 0.34146341463414637, + "business_ethics": 0.29292929292929293, + "clinical_knowledge": 0.2765151515151515, + "medical_genetics": 0.1919191919191919, + "high_school_us_history": 0.3694581280788177, + "high_school_physics": 0.26, + "high_school_world_history": 0.3050847457627119, + "virology": 0.26666666666666666, + "high_school_microeconomics": 0.28270042194092826, + "econometrics": 0.26548672566371684, + "college_computer_science": 0.2727272727272727, + "high_school_biology": 0.2977346278317152, + "abstract_algebra": 0.24242424242424243, + "professional_accounting": 0.24555160142348753, + "philosophy": 0.2903225806451613, + "professional_medicine": 0.35793357933579334, + "nutrition": 0.32131147540983607, + "global_facts": 0.20202020202020202, + "machine_learning": 0.2882882882882883, + "security_studies": 0.29098360655737704, + "public_relations": 0.3211009174311927, + "professional_psychology": 0.28805237315875615, + "prehistory": 0.29102167182662536, + "anatomy": 0.2835820895522388, + "human_sexuality": 0.3230769230769231, + "college_medicine": 0.27906976744186046, + "high_school_government_and_politics": 0.3072916666666667, + "college_chemistry": 0.24242424242424243, + "logical_fallacies": 0.35802469135802467, + "high_school_geography": 0.28426395939086296, + "elementary_mathematics": 0.27055702917771884, + "human_aging": 0.2702702702702703, + "college_mathematics": 0.2727272727272727, + "high_school_psychology": 0.3161764705882353, + "formal_logic": 0.24, + "high_school_statistics": 0.26976744186046514, + "international_law": 0.275, + "high_school_mathematics": 0.22676579925650558, + "high_school_computer_science": 0.29292929292929293, + "conceptual_physics": 0.29914529914529914, + "miscellaneous": 0.2979539641943734, + "high_school_chemistry": 0.22772277227722773, + "marketing": 0.39914163090128757, + "professional_law": 0.29158512720156554, + "management": 0.3431372549019608, + "college_physics": 0.2079207920792079, + "jurisprudence": 0.34579439252336447, + "world_religions": 0.25882352941176473, + "sociology": 0.345, + "us_foreign_policy": 0.30303030303030304, + "high_school_macroeconomics": 0.29562982005141386, + "computer_security": 0.29292929292929293, + "moral_scenarios": 0.24608501118568232, + "moral_disputes": 0.3159420289855073, + "electrical_engineering": 0.3333333333333333, + "astronomy": 0.3576158940397351, + "college_biology": 0.3076923076923077 + } + }, + "prompt_5": { + "accuracy": 0.29789059706828747, + "category_acc": { + "high_school_european_history": 0.3902439024390244, + "business_ethics": 0.2828282828282828, + "clinical_knowledge": 0.3143939393939394, + "medical_genetics": 0.25252525252525254, + "high_school_us_history": 0.35960591133004927, + "high_school_physics": 0.22666666666666666, + "high_school_world_history": 0.3093220338983051, + "virology": 0.3090909090909091, + "high_school_microeconomics": 0.2911392405063291, + "econometrics": 0.22123893805309736, + "college_computer_science": 0.25252525252525254, + "high_school_biology": 0.32038834951456313, + "abstract_algebra": 0.2222222222222222, + "professional_accounting": 0.29537366548042704, + "philosophy": 0.3193548387096774, + "professional_medicine": 0.3025830258302583, + "nutrition": 0.33114754098360655, + "global_facts": 0.3434343434343434, + "machine_learning": 0.3333333333333333, + "security_studies": 0.2459016393442623, + "public_relations": 0.3577981651376147, + "professional_psychology": 0.2864157119476268, + "prehistory": 0.32507739938080493, + "anatomy": 0.26119402985074625, + "human_sexuality": 0.2846153846153846, + "college_medicine": 0.29069767441860467, + "high_school_government_and_politics": 0.2916666666666667, + "college_chemistry": 0.21212121212121213, + "logical_fallacies": 0.36419753086419754, + "high_school_geography": 0.3147208121827411, + "elementary_mathematics": 0.22811671087533156, + "human_aging": 0.2927927927927928, + "college_mathematics": 0.2828282828282828, + "high_school_psychology": 0.32536764705882354, + "formal_logic": 0.288, + "high_school_statistics": 0.21395348837209302, + "international_law": 0.38333333333333336, + "high_school_mathematics": 0.2527881040892193, + "high_school_computer_science": 0.29292929292929293, + "conceptual_physics": 0.33760683760683763, + "miscellaneous": 0.3171355498721228, + "high_school_chemistry": 0.26732673267326734, + "marketing": 0.44635193133047213, + "professional_law": 0.2922374429223744, + "management": 0.23529411764705882, + "college_physics": 0.1782178217821782, + "jurisprudence": 0.35514018691588783, + "world_religions": 0.27647058823529413, + "sociology": 0.36, + "us_foreign_policy": 0.36363636363636365, + "high_school_macroeconomics": 0.2930591259640103, + "computer_security": 0.31313131313131315, + "moral_scenarios": 0.24720357941834453, + "moral_disputes": 0.3130434782608696, + "electrical_engineering": 0.3333333333333333, + "astronomy": 0.33112582781456956, + "college_biology": 0.2727272727272727 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.42719167904903416 + "accuracy": 0.24591381872213966 }, "prompt_2": { - "accuracy": 0.4323922734026746 + "accuracy": 0.24591381872213966 }, "prompt_3": { - "accuracy": 0.45170876671619614 + "accuracy": 0.24591381872213966 }, "prompt_4": { - "accuracy": 0.42867756315007427 + "accuracy": 0.24294205052005943 }, "prompt_5": { - "accuracy": 0.4413075780089153 + "accuracy": 0.24591381872213966 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.44084682440846823, + "accuracy": 0.25840597758405975, "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.5, - "college_programming": 0.6190476190476191, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.125, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.47619047619047616, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.3333333333333333, + "computer_network": 0.25, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.23809523809523808, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.20689655172413793, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.20833333333333334, + "high_school_chemistry": 0.375, "high_school_biology": 0.25, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.7307692307692307, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.68, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.45, - "business_administration": 0.5, - "marxism": 0.4583333333333333, - "mao_zedong_thought": 0.5517241379310345, - "education_science": 0.5, - "teacher_qualification": 0.6938775510204082, - "high_school_politics": 0.4583333333333333, - "high_school_geography": 0.5, - "middle_school_politics": 0.7307692307692307, - "middle_school_geography": 0.5294117647058824, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.6666666666666666, - "logic": 0.4074074074074074, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.56, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.40384615384615385, - "sports_science": 0.4166666666666667, - "plant_protection": 0.6296296296296297, - "basic_medicine": 0.5, - "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.49019607843137253, - "accountant": 0.4074074074074074, + "middle_school_mathematics": 0.3333333333333333, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.17857142857142858, + "college_economics": 0.2, + "business_administration": 0.13157894736842105, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.20408163265306123, + "high_school_politics": 0.16666666666666666, + "high_school_geography": 0.20833333333333334, + "middle_school_politics": 0.3076923076923077, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.37037037037037035, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.08, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.23076923076923078, + "sports_science": 0.2916666666666667, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.375, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.27450980392156865, + "accountant": 0.3333333333333333, "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.5277777777777778, - "tax_accountant": 0.3333333333333333, - "physician": 0.46296296296296297 + "environmental_impact_assessment_engineer": 0.25, + "tax_accountant": 0.2777777777777778, + "physician": 0.2962962962962963 } }, "prompt_2": { - "accuracy": 0.44209215442092153, + "accuracy": 0.2577833125778331, "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.5833333333333334, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.5714285714285714, - "college_physics": 0.5, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.2916666666666667, + "computer_network": 0.25, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.21428571428571427, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.20833333333333334, "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.5238095238095238, - "electrical_engineer": 0.5476190476190477, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.25, + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.20689655172413793, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.375, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.7692307692307693, - "middle_school_physics": 0.5416666666666666, - "middle_school_chemistry": 0.44, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.4166666666666667, - "business_administration": 0.34210526315789475, - "marxism": 0.375, - "mao_zedong_thought": 0.5517241379310345, - "education_science": 0.4411764705882353, - "teacher_qualification": 0.6938775510204082, - "high_school_politics": 0.5833333333333334, - "high_school_geography": 0.5416666666666666, - "middle_school_politics": 0.6923076923076923, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.35714285714285715, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.48148148148148145, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.4642857142857143, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.56, - "middle_school_history": 0.5925925925925926, - "civil_servant": 0.38461538461538464, - "sports_science": 0.375, - "plant_protection": 0.5185185185185185, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.5882352941176471, - "accountant": 0.4074074074074074, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.5833333333333334, - "tax_accountant": 0.37037037037037035, - "physician": 0.3888888888888889 + "high_school_biology": 0.25, + "middle_school_mathematics": 0.3333333333333333, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.17857142857142858, + "college_economics": 0.2, + "business_administration": 0.13157894736842105, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.20408163265306123, + "high_school_politics": 0.16666666666666666, + "high_school_geography": 0.20833333333333334, + "middle_school_politics": 0.3076923076923077, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.37037037037037035, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.08, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.23076923076923078, + "sports_science": 0.2916666666666667, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.375, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.27450980392156865, + "accountant": 0.3333333333333333, + "fire_engineer": 0.16666666666666666, + "environmental_impact_assessment_engineer": 0.25, + "tax_accountant": 0.2777777777777778, + "physician": 0.2962962962962963 } }, "prompt_3": { - "accuracy": 0.44209215442092153, + "accuracy": 0.25840597758405975, "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.6666666666666666, - "computer_architecture": 0.6538461538461539, - "college_programming": 0.5476190476190477, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.47619047619047616, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.5769230769230769, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.44, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.38333333333333336, - "business_administration": 0.47368421052631576, - "marxism": 0.4583333333333333, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.7142857142857143, - "high_school_politics": 0.5416666666666666, - "high_school_geography": 0.5416666666666666, - "middle_school_politics": 0.6923076923076923, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.6666666666666666, - "logic": 0.3333333333333333, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.4117647058823529, - "legal_professional": 0.25, - "high_school_chinese": 0.375, - "high_school_history": 0.6, - "middle_school_history": 0.4444444444444444, - "civil_servant": 0.4807692307692308, - "sports_science": 0.3333333333333333, - "plant_protection": 0.5925925925925926, - "basic_medicine": 0.5833333333333334, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.5294117647058824, + "computer_network": 0.25, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.23809523809523808, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.20689655172413793, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.20833333333333334, + "high_school_chemistry": 0.375, + "high_school_biology": 0.25, + "middle_school_mathematics": 0.3333333333333333, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.17857142857142858, + "college_economics": 0.2, + "business_administration": 0.13157894736842105, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.20408163265306123, + "high_school_politics": 0.16666666666666666, + "high_school_geography": 0.20833333333333334, + "middle_school_politics": 0.3076923076923077, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.37037037037037035, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.08, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.23076923076923078, + "sports_science": 0.2916666666666667, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.375, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.3333333333333333, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.5555555555555556, - "tax_accountant": 0.42592592592592593, - "physician": 0.48148148148148145 + "fire_engineer": 0.16666666666666666, + "environmental_impact_assessment_engineer": 0.25, + "tax_accountant": 0.2777777777777778, + "physician": 0.2962962962962963 } }, "prompt_4": { - "accuracy": 0.42528019925280197, + "accuracy": 0.25840597758405975, "category_acc": { - "computer_network": 0.5416666666666666, - "operating_system": 0.625, - "computer_architecture": 0.5, - "college_programming": 0.5476190476190477, - "college_physics": 0.125, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.3793103448275862, + "computer_network": 0.25, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.21428571428571427, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.20689655172413793, "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.7307692307692307, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.44, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.45, - "business_administration": 0.5, - "marxism": 0.5, - "mao_zedong_thought": 0.5172413793103449, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.6530612244897959, - "high_school_politics": 0.4583333333333333, - "high_school_geography": 0.5833333333333334, - "middle_school_politics": 0.5769230769230769, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.5, - "logic": 0.3333333333333333, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.52, - "middle_school_history": 0.5555555555555556, - "civil_servant": 0.4423076923076923, - "sports_science": 0.375, - "plant_protection": 0.5925925925925926, - "basic_medicine": 0.7083333333333334, - "clinical_medicine": 0.5185185185185185, - "urban_and_rural_planner": 0.49019607843137253, - "accountant": 0.35185185185185186, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.5277777777777778, - "tax_accountant": 0.3888888888888889, - "physician": 0.37037037037037035 + "high_school_physics": 0.25, + "high_school_chemistry": 0.375, + "high_school_biology": 0.25, + "middle_school_mathematics": 0.3333333333333333, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.17857142857142858, + "college_economics": 0.2, + "business_administration": 0.15789473684210525, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.20408163265306123, + "high_school_politics": 0.16666666666666666, + "high_school_geography": 0.20833333333333334, + "middle_school_politics": 0.3076923076923077, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.37037037037037035, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.08, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.23076923076923078, + "sports_science": 0.2916666666666667, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.375, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.27450980392156865, + "accountant": 0.3333333333333333, + "fire_engineer": 0.16666666666666666, + "environmental_impact_assessment_engineer": 0.25, + "tax_accountant": 0.2777777777777778, + "physician": 0.2962962962962963 } }, "prompt_5": { - "accuracy": 0.41718555417185554, + "accuracy": 0.25840597758405975, "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.5833333333333334, - "computer_architecture": 0.46153846153846156, - "college_programming": 0.5714285714285714, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.47619047619047616, - "metrology_engineer": 0.4827586206896552, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.375, - "high_school_chemistry": 0.25, - "high_school_biology": 0.4166666666666667, + "computer_network": 0.25, + "operating_system": 0.4166666666666667, + "computer_architecture": 0.23076923076923078, + "college_programming": 0.23809523809523808, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.20689655172413793, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.2619047619047619, + "metrology_engineer": 0.20689655172413793, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.20833333333333334, + "high_school_chemistry": 0.375, + "high_school_biology": 0.25, "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.5769230769230769, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.4, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.4, - "business_administration": 0.39473684210526316, - "marxism": 0.375, - "mao_zedong_thought": 0.5862068965517241, - "education_science": 0.5, - "teacher_qualification": 0.6938775510204082, - "high_school_politics": 0.5833333333333334, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.6538461538461539, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.35714285714285715, - "ideological_and_moral_cultivation": 0.6666666666666666, - "logic": 0.48148148148148145, - "law": 0.41379310344827586, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.4117647058823529, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.36, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.4230769230769231, - "sports_science": 0.375, - "plant_protection": 0.5925925925925926, - "basic_medicine": 0.5, - "clinical_medicine": 0.48148148148148145, - "urban_and_rural_planner": 0.37254901960784315, + "middle_school_biology": 0.34615384615384615, + "middle_school_physics": 0.2916666666666667, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.17857142857142858, + "college_economics": 0.2, + "business_administration": 0.13157894736842105, + "marxism": 0.2916666666666667, + "mao_zedong_thought": 0.1724137931034483, + "education_science": 0.23529411764705882, + "teacher_qualification": 0.20408163265306123, + "high_school_politics": 0.16666666666666666, + "high_school_geography": 0.20833333333333334, + "middle_school_politics": 0.3076923076923077, + "middle_school_geography": 0.29411764705882354, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.16666666666666666, + "logic": 0.37037037037037035, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.32142857142857145, + "art_studies": 0.18421052631578946, + "professional_tour_guide": 0.2647058823529412, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.08, + "middle_school_history": 0.3333333333333333, + "civil_servant": 0.23076923076923078, + "sports_science": 0.2916666666666667, + "plant_protection": 0.14814814814814814, + "basic_medicine": 0.375, + "clinical_medicine": 0.25925925925925924, + "urban_and_rural_planner": 0.27450980392156865, "accountant": 0.3333333333333333, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.2962962962962963, - "physician": 0.4074074074074074 + "fire_engineer": 0.16666666666666666, + "environmental_impact_assessment_engineer": 0.25, + "tax_accountant": 0.2777777777777778, + "physician": 0.2962962962962963 } } }, "cmmlu": { "prompt_1": { - "accuracy": 0.48028673835125446 + "accuracy": 0.2078853046594982 }, "prompt_2": { - "accuracy": 0.4982078853046595 + "accuracy": 0.2114695340501792 }, "prompt_3": { - "accuracy": 0.5089605734767025 + "accuracy": 0.2078853046594982 }, "prompt_4": { - "accuracy": 0.46953405017921146 + "accuracy": 0.2114695340501792 }, "prompt_5": { - "accuracy": 0.4838709677419355 + "accuracy": 0.2114695340501792 } }, "cmmlu_full": { "prompt_1": { - "accuracy": 0.4430150233120359, + "accuracy": 0.2487480573303402, "category_acc": { - "agronomy": 0.4556213017751479, - "anatomy": 0.30405405405405406, - "ancient_chinese": 0.29878048780487804, - "arts": 0.5, - "astronomy": 0.3515151515151515, - "business_ethics": 0.5119617224880383, - "chinese_civil_service_exam": 0.3875, - "chinese_driving_rule": 0.5801526717557252, - "chinese_food_culture": 0.33088235294117646, - "chinese_foreign_policy": 0.4672897196261682, - "chinese_history": 0.43962848297213625, - "chinese_literature": 0.3088235294117647, - "chinese_teacher_qualification": 0.547486033519553, - "clinical_knowledge": 0.350210970464135, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.5514018691588785, - "college_engineering_hydrology": 0.5188679245283019, - "college_law": 0.26851851851851855, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.36792452830188677, - "college_medicine": 0.3516483516483517, - "computer_science": 0.5931372549019608, - "computer_security": 0.5614035087719298, - "conceptual_physics": 0.3945578231292517, - "construction_project_management": 0.39568345323741005, - "economics": 0.5408805031446541, - "education": 0.5030674846625767, - "electrical_engineering": 0.45930232558139533, - "elementary_chinese": 0.2896825396825397, - "elementary_commonsense": 0.45454545454545453, - "elementary_information_and_technology": 0.6932773109243697, - "elementary_mathematics": 0.39565217391304347, - "ethnology": 0.4, - "food_science": 0.4825174825174825, - "genetics": 0.36363636363636365, - "global_facts": 0.5302013422818792, - "high_school_biology": 0.33136094674556216, - "high_school_chemistry": 0.3106060606060606, - "high_school_geography": 0.3728813559322034, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.39090909090909093, - "high_school_politics": 0.3706293706293706, - "human_sexuality": 0.5714285714285714, - "international_law": 0.3567567567567568, - "journalism": 0.5174418604651163, - "jurisprudence": 0.46958637469586373, - "legal_and_moral_basis": 0.7242990654205608, - "logical": 0.4146341463414634, - "machine_learning": 0.45901639344262296, - "management": 0.5904761904761905, - "marketing": 0.5833333333333334, - "marxist_theory": 0.5079365079365079, - "modern_chinese": 0.3793103448275862, - "nutrition": 0.503448275862069, - "philosophy": 0.4380952380952381, - "professional_accounting": 0.48, - "professional_law": 0.3696682464454976, - "professional_medicine": 0.29521276595744683, - "professional_psychology": 0.4698275862068966, - "public_relations": 0.4827586206896552, - "security_study": 0.5185185185185185, - "sociology": 0.5221238938053098, - "sports_science": 0.49696969696969695, - "traditional_chinese_medicine": 0.372972972972973, - "virology": 0.5029585798816568, - "world_history": 0.45962732919254656, - "world_religions": 0.525 + "agronomy": 0.23668639053254437, + "anatomy": 0.25675675675675674, + "ancient_chinese": 0.23780487804878048, + "arts": 0.25, + "astronomy": 0.2727272727272727, + "business_ethics": 0.2631578947368421, + "chinese_civil_service_exam": 0.26875, + "chinese_driving_rule": 0.21374045801526717, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.22429906542056074, + "chinese_history": 0.2476780185758514, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.2569832402234637, + "clinical_knowledge": 0.24472573839662448, + "college_actuarial_science": 0.2830188679245283, + "college_education": 0.14953271028037382, + "college_engineering_hydrology": 0.27358490566037735, + "college_law": 0.32407407407407407, + "college_mathematics": 0.2857142857142857, + "college_medical_statistics": 0.20754716981132076, + "college_medicine": 0.2271062271062271, + "computer_science": 0.24019607843137256, + "computer_security": 0.24561403508771928, + "conceptual_physics": 0.23129251700680273, + "construction_project_management": 0.2446043165467626, + "economics": 0.2578616352201258, + "education": 0.25153374233128833, + "electrical_engineering": 0.25, + "elementary_chinese": 0.20634920634920634, + "elementary_commonsense": 0.2727272727272727, + "elementary_information_and_technology": 0.226890756302521, + "elementary_mathematics": 0.24347826086956523, + "ethnology": 0.2518518518518518, + "food_science": 0.26573426573426573, + "genetics": 0.2727272727272727, + "global_facts": 0.26174496644295303, + "high_school_biology": 0.2781065088757396, + "high_school_chemistry": 0.19696969696969696, + "high_school_geography": 0.23728813559322035, + "high_school_mathematics": 0.25609756097560976, + "high_school_physics": 0.2636363636363636, + "high_school_politics": 0.24475524475524477, + "human_sexuality": 0.2698412698412698, + "international_law": 0.25405405405405407, + "journalism": 0.2558139534883721, + "jurisprudence": 0.26034063260340634, + "legal_and_moral_basis": 0.2336448598130841, + "logical": 0.23577235772357724, + "machine_learning": 0.2540983606557377, + "management": 0.24761904761904763, + "marketing": 0.2388888888888889, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.25, + "nutrition": 0.27586206896551724, + "philosophy": 0.24761904761904763, + "professional_accounting": 0.25142857142857145, + "professional_law": 0.2559241706161137, + "professional_medicine": 0.25, + "professional_psychology": 0.2543103448275862, + "public_relations": 0.23563218390804597, + "security_study": 0.2222222222222222, + "sociology": 0.25663716814159293, + "sports_science": 0.2727272727272727, + "traditional_chinese_medicine": 0.24864864864864866, + "virology": 0.23076923076923078, + "world_history": 0.2546583850931677, + "world_religions": 0.26875 } }, "prompt_2": { - "accuracy": 0.443964772923502, + "accuracy": 0.2490070799516491, "category_acc": { - "agronomy": 0.4437869822485207, - "anatomy": 0.30405405405405406, - "ancient_chinese": 0.2682926829268293, - "arts": 0.5125, - "astronomy": 0.3333333333333333, - "business_ethics": 0.5406698564593302, - "chinese_civil_service_exam": 0.325, - "chinese_driving_rule": 0.6030534351145038, - "chinese_food_culture": 0.34558823529411764, - "chinese_foreign_policy": 0.48598130841121495, - "chinese_history": 0.4458204334365325, - "chinese_literature": 0.3333333333333333, - "chinese_teacher_qualification": 0.5195530726256983, - "clinical_knowledge": 0.3459915611814346, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.5794392523364486, - "college_engineering_hydrology": 0.5, - "college_law": 0.3148148148148148, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.29245283018867924, - "college_medicine": 0.3663003663003663, - "computer_science": 0.5735294117647058, - "computer_security": 0.6023391812865497, - "conceptual_physics": 0.41496598639455784, - "construction_project_management": 0.4244604316546763, - "economics": 0.559748427672956, - "education": 0.5153374233128835, - "electrical_engineering": 0.4418604651162791, - "elementary_chinese": 0.29365079365079366, - "elementary_commonsense": 0.43434343434343436, - "elementary_information_and_technology": 0.6974789915966386, - "elementary_mathematics": 0.391304347826087, - "ethnology": 0.4222222222222222, - "food_science": 0.43356643356643354, - "genetics": 0.3977272727272727, - "global_facts": 0.5167785234899329, - "high_school_biology": 0.34911242603550297, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.3305084745762712, - "high_school_mathematics": 0.22560975609756098, - "high_school_physics": 0.45454545454545453, - "high_school_politics": 0.38461538461538464, - "human_sexuality": 0.5634920634920635, - "international_law": 0.4, - "journalism": 0.4883720930232558, - "jurisprudence": 0.4720194647201946, - "legal_and_moral_basis": 0.7429906542056075, - "logical": 0.4634146341463415, - "machine_learning": 0.4180327868852459, - "management": 0.5571428571428572, - "marketing": 0.5944444444444444, - "marxist_theory": 0.5079365079365079, - "modern_chinese": 0.33620689655172414, - "nutrition": 0.503448275862069, - "philosophy": 0.45714285714285713, - "professional_accounting": 0.4857142857142857, - "professional_law": 0.3744075829383886, - "professional_medicine": 0.32978723404255317, - "professional_psychology": 0.4698275862068966, - "public_relations": 0.4827586206896552, - "security_study": 0.5185185185185185, - "sociology": 0.47345132743362833, - "sports_science": 0.47878787878787876, - "traditional_chinese_medicine": 0.3675675675675676, - "virology": 0.5384615384615384, - "world_history": 0.5031055900621118, - "world_religions": 0.5375 + "agronomy": 0.23668639053254437, + "anatomy": 0.25675675675675674, + "ancient_chinese": 0.23780487804878048, + "arts": 0.25, + "astronomy": 0.2727272727272727, + "business_ethics": 0.2631578947368421, + "chinese_civil_service_exam": 0.26875, + "chinese_driving_rule": 0.21374045801526717, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.22429906542056074, + "chinese_history": 0.2476780185758514, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.2569832402234637, + "clinical_knowledge": 0.24472573839662448, + "college_actuarial_science": 0.29245283018867924, + "college_education": 0.14953271028037382, + "college_engineering_hydrology": 0.27358490566037735, + "college_law": 0.32407407407407407, + "college_mathematics": 0.29523809523809524, + "college_medical_statistics": 0.20754716981132076, + "college_medicine": 0.2271062271062271, + "computer_science": 0.23529411764705882, + "computer_security": 0.25146198830409355, + "conceptual_physics": 0.23129251700680273, + "construction_project_management": 0.2446043165467626, + "economics": 0.2578616352201258, + "education": 0.25153374233128833, + "electrical_engineering": 0.2558139534883721, + "elementary_chinese": 0.20634920634920634, + "elementary_commonsense": 0.2727272727272727, + "elementary_information_and_technology": 0.2184873949579832, + "elementary_mathematics": 0.24347826086956523, + "ethnology": 0.2518518518518518, + "food_science": 0.2727272727272727, + "genetics": 0.2727272727272727, + "global_facts": 0.26174496644295303, + "high_school_biology": 0.28402366863905326, + "high_school_chemistry": 0.19696969696969696, + "high_school_geography": 0.23728813559322035, + "high_school_mathematics": 0.25609756097560976, + "high_school_physics": 0.2636363636363636, + "high_school_politics": 0.24475524475524477, + "human_sexuality": 0.2698412698412698, + "international_law": 0.25405405405405407, + "journalism": 0.2558139534883721, + "jurisprudence": 0.26034063260340634, + "legal_and_moral_basis": 0.2336448598130841, + "logical": 0.23577235772357724, + "machine_learning": 0.2540983606557377, + "management": 0.24761904761904763, + "marketing": 0.2388888888888889, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.25, + "nutrition": 0.27586206896551724, + "philosophy": 0.24761904761904763, + "professional_accounting": 0.25142857142857145, + "professional_law": 0.2559241706161137, + "professional_medicine": 0.25, + "professional_psychology": 0.2543103448275862, + "public_relations": 0.23563218390804597, + "security_study": 0.2222222222222222, + "sociology": 0.25663716814159293, + "sports_science": 0.2727272727272727, + "traditional_chinese_medicine": 0.24864864864864866, + "virology": 0.23076923076923078, + "world_history": 0.2546583850931677, + "world_religions": 0.26875 } }, "prompt_3": { - "accuracy": 0.44232429632187875, + "accuracy": 0.2487480573303402, "category_acc": { - "agronomy": 0.4437869822485207, - "anatomy": 0.31756756756756754, - "ancient_chinese": 0.27439024390243905, - "arts": 0.48125, - "astronomy": 0.3515151515151515, - "business_ethics": 0.5741626794258373, - "chinese_civil_service_exam": 0.35625, - "chinese_driving_rule": 0.5877862595419847, - "chinese_food_culture": 0.3014705882352941, - "chinese_foreign_policy": 0.5607476635514018, - "chinese_history": 0.44272445820433437, - "chinese_literature": 0.28431372549019607, - "chinese_teacher_qualification": 0.5027932960893855, - "clinical_knowledge": 0.33755274261603374, - "college_actuarial_science": 0.3113207547169811, - "college_education": 0.5794392523364486, - "college_engineering_hydrology": 0.5094339622641509, - "college_law": 0.3611111111111111, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.3018867924528302, - "college_medicine": 0.3772893772893773, - "computer_science": 0.5833333333333334, - "computer_security": 0.5847953216374269, - "conceptual_physics": 0.3469387755102041, - "construction_project_management": 0.4244604316546763, - "economics": 0.5471698113207547, - "education": 0.49693251533742333, - "electrical_engineering": 0.47093023255813954, - "elementary_chinese": 0.29365079365079366, - "elementary_commonsense": 0.4393939393939394, - "elementary_information_and_technology": 0.7100840336134454, - "elementary_mathematics": 0.3826086956521739, - "ethnology": 0.42962962962962964, - "food_science": 0.45454545454545453, - "genetics": 0.375, - "global_facts": 0.5167785234899329, - "high_school_biology": 0.35502958579881655, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.4152542372881356, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.37272727272727274, - "high_school_politics": 0.3986013986013986, - "human_sexuality": 0.5476190476190477, - "international_law": 0.40540540540540543, - "journalism": 0.5, - "jurisprudence": 0.4744525547445255, - "legal_and_moral_basis": 0.6822429906542056, - "logical": 0.4634146341463415, - "machine_learning": 0.4426229508196721, - "management": 0.5619047619047619, - "marketing": 0.5833333333333334, - "marxist_theory": 0.5238095238095238, + "agronomy": 0.23668639053254437, + "anatomy": 0.25675675675675674, + "ancient_chinese": 0.23780487804878048, + "arts": 0.25, + "astronomy": 0.2727272727272727, + "business_ethics": 0.2631578947368421, + "chinese_civil_service_exam": 0.26875, + "chinese_driving_rule": 0.21374045801526717, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.22429906542056074, + "chinese_history": 0.25386996904024767, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.2569832402234637, + "clinical_knowledge": 0.24472573839662448, + "college_actuarial_science": 0.2830188679245283, + "college_education": 0.14953271028037382, + "college_engineering_hydrology": 0.27358490566037735, + "college_law": 0.32407407407407407, + "college_mathematics": 0.2761904761904762, + "college_medical_statistics": 0.20754716981132076, + "college_medicine": 0.2271062271062271, + "computer_science": 0.23529411764705882, + "computer_security": 0.25146198830409355, + "conceptual_physics": 0.23129251700680273, + "construction_project_management": 0.2446043165467626, + "economics": 0.2578616352201258, + "education": 0.25153374233128833, + "electrical_engineering": 0.2558139534883721, + "elementary_chinese": 0.20634920634920634, + "elementary_commonsense": 0.2727272727272727, + "elementary_information_and_technology": 0.22268907563025211, + "elementary_mathematics": 0.24347826086956523, + "ethnology": 0.2518518518518518, + "food_science": 0.26573426573426573, + "genetics": 0.2727272727272727, + "global_facts": 0.26174496644295303, + "high_school_biology": 0.28402366863905326, + "high_school_chemistry": 0.18181818181818182, + "high_school_geography": 0.23728813559322035, + "high_school_mathematics": 0.25609756097560976, + "high_school_physics": 0.2636363636363636, + "high_school_politics": 0.24475524475524477, + "human_sexuality": 0.2698412698412698, + "international_law": 0.25405405405405407, + "journalism": 0.2558139534883721, + "jurisprudence": 0.26034063260340634, + "legal_and_moral_basis": 0.2336448598130841, + "logical": 0.23577235772357724, + "machine_learning": 0.2540983606557377, + "management": 0.24761904761904763, + "marketing": 0.2388888888888889, + "marxist_theory": 0.23809523809523808, "modern_chinese": 0.25, - "nutrition": 0.5103448275862069, - "philosophy": 0.5238095238095238, - "professional_accounting": 0.45714285714285713, - "professional_law": 0.33175355450236965, - "professional_medicine": 0.30319148936170215, - "professional_psychology": 0.45689655172413796, - "public_relations": 0.5402298850574713, - "security_study": 0.5185185185185185, - "sociology": 0.5132743362831859, - "sports_science": 0.4, - "traditional_chinese_medicine": 0.33513513513513515, - "virology": 0.4911242603550296, - "world_history": 0.5527950310559007, - "world_religions": 0.5125 + "nutrition": 0.27586206896551724, + "philosophy": 0.24761904761904763, + "professional_accounting": 0.25142857142857145, + "professional_law": 0.2559241706161137, + "professional_medicine": 0.25, + "professional_psychology": 0.2543103448275862, + "public_relations": 0.23563218390804597, + "security_study": 0.2222222222222222, + "sociology": 0.25663716814159293, + "sports_science": 0.2727272727272727, + "traditional_chinese_medicine": 0.24864864864864866, + "virology": 0.23076923076923078, + "world_history": 0.2546583850931677, + "world_religions": 0.26875 } }, "prompt_4": { - "accuracy": 0.43438093593507165, + "accuracy": 0.2479709894664134, "category_acc": { - "agronomy": 0.41420118343195267, - "anatomy": 0.3108108108108108, - "ancient_chinese": 0.2804878048780488, - "arts": 0.45625, - "astronomy": 0.3090909090909091, - "business_ethics": 0.5215311004784688, - "chinese_civil_service_exam": 0.375, - "chinese_driving_rule": 0.5419847328244275, - "chinese_food_culture": 0.36764705882352944, - "chinese_foreign_policy": 0.4953271028037383, - "chinese_history": 0.43034055727554177, - "chinese_literature": 0.3431372549019608, - "chinese_teacher_qualification": 0.5139664804469274, - "clinical_knowledge": 0.31223628691983124, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.5046728971962616, - "college_engineering_hydrology": 0.5094339622641509, - "college_law": 0.37037037037037035, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.33962264150943394, - "college_medicine": 0.358974358974359, - "computer_science": 0.5392156862745098, - "computer_security": 0.5146198830409356, - "conceptual_physics": 0.38095238095238093, - "construction_project_management": 0.43884892086330934, - "economics": 0.5220125786163522, - "education": 0.4539877300613497, - "electrical_engineering": 0.4186046511627907, - "elementary_chinese": 0.2896825396825397, - "elementary_commonsense": 0.42424242424242425, - "elementary_information_and_technology": 0.6680672268907563, - "elementary_mathematics": 0.3869565217391304, - "ethnology": 0.3925925925925926, - "food_science": 0.4965034965034965, - "genetics": 0.3522727272727273, - "global_facts": 0.5100671140939598, - "high_school_biology": 0.3136094674556213, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.4152542372881356, - "high_school_mathematics": 0.3353658536585366, - "high_school_physics": 0.3181818181818182, - "high_school_politics": 0.3986013986013986, - "human_sexuality": 0.5714285714285714, - "international_law": 0.3945945945945946, - "journalism": 0.5058139534883721, - "jurisprudence": 0.4793187347931874, - "legal_and_moral_basis": 0.7149532710280374, - "logical": 0.4065040650406504, - "machine_learning": 0.48360655737704916, - "management": 0.5619047619047619, - "marketing": 0.6055555555555555, - "marxist_theory": 0.5396825396825397, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.4689655172413793, - "philosophy": 0.44761904761904764, - "professional_accounting": 0.4857142857142857, - "professional_law": 0.36492890995260663, - "professional_medicine": 0.324468085106383, - "professional_psychology": 0.46551724137931033, - "public_relations": 0.4942528735632184, - "security_study": 0.4666666666666667, - "sociology": 0.47345132743362833, - "sports_science": 0.45454545454545453, - "traditional_chinese_medicine": 0.34594594594594597, - "virology": 0.4970414201183432, - "world_history": 0.484472049689441, - "world_religions": 0.5125 + "agronomy": 0.23668639053254437, + "anatomy": 0.25675675675675674, + "ancient_chinese": 0.23780487804878048, + "arts": 0.25, + "astronomy": 0.26666666666666666, + "business_ethics": 0.2679425837320574, + "chinese_civil_service_exam": 0.275, + "chinese_driving_rule": 0.22137404580152673, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.22429906542056074, + "chinese_history": 0.2476780185758514, + "chinese_literature": 0.2549019607843137, + "chinese_teacher_qualification": 0.24581005586592178, + "clinical_knowledge": 0.24472573839662448, + "college_actuarial_science": 0.29245283018867924, + "college_education": 0.14953271028037382, + "college_engineering_hydrology": 0.27358490566037735, + "college_law": 0.32407407407407407, + "college_mathematics": 0.2761904761904762, + "college_medical_statistics": 0.20754716981132076, + "college_medicine": 0.2271062271062271, + "computer_science": 0.23529411764705882, + "computer_security": 0.25146198830409355, + "conceptual_physics": 0.23129251700680273, + "construction_project_management": 0.2446043165467626, + "economics": 0.2578616352201258, + "education": 0.25153374233128833, + "electrical_engineering": 0.23255813953488372, + "elementary_chinese": 0.20238095238095238, + "elementary_commonsense": 0.2727272727272727, + "elementary_information_and_technology": 0.2184873949579832, + "elementary_mathematics": 0.25217391304347825, + "ethnology": 0.24444444444444444, + "food_science": 0.25874125874125875, + "genetics": 0.26704545454545453, + "global_facts": 0.26174496644295303, + "high_school_biology": 0.2603550295857988, + "high_school_chemistry": 0.20454545454545456, + "high_school_geography": 0.23728813559322035, + "high_school_mathematics": 0.25, + "high_school_physics": 0.2727272727272727, + "high_school_politics": 0.23776223776223776, + "human_sexuality": 0.2698412698412698, + "international_law": 0.2594594594594595, + "journalism": 0.2558139534883721, + "jurisprudence": 0.26034063260340634, + "legal_and_moral_basis": 0.22897196261682243, + "logical": 0.23577235772357724, + "machine_learning": 0.2459016393442623, + "management": 0.2523809523809524, + "marketing": 0.2388888888888889, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.25, + "nutrition": 0.2827586206896552, + "philosophy": 0.2571428571428571, + "professional_accounting": 0.24571428571428572, + "professional_law": 0.24644549763033174, + "professional_medicine": 0.24202127659574468, + "professional_psychology": 0.25, + "public_relations": 0.2413793103448276, + "security_study": 0.22962962962962963, + "sociology": 0.25663716814159293, + "sports_science": 0.2727272727272727, + "traditional_chinese_medicine": 0.24864864864864866, + "virology": 0.2485207100591716, + "world_history": 0.2546583850931677, + "world_religions": 0.26875 } }, "prompt_5": { - "accuracy": 0.43610775341046454, + "accuracy": 0.24831635296149196, "category_acc": { - "agronomy": 0.39644970414201186, - "anatomy": 0.28378378378378377, - "ancient_chinese": 0.2682926829268293, - "arts": 0.53125, - "astronomy": 0.30303030303030304, - "business_ethics": 0.5406698564593302, - "chinese_civil_service_exam": 0.3625, - "chinese_driving_rule": 0.6030534351145038, - "chinese_food_culture": 0.29411764705882354, - "chinese_foreign_policy": 0.4392523364485981, - "chinese_history": 0.47368421052631576, - "chinese_literature": 0.27941176470588236, - "chinese_teacher_qualification": 0.5139664804469274, - "clinical_knowledge": 0.31223628691983124, - "college_actuarial_science": 0.3018867924528302, - "college_education": 0.5981308411214953, - "college_engineering_hydrology": 0.44339622641509435, - "college_law": 0.3333333333333333, + "agronomy": 0.23668639053254437, + "anatomy": 0.25675675675675674, + "ancient_chinese": 0.23780487804878048, + "arts": 0.25, + "astronomy": 0.2727272727272727, + "business_ethics": 0.2631578947368421, + "chinese_civil_service_exam": 0.26875, + "chinese_driving_rule": 0.21374045801526717, + "chinese_food_culture": 0.2426470588235294, + "chinese_foreign_policy": 0.22429906542056074, + "chinese_history": 0.2476780185758514, + "chinese_literature": 0.25, + "chinese_teacher_qualification": 0.2569832402234637, + "clinical_knowledge": 0.24472573839662448, + "college_actuarial_science": 0.2830188679245283, + "college_education": 0.14953271028037382, + "college_engineering_hydrology": 0.27358490566037735, + "college_law": 0.32407407407407407, "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.39622641509433965, - "college_medicine": 0.3956043956043956, - "computer_science": 0.5735294117647058, - "computer_security": 0.543859649122807, - "conceptual_physics": 0.3945578231292517, - "construction_project_management": 0.4028776978417266, - "economics": 0.5031446540880503, - "education": 0.5214723926380368, - "electrical_engineering": 0.4418604651162791, - "elementary_chinese": 0.2896825396825397, - "elementary_commonsense": 0.41414141414141414, - "elementary_information_and_technology": 0.7058823529411765, - "elementary_mathematics": 0.3695652173913043, - "ethnology": 0.4148148148148148, - "food_science": 0.44755244755244755, - "genetics": 0.38636363636363635, - "global_facts": 0.48322147651006714, - "high_school_biology": 0.3431952662721893, - "high_school_chemistry": 0.25, - "high_school_geography": 0.3813559322033898, - "high_school_mathematics": 0.3048780487804878, - "high_school_physics": 0.38181818181818183, - "high_school_politics": 0.42657342657342656, - "human_sexuality": 0.5714285714285714, - "international_law": 0.35135135135135137, - "journalism": 0.45930232558139533, - "jurisprudence": 0.45498783454987834, - "legal_and_moral_basis": 0.705607476635514, - "logical": 0.4065040650406504, - "machine_learning": 0.48360655737704916, - "management": 0.5714285714285714, - "marketing": 0.6111111111111112, - "marxist_theory": 0.5238095238095238, - "modern_chinese": 0.35344827586206895, - "nutrition": 0.503448275862069, - "philosophy": 0.4666666666666667, - "professional_accounting": 0.4342857142857143, - "professional_law": 0.33649289099526064, - "professional_medicine": 0.28191489361702127, - "professional_psychology": 0.4827586206896552, - "public_relations": 0.47701149425287354, - "security_study": 0.5555555555555556, - "sociology": 0.49557522123893805, - "sports_science": 0.4727272727272727, - "traditional_chinese_medicine": 0.33513513513513515, - "virology": 0.48520710059171596, - "world_history": 0.4782608695652174, - "world_religions": 0.50625 + "college_medical_statistics": 0.20754716981132076, + "college_medicine": 0.2271062271062271, + "computer_science": 0.23529411764705882, + "computer_security": 0.24561403508771928, + "conceptual_physics": 0.23129251700680273, + "construction_project_management": 0.2446043165467626, + "economics": 0.2578616352201258, + "education": 0.25153374233128833, + "electrical_engineering": 0.25, + "elementary_chinese": 0.20634920634920634, + "elementary_commonsense": 0.2727272727272727, + "elementary_information_and_technology": 0.2184873949579832, + "elementary_mathematics": 0.24347826086956523, + "ethnology": 0.2518518518518518, + "food_science": 0.26573426573426573, + "genetics": 0.26704545454545453, + "global_facts": 0.26174496644295303, + "high_school_biology": 0.2781065088757396, + "high_school_chemistry": 0.19696969696969696, + "high_school_geography": 0.23728813559322035, + "high_school_mathematics": 0.25609756097560976, + "high_school_physics": 0.2636363636363636, + "high_school_politics": 0.24475524475524477, + "human_sexuality": 0.2698412698412698, + "international_law": 0.25405405405405407, + "journalism": 0.2558139534883721, + "jurisprudence": 0.26034063260340634, + "legal_and_moral_basis": 0.2336448598130841, + "logical": 0.23577235772357724, + "machine_learning": 0.2540983606557377, + "management": 0.24761904761904763, + "marketing": 0.2388888888888889, + "marxist_theory": 0.23809523809523808, + "modern_chinese": 0.25, + "nutrition": 0.27586206896551724, + "philosophy": 0.24761904761904763, + "professional_accounting": 0.25142857142857145, + "professional_law": 0.2559241706161137, + "professional_medicine": 0.25, + "professional_psychology": 0.2543103448275862, + "public_relations": 0.23563218390804597, + "security_study": 0.2222222222222222, + "sociology": 0.25663716814159293, + "sports_science": 0.2727272727272727, + "traditional_chinese_medicine": 0.24864864864864866, + "virology": 0.23076923076923078, + "world_history": 0.2546583850931677, + "world_religions": 0.26875 } } }, "zbench": { "prompt_1": { - "accuracy": 0.36363636363636365 + "accuracy": 0.3939393939393939 }, "prompt_2": { - "accuracy": 0.36363636363636365 + "accuracy": 0.3939393939393939 }, "prompt_3": { - "accuracy": 0.36363636363636365 + "accuracy": 0.3939393939393939 }, "prompt_4": { - "accuracy": 0.36363636363636365 + "accuracy": 0.42424242424242425 }, "prompt_5": { - "accuracy": 0.3333333333333333 + "accuracy": 0.3939393939393939 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.6295454545454545 + "accuracy": 0.19090909090909092 }, "prompt_2": { - "accuracy": 0.6477272727272727 + "accuracy": 0.20909090909090908 }, "prompt_3": { - "accuracy": 0.6136363636363636 + "accuracy": 0.20454545454545456 }, "prompt_4": { - "accuracy": 0.5840909090909091 + "accuracy": 0.24772727272727274 }, "prompt_5": { - "accuracy": 0.625 + "accuracy": 0.2409090909090909 } }, "ocnli": { "prompt_1": { - "accuracy": 0.5220338983050847 + "accuracy": 0.34508474576271186 }, "prompt_2": { - "accuracy": 0.49627118644067797 + "accuracy": 0.32440677966101694 }, "prompt_3": { - "accuracy": 0.5027118644067796 + "accuracy": 0.3233898305084746 }, "prompt_4": { - "accuracy": 0.5257627118644068 + "accuracy": 0.3352542372881356 }, "prompt_5": { - "accuracy": 0.5376271186440678 + "accuracy": 0.3352542372881356 } }, "c3": { "prompt_1": { - "accuracy": 0.819371727748691 + "accuracy": 0.2830964846671653 }, "prompt_2": { - "accuracy": 0.8145100972326104 + "accuracy": 0.2793567688855647 }, "prompt_3": { - "accuracy": 0.8152580403889305 + "accuracy": 0.2793567688855647 }, "prompt_4": { - "accuracy": 0.8167539267015707 + "accuracy": 0.28272251308900526 }, "prompt_5": { - "accuracy": 0.818249813014211 + "accuracy": 0.2793567688855647 } }, "dream": { "prompt_1": { - "accuracy": 0.8682018618324351 + "accuracy": 0.5100440960313571 }, "prompt_2": { - "accuracy": 0.8667319941205291 + "accuracy": 0.5213130818226359 }, "prompt_3": { - "accuracy": 0.8922097011268986 + "accuracy": 0.512983831455169 }, "prompt_4": { - "accuracy": 0.8780009799118079 + "accuracy": 0.5154336109750123 }, "prompt_5": { - "accuracy": 0.9039686428221461 + "accuracy": 0.5071043606075453 } }, "samsum": { "prompt_1": { - "rouge1": 0.3382331137689176, - "rouge2": 0.13017899302289776, - "rougeL": 0.258778356990726, - "avg_rouge": 0.24239682126084713 + "rouge1": 0.43481455633608285, + "rouge2": 0.19020306710468923, + "rougeL": 0.3555281762061519, + "avg_rouge": 0.326848599882308 }, "prompt_2": { - "rouge1": 0.3673102092221032, - "rouge2": 0.1402951291875954, - "rougeL": 0.2827653301566992, - "avg_rouge": 0.2634568895221326 + "rouge1": 0.43319141489410384, + "rouge2": 0.18633011055371998, + "rougeL": 0.3530126019631636, + "avg_rouge": 0.3241780424703291 }, "prompt_3": { - "rouge1": 0.3514601845107603, - "rouge2": 0.13136466116065107, - "rougeL": 0.26731177141893436, - "avg_rouge": 0.2500455390301152 + "rouge1": 0.4303838979441285, + "rouge2": 0.18417594383860397, + "rougeL": 0.34948463192329804, + "avg_rouge": 0.32134815790201016 }, "prompt_4": { - "rouge1": 0.35426038814278493, - "rouge2": 0.13294921026096784, - "rougeL": 0.26951792086959037, - "avg_rouge": 0.25224250642444773 + "rouge1": 0.43070227421856316, + "rouge2": 0.18614737183994354, + "rougeL": 0.3529860498921421, + "avg_rouge": 0.3232785653168829 }, "prompt_5": { - "rouge1": 0.37418865756320524, - "rouge2": 0.14026981622290713, - "rougeL": 0.28802375105514655, - "avg_rouge": 0.2674940749470863 + "rouge1": 0.4308048074328594, + "rouge2": 0.18586059514617964, + "rougeL": 0.3528154914643337, + "avg_rouge": 0.3231602980144576 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.23090448981464654, - "rouge2": 0.06172830132297033, - "rougeL": 0.1701626957704241, - "avg_rouge": 0.1542651623026803 + "rouge1": 0.23467138626961317, + "rouge2": 0.06744069122334906, + "rougeL": 0.1779908580361024, + "avg_rouge": 0.16003431184302153 }, "prompt_2": { - "rouge1": 0.22862556652601945, - "rouge2": 0.05978864447172686, - "rougeL": 0.16815061855398675, - "avg_rouge": 0.15218827651724434 + "rouge1": 0.2483010108284747, + "rouge2": 0.075125300962018, + "rougeL": 0.18996631825453364, + "avg_rouge": 0.17113087668167545 }, "prompt_3": { - "rouge1": 0.2295035427211935, - "rouge2": 0.06030899594042445, - "rougeL": 0.16854350393527293, - "avg_rouge": 0.15278534753229697 + "rouge1": 0.23828052499310123, + "rouge2": 0.07068930036983678, + "rougeL": 0.18168303338974037, + "avg_rouge": 0.16355095291755947 }, "prompt_4": { - "rouge1": 0.23142813001121992, - "rouge2": 0.06210417743027356, - "rougeL": 0.1704278688968823, - "avg_rouge": 0.15465339211279194 + "rouge1": 0.2552198528902313, + "rouge2": 0.07694264335968432, + "rougeL": 0.19993614328948328, + "avg_rouge": 0.17736621317979964 }, "prompt_5": { - "rouge1": 0.22474749234365923, - "rouge2": 0.0562719722378755, - "rougeL": 0.16784908061297876, - "avg_rouge": 0.14962284839817117 + "rouge1": 0.22558914523023577, + "rouge2": 0.0636998106706898, + "rougeL": 0.1683448949296202, + "avg_rouge": 0.15254461694351526 } }, "sst2": { "prompt_1": { - "accuracy": 0.9403669724770642 + "accuracy": 0.856651376146789 }, "prompt_2": { - "accuracy": 0.9380733944954128 + "accuracy": 0.8486238532110092 }, "prompt_3": { - "accuracy": 0.944954128440367 + "accuracy": 0.8451834862385321 }, "prompt_4": { - "accuracy": 0.9357798165137615 + "accuracy": 0.8577981651376146 }, "prompt_5": { - "accuracy": 0.9094036697247706 + "accuracy": 0.8692660550458715 } }, "cola": { "prompt_1": { - "accuracy": 0.7698945349952061 + "accuracy": 0.6720997123681688 }, "prompt_2": { - "accuracy": 0.7545541706615532 + "accuracy": 0.6663470757430489 }, "prompt_3": { - "accuracy": 0.7909875359539789 + "accuracy": 0.6816874400767018 }, "prompt_4": { - "accuracy": 0.8015340364333653 + "accuracy": 0.663470757430489 }, "prompt_5": { - "accuracy": 0.7727708533077661 + "accuracy": 0.6519654841802492 } }, "qqp": { "prompt_1": { - "accuracy": 0.793 + "accuracy": 0.474 }, "prompt_2": { - "accuracy": 0.7925 + "accuracy": 0.4865 }, "prompt_3": { - "accuracy": 0.784 + "accuracy": 0.4485 }, "prompt_4": { - "accuracy": 0.779 + "accuracy": 0.4995 }, "prompt_5": { - "accuracy": 0.7795 + "accuracy": 0.4935 } }, "mnli": { "prompt_1": { - "accuracy": 0.7065 + "accuracy": 0.3 }, "prompt_2": { - "accuracy": 0.7015 + "accuracy": 0.2 }, "prompt_3": { - "accuracy": 0.7075 + "accuracy": 0.3 }, "prompt_4": { - "accuracy": 0.691 + "accuracy": 0.3 }, "prompt_5": { - "accuracy": 0.7115 + "accuracy": 0.3 } }, "qnli": { "prompt_1": { - "accuracy": 0.872 + "accuracy": 0.5 }, "prompt_2": { - "accuracy": 0.879 + "accuracy": 0.6 }, "prompt_3": { - "accuracy": 0.8755 + "accuracy": 0.5 }, "prompt_4": { - "accuracy": 0.877 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.864 + "accuracy": 0.4 } }, "wnli": { "prompt_1": { - "accuracy": 0.7183098591549296 + "accuracy": 0.7 }, "prompt_2": { - "accuracy": 0.7464788732394366 + "accuracy": 0.8 }, "prompt_3": { - "accuracy": 0.7323943661971831 + "accuracy": 0.6 }, "prompt_4": { - "accuracy": 0.7464788732394366 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.7323943661971831 + "accuracy": 0.7 } }, "rte": { "prompt_1": { - "accuracy": 0.8194945848375451 + "accuracy": 0.3 }, "prompt_2": { - "accuracy": 0.7653429602888087 + "accuracy": 0.3 }, "prompt_3": { - "accuracy": 0.8303249097472925 + "accuracy": 0.3 }, "prompt_4": { - "accuracy": 0.8158844765342961 + "accuracy": 0.3 }, "prompt_5": { - "accuracy": 0.7472924187725631 + "accuracy": 0.3 } }, "mrpc": { "prompt_1": { - "accuracy": 0.7647058823529411 + "accuracy": 0.8 }, "prompt_2": { - "accuracy": 0.7892156862745098 + "accuracy": 0.8 }, "prompt_3": { - "accuracy": 0.7696078431372549 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.7941176470588235 + "accuracy": 0.7 }, "prompt_5": { - "accuracy": 0.7965686274509803 + "accuracy": 0.5 } } }, "five_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.9191176470588235, - "language_acc": { - "English": 0.9436974789915966, - "Chinese": 0.9084033613445378, - "Spanish": 0.9243697478991597, - "Vietnamese": 0.9 - }, - "consistency_score_2": 0.9092436974789916, - "consistency_score_3": 0.8686974789915967, - "consistency_score_4": 0.8394957983193277, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.9126050420168067, - "English,Spanish": 0.9420168067226891, - "English,Vietnamese": 0.9008403361344538, - "Chinese,Spanish": 0.9084033613445378, - "Chinese,Vietnamese": 0.8932773109243698, - "Spanish,Vietnamese": 0.8983193277310925 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8857142857142857, - "English,Chinese,Vietnamese": 0.8588235294117647, - "English,Spanish,Vietnamese": 0.8739495798319328, - "Chinese,Spanish,Vietnamese": 0.8563025210084033 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8394957983193277 - } - }, - "AC3_2": 0.9141540103994069, - "AC3_3": 0.8931965852821749, - "AC3_4": 0.8775042688897892 - } + "prompt_1": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.6076190476190476, - "language_acc": { - "Spanish": 0.6333333333333333, - "English": 0.6933333333333334, - "Chinese": 0.56, - "Vietnamese": 0.5933333333333334, - "Filipino": 0.62, - "Indonesian": 0.5866666666666667, - "Malay": 0.5666666666666667 - }, - "consistency_score_2": 0.6663492063492062, - "consistency_score_3": 0.5310476190476192, - "consistency_score_4": 0.4470476190476189, - "consistency_score_5": 0.38603174603174606, - "consistency_score_6": 0.3380952380952381, - "consistency_score_7": 0.3, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.7933333333333333, - "Spanish,Chinese": 0.62, - "Spanish,Vietnamese": 0.6533333333333333, - "Spanish,Filipino": 0.68, - "Spanish,Indonesian": 0.6266666666666667, - "Spanish,Malay": 0.6866666666666666, - "English,Chinese": 0.6466666666666666, - "English,Vietnamese": 0.7133333333333334, - "English,Filipino": 0.7266666666666667, - "English,Indonesian": 0.6933333333333334, - "English,Malay": 0.7, - "Chinese,Vietnamese": 0.5733333333333334, - "Chinese,Filipino": 0.62, - "Chinese,Indonesian": 0.6066666666666667, - "Chinese,Malay": 0.6266666666666667, - "Vietnamese,Filipino": 0.66, - "Vietnamese,Indonesian": 0.68, - "Vietnamese,Malay": 0.7, - "Filipino,Indonesian": 0.6333333333333333, - "Filipino,Malay": 0.68, - "Indonesian,Malay": 0.6733333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5466666666666666, - "Spanish,English,Vietnamese": 0.5933333333333334, - "Spanish,English,Filipino": 0.62, - "Spanish,English,Indonesian": 0.5733333333333334, - "Spanish,English,Malay": 0.6133333333333333, - "Spanish,Chinese,Vietnamese": 0.46, - "Spanish,Chinese,Filipino": 0.4866666666666667, - "Spanish,Chinese,Indonesian": 0.4666666666666667, - "Spanish,Chinese,Malay": 0.5133333333333333, - "Spanish,Vietnamese,Filipino": 0.54, - "Spanish,Vietnamese,Indonesian": 0.5133333333333333, - "Spanish,Vietnamese,Malay": 0.5466666666666666, - "Spanish,Filipino,Indonesian": 0.5, - "Spanish,Filipino,Malay": 0.5533333333333333, - "Spanish,Indonesian,Malay": 0.5266666666666666, - "English,Chinese,Vietnamese": 0.5, - "English,Chinese,Filipino": 0.5133333333333333, - "English,Chinese,Indonesian": 0.49333333333333335, - "English,Chinese,Malay": 0.52, - "English,Vietnamese,Filipino": 0.58, - "English,Vietnamese,Indonesian": 0.58, - "English,Vietnamese,Malay": 0.58, - "English,Filipino,Indonesian": 0.5466666666666666, - "English,Filipino,Malay": 0.5866666666666667, - "English,Indonesian,Malay": 0.56, - "Chinese,Vietnamese,Filipino": 0.4666666666666667, - "Chinese,Vietnamese,Indonesian": 0.48, - "Chinese,Vietnamese,Malay": 0.49333333333333335, - "Chinese,Filipino,Indonesian": 0.46, - "Chinese,Filipino,Malay": 0.5066666666666667, - "Chinese,Indonesian,Malay": 0.49333333333333335, - "Vietnamese,Filipino,Indonesian": 0.5333333333333333, - "Vietnamese,Filipino,Malay": 0.56, - "Vietnamese,Indonesian,Malay": 0.5533333333333333, - "Filipino,Indonesian,Malay": 0.5266666666666666 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.43333333333333335, - "Spanish,English,Chinese,Filipino": 0.44666666666666666, - "Spanish,English,Chinese,Indonesian": 0.4266666666666667, - "Spanish,English,Chinese,Malay": 0.4666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.5, - "Spanish,English,Vietnamese,Indonesian": 0.49333333333333335, - "Spanish,English,Vietnamese,Malay": 0.5133333333333333, - "Spanish,English,Filipino,Indonesian": 0.47333333333333333, - "Spanish,English,Filipino,Malay": 0.52, - "Spanish,English,Indonesian,Malay": 0.49333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.4, - "Spanish,Chinese,Vietnamese,Indonesian": 0.3933333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.42, - "Spanish,Chinese,Filipino,Indonesian": 0.38666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.43333333333333335, - "Spanish,Chinese,Indonesian,Malay": 0.41333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.44666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.47333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.4533333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.44, - "English,Chinese,Vietnamese,Filipino": 0.42, - "English,Chinese,Vietnamese,Indonesian": 0.4266666666666667, - "English,Chinese,Vietnamese,Malay": 0.43333333333333335, - "English,Chinese,Filipino,Indonesian": 0.4066666666666667, - "English,Chinese,Filipino,Malay": 0.44, - "English,Chinese,Indonesian,Malay": 0.4266666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.4866666666666667, - "English,Vietnamese,Filipino,Malay": 0.5066666666666667, - "English,Vietnamese,Indonesian,Malay": 0.4866666666666667, - "English,Filipino,Indonesian,Malay": 0.47333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.4, - "Chinese,Vietnamese,Filipino,Malay": 0.4266666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.42, - "Chinese,Filipino,Indonesian,Malay": 0.4, - "Vietnamese,Filipino,Indonesian,Malay": 0.4666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.37333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.37333333333333335, - "Spanish,English,Chinese,Vietnamese,Malay": 0.3933333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.36, - "Spanish,English,Chinese,Filipino,Malay": 0.4, - "Spanish,English,Chinese,Indonesian,Malay": 0.38666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.4266666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.44666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.43333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.42, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.37333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.36, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.3466666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.4, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.36666666666666664, - "English,Chinese,Vietnamese,Filipino,Malay": 0.38, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "English,Chinese,Filipino,Indonesian,Malay": 0.36, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.4266666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.36 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.3466666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.34, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.32666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.38, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.32666666666666666 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3 - } - }, - "AC3_2": 0.6356303916532614, - "AC3_3": 0.566758750288769, - "AC3_4": 0.5151099529083215, - "AC3_5": 0.472118058677574, - "AC3_6": 0.434450678513501, - "AC3_7": 0.4016789086650826 - } + "prompt_1": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.5275974025974027, - "language_acc": { - "English": 0.5795454545454546, - "Filipino": 0.4602272727272727, - "Spanish": 0.5340909090909091, - "Malay": 0.5227272727272727, - "Chinese": 0.5681818181818182, - "Indonesian": 0.5397727272727273, - "Vietnamese": 0.48863636363636365 - }, - "consistency_score_2": 0.6068722943722944, - "consistency_score_3": 0.44999999999999996, - "consistency_score_4": 0.36233766233766224, - "consistency_score_5": 0.3060064935064934, - "consistency_score_6": 0.2654220779220779, - "consistency_score_7": 0.23295454545454544, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.5795454545454546, - "English,Spanish": 0.7215909090909091, - "English,Malay": 0.5909090909090909, - "English,Chinese": 0.5909090909090909, - "English,Indonesian": 0.6647727272727273, - "English,Vietnamese": 0.6306818181818182, - "Filipino,Spanish": 0.5738636363636364, - "Filipino,Malay": 0.6306818181818182, - "Filipino,Chinese": 0.5568181818181818, - "Filipino,Indonesian": 0.6306818181818182, - "Filipino,Vietnamese": 0.5454545454545454, - "Spanish,Malay": 0.6079545454545454, - "Spanish,Chinese": 0.5113636363636364, - "Spanish,Indonesian": 0.6590909090909091, - "Spanish,Vietnamese": 0.6420454545454546, - "Malay,Chinese": 0.5738636363636364, - "Malay,Indonesian": 0.6818181818181818, - "Malay,Vietnamese": 0.6193181818181818, - "Chinese,Indonesian": 0.5625, - "Chinese,Vietnamese": 0.5170454545454546, - "Indonesian,Vietnamese": 0.6534090909090909 - }, - "3_combine": { - "English,Filipino,Spanish": 0.4602272727272727, - "English,Filipino,Malay": 0.4431818181818182, - "English,Filipino,Chinese": 0.4147727272727273, - "English,Filipino,Indonesian": 0.4659090909090909, - "English,Filipino,Vietnamese": 0.4147727272727273, - "English,Spanish,Malay": 0.4943181818181818, - "English,Spanish,Chinese": 0.4431818181818182, - "English,Spanish,Indonesian": 0.5340909090909091, - "English,Spanish,Vietnamese": 0.5170454545454546, - "English,Malay,Chinese": 0.42045454545454547, - "English,Malay,Indonesian": 0.5, - "English,Malay,Vietnamese": 0.4602272727272727, - "English,Chinese,Indonesian": 0.4602272727272727, - "English,Chinese,Vietnamese": 0.4147727272727273, - "English,Indonesian,Vietnamese": 0.5056818181818182, - "Filipino,Spanish,Malay": 0.4375, - "Filipino,Spanish,Chinese": 0.3693181818181818, - "Filipino,Spanish,Indonesian": 0.4715909090909091, - "Filipino,Spanish,Vietnamese": 0.4147727272727273, - "Filipino,Malay,Chinese": 0.42045454545454547, - "Filipino,Malay,Indonesian": 0.5113636363636364, - "Filipino,Malay,Vietnamese": 0.42613636363636365, - "Filipino,Chinese,Indonesian": 0.42045454545454547, - "Filipino,Chinese,Vietnamese": 0.3693181818181818, - "Filipino,Indonesian,Vietnamese": 0.45454545454545453, - "Spanish,Malay,Chinese": 0.39204545454545453, - "Spanish,Malay,Indonesian": 0.5113636363636364, - "Spanish,Malay,Vietnamese": 0.4715909090909091, - "Spanish,Chinese,Indonesian": 0.4147727272727273, - "Spanish,Chinese,Vietnamese": 0.38636363636363635, - "Spanish,Indonesian,Vietnamese": 0.5170454545454546, - "Malay,Chinese,Indonesian": 0.4715909090909091, - "Malay,Chinese,Vietnamese": 0.39204545454545453, - "Malay,Indonesian,Vietnamese": 0.5284090909090909, - "Chinese,Indonesian,Vietnamese": 0.42045454545454547 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.375, - "English,Filipino,Spanish,Chinese": 0.3409090909090909, - "English,Filipino,Spanish,Indonesian": 0.3977272727272727, - "English,Filipino,Spanish,Vietnamese": 0.3522727272727273, - "English,Filipino,Malay,Chinese": 0.3409090909090909, - "English,Filipino,Malay,Indonesian": 0.4034090909090909, - "English,Filipino,Malay,Vietnamese": 0.3409090909090909, - "English,Filipino,Chinese,Indonesian": 0.36363636363636365, - "English,Filipino,Chinese,Vietnamese": 0.3181818181818182, - "English,Filipino,Indonesian,Vietnamese": 0.3693181818181818, - "English,Spanish,Malay,Chinese": 0.36363636363636365, - "English,Spanish,Malay,Indonesian": 0.42613636363636365, - "English,Spanish,Malay,Vietnamese": 0.4034090909090909, - "English,Spanish,Chinese,Indonesian": 0.39204545454545453, - "English,Spanish,Chinese,Vietnamese": 0.35795454545454547, - "English,Spanish,Indonesian,Vietnamese": 0.4318181818181818, - "English,Malay,Chinese,Indonesian": 0.38636363636363635, - "English,Malay,Chinese,Vietnamese": 0.32954545454545453, - "English,Malay,Indonesian,Vietnamese": 0.4147727272727273, - "English,Chinese,Indonesian,Vietnamese": 0.36363636363636365, - "Filipino,Spanish,Malay,Chinese": 0.3068181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.4034090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.3465909090909091, - "Filipino,Spanish,Chinese,Indonesian": 0.3181818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.29545454545454547, - "Filipino,Spanish,Indonesian,Vietnamese": 0.375, - "Filipino,Malay,Chinese,Indonesian": 0.36363636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.29545454545454547, - "Filipino,Malay,Indonesian,Vietnamese": 0.38636363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.3125, - "Spanish,Malay,Chinese,Indonesian": 0.35795454545454547, - "Spanish,Malay,Chinese,Vietnamese": 0.3181818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.42613636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.3465909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.35795454545454547 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.29545454545454547, - "English,Filipino,Spanish,Malay,Indonesian": 0.3522727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.3068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.3125, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.2840909090909091, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.32954545454545453, - "English,Filipino,Malay,Chinese,Indonesian": 0.3181818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.32386363636363635, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.3409090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.3068181818181818, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.36363636363636365, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.32954545454545453, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.3125, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.25, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.32954545454545453, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.30113636363636365 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.2784090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.24431818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.29545454545454547, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.25, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544 - } - }, - "AC3_2": 0.5644650483766568, - "AC3_3": 0.48571903017283247, - "AC3_4": 0.42962327707591336, - "AC3_5": 0.3873499917339046, - "AC3_6": 0.3531716491864796, - "AC3_7": 0.32320267774929623 - } + "prompt_1": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.6019417475728155 - } + "prompt_1": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.4095238095238095 - } + "prompt_1": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.6728971962616822 - } + "prompt_1": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.43, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.3, - "history": 0.2, - "literature": 0.4, - "politics": 0.8, - "culture": 0.6, - "film": 0.5, - "law": 0.3, - "geography": 0.6 - } - } + "prompt_1": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.25352917772530026 - } + "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.3608970140348058 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.2989536410056442 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.22948867258954989 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.37078727812543094 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.6382730455075846 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.5930639971397926, - "category_acc": { - "high_school_european_history": 0.7195121951219512, - "business_ethics": 0.5757575757575758, - "clinical_knowledge": 0.625, - "medical_genetics": 0.6868686868686869, - "high_school_us_history": 0.7783251231527094, - "high_school_physics": 0.36666666666666664, - "high_school_world_history": 0.8008474576271186, - "virology": 0.46060606060606063, - "high_school_microeconomics": 0.6624472573839663, - "econometrics": 0.36283185840707965, - "college_computer_science": 0.5454545454545454, - "high_school_biology": 0.7346278317152104, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.45907473309608543, - "philosophy": 0.6451612903225806, - "professional_medicine": 0.6051660516605166, - "nutrition": 0.6295081967213115, - "global_facts": 0.29292929292929293, - "machine_learning": 0.35135135135135137, - "security_studies": 0.7090163934426229, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.6088379705400983, - "prehistory": 0.6934984520123839, - "anatomy": 0.5223880597014925, - "human_sexuality": 0.6692307692307692, - "college_medicine": 0.6162790697674418, - "high_school_government_and_politics": 0.8333333333333334, - "college_chemistry": 0.43434343434343436, - "logical_fallacies": 0.691358024691358, - "high_school_geography": 0.766497461928934, - "elementary_mathematics": 0.4350132625994695, - "human_aging": 0.6531531531531531, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.7977941176470589, - "formal_logic": 0.408, - "high_school_statistics": 0.5023255813953489, - "international_law": 0.75, - "high_school_mathematics": 0.32342007434944237, - "high_school_computer_science": 0.6666666666666666, - "conceptual_physics": 0.5128205128205128, - "miscellaneous": 0.7800511508951407, - "high_school_chemistry": 0.4801980198019802, - "marketing": 0.8497854077253219, - "professional_law": 0.44944553163731243, - "management": 0.7941176470588235, - "college_physics": 0.3465346534653465, - "jurisprudence": 0.719626168224299, - "world_religions": 0.7823529411764706, - "sociology": 0.8, - "us_foreign_policy": 0.8585858585858586, - "high_school_macroeconomics": 0.5886889460154242, - "computer_security": 0.6868686868686869, - "moral_scenarios": 0.3512304250559284, - "moral_disputes": 0.7072463768115942, - "electrical_engineering": 0.5208333333333334, - "astronomy": 0.6490066225165563, - "college_biology": 0.6993006993006993 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.45690936106983654 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.46450809464508097, - "category_acc": { - "computer_network": 0.5833333333333334, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.6153846153846154, - "college_programming": 0.6190476190476191, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.42857142857142855, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.4827586206896552, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.25, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.5769230769230769, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.64, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.43333333333333335, - "business_administration": 0.39473684210526316, - "marxism": 0.5, - "mao_zedong_thought": 0.5862068965517241, - "education_science": 0.5882352941176471, - "teacher_qualification": 0.6326530612244898, - "high_school_politics": 0.5416666666666666, - "high_school_geography": 0.5833333333333334, - "middle_school_politics": 0.6923076923076923, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.48148148148148145, - "law": 0.4482758620689655, - "chinese_language_and_literature": 0.4642857142857143, - "art_studies": 0.5, - "professional_tour_guide": 0.5, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.48, - "middle_school_history": 0.48148148148148145, - "civil_servant": 0.4423076923076923, - "sports_science": 0.2916666666666667, - "plant_protection": 0.5925925925925926, - "basic_medicine": 0.5, - "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.5490196078431373, - "accountant": 0.42592592592592593, - "fire_engineer": 0.4722222222222222, - "environmental_impact_assessment_engineer": 0.5555555555555556, - "tax_accountant": 0.35185185185185186, - "physician": 0.3888888888888889 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.4767025089605735 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.45346226903816267, - "category_acc": { - "agronomy": 0.4319526627218935, - "anatomy": 0.2905405405405405, - "ancient_chinese": 0.3048780487804878, - "arts": 0.4125, - "astronomy": 0.2727272727272727, - "business_ethics": 0.5454545454545454, - "chinese_civil_service_exam": 0.425, - "chinese_driving_rule": 0.6183206106870229, - "chinese_food_culture": 0.3161764705882353, - "chinese_foreign_policy": 0.5420560747663551, - "chinese_history": 0.47368421052631576, - "chinese_literature": 0.37745098039215685, - "chinese_teacher_qualification": 0.5139664804469274, - "clinical_knowledge": 0.3881856540084388, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.5700934579439252, - "college_engineering_hydrology": 0.5094339622641509, - "college_law": 0.39814814814814814, - "college_mathematics": 0.3333333333333333, - "college_medical_statistics": 0.42452830188679247, - "college_medicine": 0.3443223443223443, - "computer_science": 0.553921568627451, - "computer_security": 0.5789473684210527, - "conceptual_physics": 0.4421768707482993, - "construction_project_management": 0.4892086330935252, - "economics": 0.5094339622641509, - "education": 0.558282208588957, - "electrical_engineering": 0.43023255813953487, - "elementary_chinese": 0.24603174603174602, - "elementary_commonsense": 0.45454545454545453, - "elementary_information_and_technology": 0.7016806722689075, - "elementary_mathematics": 0.27391304347826084, - "ethnology": 0.42962962962962964, - "food_science": 0.48951048951048953, - "genetics": 0.35795454545454547, - "global_facts": 0.5302013422818792, - "high_school_biology": 0.33727810650887574, - "high_school_chemistry": 0.3484848484848485, - "high_school_geography": 0.4576271186440678, - "high_school_mathematics": 0.3170731707317073, - "high_school_physics": 0.38181818181818183, - "high_school_politics": 0.48951048951048953, - "human_sexuality": 0.5238095238095238, - "international_law": 0.33513513513513515, - "journalism": 0.5116279069767442, - "jurisprudence": 0.4768856447688564, - "legal_and_moral_basis": 0.7476635514018691, - "logical": 0.4796747967479675, - "machine_learning": 0.4918032786885246, - "management": 0.5714285714285714, - "marketing": 0.6, - "marxist_theory": 0.5185185185185185, - "modern_chinese": 0.31896551724137934, - "nutrition": 0.5241379310344828, - "philosophy": 0.4666666666666667, - "professional_accounting": 0.5428571428571428, - "professional_law": 0.33175355450236965, - "professional_medicine": 0.3324468085106383, - "professional_psychology": 0.49137931034482757, - "public_relations": 0.5804597701149425, - "security_study": 0.5481481481481482, - "sociology": 0.5176991150442478, - "sports_science": 0.44242424242424244, - "traditional_chinese_medicine": 0.3675675675675676, - "virology": 0.4911242603550296, - "world_history": 0.5341614906832298, - "world_religions": 0.53125 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.42424242424242425 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.6704545454545454 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.536271186440678 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.8362004487658938 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.8995590396864283 - } + "prompt_1": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.42534417252625945, - "rouge2": 0.18435345325472052, - "rougeL": 0.3335734512474979, - "avg_rouge": 0.3144236923428259 - } + "prompt_1": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.3774344892901333, - "rouge2": 0.14144067198466828, - "rougeL": 0.2953664619368561, - "avg_rouge": 0.2714138744038859 - } + "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.9071100917431193 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.5906040268456376 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.7265 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.661 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.6865 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.6197183098591549 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.7003610108303249 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.6887254901960784 - } + "prompt_1": -1 } } }, - "gemma_2b": { - "model_size": "7B", - "model_link": "https://huggingface.co/google/gemma-2b", + "mt0_xxl": { + "model_size": "13B", + "model_link": "https://huggingface.co/bigscience/mt0-xxl", "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.46680672268907564, + "overall_acc": 0.9678571428571429, "language_acc": { - "English": 0.4966386554621849, - "Chinese": 0.47394957983193275, - "Spanish": 0.4336134453781513, - "Vietnamese": 0.46302521008403363 - }, - "consistency_score_2": 0.5715686274509804, - "consistency_score_3": 0.4147058823529412, - "consistency_score_4": 0.3310924369747899, + "Spanish": 0.9705882352941176, + "English": 0.9722689075630252, + "Chinese": 0.957983193277311, + "Vietnamese": 0.9705882352941176 + }, + "consistency_score_2": 0.9642857142857143, + "consistency_score_3": 0.9474789915966386, + "consistency_score_4": 0.9352941176470588, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.5882352941176471, - "English,Spanish": 0.6991596638655462, - "English,Vietnamese": 0.5747899159663865, - "Chinese,Spanish": 0.5361344537815126, - "Chinese,Vietnamese": 0.5058823529411764, - "Spanish,Vietnamese": 0.5252100840336135 + "Spanish,English": 0.9773109243697479, + "Spanish,Chinese": 0.957983193277311, + "Spanish,Vietnamese": 0.9680672268907563, + "English,Chinese": 0.9596638655462185, + "English,Vietnamese": 0.9680672268907563, + "Chinese,Vietnamese": 0.9546218487394958 }, "3_combine": { - "English,Chinese,Spanish": 0.44873949579831934, - "English,Chinese,Vietnamese": 0.39915966386554624, - "English,Spanish,Vietnamese": 0.44201680672268906, - "Chinese,Spanish,Vietnamese": 0.3689075630252101 + "Spanish,English,Chinese": 0.9487394957983193, + "Spanish,English,Vietnamese": 0.957983193277311, + "Spanish,Chinese,Vietnamese": 0.9411764705882353, + "English,Chinese,Vietnamese": 0.9420168067226891 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3310924369747899 + "Spanish,English,Chinese,Vietnamese": 0.9352941176470588 } }, - "AC3_2": 0.5139029498545062, - "AC3_3": 0.4392166208215929, - "AC3_4": 0.3874077908719221 + "AC3_2": 0.9660681277556511, + "AC3_3": 0.9575596608660338, + "AC3_4": 0.9512970526048331 }, "prompt_2": { - "overall_acc": 0.46911764705882353, + "overall_acc": 0.969327731092437, "language_acc": { - "English": 0.5319327731092437, - "Chinese": 0.46638655462184875, - "Spanish": 0.46554621848739497, - "Vietnamese": 0.4126050420168067 + "Spanish": 0.9722689075630252, + "English": 0.9747899159663865, + "Chinese": 0.957983193277311, + "Vietnamese": 0.9722689075630252 }, - "consistency_score_2": 0.4997198879551821, - "consistency_score_3": 0.3224789915966387, - "consistency_score_4": 0.22773109243697479, + "consistency_score_2": 0.9656862745098039, + "consistency_score_3": 0.9493697478991596, + "consistency_score_4": 0.9369747899159664, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.5, - "English,Spanish": 0.5915966386554622, - "English,Vietnamese": 0.4957983193277311, - "Chinese,Spanish": 0.4823529411764706, - "Chinese,Vietnamese": 0.4394957983193277, - "Spanish,Vietnamese": 0.4890756302521008 + "Spanish,English": 0.9781512605042016, + "Spanish,Chinese": 0.9571428571428572, + "Spanish,Vietnamese": 0.9722689075630252, + "English,Chinese": 0.9588235294117647, + "English,Vietnamese": 0.9714285714285714, + "Chinese,Vietnamese": 0.9563025210084034 }, "3_combine": { - "English,Chinese,Spanish": 0.346218487394958, - "English,Chinese,Vietnamese": 0.3, - "English,Spanish,Vietnamese": 0.35378151260504204, - "Chinese,Spanish,Vietnamese": 0.28991596638655465 + "Spanish,English,Chinese": 0.9478991596638655, + "Spanish,English,Vietnamese": 0.9621848739495799, + "Spanish,Chinese,Vietnamese": 0.9428571428571428, + "English,Chinese,Vietnamese": 0.9445378151260504 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.22773109243697479 + "Spanish,English,Chinese,Vietnamese": 0.9369747899159664 } }, - "AC3_2": 0.48393545776165303, - "AC3_3": 0.3822163419025088, - "AC3_4": 0.30661653864262284 + "AC3_2": 0.9675035763660907, + "AC3_3": 0.9592449395770114, + "AC3_4": 0.9528767203979467 }, "prompt_3": { - "overall_acc": 0.4384453781512605, + "overall_acc": 0.9689075630252101, "language_acc": { - "English": 0.4890756302521008, - "Chinese": 0.46974789915966386, - "Spanish": 0.44789915966386556, - "Vietnamese": 0.34705882352941175 - }, - "consistency_score_2": 0.44257703081232497, - "consistency_score_3": 0.2584033613445378, - "consistency_score_4": 0.1689075630252101, + "Spanish": 0.9722689075630252, + "English": 0.9756302521008403, + "Chinese": 0.9554621848739496, + "Vietnamese": 0.9722689075630252 + }, + "consistency_score_2": 0.965546218487395, + "consistency_score_3": 0.9493697478991596, + "consistency_score_4": 0.9369747899159664, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.5042016806722689, - "English,Spanish": 0.6142857142857143, - "English,Vietnamese": 0.36470588235294116, - "Chinese,Spanish": 0.4764705882352941, - "Chinese,Vietnamese": 0.3403361344537815, - "Spanish,Vietnamese": 0.3554621848739496 + "Spanish,English": 0.9815126050420168, + "Spanish,Chinese": 0.9537815126050421, + "Spanish,Vietnamese": 0.973109243697479, + "English,Chinese": 0.9554621848739496, + "English,Vietnamese": 0.9739495798319328, + "Chinese,Vietnamese": 0.9554621848739496 }, "3_combine": { - "English,Chinese,Spanish": 0.36050420168067226, - "English,Chinese,Vietnamese": 0.21008403361344538, - "English,Spanish,Vietnamese": 0.2546218487394958, - "Chinese,Spanish,Vietnamese": 0.20840336134453782 + "Spanish,English,Chinese": 0.946218487394958, + "Spanish,English,Vietnamese": 0.965546218487395, + "Spanish,Chinese,Vietnamese": 0.9420168067226891, + "English,Chinese,Vietnamese": 0.9436974789915966 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.1689075630252101 + "Spanish,English,Chinese,Vietnamese": 0.9369747899159664 } }, - "AC3_2": 0.44050151650925184, - "AC3_3": 0.3251660024411582, - "AC3_4": 0.24386723208272088 + "AC3_2": 0.9672239703374601, + "AC3_3": 0.9590391582702165, + "AC3_4": 0.9526736620074296 }, "prompt_4": { - "overall_acc": 0.49411764705882355, + "overall_acc": 0.9686974789915966, "language_acc": { - "English": 0.5319327731092437, - "Chinese": 0.5025210084033613, - "Spanish": 0.4647058823529412, - "Vietnamese": 0.4773109243697479 - }, - "consistency_score_2": 0.6410364145658264, - "consistency_score_3": 0.5069327731092437, - "consistency_score_4": 0.4319327731092437, + "Spanish": 0.9722689075630252, + "English": 0.9747899159663865, + "Chinese": 0.9563025210084034, + "Vietnamese": 0.9714285714285714 + }, + "consistency_score_2": 0.9656862745098039, + "consistency_score_3": 0.9495798319327731, + "consistency_score_4": 0.9369747899159664, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.6764705882352942, - "English,Spanish": 0.7596638655462185, - "English,Vietnamese": 0.6084033613445378, - "Chinese,Spanish": 0.6546218487394958, - "Chinese,Vietnamese": 0.5605042016806723, - "Spanish,Vietnamese": 0.5865546218487395 + "Spanish,English": 0.9798319327731092, + "Spanish,Chinese": 0.9563025210084034, + "Spanish,Vietnamese": 0.9722689075630252, + "English,Chinese": 0.9596638655462185, + "English,Vietnamese": 0.9705882352941176, + "Chinese,Vietnamese": 0.9554621848739496 }, "3_combine": { - "English,Chinese,Spanish": 0.5756302521008403, - "English,Chinese,Vietnamese": 0.473109243697479, - "English,Spanish,Vietnamese": 0.5168067226890757, - "Chinese,Spanish,Vietnamese": 0.46218487394957986 + "Spanish,English,Chinese": 0.9487394957983193, + "Spanish,English,Vietnamese": 0.9630252100840336, + "Spanish,Chinese,Vietnamese": 0.9428571428571428, + "English,Chinese,Vietnamese": 0.9436974789915966 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.4319327731092437 + "Spanish,English,Chinese,Vietnamese": 0.9369747899159664 } }, - "AC3_2": 0.5580694559874164, - "AC3_3": 0.5004431824696057, - "AC3_4": 0.4609373331412151 + "AC3_2": 0.9671895329691594, + "AC3_3": 0.959043391701624, + "AC3_4": 0.9525720992340004 }, "prompt_5": { - "overall_acc": 0.4670168067226891, + "overall_acc": 0.9697478991596639, "language_acc": { - "English": 0.48739495798319327, - "Chinese": 0.49411764705882355, - "Spanish": 0.44873949579831934, - "Vietnamese": 0.43781512605042017 - }, - "consistency_score_2": 0.5677871148459385, - "consistency_score_3": 0.4119747899159664, - "consistency_score_4": 0.3243697478991597, + "Spanish": 0.9714285714285714, + "English": 0.9747899159663865, + "Chinese": 0.9588235294117647, + "Vietnamese": 0.9739495798319328 + }, + "consistency_score_2": 0.9666666666666667, + "consistency_score_3": 0.9508403361344537, + "consistency_score_4": 0.938655462184874, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.6134453781512605, - "English,Spanish": 0.7310924369747899, - "English,Vietnamese": 0.5084033613445378, - "Chinese,Spanish": 0.5915966386554622, - "Chinese,Vietnamese": 0.46218487394957986, - "Spanish,Vietnamese": 0.5 + "Spanish,English": 0.9798319327731092, + "Spanish,Chinese": 0.9571428571428572, + "Spanish,Vietnamese": 0.9705882352941176, + "English,Chinese": 0.9588235294117647, + "English,Vietnamese": 0.9739495798319328, + "Chinese,Vietnamese": 0.9596638655462185 }, "3_combine": { - "English,Chinese,Spanish": 0.5016806722689076, - "English,Chinese,Vietnamese": 0.373109243697479, - "English,Spanish,Vietnamese": 0.4142857142857143, - "Chinese,Spanish,Vietnamese": 0.3588235294117647 + "Spanish,English,Chinese": 0.9487394957983193, + "Spanish,English,Vietnamese": 0.9630252100840336, + "Spanish,Chinese,Vietnamese": 0.9445378151260504, + "English,Chinese,Vietnamese": 0.9470588235294117 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3243697478991597 + "Spanish,English,Chinese,Vietnamese": 0.938655462184874 } }, - "AC3_2": 0.5124954007635706, - "AC3_3": 0.43777244640487745, - "AC3_4": 0.38283724422018406 + "AC3_2": 0.9682048314269277, + "AC3_3": 0.960201048210044, + "AC3_4": 0.9539483956887579 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.35809523809523813, + "overall_acc": 0.45142857142857146, "language_acc": { - "Spanish": 0.32666666666666666, - "English": 0.37333333333333335, - "Chinese": 0.3466666666666667, - "Vietnamese": 0.38, - "Filipino": 0.38666666666666666, - "Indonesian": 0.36, - "Malay": 0.3333333333333333 + "Filipino": 0.4533333333333333, + "Vietnamese": 0.43333333333333335, + "Chinese": 0.4533333333333333, + "Spanish": 0.42, + "Malay": 0.48, + "Indonesian": 0.46, + "English": 0.46 }, - "consistency_score_2": 0.5409523809523809, - "consistency_score_3": 0.3622857142857142, - "consistency_score_4": 0.2666666666666666, - "consistency_score_5": 0.20825396825396825, - "consistency_score_6": 0.17047619047619048, - "consistency_score_7": 0.14666666666666667, + "consistency_score_2": 0.6977777777777778, + "consistency_score_3": 0.570857142857143, + "consistency_score_4": 0.49542857142857144, + "consistency_score_5": 0.44317460317460317, + "consistency_score_6": 0.40380952380952373, + "consistency_score_7": 0.37333333333333335, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.64, - "Spanish,Chinese": 0.58, - "Spanish,Vietnamese": 0.5, - "Spanish,Filipino": 0.6333333333333333, - "Spanish,Indonesian": 0.5, - "Spanish,Malay": 0.5666666666666667, - "English,Chinese": 0.54, - "English,Vietnamese": 0.43333333333333335, - "English,Filipino": 0.56, - "English,Indonesian": 0.4866666666666667, - "English,Malay": 0.5266666666666666, - "Chinese,Vietnamese": 0.46, - "Chinese,Filipino": 0.4666666666666667, - "Chinese,Indonesian": 0.5, - "Chinese,Malay": 0.56, - "Vietnamese,Filipino": 0.52, - "Vietnamese,Indonesian": 0.52, - "Vietnamese,Malay": 0.49333333333333335, - "Filipino,Indonesian": 0.6333333333333333, - "Filipino,Malay": 0.62, - "Indonesian,Malay": 0.62 + "Filipino,Vietnamese": 0.7066666666666667, + "Filipino,Chinese": 0.5933333333333334, + "Filipino,Spanish": 0.7266666666666667, + "Filipino,Malay": 0.74, + "Filipino,Indonesian": 0.7, + "Filipino,English": 0.7733333333333333, + "Vietnamese,Chinese": 0.6266666666666667, + "Vietnamese,Spanish": 0.6733333333333333, + "Vietnamese,Malay": 0.7066666666666667, + "Vietnamese,Indonesian": 0.7, + "Vietnamese,English": 0.7133333333333334, + "Chinese,Spanish": 0.6333333333333333, + "Chinese,Malay": 0.5733333333333334, + "Chinese,Indonesian": 0.6533333333333333, + "Chinese,English": 0.6466666666666666, + "Spanish,Malay": 0.7266666666666667, + "Spanish,Indonesian": 0.7333333333333333, + "Spanish,English": 0.7733333333333333, + "Malay,Indonesian": 0.76, + "Malay,English": 0.76, + "Indonesian,English": 0.7333333333333333 }, "3_combine": { - "Spanish,English,Chinese": 0.4266666666666667, - "Spanish,English,Vietnamese": 0.3333333333333333, - "Spanish,English,Filipino": 0.44666666666666666, - "Spanish,English,Indonesian": 0.34, - "Spanish,English,Malay": 0.4066666666666667, - "Spanish,Chinese,Vietnamese": 0.3333333333333333, - "Spanish,Chinese,Filipino": 0.38, - "Spanish,Chinese,Indonesian": 0.36, - "Spanish,Chinese,Malay": 0.4066666666666667, - "Spanish,Vietnamese,Filipino": 0.36666666666666664, - "Spanish,Vietnamese,Indonesian": 0.32, - "Spanish,Vietnamese,Malay": 0.35333333333333333, - "Spanish,Filipino,Indonesian": 0.41333333333333333, - "Spanish,Filipino,Malay": 0.44666666666666666, - "Spanish,Indonesian,Malay": 0.3933333333333333, - "English,Chinese,Vietnamese": 0.2866666666666667, - "English,Chinese,Filipino": 0.3333333333333333, - "English,Chinese,Indonesian": 0.31333333333333335, - "English,Chinese,Malay": 0.3933333333333333, - "English,Vietnamese,Filipino": 0.29333333333333333, - "English,Vietnamese,Indonesian": 0.28, - "English,Vietnamese,Malay": 0.2866666666666667, - "English,Filipino,Indonesian": 0.37333333333333335, - "English,Filipino,Malay": 0.3933333333333333, - "English,Indonesian,Malay": 0.36, - "Chinese,Vietnamese,Filipino": 0.29333333333333333, - "Chinese,Vietnamese,Indonesian": 0.32, - "Chinese,Vietnamese,Malay": 0.32, - "Chinese,Filipino,Indonesian": 0.35333333333333333, - "Chinese,Filipino,Malay": 0.38666666666666666, - "Chinese,Indonesian,Malay": 0.38666666666666666, - "Vietnamese,Filipino,Indonesian": 0.36, - "Vietnamese,Filipino,Malay": 0.36, - "Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Filipino,Indonesian,Malay": 0.47333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.24, - "Spanish,English,Chinese,Filipino": 0.30666666666666664, - "Spanish,English,Chinese,Indonesian": 0.26, - "Spanish,English,Chinese,Malay": 0.32666666666666666, - "Spanish,English,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.21333333333333335, - "Spanish,English,Vietnamese,Malay": 0.24, - "Spanish,English,Filipino,Indonesian": 0.3, - "Spanish,English,Filipino,Malay": 0.32666666666666666, - "Spanish,English,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.24666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.3, - "Spanish,Chinese,Filipino,Malay": 0.32666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.3, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.28, - "Spanish,Vietnamese,Indonesian,Malay": 0.28, - "Spanish,Filipino,Indonesian,Malay": 0.34, - "English,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.2, - "English,Chinese,Vietnamese,Malay": 0.21333333333333335, - "English,Chinese,Filipino,Indonesian": 0.24, - "English,Chinese,Filipino,Malay": 0.2866666666666667, - "English,Chinese,Indonesian,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.22, - "English,Vietnamese,Filipino,Malay": 0.22, - "English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "English,Filipino,Indonesian,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Filipino,Indonesian": 0.24, - "Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.31333333333333335, - "Vietnamese,Filipino,Indonesian,Malay": 0.3 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.18666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "English,Chinese,Filipino,Indonesian,Malay": 0.22, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 - } - }, - "AC3_2": 0.4309281678293888, - "AC3_3": 0.3601782880817536, - "AC3_4": 0.30569105686163983, - "AC3_5": 0.26335255173654504, - "AC3_6": 0.23098755894385867, - "AC3_7": 0.20810062888959016 - }, - "prompt_2": { - "overall_acc": 0.37142857142857144, + "Filipino,Vietnamese,Chinese": 0.49333333333333335, + "Filipino,Vietnamese,Spanish": 0.5733333333333334, + "Filipino,Vietnamese,Malay": 0.6, + "Filipino,Vietnamese,Indonesian": 0.5866666666666667, + "Filipino,Vietnamese,English": 0.6133333333333333, + "Filipino,Chinese,Spanish": 0.5133333333333333, + "Filipino,Chinese,Malay": 0.4866666666666667, + "Filipino,Chinese,Indonesian": 0.49333333333333335, + "Filipino,Chinese,English": 0.5333333333333333, + "Filipino,Spanish,Malay": 0.6133333333333333, + "Filipino,Spanish,Indonesian": 0.6, + "Filipino,Spanish,English": 0.6666666666666666, + "Filipino,Malay,Indonesian": 0.62, + "Filipino,Malay,English": 0.6466666666666666, + "Filipino,Indonesian,English": 0.62, + "Vietnamese,Chinese,Spanish": 0.5066666666666667, + "Vietnamese,Chinese,Malay": 0.4866666666666667, + "Vietnamese,Chinese,Indonesian": 0.52, + "Vietnamese,Chinese,English": 0.5133333333333333, + "Vietnamese,Spanish,Malay": 0.5666666666666667, + "Vietnamese,Spanish,Indonesian": 0.5866666666666667, + "Vietnamese,Spanish,English": 0.6066666666666667, + "Vietnamese,Malay,Indonesian": 0.6066666666666667, + "Vietnamese,Malay,English": 0.6066666666666667, + "Vietnamese,Indonesian,English": 0.5933333333333334, + "Chinese,Spanish,Malay": 0.5, + "Chinese,Spanish,Indonesian": 0.5466666666666666, + "Chinese,Spanish,English": 0.5466666666666666, + "Chinese,Malay,Indonesian": 0.52, + "Chinese,Malay,English": 0.5066666666666667, + "Chinese,Indonesian,English": 0.5333333333333333, + "Spanish,Malay,Indonesian": 0.6333333333333333, + "Spanish,Malay,English": 0.6533333333333333, + "Spanish,Indonesian,English": 0.64, + "Malay,Indonesian,English": 0.6466666666666666 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.44, + "Filipino,Vietnamese,Chinese,Malay": 0.44, + "Filipino,Vietnamese,Chinese,Indonesian": 0.43333333333333335, + "Filipino,Vietnamese,Chinese,English": 0.46, + "Filipino,Vietnamese,Spanish,Malay": 0.5133333333333333, + "Filipino,Vietnamese,Spanish,Indonesian": 0.52, + "Filipino,Vietnamese,Spanish,English": 0.5466666666666666, + "Filipino,Vietnamese,Malay,Indonesian": 0.54, + "Filipino,Vietnamese,Malay,English": 0.5533333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.5333333333333333, + "Filipino,Chinese,Spanish,Malay": 0.43333333333333335, + "Filipino,Chinese,Spanish,Indonesian": 0.4533333333333333, + "Filipino,Chinese,Spanish,English": 0.48, + "Filipino,Chinese,Malay,Indonesian": 0.4533333333333333, + "Filipino,Chinese,Malay,English": 0.4533333333333333, + "Filipino,Chinese,Indonesian,English": 0.46, + "Filipino,Spanish,Malay,Indonesian": 0.5466666666666666, + "Filipino,Spanish,Malay,English": 0.58, + "Filipino,Spanish,Indonesian,English": 0.5666666666666667, + "Filipino,Malay,Indonesian,English": 0.5666666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.44, + "Vietnamese,Chinese,Spanish,Indonesian": 0.4533333333333333, + "Vietnamese,Chinese,Spanish,English": 0.46, + "Vietnamese,Chinese,Malay,Indonesian": 0.4533333333333333, + "Vietnamese,Chinese,Malay,English": 0.44666666666666666, + "Vietnamese,Chinese,Indonesian,English": 0.44666666666666666, + "Vietnamese,Spanish,Malay,Indonesian": 0.54, + "Vietnamese,Spanish,Malay,English": 0.54, + "Vietnamese,Spanish,Indonesian,English": 0.5466666666666666, + "Vietnamese,Malay,Indonesian,English": 0.5533333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.48, + "Chinese,Spanish,Malay,English": 0.46, + "Chinese,Spanish,Indonesian,English": 0.4866666666666667, + "Chinese,Malay,Indonesian,English": 0.47333333333333333, + "Spanish,Malay,Indonesian,English": 0.5866666666666667 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.4, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.4, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.42, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.41333333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.42, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.4066666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.4866666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.5, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.5, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.5066666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.42, + "Filipino,Chinese,Spanish,Malay,English": 0.41333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.43333333333333335, + "Filipino,Chinese,Malay,Indonesian,English": 0.4266666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.52, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4266666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.41333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.42, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.42, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.5133333333333333, + "Chinese,Spanish,Malay,Indonesian,English": 0.44666666666666666 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.38666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.38666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.38666666666666666, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.3933333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.47333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.4, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.37333333333333335 + } + }, + "AC3_2": 0.5481988949799219, + "AC3_3": 0.5041667331615745, + "AC3_4": 0.47240624186751473, + "AC3_5": 0.4472635100382127, + "AC3_6": 0.426293350252414, + "AC3_7": 0.40868360272181087 + }, + "prompt_2": { + "overall_acc": 0.44476190476190475, "language_acc": { - "Spanish": 0.42, - "English": 0.46, - "Chinese": 0.36, - "Vietnamese": 0.36, - "Filipino": 0.31333333333333335, - "Indonesian": 0.34, - "Malay": 0.3466666666666667 - }, - "consistency_score_2": 0.480952380952381, - "consistency_score_3": 0.2883809523809524, - "consistency_score_4": 0.1927619047619048, - "consistency_score_5": 0.13904761904761903, - "consistency_score_6": 0.10666666666666669, - "consistency_score_7": 0.08666666666666667, + "Filipino": 0.4266666666666667, + "Vietnamese": 0.44, + "Chinese": 0.46, + "Spanish": 0.4, + "Malay": 0.4866666666666667, + "Indonesian": 0.43333333333333335, + "English": 0.4666666666666667 + }, + "consistency_score_2": 0.7069841269841268, + "consistency_score_3": 0.584190476190476, + "consistency_score_4": 0.5085714285714285, + "consistency_score_5": 0.45396825396825397, + "consistency_score_6": 0.41238095238095235, + "consistency_score_7": 0.38, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.56, - "Spanish,Chinese": 0.5133333333333333, - "Spanish,Vietnamese": 0.5, - "Spanish,Filipino": 0.4666666666666667, - "Spanish,Indonesian": 0.42, - "Spanish,Malay": 0.49333333333333335, - "English,Chinese": 0.47333333333333333, - "English,Vietnamese": 0.43333333333333335, - "English,Filipino": 0.48, - "English,Indonesian": 0.46, - "English,Malay": 0.5066666666666667, - "Chinese,Vietnamese": 0.4, - "Chinese,Filipino": 0.4266666666666667, - "Chinese,Indonesian": 0.43333333333333335, - "Chinese,Malay": 0.47333333333333333, - "Vietnamese,Filipino": 0.52, - "Vietnamese,Indonesian": 0.43333333333333335, - "Vietnamese,Malay": 0.4533333333333333, - "Filipino,Indonesian": 0.49333333333333335, - "Filipino,Malay": 0.5733333333333334, - "Indonesian,Malay": 0.5866666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.32666666666666666, - "Spanish,English,Vietnamese": 0.29333333333333333, - "Spanish,English,Filipino": 0.31333333333333335, - "Spanish,English,Indonesian": 0.29333333333333333, - "Spanish,English,Malay": 0.32666666666666666, - "Spanish,Chinese,Vietnamese": 0.26666666666666666, - "Spanish,Chinese,Filipino": 0.2733333333333333, - "Spanish,Chinese,Indonesian": 0.28, - "Spanish,Chinese,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino": 0.32666666666666666, - "Spanish,Vietnamese,Indonesian": 0.26, - "Spanish,Vietnamese,Malay": 0.29333333333333333, - "Spanish,Filipino,Indonesian": 0.26, - "Spanish,Filipino,Malay": 0.32666666666666666, - "Spanish,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese": 0.22, - "English,Chinese,Filipino": 0.26, - "English,Chinese,Indonesian": 0.28, - "English,Chinese,Malay": 0.3, - "English,Vietnamese,Filipino": 0.2866666666666667, - "English,Vietnamese,Indonesian": 0.24666666666666667, - "English,Vietnamese,Malay": 0.25333333333333335, - "English,Filipino,Indonesian": 0.2733333333333333, - "English,Filipino,Malay": 0.34, - "English,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino": 0.25333333333333335, - "Chinese,Vietnamese,Indonesian": 0.24, - "Chinese,Vietnamese,Malay": 0.22666666666666666, - "Chinese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Filipino,Malay": 0.3, - "Chinese,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian": 0.28, - "Vietnamese,Filipino,Malay": 0.31333333333333335, - "Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Filipino,Indonesian,Malay": 0.35333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.17333333333333334, - "Spanish,English,Chinese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Indonesian": 0.21333333333333335, - "Spanish,English,Chinese,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.18, - "Spanish,English,Vietnamese,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Malay": 0.17333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Filipino,Malay": 0.22, - "Spanish,Chinese,Indonesian,Malay": 0.22, - "Spanish,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.16, - "English,Chinese,Vietnamese,Indonesian": 0.16, - "English,Chinese,Vietnamese,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian": 0.16666666666666666, - "English,Chinese,Filipino,Malay": 0.2, - "English,Chinese,Indonesian,Malay": 0.22, - "English,Vietnamese,Filipino,Indonesian": 0.18, - "English,Vietnamese,Filipino,Malay": 0.2, - "English,Vietnamese,Indonesian,Malay": 0.18, - "English,Filipino,Indonesian,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Chinese,Filipino,Indonesian,Malay": 0.22, - "Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian": 0.14, - "Spanish,English,Chinese,Filipino,Malay": 0.16, - "Spanish,English,Chinese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.12, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 - } - }, - "AC3_2": 0.41915403027804027, - "AC3_3": 0.3246783239362755, - "AC3_4": 0.2538053438346323, - "AC3_5": 0.202345415738613, - "AC3_6": 0.1657370517581626, - "AC3_7": 0.14054054050986123 + "Filipino,Vietnamese": 0.7066666666666667, + "Filipino,Chinese": 0.62, + "Filipino,Spanish": 0.7466666666666667, + "Filipino,Malay": 0.74, + "Filipino,Indonesian": 0.7333333333333333, + "Filipino,English": 0.7666666666666667, + "Vietnamese,Chinese": 0.68, + "Vietnamese,Spanish": 0.66, + "Vietnamese,Malay": 0.7133333333333334, + "Vietnamese,Indonesian": 0.7266666666666667, + "Vietnamese,English": 0.7, + "Chinese,Spanish": 0.64, + "Chinese,Malay": 0.6133333333333333, + "Chinese,Indonesian": 0.6666666666666666, + "Chinese,English": 0.6466666666666666, + "Spanish,Malay": 0.7333333333333333, + "Spanish,Indonesian": 0.7133333333333334, + "Spanish,English": 0.74, + "Malay,Indonesian": 0.78, + "Malay,English": 0.7733333333333333, + "Indonesian,English": 0.7466666666666667 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.5266666666666666, + "Filipino,Vietnamese,Spanish": 0.58, + "Filipino,Vietnamese,Malay": 0.6066666666666667, + "Filipino,Vietnamese,Indonesian": 0.62, + "Filipino,Vietnamese,English": 0.6066666666666667, + "Filipino,Chinese,Spanish": 0.5333333333333333, + "Filipino,Chinese,Malay": 0.5, + "Filipino,Chinese,Indonesian": 0.5333333333333333, + "Filipino,Chinese,English": 0.5466666666666666, + "Filipino,Spanish,Malay": 0.6266666666666667, + "Filipino,Spanish,Indonesian": 0.6266666666666667, + "Filipino,Spanish,English": 0.64, + "Filipino,Malay,Indonesian": 0.6466666666666666, + "Filipino,Malay,English": 0.6533333333333333, + "Filipino,Indonesian,English": 0.6466666666666666, + "Vietnamese,Chinese,Spanish": 0.5266666666666666, + "Vietnamese,Chinese,Malay": 0.5333333333333333, + "Vietnamese,Chinese,Indonesian": 0.5666666666666667, + "Vietnamese,Chinese,English": 0.5466666666666666, + "Vietnamese,Spanish,Malay": 0.5733333333333334, + "Vietnamese,Spanish,Indonesian": 0.5866666666666667, + "Vietnamese,Spanish,English": 0.5866666666666667, + "Vietnamese,Malay,Indonesian": 0.6333333333333333, + "Vietnamese,Malay,English": 0.6066666666666667, + "Vietnamese,Indonesian,English": 0.6133333333333333, + "Chinese,Spanish,Malay": 0.5066666666666667, + "Chinese,Spanish,Indonesian": 0.54, + "Chinese,Spanish,English": 0.54, + "Chinese,Malay,Indonesian": 0.5466666666666666, + "Chinese,Malay,English": 0.54, + "Chinese,Indonesian,English": 0.56, + "Spanish,Malay,Indonesian": 0.6333333333333333, + "Spanish,Malay,English": 0.6333333333333333, + "Spanish,Indonesian,English": 0.62, + "Malay,Indonesian,English": 0.66 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.4666666666666667, + "Filipino,Vietnamese,Chinese,Malay": 0.46, + "Filipino,Vietnamese,Chinese,Indonesian": 0.48, + "Filipino,Vietnamese,Chinese,English": 0.48, + "Filipino,Vietnamese,Spanish,Malay": 0.5266666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.5333333333333333, + "Filipino,Vietnamese,Spanish,English": 0.5333333333333333, + "Filipino,Vietnamese,Malay,Indonesian": 0.5733333333333334, + "Filipino,Vietnamese,Malay,English": 0.5533333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.56, + "Filipino,Chinese,Spanish,Malay": 0.44, + "Filipino,Chinese,Spanish,Indonesian": 0.47333333333333333, + "Filipino,Chinese,Spanish,English": 0.4866666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.4666666666666667, + "Filipino,Chinese,Malay,English": 0.47333333333333333, + "Filipino,Chinese,Indonesian,English": 0.49333333333333335, + "Filipino,Spanish,Malay,Indonesian": 0.56, + "Filipino,Spanish,Malay,English": 0.5733333333333334, + "Filipino,Spanish,Indonesian,English": 0.5666666666666667, + "Filipino,Malay,Indonesian,English": 0.5866666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.4533333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.47333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.47333333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.49333333333333335, + "Vietnamese,Chinese,Malay,English": 0.48, + "Vietnamese,Chinese,Indonesian,English": 0.5, + "Vietnamese,Spanish,Malay,Indonesian": 0.54, + "Vietnamese,Spanish,Malay,English": 0.52, + "Vietnamese,Spanish,Indonesian,English": 0.5333333333333333, + "Vietnamese,Malay,Indonesian,English": 0.5666666666666667, + "Chinese,Spanish,Malay,Indonesian": 0.4666666666666667, + "Chinese,Spanish,Malay,English": 0.46, + "Chinese,Spanish,Indonesian,English": 0.49333333333333335, + "Chinese,Malay,Indonesian,English": 0.5, + "Spanish,Malay,Indonesian,English": 0.56 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.41333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.4266666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.43333333333333335, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.44, + "Filipino,Vietnamese,Chinese,Malay,English": 0.43333333333333335, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.44666666666666666, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.5, + "Filipino,Vietnamese,Spanish,Malay,English": 0.49333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.5, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.5266666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, + "Filipino,Chinese,Spanish,Malay,English": 0.42, + "Filipino,Chinese,Spanish,Indonesian,English": 0.44666666666666666, + "Filipino,Chinese,Malay,Indonesian,English": 0.44666666666666666, + "Filipino,Spanish,Malay,Indonesian,English": 0.52, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.4266666666666667, + "Vietnamese,Chinese,Spanish,Malay,English": 0.42, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.44, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.46, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.49333333333333335, + "Chinese,Spanish,Malay,Indonesian,English": 0.43333333333333335 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.3933333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.3933333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.4066666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.42, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.47333333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.4, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.38 + } + }, + "AC3_2": 0.5460224706828906, + "AC3_3": 0.5050295304599006, + "AC3_4": 0.47453118305283565, + "AC3_5": 0.4493179256367052, + "AC3_6": 0.4279597883098597, + "AC3_7": 0.4098383371327563 }, "prompt_3": { - "overall_acc": 0.36, + "overall_acc": 0.4371428571428571, "language_acc": { + "Filipino": 0.4066666666666667, + "Vietnamese": 0.4533333333333333, + "Chinese": 0.47333333333333333, "Spanish": 0.37333333333333335, - "English": 0.43333333333333335, - "Chinese": 0.32666666666666666, - "Vietnamese": 0.38666666666666666, - "Filipino": 0.32, - "Indonesian": 0.3466666666666667, - "Malay": 0.3333333333333333 - }, - "consistency_score_2": 0.47492063492063497, - "consistency_score_3": 0.2889523809523809, - "consistency_score_4": 0.1977142857142857, - "consistency_score_5": 0.14349206349206348, - "consistency_score_6": 0.10666666666666666, - "consistency_score_7": 0.08, + "Malay": 0.47333333333333333, + "Indonesian": 0.4266666666666667, + "English": 0.4533333333333333 + }, + "consistency_score_2": 0.6971428571428572, + "consistency_score_3": 0.5697142857142856, + "consistency_score_4": 0.4956190476190476, + "consistency_score_5": 0.4453968253968254, + "consistency_score_6": 0.40857142857142853, + "consistency_score_7": 0.38, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.58, - "Spanish,Chinese": 0.4533333333333333, - "Spanish,Vietnamese": 0.43333333333333335, - "Spanish,Filipino": 0.49333333333333335, - "Spanish,Indonesian": 0.4266666666666667, - "Spanish,Malay": 0.42, - "English,Chinese": 0.44666666666666666, - "English,Vietnamese": 0.42, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.4866666666666667, - "English,Malay": 0.44, - "Chinese,Vietnamese": 0.35333333333333333, - "Chinese,Filipino": 0.4, - "Chinese,Indonesian": 0.4, - "Chinese,Malay": 0.4533333333333333, - "Vietnamese,Filipino": 0.5066666666666667, - "Vietnamese,Indonesian": 0.48, - "Vietnamese,Malay": 0.5133333333333333, - "Filipino,Indonesian": 0.56, - "Filipino,Malay": 0.58, - "Indonesian,Malay": 0.66 + "Filipino,Vietnamese": 0.6666666666666666, + "Filipino,Chinese": 0.6266666666666667, + "Filipino,Spanish": 0.72, + "Filipino,Malay": 0.72, + "Filipino,Indonesian": 0.6933333333333334, + "Filipino,English": 0.7666666666666667, + "Vietnamese,Chinese": 0.68, + "Vietnamese,Spanish": 0.64, + "Vietnamese,Malay": 0.7, + "Vietnamese,Indonesian": 0.7333333333333333, + "Vietnamese,English": 0.7133333333333334, + "Chinese,Spanish": 0.6333333333333333, + "Chinese,Malay": 0.58, + "Chinese,Indonesian": 0.6866666666666666, + "Chinese,English": 0.6933333333333334, + "Spanish,Malay": 0.7, + "Spanish,Indonesian": 0.7266666666666667, + "Spanish,English": 0.74, + "Malay,Indonesian": 0.7666666666666667, + "Malay,English": 0.76, + "Indonesian,English": 0.6933333333333334 }, "3_combine": { - "Spanish,English,Chinese": 0.31333333333333335, - "Spanish,English,Vietnamese": 0.30666666666666664, - "Spanish,English,Filipino": 0.34, - "Spanish,English,Indonesian": 0.3, - "Spanish,English,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese": 0.22, - "Spanish,Chinese,Filipino": 0.25333333333333335, - "Spanish,Chinese,Indonesian": 0.24, - "Spanish,Chinese,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Filipino": 0.3, - "Spanish,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Malay": 0.2733333333333333, - "Spanish,Filipino,Indonesian": 0.32, - "Spanish,Filipino,Malay": 0.31333333333333335, - "Spanish,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese": 0.19333333333333333, - "English,Chinese,Filipino": 0.23333333333333334, - "English,Chinese,Indonesian": 0.23333333333333334, - "English,Chinese,Malay": 0.24666666666666667, - "English,Vietnamese,Filipino": 0.28, - "English,Vietnamese,Indonesian": 0.26666666666666666, - "English,Vietnamese,Malay": 0.26666666666666666, - "English,Filipino,Indonesian": 0.32666666666666666, - "English,Filipino,Malay": 0.30666666666666664, - "English,Indonesian,Malay": 0.34, - "Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "Chinese,Vietnamese,Malay": 0.25333333333333335, - "Chinese,Filipino,Indonesian": 0.2733333333333333, - "Chinese,Filipino,Malay": 0.29333333333333333, - "Chinese,Indonesian,Malay": 0.3333333333333333, - "Vietnamese,Filipino,Indonesian": 0.34, - "Vietnamese,Filipino,Malay": 0.34, - "Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Filipino,Indonesian,Malay": 0.44 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.15333333333333332, - "Spanish,English,Chinese,Filipino": 0.18, - "Spanish,English,Chinese,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino": 0.22, - "Spanish,English,Vietnamese,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Filipino,Indonesian": 0.23333333333333334, - "Spanish,English,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino": 0.16, - "Spanish,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.18, - "Spanish,Chinese,Indonesian,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.26, - "English,Chinese,Vietnamese,Filipino": 0.14, - "English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "English,Chinese,Vietnamese,Malay": 0.14, - "English,Chinese,Filipino,Indonesian": 0.17333333333333334, - "English,Chinese,Filipino,Malay": 0.17333333333333334, - "English,Chinese,Indonesian,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "English,Vietnamese,Filipino,Malay": 0.21333333333333335, - "English,Vietnamese,Indonesian,Malay": 0.22, - "English,Filipino,Indonesian,Malay": 0.26, - "Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.2, - "Chinese,Filipino,Indonesian,Malay": 0.24, - "Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 + "Filipino,Vietnamese,Chinese": 0.5066666666666667, + "Filipino,Vietnamese,Spanish": 0.54, + "Filipino,Vietnamese,Malay": 0.5733333333333334, + "Filipino,Vietnamese,Indonesian": 0.5866666666666667, + "Filipino,Vietnamese,English": 0.5933333333333334, + "Filipino,Chinese,Spanish": 0.5133333333333333, + "Filipino,Chinese,Malay": 0.4866666666666667, + "Filipino,Chinese,Indonesian": 0.5266666666666666, + "Filipino,Chinese,English": 0.5666666666666667, + "Filipino,Spanish,Malay": 0.5933333333333334, + "Filipino,Spanish,Indonesian": 0.5933333333333334, + "Filipino,Spanish,English": 0.6266666666666667, + "Filipino,Malay,Indonesian": 0.6133333333333333, + "Filipino,Malay,English": 0.6333333333333333, + "Filipino,Indonesian,English": 0.6066666666666667, + "Vietnamese,Chinese,Spanish": 0.5066666666666667, + "Vietnamese,Chinese,Malay": 0.5066666666666667, + "Vietnamese,Chinese,Indonesian": 0.5733333333333334, + "Vietnamese,Chinese,English": 0.5666666666666667, + "Vietnamese,Spanish,Malay": 0.54, + "Vietnamese,Spanish,Indonesian": 0.58, + "Vietnamese,Spanish,English": 0.58, + "Vietnamese,Malay,Indonesian": 0.6266666666666667, + "Vietnamese,Malay,English": 0.6, + "Vietnamese,Indonesian,English": 0.6, + "Chinese,Spanish,Malay": 0.48, + "Chinese,Spanish,Indonesian": 0.56, + "Chinese,Spanish,English": 0.5533333333333333, + "Chinese,Malay,Indonesian": 0.54, + "Chinese,Malay,English": 0.54, + "Chinese,Indonesian,English": 0.5666666666666667, + "Spanish,Malay,Indonesian": 0.6133333333333333, + "Spanish,Malay,English": 0.6133333333333333, + "Spanish,Indonesian,English": 0.6066666666666667, + "Malay,Indonesian,English": 0.6266666666666667 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.43333333333333335, + "Filipino,Vietnamese,Chinese,Malay": 0.43333333333333335, + "Filipino,Vietnamese,Chinese,Indonesian": 0.4533333333333333, + "Filipino,Vietnamese,Chinese,English": 0.47333333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.49333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian": 0.5066666666666667, + "Filipino,Vietnamese,Spanish,English": 0.5066666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.5466666666666666, + "Filipino,Vietnamese,Malay,English": 0.54, + "Filipino,Vietnamese,Indonesian,English": 0.5333333333333333, + "Filipino,Chinese,Spanish,Malay": 0.4266666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.4666666666666667, + "Filipino,Chinese,Spanish,English": 0.4866666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.4666666666666667, + "Filipino,Chinese,Malay,English": 0.48, + "Filipino,Chinese,Indonesian,English": 0.49333333333333335, + "Filipino,Spanish,Malay,Indonesian": 0.54, + "Filipino,Spanish,Malay,English": 0.5533333333333333, + "Filipino,Spanish,Indonesian,English": 0.5466666666666666, + "Filipino,Malay,Indonesian,English": 0.5666666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.42, + "Vietnamese,Chinese,Spanish,Indonesian": 0.47333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.4666666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.48, + "Vietnamese,Chinese,Malay,English": 0.47333333333333333, + "Vietnamese,Chinese,Indonesian,English": 0.49333333333333335, + "Vietnamese,Spanish,Malay,Indonesian": 0.5133333333333333, + "Vietnamese,Spanish,Malay,English": 0.5133333333333333, + "Vietnamese,Spanish,Indonesian,English": 0.5266666666666666, + "Vietnamese,Malay,Indonesian,English": 0.5533333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.4666666666666667, + "Chinese,Spanish,Malay,English": 0.46, + "Chinese,Spanish,Indonesian,English": 0.5066666666666667, + "Chinese,Malay,Indonesian,English": 0.5066666666666667, + "Spanish,Malay,Indonesian,English": 0.5466666666666666 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.38666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.4066666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.41333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.42, + "Filipino,Vietnamese,Chinese,Malay,English": 0.4266666666666667, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.4266666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.48, + "Filipino,Vietnamese,Spanish,Malay,English": 0.48, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.48, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.5133333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.42, + "Filipino,Chinese,Spanish,Malay,English": 0.4266666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.4533333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.46, + "Filipino,Spanish,Malay,Indonesian,English": 0.5133333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, + "Vietnamese,Chinese,Spanish,Malay,English": 0.4066666666666667, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.44, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.4533333333333333, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.4866666666666667, + "Chinese,Spanish,Malay,Indonesian,English": 0.44666666666666666 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.38, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.38666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.3933333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.41333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.4666666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.42, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.38 } }, - "AC3_2": 0.40955133074942635, - "AC3_3": 0.3205870266603145, - "AC3_4": 0.25524590159357785, - "AC3_5": 0.20519546023667304, - "AC3_6": 0.16457142853616324, - "AC3_7": 0.13090909087933883 + "AC3_2": 0.5373443684304968, + "AC3_3": 0.49470082693319206, + "AC3_4": 0.46454797392843394, + "AC3_5": 0.44123124352657167, + "AC3_6": 0.42237451732457443, + "AC3_7": 0.40657342652367107 }, "prompt_4": { - "overall_acc": 0.38, + "overall_acc": 0.439047619047619, "language_acc": { - "Spanish": 0.38, - "English": 0.43333333333333335, - "Chinese": 0.36, - "Vietnamese": 0.41333333333333333, - "Filipino": 0.3333333333333333, - "Indonesian": 0.36666666666666664, - "Malay": 0.37333333333333335 - }, - "consistency_score_2": 0.54, - "consistency_score_3": 0.3615238095238095, - "consistency_score_4": 0.27276190476190476, - "consistency_score_5": 0.22222222222222224, - "consistency_score_6": 0.1895238095238095, - "consistency_score_7": 0.16666666666666666, + "Filipino": 0.4266666666666667, + "Vietnamese": 0.46, + "Chinese": 0.43333333333333335, + "Spanish": 0.4, + "Malay": 0.48, + "Indonesian": 0.41333333333333333, + "English": 0.46 + }, + "consistency_score_2": 0.6946031746031746, + "consistency_score_3": 0.5664761904761904, + "consistency_score_4": 0.49085714285714277, + "consistency_score_5": 0.43936507936507946, + "consistency_score_6": 0.40190476190476193, + "consistency_score_7": 0.37333333333333335, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6333333333333333, - "Spanish,Chinese": 0.5933333333333334, - "Spanish,Vietnamese": 0.5466666666666666, - "Spanish,Filipino": 0.5466666666666666, - "Spanish,Indonesian": 0.47333333333333333, - "Spanish,Malay": 0.56, - "English,Chinese": 0.5533333333333333, - "English,Vietnamese": 0.5, - "English,Filipino": 0.5333333333333333, - "English,Indonesian": 0.46, - "English,Malay": 0.5066666666666667, - "Chinese,Vietnamese": 0.52, - "Chinese,Filipino": 0.48, - "Chinese,Indonesian": 0.4666666666666667, - "Chinese,Malay": 0.5, - "Vietnamese,Filipino": 0.47333333333333333, - "Vietnamese,Indonesian": 0.6133333333333333, - "Vietnamese,Malay": 0.5866666666666667, - "Filipino,Indonesian": 0.54, - "Filipino,Malay": 0.5933333333333334, - "Indonesian,Malay": 0.66 + "Filipino,Vietnamese": 0.6733333333333333, + "Filipino,Chinese": 0.6, + "Filipino,Spanish": 0.7333333333333333, + "Filipino,Malay": 0.6933333333333334, + "Filipino,Indonesian": 0.68, + "Filipino,English": 0.7666666666666667, + "Vietnamese,Chinese": 0.68, + "Vietnamese,Spanish": 0.66, + "Vietnamese,Malay": 0.7066666666666667, + "Vietnamese,Indonesian": 0.72, + "Vietnamese,English": 0.72, + "Chinese,Spanish": 0.6266666666666667, + "Chinese,Malay": 0.5733333333333334, + "Chinese,Indonesian": 0.6666666666666666, + "Chinese,English": 0.66, + "Spanish,Malay": 0.6933333333333334, + "Spanish,Indonesian": 0.74, + "Spanish,English": 0.76, + "Malay,Indonesian": 0.7266666666666667, + "Malay,English": 0.76, + "Indonesian,English": 0.7466666666666667 }, "3_combine": { - "Spanish,English,Chinese": 0.43333333333333335, - "Spanish,English,Vietnamese": 0.38666666666666666, - "Spanish,English,Filipino": 0.3933333333333333, - "Spanish,English,Indonesian": 0.3333333333333333, - "Spanish,English,Malay": 0.4, - "Spanish,Chinese,Vietnamese": 0.35333333333333333, - "Spanish,Chinese,Filipino": 0.35333333333333333, - "Spanish,Chinese,Indonesian": 0.30666666666666664, - "Spanish,Chinese,Malay": 0.36666666666666664, - "Spanish,Vietnamese,Filipino": 0.36, - "Spanish,Vietnamese,Indonesian": 0.37333333333333335, - "Spanish,Vietnamese,Malay": 0.41333333333333333, - "Spanish,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Filipino,Malay": 0.4266666666666667, - "Spanish,Indonesian,Malay": 0.3933333333333333, - "English,Chinese,Vietnamese": 0.3466666666666667, - "English,Chinese,Filipino": 0.34, - "English,Chinese,Indonesian": 0.3, - "English,Chinese,Malay": 0.34, - "English,Vietnamese,Filipino": 0.31333333333333335, - "English,Vietnamese,Indonesian": 0.32666666666666666, - "English,Vietnamese,Malay": 0.35333333333333333, - "English,Filipino,Indonesian": 0.32, - "English,Filipino,Malay": 0.38666666666666666, - "English,Indonesian,Malay": 0.36, - "Chinese,Vietnamese,Filipino": 0.30666666666666664, - "Chinese,Vietnamese,Indonesian": 0.36, - "Chinese,Vietnamese,Malay": 0.35333333333333333, - "Chinese,Filipino,Indonesian": 0.29333333333333333, - "Chinese,Filipino,Malay": 0.3333333333333333, - "Chinese,Indonesian,Malay": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Vietnamese,Filipino,Malay": 0.36666666666666664, - "Vietnamese,Indonesian,Malay": 0.4533333333333333, - "Filipino,Indonesian,Malay": 0.43333333333333335 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.2866666666666667, - "Spanish,English,Chinese,Filipino": 0.28, - "Spanish,English,Chinese,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Malay": 0.30666666666666664, - "Spanish,English,Vietnamese,Filipino": 0.2733333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,English,Vietnamese,Malay": 0.3, - "Spanish,English,Filipino,Indonesian": 0.2733333333333333, - "Spanish,English,Filipino,Malay": 0.3333333333333333, - "Spanish,English,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.2866666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.2866666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.32, - "Spanish,Vietnamese,Indonesian,Malay": 0.32, - "Spanish,Filipino,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese,Filipino": 0.24666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.24, - "English,Chinese,Vietnamese,Malay": 0.2733333333333333, - "English,Chinese,Filipino,Indonesian": 0.22666666666666666, - "English,Chinese,Filipino,Malay": 0.26, - "English,Chinese,Indonesian,Malay": 0.25333333333333335, - "English,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "English,Filipino,Indonesian,Malay": 0.2866666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.28, - "Chinese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.22, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.2, - "Spanish,English,Chinese,Vietnamese,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.2, - "Spanish,English,Chinese,Filipino,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.22, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.26, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.2, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666 + "Filipino,Vietnamese,Chinese": 0.5066666666666667, + "Filipino,Vietnamese,Spanish": 0.56, + "Filipino,Vietnamese,Malay": 0.5733333333333334, + "Filipino,Vietnamese,Indonesian": 0.5733333333333334, + "Filipino,Vietnamese,English": 0.5933333333333334, + "Filipino,Chinese,Spanish": 0.5133333333333333, + "Filipino,Chinese,Malay": 0.4533333333333333, + "Filipino,Chinese,Indonesian": 0.5066666666666667, + "Filipino,Chinese,English": 0.5333333333333333, + "Filipino,Spanish,Malay": 0.5866666666666667, + "Filipino,Spanish,Indonesian": 0.6, + "Filipino,Spanish,English": 0.6466666666666666, + "Filipino,Malay,Indonesian": 0.58, + "Filipino,Malay,English": 0.6266666666666667, + "Filipino,Indonesian,English": 0.62, + "Vietnamese,Chinese,Spanish": 0.52, + "Vietnamese,Chinese,Malay": 0.5133333333333333, + "Vietnamese,Chinese,Indonesian": 0.5533333333333333, + "Vietnamese,Chinese,English": 0.5533333333333333, + "Vietnamese,Spanish,Malay": 0.5466666666666666, + "Vietnamese,Spanish,Indonesian": 0.5933333333333334, + "Vietnamese,Spanish,English": 0.5933333333333334, + "Vietnamese,Malay,Indonesian": 0.6066666666666667, + "Vietnamese,Malay,English": 0.6066666666666667, + "Vietnamese,Indonesian,English": 0.62, + "Chinese,Spanish,Malay": 0.4666666666666667, + "Chinese,Spanish,Indonesian": 0.5466666666666666, + "Chinese,Spanish,English": 0.54, + "Chinese,Malay,Indonesian": 0.5066666666666667, + "Chinese,Malay,English": 0.5133333333333333, + "Chinese,Indonesian,English": 0.56, + "Spanish,Malay,Indonesian": 0.6066666666666667, + "Spanish,Malay,English": 0.6266666666666667, + "Spanish,Indonesian,English": 0.6533333333333333, + "Malay,Indonesian,English": 0.6266666666666667 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.4533333333333333, + "Filipino,Vietnamese,Chinese,Malay": 0.43333333333333335, + "Filipino,Vietnamese,Chinese,Indonesian": 0.4533333333333333, + "Filipino,Vietnamese,Chinese,English": 0.47333333333333333, + "Filipino,Vietnamese,Spanish,Malay": 0.49333333333333335, + "Filipino,Vietnamese,Spanish,Indonesian": 0.5133333333333333, + "Filipino,Vietnamese,Spanish,English": 0.5266666666666666, + "Filipino,Vietnamese,Malay,Indonesian": 0.52, + "Filipino,Vietnamese,Malay,English": 0.5333333333333333, + "Filipino,Vietnamese,Indonesian,English": 0.5333333333333333, + "Filipino,Chinese,Spanish,Malay": 0.4066666666666667, + "Filipino,Chinese,Spanish,Indonesian": 0.4533333333333333, + "Filipino,Chinese,Spanish,English": 0.47333333333333333, + "Filipino,Chinese,Malay,Indonesian": 0.42, + "Filipino,Chinese,Malay,English": 0.44, + "Filipino,Chinese,Indonesian,English": 0.48, + "Filipino,Spanish,Malay,Indonesian": 0.52, + "Filipino,Spanish,Malay,English": 0.5533333333333333, + "Filipino,Spanish,Indonesian,English": 0.5666666666666667, + "Filipino,Malay,Indonesian,English": 0.54, + "Vietnamese,Chinese,Spanish,Malay": 0.43333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian": 0.47333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.48, + "Vietnamese,Chinese,Malay,Indonesian": 0.4666666666666667, + "Vietnamese,Chinese,Malay,English": 0.47333333333333333, + "Vietnamese,Chinese,Indonesian,English": 0.49333333333333335, + "Vietnamese,Spanish,Malay,Indonesian": 0.52, + "Vietnamese,Spanish,Malay,English": 0.5133333333333333, + "Vietnamese,Spanish,Indonesian,English": 0.5533333333333333, + "Vietnamese,Malay,Indonesian,English": 0.5533333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, + "Chinese,Spanish,Malay,English": 0.44666666666666666, + "Chinese,Spanish,Indonesian,English": 0.5066666666666667, + "Chinese,Malay,Indonesian,English": 0.47333333333333333, + "Spanish,Malay,Indonesian,English": 0.56 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.3933333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.41333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.43333333333333335, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.4066666666666667, + "Filipino,Vietnamese,Chinese,Malay,English": 0.42, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.43333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.4666666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.47333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.49333333333333335, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.49333333333333335, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.38666666666666666, + "Filipino,Chinese,Spanish,Malay,English": 0.4, + "Filipino,Chinese,Spanish,Indonesian,English": 0.44, + "Filipino,Chinese,Malay,Indonesian,English": 0.41333333333333333, + "Filipino,Spanish,Malay,Indonesian,English": 0.5, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, + "Vietnamese,Chinese,Spanish,Malay,English": 0.42, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4533333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.44666666666666666, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.49333333333333335, + "Chinese,Spanish,Malay,Indonesian,English": 0.43333333333333335 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.37333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.38666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.4066666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.4, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.4533333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.38666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4066666666666667 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.37333333333333335 } }, - "AC3_2": 0.4460869564732514, - "AC3_3": 0.3705317235565042, - "AC3_4": 0.3175722205524583, - "AC3_5": 0.2804428043814763, - "AC3_6": 0.2529096989522483, - "AC3_7": 0.23170731703078523 + "AC3_2": 0.5380208291187075, + "AC3_3": 0.4946874858563225, + "AC3_4": 0.46350910043590454, + "AC3_5": 0.4392062917909142, + "AC3_6": 0.41965593480422037, + "AC3_7": 0.40353262988389493 }, "prompt_5": { - "overall_acc": 0.36, + "overall_acc": 0.43619047619047624, "language_acc": { - "Spanish": 0.36666666666666664, - "English": 0.3933333333333333, - "Chinese": 0.34, - "Vietnamese": 0.36, - "Filipino": 0.31333333333333335, - "Indonesian": 0.38, - "Malay": 0.36666666666666664 + "Filipino": 0.44, + "Vietnamese": 0.44, + "Chinese": 0.44666666666666666, + "Spanish": 0.38666666666666666, + "Malay": 0.4666666666666667, + "Indonesian": 0.4066666666666667, + "English": 0.4666666666666667 }, - "consistency_score_2": 0.5206349206349207, - "consistency_score_3": 0.34228571428571425, - "consistency_score_4": 0.24952380952380956, - "consistency_score_5": 0.1911111111111111, - "consistency_score_6": 0.15047619047619049, - "consistency_score_7": 0.12, + "consistency_score_2": 0.7000000000000002, + "consistency_score_3": 0.5779047619047618, + "consistency_score_4": 0.5095238095238095, + "consistency_score_5": 0.4647619047619048, + "consistency_score_6": 0.4323809523809525, + "consistency_score_7": 0.4066666666666667, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5933333333333334, - "Spanish,Chinese": 0.5933333333333334, - "Spanish,Vietnamese": 0.46, - "Spanish,Filipino": 0.5133333333333333, - "Spanish,Indonesian": 0.49333333333333335, - "Spanish,Malay": 0.5533333333333333, - "English,Chinese": 0.6333333333333333, - "English,Vietnamese": 0.4, - "English,Filipino": 0.5, - "English,Indonesian": 0.42, - "English,Malay": 0.5266666666666666, - "Chinese,Vietnamese": 0.3933333333333333, - "Chinese,Filipino": 0.48, - "Chinese,Indonesian": 0.4666666666666667, - "Chinese,Malay": 0.5, - "Vietnamese,Filipino": 0.5, - "Vietnamese,Indonesian": 0.5333333333333333, - "Vietnamese,Malay": 0.56, - "Filipino,Indonesian": 0.5866666666666667, - "Filipino,Malay": 0.5733333333333334, - "Indonesian,Malay": 0.6533333333333333 + "Filipino,Vietnamese": 0.6733333333333333, + "Filipino,Chinese": 0.6066666666666667, + "Filipino,Spanish": 0.7066666666666667, + "Filipino,Malay": 0.7333333333333333, + "Filipino,Indonesian": 0.6933333333333334, + "Filipino,English": 0.7533333333333333, + "Vietnamese,Chinese": 0.6533333333333333, + "Vietnamese,Spanish": 0.6533333333333333, + "Vietnamese,Malay": 0.7066666666666667, + "Vietnamese,Indonesian": 0.7266666666666667, + "Vietnamese,English": 0.72, + "Chinese,Spanish": 0.6533333333333333, + "Chinese,Malay": 0.5733333333333334, + "Chinese,Indonesian": 0.6466666666666666, + "Chinese,English": 0.66, + "Spanish,Malay": 0.7266666666666667, + "Spanish,Indonesian": 0.7533333333333333, + "Spanish,English": 0.7666666666666667, + "Malay,Indonesian": 0.7666666666666667, + "Malay,English": 0.78, + "Indonesian,English": 0.7466666666666667 }, "3_combine": { - "Spanish,English,Chinese": 0.4533333333333333, - "Spanish,English,Vietnamese": 0.3, - "Spanish,English,Filipino": 0.3466666666666667, - "Spanish,English,Indonesian": 0.31333333333333335, - "Spanish,English,Malay": 0.38666666666666666, - "Spanish,Chinese,Vietnamese": 0.29333333333333333, - "Spanish,Chinese,Filipino": 0.3466666666666667, - "Spanish,Chinese,Indonesian": 0.3333333333333333, - "Spanish,Chinese,Malay": 0.38666666666666666, - "Spanish,Vietnamese,Filipino": 0.3, - "Spanish,Vietnamese,Indonesian": 0.30666666666666664, - "Spanish,Vietnamese,Malay": 0.36666666666666664, - "Spanish,Filipino,Indonesian": 0.36, - "Spanish,Filipino,Malay": 0.3933333333333333, - "Spanish,Indonesian,Malay": 0.42, - "English,Chinese,Vietnamese": 0.3, - "English,Chinese,Filipino": 0.36666666666666664, - "English,Chinese,Indonesian": 0.34, - "English,Chinese,Malay": 0.38666666666666666, - "English,Vietnamese,Filipino": 0.2733333333333333, - "English,Vietnamese,Indonesian": 0.24666666666666667, - "English,Vietnamese,Malay": 0.31333333333333335, - "English,Filipino,Indonesian": 0.32, - "English,Filipino,Malay": 0.35333333333333333, - "English,Indonesian,Malay": 0.35333333333333333, - "Chinese,Vietnamese,Filipino": 0.25333333333333335, - "Chinese,Vietnamese,Indonesian": 0.28, - "Chinese,Vietnamese,Malay": 0.30666666666666664, - "Chinese,Filipino,Indonesian": 0.32666666666666666, - "Chinese,Filipino,Malay": 0.32666666666666666, - "Chinese,Indonesian,Malay": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian": 0.34, - "Vietnamese,Filipino,Malay": 0.37333333333333335, - "Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Filipino,Indonesian,Malay": 0.44 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.24, - "Spanish,English,Chinese,Filipino": 0.2733333333333333, - "Spanish,English,Chinese,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Malay": 0.32, - "Spanish,English,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.2, - "Spanish,English,Vietnamese,Malay": 0.25333333333333335, - "Spanish,English,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Filipino,Malay": 0.28, - "Spanish,English,Indonesian,Malay": 0.2733333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.2, - "Spanish,Chinese,Vietnamese,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.2733333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,Vietnamese,Filipino,Malay": 0.26, - "Spanish,Vietnamese,Indonesian,Malay": 0.28, - "Spanish,Filipino,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese,Filipino": 0.2, - "English,Chinese,Vietnamese,Indonesian": 0.20666666666666667, - "English,Chinese,Vietnamese,Malay": 0.24666666666666667, - "English,Chinese,Filipino,Indonesian": 0.26666666666666666, - "English,Chinese,Filipino,Malay": 0.26666666666666666, - "English,Chinese,Indonesian,Malay": 0.2866666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.22, - "English,Filipino,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2, - "Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian,Malay": 0.24, - "Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Chinese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.24, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,English,Filipino,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "English,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - } - }, - "AC3_2": 0.42566690694517484, - "AC3_3": 0.35091944665466973, - "AC3_4": 0.2947499999516426, - "AC3_5": 0.2496774193095343, - "AC3_6": 0.2122388059285726, - "AC3_7": 0.1799999999625 + "Filipino,Vietnamese,Chinese": 0.5066666666666667, + "Filipino,Vietnamese,Spanish": 0.5533333333333333, + "Filipino,Vietnamese,Malay": 0.5866666666666667, + "Filipino,Vietnamese,Indonesian": 0.5933333333333334, + "Filipino,Vietnamese,English": 0.6, + "Filipino,Chinese,Spanish": 0.52, + "Filipino,Chinese,Malay": 0.48, + "Filipino,Chinese,Indonesian": 0.52, + "Filipino,Chinese,English": 0.54, + "Filipino,Spanish,Malay": 0.6066666666666667, + "Filipino,Spanish,Indonesian": 0.6, + "Filipino,Spanish,English": 0.6266666666666667, + "Filipino,Malay,Indonesian": 0.62, + "Filipino,Malay,English": 0.64, + "Filipino,Indonesian,English": 0.6133333333333333, + "Vietnamese,Chinese,Spanish": 0.52, + "Vietnamese,Chinese,Malay": 0.5066666666666667, + "Vietnamese,Chinese,Indonesian": 0.5466666666666666, + "Vietnamese,Chinese,English": 0.54, + "Vietnamese,Spanish,Malay": 0.5666666666666667, + "Vietnamese,Spanish,Indonesian": 0.6, + "Vietnamese,Spanish,English": 0.6, + "Vietnamese,Malay,Indonesian": 0.6333333333333333, + "Vietnamese,Malay,English": 0.62, + "Vietnamese,Indonesian,English": 0.6333333333333333, + "Chinese,Spanish,Malay": 0.5066666666666667, + "Chinese,Spanish,Indonesian": 0.5533333333333333, + "Chinese,Spanish,English": 0.56, + "Chinese,Malay,Indonesian": 0.5333333333333333, + "Chinese,Malay,English": 0.5333333333333333, + "Chinese,Indonesian,English": 0.56, + "Spanish,Malay,Indonesian": 0.6333333333333333, + "Spanish,Malay,English": 0.6533333333333333, + "Spanish,Indonesian,English": 0.66, + "Malay,Indonesian,English": 0.66 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.46, + "Filipino,Vietnamese,Chinese,Malay": 0.4533333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.47333333333333333, + "Filipino,Vietnamese,Chinese,English": 0.48, + "Filipino,Vietnamese,Spanish,Malay": 0.5133333333333333, + "Filipino,Vietnamese,Spanish,Indonesian": 0.5133333333333333, + "Filipino,Vietnamese,Spanish,English": 0.5333333333333333, + "Filipino,Vietnamese,Malay,Indonesian": 0.56, + "Filipino,Vietnamese,Malay,English": 0.5466666666666666, + "Filipino,Vietnamese,Indonesian,English": 0.5466666666666666, + "Filipino,Chinese,Spanish,Malay": 0.44666666666666666, + "Filipino,Chinese,Spanish,Indonesian": 0.48, + "Filipino,Chinese,Spanish,English": 0.4866666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.47333333333333333, + "Filipino,Chinese,Malay,English": 0.4666666666666667, + "Filipino,Chinese,Indonesian,English": 0.49333333333333335, + "Filipino,Spanish,Malay,Indonesian": 0.5533333333333333, + "Filipino,Spanish,Malay,English": 0.5666666666666667, + "Filipino,Spanish,Indonesian,English": 0.5666666666666667, + "Filipino,Malay,Indonesian,English": 0.5666666666666667, + "Vietnamese,Chinese,Spanish,Malay": 0.4533333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.48, + "Vietnamese,Chinese,Spanish,English": 0.48, + "Vietnamese,Chinese,Malay,Indonesian": 0.48, + "Vietnamese,Chinese,Malay,English": 0.47333333333333333, + "Vietnamese,Chinese,Indonesian,English": 0.49333333333333335, + "Vietnamese,Spanish,Malay,Indonesian": 0.54, + "Vietnamese,Spanish,Malay,English": 0.54, + "Vietnamese,Spanish,Indonesian,English": 0.56, + "Vietnamese,Malay,Indonesian,English": 0.5733333333333334, + "Chinese,Spanish,Malay,Indonesian": 0.48, + "Chinese,Spanish,Malay,English": 0.49333333333333335, + "Chinese,Spanish,Indonesian,English": 0.5133333333333333, + "Chinese,Malay,Indonesian,English": 0.5066666666666667, + "Spanish,Malay,Indonesian,English": 0.5866666666666667 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.42, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.43333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.44666666666666666, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.44666666666666666, + "Filipino,Vietnamese,Chinese,Malay,English": 0.44, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.4533333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.49333333333333335, + "Filipino,Vietnamese,Spanish,Malay,English": 0.5, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.5, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.52, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.44, + "Filipino,Chinese,Spanish,Malay,English": 0.44, + "Filipino,Chinese,Spanish,Indonesian,English": 0.4666666666666667, + "Filipino,Chinese,Malay,Indonesian,English": 0.46, + "Filipino,Spanish,Malay,Indonesian,English": 0.5266666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.44, + "Vietnamese,Chinese,Spanish,Malay,English": 0.44, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4533333333333333, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.46, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.5133333333333333, + "Chinese,Spanish,Malay,Indonesian,English": 0.4666666666666667 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.41333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.4266666666666667, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.43333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.48, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.43333333333333335, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4266666666666667 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.4066666666666667 + } + }, + "AC3_2": 0.5374685665914216, + "AC3_3": 0.4971457192454561, + "AC3_4": 0.47001390682221356, + "AC3_5": 0.45002315508948454, + "AC3_6": 0.4342773600168347, + "AC3_7": 0.4209114877090067 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.30032467532467527, + "overall_acc": 0.4862012987012987, "language_acc": { - "English": 0.32386363636363635, - "Filipino": 0.2784090909090909, - "Spanish": 0.2556818181818182, - "Malay": 0.3409090909090909, - "Chinese": 0.26704545454545453, - "Indonesian": 0.2727272727272727, - "Vietnamese": 0.36363636363636365 + "Indonesian": 0.4715909090909091, + "English": 0.5056818181818182, + "Filipino": 0.5113636363636364, + "Spanish": 0.4772727272727273, + "Chinese": 0.48863636363636365, + "Malay": 0.4772727272727273, + "Vietnamese": 0.4715909090909091 }, - "consistency_score_2": 0.450487012987013, - "consistency_score_3": 0.24951298701298705, - "consistency_score_4": 0.14967532467532466, - "consistency_score_5": 0.0928030303030303, - "consistency_score_6": 0.05925324675324676, - "consistency_score_7": 0.03977272727272727, + "consistency_score_2": 0.7167207792207793, + "consistency_score_3": 0.5917207792207791, + "consistency_score_4": 0.5137987012987014, + "consistency_score_5": 0.4583333333333333, + "consistency_score_6": 0.4155844155844156, + "consistency_score_7": 0.3806818181818182, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.42045454545454547, - "English,Spanish": 0.5113636363636364, - "English,Malay": 0.4715909090909091, - "English,Chinese": 0.4772727272727273, - "English,Indonesian": 0.4318181818181818, - "English,Vietnamese": 0.3522727272727273, - "Filipino,Spanish": 0.4034090909090909, - "Filipino,Malay": 0.6363636363636364, - "Filipino,Chinese": 0.4318181818181818, - "Filipino,Indonesian": 0.5852272727272727, - "Filipino,Vietnamese": 0.5965909090909091, - "Spanish,Malay": 0.3465909090909091, - "Spanish,Chinese": 0.3693181818181818, - "Spanish,Indonesian": 0.35795454545454547, - "Spanish,Vietnamese": 0.26704545454545453, - "Malay,Chinese": 0.44886363636363635, - "Malay,Indonesian": 0.5965909090909091, - "Malay,Vietnamese": 0.5, - "Chinese,Indonesian": 0.4375, - "Chinese,Vietnamese": 0.32954545454545453, - "Indonesian,Vietnamese": 0.48863636363636365 - }, - "3_combine": { - "English,Filipino,Spanish": 0.23295454545454544, - "English,Filipino,Malay": 0.3068181818181818, - "English,Filipino,Chinese": 0.24431818181818182, - "English,Filipino,Indonesian": 0.2784090909090909, - "English,Filipino,Vietnamese": 0.23863636363636365, - "English,Spanish,Malay": 0.2215909090909091, - "English,Spanish,Chinese": 0.24431818181818182, - "English,Spanish,Indonesian": 0.21022727272727273, - "English,Spanish,Vietnamese": 0.1590909090909091, - "English,Malay,Chinese": 0.2784090909090909, - "English,Malay,Indonesian": 0.3125, - "English,Malay,Vietnamese": 0.2159090909090909, - "English,Chinese,Indonesian": 0.26704545454545453, - "English,Chinese,Vietnamese": 0.19318181818181818, - "English,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay": 0.26136363636363635, - "Filipino,Spanish,Chinese": 0.20454545454545456, - "Filipino,Spanish,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese": 0.3125, - "Filipino,Malay,Indonesian": 0.44886363636363635, - "Filipino,Malay,Vietnamese": 0.3977272727272727, - "Filipino,Chinese,Indonesian": 0.3068181818181818, - "Filipino,Chinese,Vietnamese": 0.2556818181818182, - "Filipino,Indonesian,Vietnamese": 0.3693181818181818, - "Spanish,Malay,Chinese": 0.21022727272727273, - "Spanish,Malay,Indonesian": 0.2215909090909091, - "Spanish,Malay,Vietnamese": 0.14772727272727273, - "Spanish,Chinese,Indonesian": 0.17613636363636365, - "Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "Malay,Chinese,Indonesian": 0.32386363636363635, - "Malay,Chinese,Vietnamese": 0.22727272727272727, - "Malay,Indonesian,Vietnamese": 0.32954545454545453, - "Chinese,Indonesian,Vietnamese": 0.2159090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.1590909090909091, - "English,Filipino,Spanish,Chinese": 0.13068181818181818, - "English,Filipino,Spanish,Indonesian": 0.125, - "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese": 0.18181818181818182, - "English,Filipino,Malay,Indonesian": 0.23295454545454544, - "English,Filipino,Malay,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian": 0.17613636363636365, - "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Indonesian,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Spanish,Malay,Indonesian": 0.14772727272727273, - "English,Spanish,Malay,Vietnamese": 0.08522727272727272, - "English,Spanish,Chinese,Indonesian": 0.125, - "English,Spanish,Chinese,Vietnamese": 0.09090909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.09090909090909091, - "English,Malay,Chinese,Indonesian": 0.19886363636363635, - "English,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese": 0.1534090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.24431818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.1875, - "Filipino,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1875, - "Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.0625, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 + "Indonesian,English": 0.7329545454545454, + "Indonesian,Filipino": 0.7272727272727273, + "Indonesian,Spanish": 0.7159090909090909, + "Indonesian,Chinese": 0.6761363636363636, + "Indonesian,Malay": 0.7159090909090909, + "Indonesian,Vietnamese": 0.7613636363636364, + "English,Filipino": 0.75, + "English,Spanish": 0.7897727272727273, + "English,Chinese": 0.6761363636363636, + "English,Malay": 0.7102272727272727, + "English,Vietnamese": 0.7954545454545454, + "Filipino,Spanish": 0.7443181818181818, + "Filipino,Chinese": 0.6590909090909091, + "Filipino,Malay": 0.6704545454545454, + "Filipino,Vietnamese": 0.7159090909090909, + "Spanish,Chinese": 0.6306818181818182, + "Spanish,Malay": 0.7386363636363636, + "Spanish,Vietnamese": 0.7727272727272727, + "Chinese,Malay": 0.6363636363636364, + "Chinese,Vietnamese": 0.6931818181818182, + "Malay,Vietnamese": 0.7386363636363636 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.6136363636363636, + "Indonesian,English,Spanish": 0.6306818181818182, + "Indonesian,English,Chinese": 0.5568181818181818, + "Indonesian,English,Malay": 0.5965909090909091, + "Indonesian,English,Vietnamese": 0.6590909090909091, + "Indonesian,Filipino,Spanish": 0.6079545454545454, + "Indonesian,Filipino,Chinese": 0.5454545454545454, + "Indonesian,Filipino,Malay": 0.5738636363636364, + "Indonesian,Filipino,Vietnamese": 0.6193181818181818, + "Indonesian,Spanish,Chinese": 0.5227272727272727, + "Indonesian,Spanish,Malay": 0.6022727272727273, + "Indonesian,Spanish,Vietnamese": 0.6363636363636364, + "Indonesian,Chinese,Malay": 0.5340909090909091, + "Indonesian,Chinese,Vietnamese": 0.5852272727272727, + "Indonesian,Malay,Vietnamese": 0.6363636363636364, + "English,Filipino,Spanish": 0.6477272727272727, + "English,Filipino,Chinese": 0.5568181818181818, + "English,Filipino,Malay": 0.5795454545454546, + "English,Filipino,Vietnamese": 0.6420454545454546, + "English,Spanish,Chinese": 0.5681818181818182, + "English,Spanish,Malay": 0.6306818181818182, + "English,Spanish,Vietnamese": 0.6931818181818182, + "English,Chinese,Malay": 0.5284090909090909, + "English,Chinese,Vietnamese": 0.6022727272727273, + "English,Malay,Vietnamese": 0.6477272727272727, + "Filipino,Spanish,Chinese": 0.5340909090909091, + "Filipino,Spanish,Malay": 0.5965909090909091, + "Filipino,Spanish,Vietnamese": 0.6306818181818182, + "Filipino,Chinese,Malay": 0.5056818181818182, + "Filipino,Chinese,Vietnamese": 0.5511363636363636, + "Filipino,Malay,Vietnamese": 0.5909090909090909, + "Spanish,Chinese,Malay": 0.5170454545454546, + "Spanish,Chinese,Vietnamese": 0.5625, + "Spanish,Malay,Vietnamese": 0.6420454545454546, + "Chinese,Malay,Vietnamese": 0.5625 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.5568181818181818, + "Indonesian,English,Filipino,Chinese": 0.48863636363636365, + "Indonesian,English,Filipino,Malay": 0.5056818181818182, + "Indonesian,English,Filipino,Vietnamese": 0.5681818181818182, + "Indonesian,English,Spanish,Chinese": 0.48863636363636365, + "Indonesian,English,Spanish,Malay": 0.5454545454545454, + "Indonesian,English,Spanish,Vietnamese": 0.5909090909090909, + "Indonesian,English,Chinese,Malay": 0.4659090909090909, + "Indonesian,English,Chinese,Vietnamese": 0.5340909090909091, + "Indonesian,English,Malay,Vietnamese": 0.5568181818181818, + "Indonesian,Filipino,Spanish,Chinese": 0.45454545454545453, + "Indonesian,Filipino,Spanish,Malay": 0.5170454545454546, + "Indonesian,Filipino,Spanish,Vietnamese": 0.5625, + "Indonesian,Filipino,Chinese,Malay": 0.4431818181818182, + "Indonesian,Filipino,Chinese,Vietnamese": 0.48295454545454547, + "Indonesian,Filipino,Malay,Vietnamese": 0.5284090909090909, + "Indonesian,Spanish,Chinese,Malay": 0.45454545454545453, + "Indonesian,Spanish,Chinese,Vietnamese": 0.4943181818181818, + "Indonesian,Spanish,Malay,Vietnamese": 0.5568181818181818, + "Indonesian,Chinese,Malay,Vietnamese": 0.5, + "English,Filipino,Spanish,Chinese": 0.4943181818181818, + "English,Filipino,Spanish,Malay": 0.5340909090909091, + "English,Filipino,Spanish,Vietnamese": 0.5852272727272727, + "English,Filipino,Chinese,Malay": 0.45454545454545453, + "English,Filipino,Chinese,Vietnamese": 0.5170454545454546, + "English,Filipino,Malay,Vietnamese": 0.5397727272727273, + "English,Spanish,Chinese,Malay": 0.4715909090909091, + "English,Spanish,Chinese,Vietnamese": 0.5340909090909091, + "English,Spanish,Malay,Vietnamese": 0.5909090909090909, + "English,Chinese,Malay,Vietnamese": 0.5170454545454546, + "Filipino,Spanish,Chinese,Malay": 0.45454545454545453, + "Filipino,Spanish,Chinese,Vietnamese": 0.48863636363636365, + "Filipino,Spanish,Malay,Vietnamese": 0.5397727272727273, + "Filipino,Chinese,Malay,Vietnamese": 0.4715909090909091, + "Spanish,Chinese,Malay,Vietnamese": 0.4943181818181818 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.4375, + "Indonesian,English,Filipino,Spanish,Malay": 0.4772727272727273, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.5284090909090909, + "Indonesian,English,Filipino,Chinese,Malay": 0.4090909090909091, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.4659090909090909, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.48295454545454547, + "Indonesian,English,Spanish,Chinese,Malay": 0.42613636363636365, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4772727272727273, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.5170454545454546, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.4602272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.3977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4375, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.48863636363636365, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.42045454545454547, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.4431818181818182, + "English,Filipino,Spanish,Chinese,Malay": 0.42045454545454547, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.4715909090909091, + "English,Filipino,Spanish,Malay,Vietnamese": 0.5056818181818182, + "English,Filipino,Chinese,Malay,Vietnamese": 0.44886363636363635, + "English,Spanish,Chinese,Malay,Vietnamese": 0.4715909090909091, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4375 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.3806818181818182, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.42613636363636365, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4602272727272727, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.4034090909090909, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.42613636363636365, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.39204545454545453, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.42045454545454547 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3806818181818182 } }, - "AC3_2": 0.3603896103416104, - "AC3_3": 0.27257102207857936, - "AC3_4": 0.19978308123219146, - "AC3_5": 0.1417912781970552, - "AC3_6": 0.09897833540679112, - "AC3_7": 0.07024300279991466 + "AC3_2": 0.5793734773516294, + "AC3_3": 0.5337963053017544, + "AC3_4": 0.49961919163497664, + "AC3_5": 0.4718562016114586, + "AC3_6": 0.44812792962639114, + "AC3_7": 0.4270194926303459 }, "prompt_2": { - "overall_acc": 0.31006493506493504, + "overall_acc": 0.49350649350649345, "language_acc": { - "English": 0.3352272727272727, - "Filipino": 0.3068181818181818, - "Spanish": 0.30113636363636365, - "Malay": 0.3522727272727273, - "Chinese": 0.3125, - "Indonesian": 0.3068181818181818, - "Vietnamese": 0.2556818181818182 + "Indonesian": 0.48295454545454547, + "English": 0.5227272727272727, + "Filipino": 0.4943181818181818, + "Spanish": 0.48863636363636365, + "Chinese": 0.4943181818181818, + "Malay": 0.4943181818181818, + "Vietnamese": 0.4772727272727273 }, - "consistency_score_2": 0.39529220779220775, - "consistency_score_3": 0.19366883116883118, - "consistency_score_4": 0.10194805194805193, - "consistency_score_5": 0.052759740259740256, - "consistency_score_6": 0.023538961038961036, - "consistency_score_7": 0.005681818181818182, + "consistency_score_2": 0.7088744588744588, + "consistency_score_3": 0.5788961038961039, + "consistency_score_4": 0.5003246753246753, + "consistency_score_5": 0.4458874458874459, + "consistency_score_6": 0.40422077922077915, + "consistency_score_7": 0.3693181818181818, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4659090909090909, - "English,Spanish": 0.5227272727272727, - "English,Malay": 0.48295454545454547, - "English,Chinese": 0.3977272727272727, - "English,Indonesian": 0.4659090909090909, - "English,Vietnamese": 0.2727272727272727, - "Filipino,Spanish": 0.4090909090909091, - "Filipino,Malay": 0.48295454545454547, - "Filipino,Chinese": 0.3806818181818182, - "Filipino,Indonesian": 0.4602272727272727, - "Filipino,Vietnamese": 0.2840909090909091, - "Spanish,Malay": 0.5227272727272727, - "Spanish,Chinese": 0.375, - "Spanish,Indonesian": 0.4375, - "Spanish,Vietnamese": 0.2727272727272727, - "Malay,Chinese": 0.4090909090909091, - "Malay,Indonesian": 0.5170454545454546, - "Malay,Vietnamese": 0.23863636363636365, - "Chinese,Indonesian": 0.4431818181818182, - "Chinese,Vietnamese": 0.19318181818181818, - "Indonesian,Vietnamese": 0.26704545454545453 + "Indonesian,English": 0.7159090909090909, + "Indonesian,Filipino": 0.7159090909090909, + "Indonesian,Spanish": 0.7159090909090909, + "Indonesian,Chinese": 0.6647727272727273, + "Indonesian,Malay": 0.7556818181818182, + "Indonesian,Vietnamese": 0.7556818181818182, + "English,Filipino": 0.75, + "English,Spanish": 0.7727272727272727, + "English,Chinese": 0.6818181818181818, + "English,Malay": 0.6818181818181818, + "English,Vietnamese": 0.7556818181818182, + "Filipino,Spanish": 0.7159090909090909, + "Filipino,Chinese": 0.6704545454545454, + "Filipino,Malay": 0.7045454545454546, + "Filipino,Vietnamese": 0.6988636363636364, + "Spanish,Chinese": 0.6193181818181818, + "Spanish,Malay": 0.7329545454545454, + "Spanish,Vietnamese": 0.7556818181818182, + "Chinese,Malay": 0.6306818181818182, + "Chinese,Vietnamese": 0.6590909090909091, + "Malay,Vietnamese": 0.7329545454545454 }, "3_combine": { - "English,Filipino,Spanish": 0.26136363636363635, - "English,Filipino,Malay": 0.26136363636363635, - "English,Filipino,Chinese": 0.21022727272727273, - "English,Filipino,Indonesian": 0.24431818181818182, - "English,Filipino,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay": 0.32954545454545453, - "English,Spanish,Chinese": 0.23863636363636365, - "English,Spanish,Indonesian": 0.2727272727272727, - "English,Spanish,Vietnamese": 0.14204545454545456, - "English,Malay,Chinese": 0.23863636363636365, - "English,Malay,Indonesian": 0.29545454545454547, - "English,Malay,Vietnamese": 0.13068181818181818, - "English,Chinese,Indonesian": 0.25, - "English,Chinese,Vietnamese": 0.08522727272727272, - "English,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay": 0.2784090909090909, - "Filipino,Spanish,Chinese": 0.19318181818181818, - "Filipino,Spanish,Indonesian": 0.23863636363636365, - "Filipino,Spanish,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese": 0.22727272727272727, - "Filipino,Malay,Indonesian": 0.2784090909090909, - "Filipino,Malay,Vietnamese": 0.125, - "Filipino,Chinese,Indonesian": 0.24431818181818182, - "Filipino,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese": 0.23863636363636365, - "Spanish,Malay,Indonesian": 0.30113636363636365, - "Spanish,Malay,Vietnamese": 0.13636363636363635, - "Spanish,Chinese,Indonesian": 0.20454545454545456, - "Spanish,Chinese,Vietnamese": 0.10227272727272728, - "Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "Malay,Chinese,Indonesian": 0.2556818181818182, - "Malay,Chinese,Vietnamese": 0.09090909090909091, - "Malay,Indonesian,Vietnamese": 0.125, - "Chinese,Indonesian,Vietnamese": 0.09659090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Chinese": 0.14204545454545456, - "English,Filipino,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Malay,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese": 0.17045454545454544, - "English,Spanish,Malay,Indonesian": 0.19886363636363635, - "English,Spanish,Malay,Vietnamese": 0.07386363636363637, - "English,Spanish,Chinese,Indonesian": 0.14204545454545456, - "English,Spanish,Chinese,Vietnamese": 0.05113636363636364, - "English,Spanish,Indonesian,Vietnamese": 0.05113636363636364, - "English,Malay,Chinese,Indonesian": 0.17045454545454544, - "English,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Malay,Indonesian,Vietnamese": 0.056818181818181816, - "English,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Malay,Chinese": 0.14204545454545456, - "Filipino,Spanish,Malay,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Indonesian,Vietnamese": 0.0625, - "Spanish,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.03409090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 + "Indonesian,English,Filipino": 0.6022727272727273, + "Indonesian,English,Spanish": 0.6079545454545454, + "Indonesian,English,Chinese": 0.5397727272727273, + "Indonesian,English,Malay": 0.5909090909090909, + "Indonesian,English,Vietnamese": 0.625, + "Indonesian,Filipino,Spanish": 0.5852272727272727, + "Indonesian,Filipino,Chinese": 0.5397727272727273, + "Indonesian,Filipino,Malay": 0.6079545454545454, + "Indonesian,Filipino,Vietnamese": 0.6022727272727273, + "Indonesian,Spanish,Chinese": 0.5113636363636364, + "Indonesian,Spanish,Malay": 0.6136363636363636, + "Indonesian,Spanish,Vietnamese": 0.625, + "Indonesian,Chinese,Malay": 0.5454545454545454, + "Indonesian,Chinese,Vietnamese": 0.5625, + "Indonesian,Malay,Vietnamese": 0.6420454545454546, + "English,Filipino,Spanish": 0.625, + "English,Filipino,Chinese": 0.5681818181818182, + "English,Filipino,Malay": 0.5795454545454546, + "English,Filipino,Vietnamese": 0.6193181818181818, + "English,Spanish,Chinese": 0.5511363636363636, + "English,Spanish,Malay": 0.6022727272727273, + "English,Spanish,Vietnamese": 0.6590909090909091, + "English,Chinese,Malay": 0.5113636363636364, + "English,Chinese,Vietnamese": 0.5681818181818182, + "English,Malay,Vietnamese": 0.6079545454545454, + "Filipino,Spanish,Chinese": 0.5284090909090909, + "Filipino,Spanish,Malay": 0.5909090909090909, + "Filipino,Spanish,Vietnamese": 0.6079545454545454, + "Filipino,Chinese,Malay": 0.5284090909090909, + "Filipino,Chinese,Vietnamese": 0.5284090909090909, + "Filipino,Malay,Vietnamese": 0.5965909090909091, + "Spanish,Chinese,Malay": 0.5, + "Spanish,Chinese,Vietnamese": 0.5284090909090909, + "Spanish,Malay,Vietnamese": 0.625, + "Chinese,Malay,Vietnamese": 0.5340909090909091 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.5284090909090909, + "Indonesian,English,Filipino,Chinese": 0.4772727272727273, + "Indonesian,English,Filipino,Malay": 0.5227272727272727, + "Indonesian,English,Filipino,Vietnamese": 0.5511363636363636, + "Indonesian,English,Spanish,Chinese": 0.4659090909090909, + "Indonesian,English,Spanish,Malay": 0.5340909090909091, + "Indonesian,English,Spanish,Vietnamese": 0.5681818181818182, + "Indonesian,English,Chinese,Malay": 0.4659090909090909, + "Indonesian,English,Chinese,Vietnamese": 0.5056818181818182, + "Indonesian,English,Malay,Vietnamese": 0.5397727272727273, + "Indonesian,Filipino,Spanish,Chinese": 0.44886363636363635, + "Indonesian,Filipino,Spanish,Malay": 0.5284090909090909, + "Indonesian,Filipino,Spanish,Vietnamese": 0.5284090909090909, + "Indonesian,Filipino,Chinese,Malay": 0.4715909090909091, + "Indonesian,Filipino,Chinese,Vietnamese": 0.4715909090909091, + "Indonesian,Filipino,Malay,Vietnamese": 0.5397727272727273, + "Indonesian,Spanish,Chinese,Malay": 0.4602272727272727, + "Indonesian,Spanish,Chinese,Vietnamese": 0.4659090909090909, + "Indonesian,Spanish,Malay,Vietnamese": 0.5454545454545454, + "Indonesian,Chinese,Malay,Vietnamese": 0.4943181818181818, + "English,Filipino,Spanish,Chinese": 0.4772727272727273, + "English,Filipino,Spanish,Malay": 0.5113636363636364, + "English,Filipino,Spanish,Vietnamese": 0.5568181818181818, + "English,Filipino,Chinese,Malay": 0.4659090909090909, + "English,Filipino,Chinese,Vietnamese": 0.4943181818181818, + "English,Filipino,Malay,Vietnamese": 0.5284090909090909, + "English,Spanish,Chinese,Malay": 0.44886363636363635, + "English,Spanish,Chinese,Vietnamese": 0.5, + "English,Spanish,Malay,Vietnamese": 0.5568181818181818, + "English,Chinese,Malay,Vietnamese": 0.48295454545454547, + "Filipino,Spanish,Chinese,Malay": 0.45454545454545453, + "Filipino,Spanish,Chinese,Vietnamese": 0.4659090909090909, + "Filipino,Spanish,Malay,Vietnamese": 0.5284090909090909, + "Filipino,Chinese,Malay,Vietnamese": 0.4659090909090909, + "Spanish,Chinese,Malay,Vietnamese": 0.4602272727272727 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.4147727272727273, + "Indonesian,English,Filipino,Spanish,Malay": 0.4715909090909091, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.5, + "Indonesian,English,Filipino,Chinese,Malay": 0.42613636363636365, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.44886363636363635, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.48863636363636365, + "Indonesian,English,Spanish,Chinese,Malay": 0.42045454545454547, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.44886363636363635, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.5, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.44886363636363635, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.4147727272727273, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4147727272727273, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4772727272727273, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.4318181818181818, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.42613636363636365, + "English,Filipino,Spanish,Chinese,Malay": 0.4090909090909091, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.4431818181818182, + "English,Filipino,Spanish,Malay,Vietnamese": 0.48295454545454547, + "English,Filipino,Chinese,Malay,Vietnamese": 0.4375, + "English,Spanish,Chinese,Malay,Vietnamese": 0.4375, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.42045454545454547 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.3806818181818182, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.3977272727272727, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.44886363636363635, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.4090909090909091, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.4090909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.38636363636363635, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3693181818181818 } }, - "AC3_2": 0.3475296279751796, - "AC3_3": 0.2384192506987009, - "AC3_4": 0.15344427042830447, - "AC3_5": 0.09017548444593274, - "AC3_6": 0.043756122210159315, - "AC3_7": 0.011159149330422323 + "AC3_2": 0.5819023460303678, + "AC3_3": 0.5328017425613238, + "AC3_4": 0.4968921962868246, + "AC3_5": 0.46849003526044025, + "AC3_6": 0.4444235691480239, + "AC3_7": 0.42247498498481667 }, "prompt_3": { - "overall_acc": 0.31006493506493504, + "overall_acc": 0.4983766233766234, "language_acc": { - "English": 0.36363636363636365, - "Filipino": 0.3181818181818182, - "Spanish": 0.3181818181818182, - "Malay": 0.32954545454545453, - "Chinese": 0.2784090909090909, - "Indonesian": 0.2897727272727273, - "Vietnamese": 0.2727272727272727 + "Indonesian": 0.48295454545454547, + "English": 0.5284090909090909, + "Filipino": 0.4943181818181818, + "Spanish": 0.48295454545454547, + "Chinese": 0.5056818181818182, + "Malay": 0.5284090909090909, + "Vietnamese": 0.4659090909090909 }, - "consistency_score_2": 0.3430735930735931, - "consistency_score_3": 0.14025974025974022, - "consistency_score_4": 0.06655844155844155, - "consistency_score_5": 0.0349025974025974, - "consistency_score_6": 0.01948051948051948, - "consistency_score_7": 0.011363636363636364, + "consistency_score_2": 0.7110389610389609, + "consistency_score_3": 0.5818181818181818, + "consistency_score_4": 0.5043831168831169, + "consistency_score_5": 0.4510281385281385, + "consistency_score_6": 0.4099025974025974, + "consistency_score_7": 0.375, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.3977272727272727, - "English,Spanish": 0.5056818181818182, - "English,Malay": 0.39204545454545453, - "English,Chinese": 0.32386363636363635, - "English,Indonesian": 0.32954545454545453, - "English,Vietnamese": 0.2727272727272727, - "Filipino,Spanish": 0.3181818181818182, - "Filipino,Malay": 0.3465909090909091, - "Filipino,Chinese": 0.32386363636363635, - "Filipino,Indonesian": 0.35795454545454547, - "Filipino,Vietnamese": 0.36363636363636365, - "Spanish,Malay": 0.39204545454545453, - "Spanish,Chinese": 0.3068181818181818, - "Spanish,Indonesian": 0.26136363636363635, - "Spanish,Vietnamese": 0.24431818181818182, - "Malay,Chinese": 0.3352272727272727, - "Malay,Indonesian": 0.4090909090909091, - "Malay,Vietnamese": 0.35795454545454547, - "Chinese,Indonesian": 0.2784090909090909, - "Chinese,Vietnamese": 0.22727272727272727, - "Indonesian,Vietnamese": 0.4602272727272727 - }, - "3_combine": { - "English,Filipino,Spanish": 0.19886363636363635, - "English,Filipino,Malay": 0.17045454545454544, - "English,Filipino,Chinese": 0.14204545454545456, - "English,Filipino,Indonesian": 0.17613636363636365, - "English,Filipino,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay": 0.24431818181818182, - "English,Spanish,Chinese": 0.17613636363636365, - "English,Spanish,Indonesian": 0.1534090909090909, - "English,Spanish,Vietnamese": 0.11363636363636363, - "English,Malay,Chinese": 0.1590909090909091, - "English,Malay,Indonesian": 0.1534090909090909, - "English,Malay,Vietnamese": 0.14772727272727273, - "English,Chinese,Indonesian": 0.11931818181818182, - "English,Chinese,Vietnamese": 0.07954545454545454, - "English,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay": 0.14204545454545456, - "Filipino,Spanish,Chinese": 0.10795454545454546, - "Filipino,Spanish,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Vietnamese": 0.09659090909090909, - "Filipino,Malay,Chinese": 0.11931818181818182, - "Filipino,Malay,Indonesian": 0.17613636363636365, - "Filipino,Malay,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian": 0.125, - "Filipino,Chinese,Vietnamese": 0.10227272727272728, - "Filipino,Indonesian,Vietnamese": 0.19886363636363635, - "Spanish,Malay,Chinese": 0.13068181818181818, - "Spanish,Malay,Indonesian": 0.14204545454545456, - "Spanish,Malay,Vietnamese": 0.125, - "Spanish,Chinese,Indonesian": 0.09090909090909091, - "Spanish,Chinese,Vietnamese": 0.045454545454545456, - "Spanish,Indonesian,Vietnamese": 0.09659090909090909, - "Malay,Chinese,Indonesian": 0.14204545454545456, - "Malay,Chinese,Vietnamese": 0.11363636363636363, - "Malay,Indonesian,Vietnamese": 0.2159090909090909, - "Chinese,Indonesian,Vietnamese": 0.11363636363636363 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.09090909090909091, - "English,Filipino,Spanish,Chinese": 0.07954545454545454, - "English,Filipino,Spanish,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Vietnamese": 0.0625, - "English,Filipino,Malay,Chinese": 0.056818181818181816, - "English,Filipino,Malay,Indonesian": 0.09659090909090909, - "English,Filipino,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese": 0.08522727272727272, - "English,Spanish,Malay,Indonesian": 0.09090909090909091, - "English,Spanish,Malay,Vietnamese": 0.07386363636363637, - "English,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Spanish,Indonesian,Vietnamese": 0.056818181818181816, - "English,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese": 0.05113636363636364, - "Filipino,Spanish,Malay,Indonesian": 0.07386363636363637, - "Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Chinese,Indonesian": 0.03977272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Chinese,Indonesian": 0.06818181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.0625, - "Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "Spanish,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Vietnamese": 0.045454545454545456, - "English,Filipino,Spanish,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.045454545454545456, - "English,Filipino,Malay,Chinese,Indonesian": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Indonesian": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.045454545454545456, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 + "Indonesian,English": 0.7272727272727273, + "Indonesian,Filipino": 0.7272727272727273, + "Indonesian,Spanish": 0.7272727272727273, + "Indonesian,Chinese": 0.6306818181818182, + "Indonesian,Malay": 0.7556818181818182, + "Indonesian,Vietnamese": 0.7443181818181818, + "English,Filipino": 0.7670454545454546, + "English,Spanish": 0.7954545454545454, + "English,Chinese": 0.6818181818181818, + "English,Malay": 0.7102272727272727, + "English,Vietnamese": 0.7670454545454546, + "Filipino,Spanish": 0.7215909090909091, + "Filipino,Chinese": 0.6704545454545454, + "Filipino,Malay": 0.7272727272727273, + "Filipino,Vietnamese": 0.6931818181818182, + "Spanish,Chinese": 0.6136363636363636, + "Spanish,Malay": 0.7443181818181818, + "Spanish,Vietnamese": 0.7443181818181818, + "Chinese,Malay": 0.6306818181818182, + "Chinese,Vietnamese": 0.6363636363636364, + "Malay,Vietnamese": 0.7159090909090909 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.6193181818181818, + "Indonesian,English,Spanish": 0.6306818181818182, + "Indonesian,English,Chinese": 0.5284090909090909, + "Indonesian,English,Malay": 0.6079545454545454, + "Indonesian,English,Vietnamese": 0.6363636363636364, + "Indonesian,Filipino,Spanish": 0.5965909090909091, + "Indonesian,Filipino,Chinese": 0.5340909090909091, + "Indonesian,Filipino,Malay": 0.625, + "Indonesian,Filipino,Vietnamese": 0.6022727272727273, + "Indonesian,Spanish,Chinese": 0.4943181818181818, + "Indonesian,Spanish,Malay": 0.625, + "Indonesian,Spanish,Vietnamese": 0.6193181818181818, + "Indonesian,Chinese,Malay": 0.5227272727272727, + "Indonesian,Chinese,Vietnamese": 0.5227272727272727, + "Indonesian,Malay,Vietnamese": 0.6306818181818182, + "English,Filipino,Spanish": 0.6477272727272727, + "English,Filipino,Chinese": 0.5795454545454546, + "English,Filipino,Malay": 0.6193181818181818, + "English,Filipino,Vietnamese": 0.6306818181818182, + "English,Spanish,Chinese": 0.5568181818181818, + "English,Spanish,Malay": 0.6306818181818182, + "English,Spanish,Vietnamese": 0.6704545454545454, + "English,Chinese,Malay": 0.5227272727272727, + "English,Chinese,Vietnamese": 0.5568181818181818, + "English,Malay,Vietnamese": 0.6193181818181818, + "Filipino,Spanish,Chinese": 0.5227272727272727, + "Filipino,Spanish,Malay": 0.6136363636363636, + "Filipino,Spanish,Vietnamese": 0.6022727272727273, + "Filipino,Chinese,Malay": 0.5340909090909091, + "Filipino,Chinese,Vietnamese": 0.5170454545454546, + "Filipino,Malay,Vietnamese": 0.6022727272727273, + "Spanish,Chinese,Malay": 0.5, + "Spanish,Chinese,Vietnamese": 0.5113636363636364, + "Spanish,Malay,Vietnamese": 0.6136363636363636, + "Chinese,Malay,Vietnamese": 0.5170454545454546 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.5511363636363636, + "Indonesian,English,Filipino,Chinese": 0.48295454545454547, + "Indonesian,English,Filipino,Malay": 0.5454545454545454, + "Indonesian,English,Filipino,Vietnamese": 0.5568181818181818, + "Indonesian,English,Spanish,Chinese": 0.4602272727272727, + "Indonesian,English,Spanish,Malay": 0.5511363636363636, + "Indonesian,English,Spanish,Vietnamese": 0.5738636363636364, + "Indonesian,English,Chinese,Malay": 0.4659090909090909, + "Indonesian,English,Chinese,Vietnamese": 0.48295454545454547, + "Indonesian,English,Malay,Vietnamese": 0.5454545454545454, + "Indonesian,Filipino,Spanish,Chinese": 0.4431818181818182, + "Indonesian,Filipino,Spanish,Malay": 0.5454545454545454, + "Indonesian,Filipino,Spanish,Vietnamese": 0.5340909090909091, + "Indonesian,Filipino,Chinese,Malay": 0.4715909090909091, + "Indonesian,Filipino,Chinese,Vietnamese": 0.45454545454545453, + "Indonesian,Filipino,Malay,Vietnamese": 0.5511363636363636, + "Indonesian,Spanish,Chinese,Malay": 0.44886363636363635, + "Indonesian,Spanish,Chinese,Vietnamese": 0.4431818181818182, + "Indonesian,Spanish,Malay,Vietnamese": 0.5454545454545454, + "Indonesian,Chinese,Malay,Vietnamese": 0.4659090909090909, + "English,Filipino,Spanish,Chinese": 0.48863636363636365, + "English,Filipino,Spanish,Malay": 0.5568181818181818, + "English,Filipino,Spanish,Vietnamese": 0.5738636363636364, + "English,Filipino,Chinese,Malay": 0.48863636363636365, + "English,Filipino,Chinese,Vietnamese": 0.48863636363636365, + "English,Filipino,Malay,Vietnamese": 0.5568181818181818, + "English,Spanish,Chinese,Malay": 0.4602272727272727, + "English,Spanish,Chinese,Vietnamese": 0.4943181818181818, + "English,Spanish,Malay,Vietnamese": 0.5681818181818182, + "English,Chinese,Malay,Vietnamese": 0.48295454545454547, + "Filipino,Spanish,Chinese,Malay": 0.4659090909090909, + "Filipino,Spanish,Chinese,Vietnamese": 0.45454545454545453, + "Filipino,Spanish,Malay,Vietnamese": 0.5397727272727273, + "Filipino,Chinese,Malay,Vietnamese": 0.4659090909090909, + "Spanish,Chinese,Malay,Vietnamese": 0.44886363636363635 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.42045454545454547, + "Indonesian,English,Filipino,Spanish,Malay": 0.5, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.5113636363636364, + "Indonesian,English,Filipino,Chinese,Malay": 0.4375, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.4375, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.5056818181818182, + "Indonesian,English,Spanish,Chinese,Malay": 0.42045454545454547, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4318181818181818, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.5056818181818182, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.4375, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.42045454545454547, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4034090909090909, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4943181818181818, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.42613636363636365, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.4090909090909091, + "English,Filipino,Spanish,Chinese,Malay": 0.4375, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.4431818181818182, + "English,Filipino,Spanish,Malay,Vietnamese": 0.5170454545454546, + "English,Filipino,Chinese,Malay,Vietnamese": 0.44886363636363635, + "English,Spanish,Chinese,Malay,Vietnamese": 0.4375, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.42613636363636365 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.3977272727272727, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.39204545454545453, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4715909090909091, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.4090909090909091, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.38636363636363635, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4147727272727273 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.375 } }, - "AC3_2": 0.3257351595709543, - "AC3_3": 0.19314787587355034, - "AC3_4": 0.10959191667488712, - "AC3_5": 0.06274255154789302, - "AC3_6": 0.03665792334630696, - "AC3_7": 0.021923783280598918 + "AC3_2": 0.5860106336131864, + "AC3_3": 0.5368746071585395, + "AC3_4": 0.5013618807436455, + "AC3_5": 0.4735217048055425, + "AC3_6": 0.44983055366842767, + "AC3_7": 0.4279739776461651 }, "prompt_4": { - "overall_acc": 0.32792207792207784, + "overall_acc": 0.49269480519480524, "language_acc": { - "English": 0.3181818181818182, - "Filipino": 0.3352272727272727, - "Spanish": 0.32386363636363635, - "Malay": 0.32386363636363635, - "Chinese": 0.3409090909090909, - "Indonesian": 0.30113636363636365, - "Vietnamese": 0.3522727272727273 + "Indonesian": 0.48295454545454547, + "English": 0.5113636363636364, + "Filipino": 0.5, + "Spanish": 0.48295454545454547, + "Chinese": 0.5, + "Malay": 0.5056818181818182, + "Vietnamese": 0.4659090909090909 }, - "consistency_score_2": 0.509469696969697, - "consistency_score_3": 0.32272727272727264, - "consistency_score_4": 0.22759740259740258, - "consistency_score_5": 0.17153679653679652, - "consistency_score_6": 0.13474025974025974, - "consistency_score_7": 0.10795454545454546, + "consistency_score_2": 0.7058982683982685, + "consistency_score_3": 0.5751623376623378, + "consistency_score_4": 0.4956168831168831, + "consistency_score_5": 0.439935064935065, + "consistency_score_6": 0.39772727272727276, + "consistency_score_7": 0.36363636363636365, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4659090909090909, - "English,Spanish": 0.5568181818181818, - "English,Malay": 0.4431818181818182, - "English,Chinese": 0.5284090909090909, - "English,Indonesian": 0.4659090909090909, - "English,Vietnamese": 0.39204545454545453, - "Filipino,Spanish": 0.4431818181818182, - "Filipino,Malay": 0.625, - "Filipino,Chinese": 0.4772727272727273, - "Filipino,Indonesian": 0.5965909090909091, - "Filipino,Vietnamese": 0.6420454545454546, - "Spanish,Malay": 0.4034090909090909, - "Spanish,Chinese": 0.5113636363636364, - "Spanish,Indonesian": 0.375, - "Spanish,Vietnamese": 0.36363636363636365, - "Malay,Chinese": 0.5340909090909091, - "Malay,Indonesian": 0.6931818181818182, - "Malay,Vietnamese": 0.625, - "Chinese,Indonesian": 0.48295454545454547, - "Chinese,Vietnamese": 0.44886363636363635, - "Indonesian,Vietnamese": 0.625 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Malay": 0.3068181818181818, - "English,Filipino,Chinese": 0.3068181818181818, - "English,Filipino,Indonesian": 0.32386363636363635, - "English,Filipino,Vietnamese": 0.29545454545454547, - "English,Spanish,Malay": 0.2727272727272727, - "English,Spanish,Chinese": 0.3352272727272727, - "English,Spanish,Indonesian": 0.2727272727272727, - "English,Spanish,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese": 0.3181818181818182, - "English,Malay,Indonesian": 0.3409090909090909, - "English,Malay,Vietnamese": 0.26704545454545453, - "English,Chinese,Indonesian": 0.3125, - "English,Chinese,Vietnamese": 0.26136363636363635, - "English,Indonesian,Vietnamese": 0.3125, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.2840909090909091, - "Filipino,Spanish,Indonesian": 0.2784090909090909, - "Filipino,Spanish,Vietnamese": 0.2840909090909091, - "Filipino,Malay,Chinese": 0.3693181818181818, - "Filipino,Malay,Indonesian": 0.48295454545454547, - "Filipino,Malay,Vietnamese": 0.4772727272727273, - "Filipino,Chinese,Indonesian": 0.3409090909090909, - "Filipino,Chinese,Vietnamese": 0.3352272727272727, - "Filipino,Indonesian,Vietnamese": 0.45454545454545453, - "Spanish,Malay,Chinese": 0.3181818181818182, - "Spanish,Malay,Indonesian": 0.2840909090909091, - "Spanish,Malay,Vietnamese": 0.24431818181818182, - "Spanish,Chinese,Indonesian": 0.2840909090909091, - "Spanish,Chinese,Vietnamese": 0.26704545454545453, - "Spanish,Indonesian,Vietnamese": 0.25, - "Malay,Chinese,Indonesian": 0.4090909090909091, - "Malay,Chinese,Vietnamese": 0.3465909090909091, - "Malay,Indonesian,Vietnamese": 0.5, - "Chinese,Indonesian,Vietnamese": 0.35795454545454547 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.1875, - "English,Filipino,Spanish,Chinese": 0.19318181818181818, - "English,Filipino,Spanish,Indonesian": 0.20454545454545456, - "English,Filipino,Spanish,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese": 0.2215909090909091, - "English,Filipino,Malay,Indonesian": 0.2727272727272727, - "English,Filipino,Malay,Vietnamese": 0.22727272727272727, - "English,Filipino,Chinese,Indonesian": 0.22727272727272727, - "English,Filipino,Chinese,Vietnamese": 0.21022727272727273, - "English,Filipino,Indonesian,Vietnamese": 0.25, - "English,Spanish,Malay,Chinese": 0.2215909090909091, - "English,Spanish,Malay,Indonesian": 0.20454545454545456, - "English,Spanish,Malay,Vietnamese": 0.17045454545454544, - "English,Spanish,Chinese,Indonesian": 0.21022727272727273, - "English,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Malay,Chinese,Indonesian": 0.25, - "English,Malay,Chinese,Vietnamese": 0.20454545454545456, - "English,Malay,Indonesian,Vietnamese": 0.25, - "English,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Malay,Chinese": 0.2159090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.21022727272727273, - "Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.3068181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.2727272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.3806818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Chinese,Indonesian": 0.23863636363636365, - "Spanish,Malay,Chinese,Vietnamese": 0.19886363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.32954545454545453 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.19886363636363635, - "English,Filipino,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.1875, - "English,Spanish,Malay,Chinese,Indonesian": 0.17045454545454544, - "English,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818 + "Indonesian,English": 0.7272727272727273, + "Indonesian,Filipino": 0.7215909090909091, + "Indonesian,Spanish": 0.7443181818181818, + "Indonesian,Chinese": 0.6590909090909091, + "Indonesian,Malay": 0.7386363636363636, + "Indonesian,Vietnamese": 0.7386363636363636, + "English,Filipino": 0.7556818181818182, + "English,Spanish": 0.7784090909090909, + "English,Chinese": 0.6875, + "English,Malay": 0.6647727272727273, + "English,Vietnamese": 0.75, + "Filipino,Spanish": 0.7329545454545454, + "Filipino,Chinese": 0.6761363636363636, + "Filipino,Malay": 0.6988636363636364, + "Filipino,Vietnamese": 0.6875, + "Spanish,Chinese": 0.6363636363636364, + "Spanish,Malay": 0.7159090909090909, + "Spanish,Vietnamese": 0.7386363636363636, + "Chinese,Malay": 0.6193181818181818, + "Chinese,Vietnamese": 0.6534090909090909, + "Malay,Vietnamese": 0.6988636363636364 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.6079545454545454, + "Indonesian,English,Spanish": 0.6306818181818182, + "Indonesian,English,Chinese": 0.5454545454545454, + "Indonesian,English,Malay": 0.5795454545454546, + "Indonesian,English,Vietnamese": 0.625, + "Indonesian,Filipino,Spanish": 0.6079545454545454, + "Indonesian,Filipino,Chinese": 0.5511363636363636, + "Indonesian,Filipino,Malay": 0.5965909090909091, + "Indonesian,Filipino,Vietnamese": 0.6022727272727273, + "Indonesian,Spanish,Chinese": 0.5284090909090909, + "Indonesian,Spanish,Malay": 0.6136363636363636, + "Indonesian,Spanish,Vietnamese": 0.6193181818181818, + "Indonesian,Chinese,Malay": 0.5227272727272727, + "Indonesian,Chinese,Vietnamese": 0.5454545454545454, + "Indonesian,Malay,Vietnamese": 0.6079545454545454, + "English,Filipino,Spanish": 0.6363636363636364, + "English,Filipino,Chinese": 0.5795454545454546, + "English,Filipino,Malay": 0.5738636363636364, + "English,Filipino,Vietnamese": 0.6193181818181818, + "English,Spanish,Chinese": 0.5681818181818182, + "English,Spanish,Malay": 0.5909090909090909, + "English,Spanish,Vietnamese": 0.6534090909090909, + "English,Chinese,Malay": 0.5, + "English,Chinese,Vietnamese": 0.5625, + "English,Malay,Vietnamese": 0.5795454545454546, + "Filipino,Spanish,Chinese": 0.5454545454545454, + "Filipino,Spanish,Malay": 0.5909090909090909, + "Filipino,Spanish,Vietnamese": 0.6022727272727273, + "Filipino,Chinese,Malay": 0.5170454545454546, + "Filipino,Chinese,Vietnamese": 0.5227272727272727, + "Filipino,Malay,Vietnamese": 0.5738636363636364, + "Spanish,Chinese,Malay": 0.4943181818181818, + "Spanish,Chinese,Vietnamese": 0.5284090909090909, + "Spanish,Malay,Vietnamese": 0.5965909090909091, + "Chinese,Malay,Vietnamese": 0.5113636363636364 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.5511363636363636, + "Indonesian,English,Filipino,Chinese": 0.4943181818181818, + "Indonesian,English,Filipino,Malay": 0.5113636363636364, + "Indonesian,English,Filipino,Vietnamese": 0.5511363636363636, + "Indonesian,English,Spanish,Chinese": 0.48295454545454547, + "Indonesian,English,Spanish,Malay": 0.5227272727272727, + "Indonesian,English,Spanish,Vietnamese": 0.5625, + "Indonesian,English,Chinese,Malay": 0.44886363636363635, + "Indonesian,English,Chinese,Vietnamese": 0.5, + "Indonesian,English,Malay,Vietnamese": 0.5113636363636364, + "Indonesian,Filipino,Spanish,Chinese": 0.4715909090909091, + "Indonesian,Filipino,Spanish,Malay": 0.5284090909090909, + "Indonesian,Filipino,Spanish,Vietnamese": 0.5340909090909091, + "Indonesian,Filipino,Chinese,Malay": 0.4602272727272727, + "Indonesian,Filipino,Chinese,Vietnamese": 0.4659090909090909, + "Indonesian,Filipino,Malay,Vietnamese": 0.5227272727272727, + "Indonesian,Spanish,Chinese,Malay": 0.44886363636363635, + "Indonesian,Spanish,Chinese,Vietnamese": 0.4602272727272727, + "Indonesian,Spanish,Malay,Vietnamese": 0.5284090909090909, + "Indonesian,Chinese,Malay,Vietnamese": 0.4602272727272727, + "English,Filipino,Spanish,Chinese": 0.5, + "English,Filipino,Spanish,Malay": 0.5170454545454546, + "English,Filipino,Spanish,Vietnamese": 0.5625, + "English,Filipino,Chinese,Malay": 0.4659090909090909, + "English,Filipino,Chinese,Vietnamese": 0.4943181818181818, + "English,Filipino,Malay,Vietnamese": 0.5170454545454546, + "English,Spanish,Chinese,Malay": 0.44886363636363635, + "English,Spanish,Chinese,Vietnamese": 0.5, + "English,Spanish,Malay,Vietnamese": 0.5397727272727273, + "English,Chinese,Malay,Vietnamese": 0.4602272727272727, + "Filipino,Spanish,Chinese,Malay": 0.45454545454545453, + "Filipino,Spanish,Chinese,Vietnamese": 0.4659090909090909, + "Filipino,Spanish,Malay,Vietnamese": 0.5170454545454546, + "Filipino,Chinese,Malay,Vietnamese": 0.4431818181818182, + "Spanish,Chinese,Malay,Vietnamese": 0.4431818181818182 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.4431818181818182, + "Indonesian,English,Filipino,Spanish,Malay": 0.4715909090909091, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.5056818181818182, + "Indonesian,English,Filipino,Chinese,Malay": 0.42045454545454547, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.44886363636363635, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.4715909090909091, + "Indonesian,English,Spanish,Chinese,Malay": 0.4090909090909091, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.4431818181818182, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.4772727272727273, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.42045454545454547, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.4147727272727273, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4147727272727273, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4715909090909091, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.4090909090909091, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.4034090909090909, + "English,Filipino,Spanish,Chinese,Malay": 0.42045454545454547, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.44886363636363635, + "English,Filipino,Spanish,Malay,Vietnamese": 0.48295454545454547, + "English,Filipino,Chinese,Malay,Vietnamese": 0.42613636363636365, + "English,Spanish,Chinese,Malay,Vietnamese": 0.42613636363636365, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.4090909090909091 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.38636363636363635, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.4034090909090909, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4431818181818182, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.39204545454545453, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.38636363636363635, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.375, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.36363636363636365 } }, - "AC3_2": 0.399016008176759, - "AC3_3": 0.3253039375293898, - "AC3_4": 0.26870061552541985, - "AC3_5": 0.2252465843564158, - "AC3_6": 0.19100022780105022, - "AC3_7": 0.16243439982730132 + "AC3_2": 0.5803344228650118, + "AC3_3": 0.5307442063367972, + "AC3_4": 0.49415152434544735, + "AC3_5": 0.46482260119573776, + "AC3_6": 0.4401466809814796, + "AC3_7": 0.4184403273962025 }, "prompt_5": { - "overall_acc": 0.30032467532467527, + "overall_acc": 0.4886363636363637, "language_acc": { - "English": 0.3181818181818182, - "Filipino": 0.30113636363636365, - "Spanish": 0.3181818181818182, - "Malay": 0.2784090909090909, - "Chinese": 0.3125, - "Indonesian": 0.2727272727272727, - "Vietnamese": 0.30113636363636365 + "Indonesian": 0.45454545454545453, + "English": 0.5284090909090909, + "Filipino": 0.48863636363636365, + "Spanish": 0.48295454545454547, + "Chinese": 0.5056818181818182, + "Malay": 0.5056818181818182, + "Vietnamese": 0.45454545454545453 }, - "consistency_score_2": 0.4978354978354978, - "consistency_score_3": 0.300974025974026, - "consistency_score_4": 0.20227272727272727, - "consistency_score_5": 0.1471861471861472, - "consistency_score_6": 0.11525974025974027, - "consistency_score_7": 0.09659090909090909, + "consistency_score_2": 0.6999458874458876, + "consistency_score_3": 0.5681818181818182, + "consistency_score_4": 0.48814935064935044, + "consistency_score_5": 0.4320887445887445, + "consistency_score_6": 0.3887987012987013, + "consistency_score_7": 0.3522727272727273, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.36363636363636365, - "English,Spanish": 0.5511363636363636, - "English,Malay": 0.375, - "English,Chinese": 0.5340909090909091, - "English,Indonesian": 0.4431818181818182, - "English,Vietnamese": 0.3125, - "Filipino,Spanish": 0.4772727272727273, - "Filipino,Malay": 0.6818181818181818, - "Filipino,Chinese": 0.44886363636363635, - "Filipino,Indonesian": 0.6079545454545454, - "Filipino,Vietnamese": 0.6590909090909091, - "Spanish,Malay": 0.4375, - "Spanish,Chinese": 0.5227272727272727, - "Spanish,Indonesian": 0.4375, - "Spanish,Vietnamese": 0.42613636363636365, - "Malay,Chinese": 0.4602272727272727, - "Malay,Indonesian": 0.6704545454545454, - "Malay,Vietnamese": 0.6818181818181818, - "Chinese,Indonesian": 0.42613636363636365, - "Chinese,Vietnamese": 0.35795454545454547, - "Indonesian,Vietnamese": 0.5795454545454546 + "Indonesian,English": 0.6761363636363636, + "Indonesian,Filipino": 0.7272727272727273, + "Indonesian,Spanish": 0.7159090909090909, + "Indonesian,Chinese": 0.6136363636363636, + "Indonesian,Malay": 0.7159090909090909, + "Indonesian,Vietnamese": 0.7443181818181818, + "English,Filipino": 0.75, + "English,Spanish": 0.7613636363636364, + "English,Chinese": 0.6647727272727273, + "English,Malay": 0.7045454545454546, + "English,Vietnamese": 0.7443181818181818, + "Filipino,Spanish": 0.7386363636363636, + "Filipino,Chinese": 0.6420454545454546, + "Filipino,Malay": 0.7159090909090909, + "Filipino,Vietnamese": 0.7045454545454546, + "Spanish,Chinese": 0.6193181818181818, + "Spanish,Malay": 0.7329545454545454, + "Spanish,Vietnamese": 0.7443181818181818, + "Chinese,Malay": 0.6136363636363636, + "Chinese,Vietnamese": 0.6534090909090909, + "Malay,Vietnamese": 0.7159090909090909 }, "3_combine": { - "English,Filipino,Spanish": 0.23863636363636365, - "English,Filipino,Malay": 0.26136363636363635, - "English,Filipino,Chinese": 0.24431818181818182, - "English,Filipino,Indonesian": 0.2556818181818182, - "English,Filipino,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay": 0.23863636363636365, - "English,Spanish,Chinese": 0.3465909090909091, - "English,Spanish,Indonesian": 0.2727272727272727, - "English,Spanish,Vietnamese": 0.21022727272727273, - "English,Malay,Chinese": 0.2727272727272727, - "English,Malay,Indonesian": 0.2840909090909091, - "English,Malay,Vietnamese": 0.23295454545454544, - "English,Chinese,Indonesian": 0.2840909090909091, - "English,Chinese,Vietnamese": 0.20454545454545456, - "English,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay": 0.3465909090909091, - "Filipino,Spanish,Chinese": 0.2784090909090909, - "Filipino,Spanish,Indonesian": 0.3181818181818182, - "Filipino,Spanish,Vietnamese": 0.32386363636363635, - "Filipino,Malay,Chinese": 0.3409090909090909, - "Filipino,Malay,Indonesian": 0.4943181818181818, - "Filipino,Malay,Vietnamese": 0.5284090909090909, - "Filipino,Chinese,Indonesian": 0.30113636363636365, - "Filipino,Chinese,Vietnamese": 0.2840909090909091, - "Filipino,Indonesian,Vietnamese": 0.4431818181818182, - "Spanish,Malay,Chinese": 0.2784090909090909, - "Spanish,Malay,Indonesian": 0.32954545454545453, - "Spanish,Malay,Vietnamese": 0.29545454545454547, - "Spanish,Chinese,Indonesian": 0.2784090909090909, - "Spanish,Chinese,Vietnamese": 0.23863636363636365, - "Spanish,Indonesian,Vietnamese": 0.2784090909090909, - "Malay,Chinese,Indonesian": 0.3352272727272727, - "Malay,Chinese,Vietnamese": 0.2897727272727273, - "Malay,Indonesian,Vietnamese": 0.48295454545454547, - "Chinese,Indonesian,Vietnamese": 0.26136363636363635 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.16477272727272727, - "English,Filipino,Spanish,Chinese": 0.17613636363636365, - "English,Filipino,Spanish,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese": 0.1875, - "English,Filipino,Malay,Indonesian": 0.20454545454545456, - "English,Filipino,Malay,Vietnamese": 0.19886363636363635, - "English,Filipino,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay,Chinese": 0.19886363636363635, - "English,Spanish,Malay,Indonesian": 0.19318181818181818, - "English,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Spanish,Chinese,Indonesian": 0.21022727272727273, - "English,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "English,Malay,Chinese,Indonesian": 0.2159090909090909, - "English,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese": 0.21022727272727273, - "Filipino,Spanish,Malay,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Malay,Vietnamese": 0.2556818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.26704545454545453, - "Filipino,Malay,Chinese,Vietnamese": 0.24431818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.39204545454545453, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Chinese,Indonesian": 0.21022727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "English,Spanish,Malay,Chinese,Vietnamese": 0.125, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273 + "Indonesian,English,Filipino": 0.5909090909090909, + "Indonesian,English,Spanish": 0.5909090909090909, + "Indonesian,English,Chinese": 0.5, + "Indonesian,English,Malay": 0.5681818181818182, + "Indonesian,English,Vietnamese": 0.6022727272727273, + "Indonesian,Filipino,Spanish": 0.5965909090909091, + "Indonesian,Filipino,Chinese": 0.5056818181818182, + "Indonesian,Filipino,Malay": 0.5909090909090909, + "Indonesian,Filipino,Vietnamese": 0.6079545454545454, + "Indonesian,Spanish,Chinese": 0.48863636363636365, + "Indonesian,Spanish,Malay": 0.5965909090909091, + "Indonesian,Spanish,Vietnamese": 0.6136363636363636, + "Indonesian,Chinese,Malay": 0.4943181818181818, + "Indonesian,Chinese,Vietnamese": 0.5284090909090909, + "Indonesian,Malay,Vietnamese": 0.6136363636363636, + "English,Filipino,Spanish": 0.6306818181818182, + "English,Filipino,Chinese": 0.5511363636363636, + "English,Filipino,Malay": 0.6022727272727273, + "English,Filipino,Vietnamese": 0.6193181818181818, + "English,Spanish,Chinese": 0.5397727272727273, + "English,Spanish,Malay": 0.6136363636363636, + "English,Spanish,Vietnamese": 0.6420454545454546, + "English,Chinese,Malay": 0.5056818181818182, + "English,Chinese,Vietnamese": 0.5568181818181818, + "English,Malay,Vietnamese": 0.6079545454545454, + "Filipino,Spanish,Chinese": 0.5227272727272727, + "Filipino,Spanish,Malay": 0.6079545454545454, + "Filipino,Spanish,Vietnamese": 0.6136363636363636, + "Filipino,Chinese,Malay": 0.5113636363636364, + "Filipino,Chinese,Vietnamese": 0.5227272727272727, + "Filipino,Malay,Vietnamese": 0.5965909090909091, + "Spanish,Chinese,Malay": 0.4943181818181818, + "Spanish,Chinese,Vietnamese": 0.5284090909090909, + "Spanish,Malay,Vietnamese": 0.6136363636363636, + "Chinese,Malay,Vietnamese": 0.5170454545454546 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.5284090909090909, + "Indonesian,English,Filipino,Chinese": 0.45454545454545453, + "Indonesian,English,Filipino,Malay": 0.5, + "Indonesian,English,Filipino,Vietnamese": 0.5397727272727273, + "Indonesian,English,Spanish,Chinese": 0.4431818181818182, + "Indonesian,English,Spanish,Malay": 0.5113636363636364, + "Indonesian,English,Spanish,Vietnamese": 0.5397727272727273, + "Indonesian,English,Chinese,Malay": 0.4318181818181818, + "Indonesian,English,Chinese,Vietnamese": 0.4772727272727273, + "Indonesian,English,Malay,Vietnamese": 0.5284090909090909, + "Indonesian,Filipino,Spanish,Chinese": 0.4431818181818182, + "Indonesian,Filipino,Spanish,Malay": 0.5170454545454546, + "Indonesian,Filipino,Spanish,Vietnamese": 0.5340909090909091, + "Indonesian,Filipino,Chinese,Malay": 0.4318181818181818, + "Indonesian,Filipino,Chinese,Vietnamese": 0.45454545454545453, + "Indonesian,Filipino,Malay,Vietnamese": 0.5284090909090909, + "Indonesian,Spanish,Chinese,Malay": 0.42613636363636365, + "Indonesian,Spanish,Chinese,Vietnamese": 0.4431818181818182, + "Indonesian,Spanish,Malay,Vietnamese": 0.5340909090909091, + "Indonesian,Chinese,Malay,Vietnamese": 0.44886363636363635, + "English,Filipino,Spanish,Chinese": 0.4715909090909091, + "English,Filipino,Spanish,Malay": 0.5397727272727273, + "English,Filipino,Spanish,Vietnamese": 0.5568181818181818, + "English,Filipino,Chinese,Malay": 0.45454545454545453, + "English,Filipino,Chinese,Vietnamese": 0.48295454545454547, + "English,Filipino,Malay,Vietnamese": 0.5397727272727273, + "English,Spanish,Chinese,Malay": 0.4431818181818182, + "English,Spanish,Chinese,Vietnamese": 0.48863636363636365, + "English,Spanish,Malay,Vietnamese": 0.5511363636363636, + "English,Chinese,Malay,Vietnamese": 0.4772727272727273, + "Filipino,Spanish,Chinese,Malay": 0.44886363636363635, + "Filipino,Spanish,Chinese,Vietnamese": 0.4715909090909091, + "Filipino,Spanish,Malay,Vietnamese": 0.5397727272727273, + "Filipino,Chinese,Malay,Vietnamese": 0.44886363636363635, + "Spanish,Chinese,Malay,Vietnamese": 0.45454545454545453 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.4090909090909091, + "Indonesian,English,Filipino,Spanish,Malay": 0.4602272727272727, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.48863636363636365, + "Indonesian,English,Filipino,Chinese,Malay": 0.39204545454545453, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.4318181818181818, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.4772727272727273, + "Indonesian,English,Spanish,Chinese,Malay": 0.39204545454545453, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.42045454545454547, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.48295454545454547, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.42045454545454547, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.39204545454545453, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.4090909090909091, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.4772727272727273, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.3977272727272727, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727, + "English,Filipino,Spanish,Chinese,Malay": 0.4090909090909091, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.4375, + "English,Filipino,Spanish,Malay,Vietnamese": 0.5, + "English,Filipino,Chinese,Malay,Vietnamese": 0.42613636363636365, + "English,Spanish,Chinese,Malay,Vietnamese": 0.4318181818181818, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.42045454545454547 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.36363636363636365, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.38636363636363635, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.4431818181818182, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.3806818181818182, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.3806818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3693181818181818, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3977272727272727 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.3522727272727273 } }, - "AC3_2": 0.3746423067987584, - "AC3_3": 0.3006489999780498, - "AC3_4": 0.24173420244191945, - "AC3_5": 0.19755335349469832, - "AC3_6": 0.166586343304071, - "AC3_7": 0.14617029183898678 + "AC3_2": 0.575507522249451, + "AC3_3": 0.5254154447205668, + "AC3_4": 0.4883927356836032, + "AC3_5": 0.4586260785439428, + "AC3_6": 0.4330375914063701, + "AC3_7": 0.4093980343493492 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.42718446601941745 + "accuracy": 0.5048543689320388 }, "prompt_2": { - "accuracy": 0.3592233009708738 + "accuracy": 0.49514563106796117 }, "prompt_3": { - "accuracy": 0.3106796116504854 + "accuracy": 0.5242718446601942 }, "prompt_4": { - "accuracy": 0.2912621359223301 + "accuracy": 0.5145631067961165 }, "prompt_5": { - "accuracy": 0.42718446601941745 + "accuracy": 0.5339805825242718 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.18095238095238095 + "accuracy": 0.3047619047619048 }, "prompt_2": { - "accuracy": 0.24761904761904763 + "accuracy": 0.3142857142857143 }, "prompt_3": { - "accuracy": 0.2761904761904762 + "accuracy": 0.29523809523809524 }, "prompt_4": { - "accuracy": 0.24761904761904763 + "accuracy": 0.3142857142857143 }, "prompt_5": { - "accuracy": 0.2571428571428571 + "accuracy": 0.3238095238095238 } }, "us_eval": { "prompt_1": { - "accuracy": 0.37383177570093457 + "accuracy": 0.2523364485981308 }, "prompt_2": { - "accuracy": 0.3364485981308411 + "accuracy": 0.2336448598130841 }, "prompt_3": { - "accuracy": 0.29906542056074764 + "accuracy": 0.24299065420560748 }, "prompt_4": { - "accuracy": 0.3177570093457944 + "accuracy": 0.2523364485981308 }, "prompt_5": { - "accuracy": 0.3177570093457944 + "accuracy": 0.24299065420560748 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.24, + "accuracy": 0.3, "category_acc": { - "brand": 0.2, - "demographics": 0.0, - "biology": 0.3, + "brand": 0.3, + "demographics": 0.2, + "biology": 0.6, "history": 0.2, - "literature": 0.2, - "politics": 0.3, + "literature": 0.0, + "politics": 0.5, "culture": 0.3, - "film": 0.3, - "law": 0.2, - "geography": 0.3 + "film": 0.1, + "law": 0.3, + "geography": 0.5 } }, "prompt_2": { - "accuracy": 0.26, + "accuracy": 0.33, "category_acc": { "brand": 0.3, - "demographics": 0.0, - "biology": 0.2, + "demographics": 0.2, + "biology": 0.6, "history": 0.26666666666666666, - "literature": 0.1, - "politics": 0.4, - "culture": 0.4, - "film": 0.3, - "law": 0.2, - "geography": 0.3 + "literature": 0.3, + "politics": 0.5, + "culture": 0.2, + "film": 0.1, + "law": 0.3, + "geography": 0.5 } }, "prompt_3": { "accuracy": 0.31, "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.4, - "history": 0.26666666666666666, + "brand": 0.4, + "demographics": 0.0, + "biology": 0.5, + "history": 0.2, "literature": 0.2, - "politics": 0.5, - "culture": 0.5, - "film": 0.3, - "law": 0.2, - "geography": 0.4 + "politics": 0.4, + "culture": 0.4, + "film": 0.1, + "law": 0.3, + "geography": 0.5 } }, "prompt_4": { - "accuracy": 0.25, + "accuracy": 0.31, "category_acc": { - "brand": 0.1, - "demographics": 0.0, - "biology": 0.1, + "brand": 0.3, + "demographics": 0.2, + "biology": 0.5, "history": 0.2, - "literature": 0.1, - "politics": 0.6, - "culture": 0.4, - "film": 0.3, + "literature": 0.2, + "politics": 0.5, + "culture": 0.3, + "film": 0.1, "law": 0.3, - "geography": 0.3 + "geography": 0.5 } }, "prompt_5": { - "accuracy": 0.26, + "accuracy": 0.32, "category_acc": { - "brand": 0.1, + "brand": 0.4, "demographics": 0.0, - "biology": 0.2, - "history": 0.26666666666666666, - "literature": 0.2, - "politics": 0.3, - "culture": 0.5, - "film": 0.3, + "biology": 0.5, + "history": 0.2, + "literature": 0.3, + "politics": 0.4, + "culture": 0.3, + "film": 0.1, "law": 0.3, - "geography": 0.3 + "geography": 0.6 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.1356886557716033 + "bleu_score": 0.1415726644994035 }, "prompt_2": { - "bleu_score": 0.10806061464113383 + "bleu_score": 0.13732215511867826 }, "prompt_3": { - "bleu_score": 0.0758690303307498 + "bleu_score": 0.1372456478046343 }, "prompt_4": { - "bleu_score": 0.10734329848134226 + "bleu_score": 0.13881717658413173 }, "prompt_5": { - "bleu_score": 0.04933226086349954 + "bleu_score": 0.1359329437121942 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "prompt_1": { + "accuracy": 0.5062420722344616, + "category_acc": { + "History": 0.3714859437751004, + "Geography": 0.4326530612244898, + "Lampungic": 0.4557823129251701, + "Social science": 0.672787979966611, + "Balinese": 0.4309978768577495, + "Makassarese": 0.3924731182795699, + "Banjarese": 0.4166666666666667, + "Chemistry": 0.22773722627737225, + "Biology": 0.3562130177514793, + "Science": 0.5521155830753354, + "Christian religion": 0.6069651741293532, + "Art": 0.5640599001663894, + "Islam religion": 0.6173541963015647, + "Hindu religion": 0.5533333333333333, + "Madurese": 0.3152542372881356, + "Sport": 0.4594594594594595, + "Indonesian language": 0.573785803237858, + "Physics": 0.37373737373737376, + "Minangkabau culture": 0.4020100502512563, + "Dayak language": 0.3577981651376147, + "Sociology": 0.4798387096774194, + "Economy": 0.4733606557377049, + "Sundanese": 0.6076058772687987, + "Javanese": 0.5362903225806451, + "Civic education": 0.5693848354792561 + } + }, + "prompt_2": { + "accuracy": 0.5037719473930169, + "category_acc": { + "History": 0.37751004016064255, + "Geography": 0.4489795918367347, + "Lampungic": 0.40816326530612246, + "Social science": 0.671118530884808, + "Balinese": 0.42250530785562634, + "Makassarese": 0.3870967741935484, + "Banjarese": 0.4166666666666667, + "Chemistry": 0.22335766423357664, + "Biology": 0.3562130177514793, + "Science": 0.542827657378741, + "Christian religion": 0.5870646766169154, + "Art": 0.5490848585690515, + "Islam religion": 0.6145092460881935, + "Hindu religion": 0.54, + "Madurese": 0.3050847457627119, + "Sport": 0.46621621621621623, + "Indonesian language": 0.5688044831880449, + "Physics": 0.37777777777777777, + "Minangkabau culture": 0.39195979899497485, + "Dayak language": 0.3302752293577982, + "Sociology": 0.48185483870967744, + "Economy": 0.48565573770491804, + "Sundanese": 0.6093344857389801, + "Javanese": 0.5383064516129032, + "Civic education": 0.575107296137339 + } + }, + "prompt_3": { + "accuracy": 0.5051739101408639, + "category_acc": { + "History": 0.3714859437751004, + "Geography": 0.43673469387755104, + "Lampungic": 0.41496598639455784, + "Social science": 0.667779632721202, + "Balinese": 0.4309978768577495, + "Makassarese": 0.3978494623655914, + "Banjarese": 0.4166666666666667, + "Chemistry": 0.22773722627737225, + "Biology": 0.35502958579881655, + "Science": 0.543859649122807, + "Christian religion": 0.6069651741293532, + "Art": 0.5490848585690515, + "Islam religion": 0.6173541963015647, + "Hindu religion": 0.5533333333333333, + "Madurese": 0.3050847457627119, + "Sport": 0.43243243243243246, + "Indonesian language": 0.5712951432129514, + "Physics": 0.3717171717171717, + "Minangkabau culture": 0.40703517587939697, + "Dayak language": 0.3577981651376147, + "Sociology": 0.4899193548387097, + "Economy": 0.48155737704918034, + "Sundanese": 0.6093344857389801, + "Javanese": 0.5352822580645161, + "Civic education": 0.5879828326180258 + } + }, + "prompt_4": { + "accuracy": 0.5062420722344616, + "category_acc": { + "History": 0.3855421686746988, + "Geography": 0.4448979591836735, + "Lampungic": 0.41496598639455784, + "Social science": 0.674457429048414, + "Balinese": 0.42250530785562634, + "Makassarese": 0.3978494623655914, + "Banjarese": 0.4027777777777778, + "Chemistry": 0.22189781021897811, + "Biology": 0.3668639053254438, + "Science": 0.5469556243550051, + "Christian religion": 0.5920398009950248, + "Art": 0.5391014975041597, + "Islam religion": 0.6187766714082503, + "Hindu religion": 0.54, + "Madurese": 0.33559322033898303, + "Sport": 0.4594594594594595, + "Indonesian language": 0.5669364881693649, + "Physics": 0.3717171717171717, + "Minangkabau culture": 0.39195979899497485, + "Dayak language": 0.3394495412844037, + "Sociology": 0.4899193548387097, + "Economy": 0.4918032786885246, + "Sundanese": 0.6093344857389801, + "Javanese": 0.5433467741935484, + "Civic education": 0.5894134477825465 + } + }, + "prompt_5": { + "accuracy": 0.5023699846451699, + "category_acc": { + "History": 0.3895582329317269, + "Geography": 0.4489795918367347, + "Lampungic": 0.38095238095238093, + "Social science": 0.672787979966611, + "Balinese": 0.4118895966029724, + "Makassarese": 0.3870967741935484, + "Banjarese": 0.4236111111111111, + "Chemistry": 0.2321167883211679, + "Biology": 0.35502958579881655, + "Science": 0.5345717234262126, + "Christian religion": 0.6019900497512438, + "Art": 0.5440931780366056, + "Islam religion": 0.604551920341394, + "Hindu religion": 0.54, + "Madurese": 0.3152542372881356, + "Sport": 0.44594594594594594, + "Indonesian language": 0.5647571606475716, + "Physics": 0.37777777777777777, + "Minangkabau culture": 0.4020100502512563, + "Dayak language": 0.3486238532110092, + "Sociology": 0.49798387096774194, + "Economy": 0.47540983606557374, + "Sundanese": 0.6101987899740708, + "Javanese": 0.5282258064516129, + "Civic education": 0.5822603719599427 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.2444200671172596 + "bleu_score": 0.3702499286280674 }, "prompt_2": { - "bleu_score": 0.08485335693577586 + "bleu_score": 0.3725328734892377 }, "prompt_3": { - "bleu_score": 0.12313592712466565 + "bleu_score": 0.3682673142036171 }, "prompt_4": { - "bleu_score": 0.0686180575032872 + "bleu_score": 0.3729836664115557 }, "prompt_5": { - "bleu_score": 0.08952795856095064 + "bleu_score": 0.3738645804241501 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.16673367017296867 + "bleu_score": 0.3331210760216344 }, "prompt_2": { - "bleu_score": 0.10096533644997745 + "bleu_score": 0.3385870277660897 }, "prompt_3": { - "bleu_score": 0.18339451707341273 + "bleu_score": 0.3340723812360414 }, "prompt_4": { - "bleu_score": 0.05309273703256413 + "bleu_score": 0.34067568908971596 }, "prompt_5": { - "bleu_score": 0.08221664133321595 + "bleu_score": 0.3404965901668535 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.07418108303507828 + "bleu_score": 0.2777190863401374 }, "prompt_2": { - "bleu_score": 0.06940439628579004 + "bleu_score": 0.28465043692335784 }, "prompt_3": { - "bleu_score": 0.07168109252099726 + "bleu_score": 0.27745720654550554 }, "prompt_4": { - "bleu_score": 0.051208855779101045 + "bleu_score": 0.28738979676124116 }, "prompt_5": { - "bleu_score": 0.05649922074287873 + "bleu_score": 0.28484775494547243 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.11055368477566999 + "bleu_score": 0.37879242173955846 }, "prompt_2": { - "bleu_score": 0.0987530503033376 + "bleu_score": 0.3834367453020953 }, "prompt_3": { - "bleu_score": 0.11771323624007263 + "bleu_score": 0.37836560188389307 }, "prompt_4": { - "bleu_score": 0.06738875959390361 + "bleu_score": 0.38465772917800367 }, "prompt_5": { - "bleu_score": 0.0797475892868344 + "bleu_score": 0.382863345203959 } }, "mmlu": { "prompt_1": { - "accuracy": 0.3687281213535589 + "accuracy": 0.4515752625437573 }, "prompt_2": { - "accuracy": 0.3570595099183197 + "accuracy": 0.4492415402567094 }, "prompt_3": { - "accuracy": 0.34772462077012833 + "accuracy": 0.44807467911318555 }, "prompt_4": { - "accuracy": 0.352392065344224 + "accuracy": 0.4457409568261377 }, "prompt_5": { - "accuracy": 0.3570595099183197 + "accuracy": 0.45274212368728123 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.35366464068644976, + "accuracy": 0.42710046478369684, "category_acc": { - "high_school_european_history": 0.36585365853658536, - "business_ethics": 0.36363636363636365, - "clinical_knowledge": 0.3484848484848485, - "medical_genetics": 0.36363636363636365, - "high_school_us_history": 0.3842364532019704, - "high_school_physics": 0.22, - "high_school_world_history": 0.4025423728813559, - "virology": 0.40606060606060607, - "high_school_microeconomics": 0.29957805907172996, + "high_school_european_history": 0.6646341463414634, + "business_ethics": 0.5454545454545454, + "clinical_knowledge": 0.4393939393939394, + "medical_genetics": 0.3939393939393939, + "high_school_us_history": 0.47783251231527096, + "high_school_physics": 0.29333333333333333, + "high_school_world_history": 0.6483050847457628, + "virology": 0.3878787878787879, + "high_school_microeconomics": 0.4345991561181435, "econometrics": 0.26548672566371684, - "college_computer_science": 0.21212121212121213, - "high_school_biology": 0.37216828478964403, - "abstract_algebra": 0.3434343434343434, - "professional_accounting": 0.2918149466192171, - "philosophy": 0.3903225806451613, - "professional_medicine": 0.22878228782287824, - "nutrition": 0.4459016393442623, - "global_facts": 0.31313131313131315, - "machine_learning": 0.34234234234234234, - "security_studies": 0.45491803278688525, - "public_relations": 0.3302752293577982, - "professional_psychology": 0.3567921440261866, - "prehistory": 0.43343653250773995, - "anatomy": 0.39552238805970147, - "human_sexuality": 0.4076923076923077, - "college_medicine": 0.26744186046511625, - "high_school_government_and_politics": 0.4270833333333333, - "college_chemistry": 0.2222222222222222, - "logical_fallacies": 0.36419753086419754, - "high_school_geography": 0.3553299492385787, - "elementary_mathematics": 0.26790450928381965, - "human_aging": 0.38288288288288286, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.38786764705882354, - "formal_logic": 0.2, - "high_school_statistics": 0.22790697674418606, - "international_law": 0.5, - "high_school_mathematics": 0.26022304832713755, - "high_school_computer_science": 0.43434343434343436, - "conceptual_physics": 0.36752136752136755, - "miscellaneous": 0.45524296675191817, - "high_school_chemistry": 0.3465346534653465, - "marketing": 0.5364806866952789, - "professional_law": 0.3333333333333333, - "management": 0.3333333333333333, - "college_physics": 0.21782178217821782, - "jurisprudence": 0.42990654205607476, - "world_religions": 0.4823529411764706, - "sociology": 0.38, - "us_foreign_policy": 0.5454545454545454, - "high_school_macroeconomics": 0.3496143958868895, - "computer_security": 0.494949494949495, - "moral_scenarios": 0.2360178970917226, - "moral_disputes": 0.3681159420289855, - "electrical_engineering": 0.3680555555555556, - "astronomy": 0.3443708609271523, - "college_biology": 0.3776223776223776 + "college_computer_science": 0.40404040404040403, + "high_school_biology": 0.45307443365695793, + "abstract_algebra": 0.2828282828282828, + "professional_accounting": 0.3202846975088968, + "philosophy": 0.44516129032258067, + "professional_medicine": 0.4095940959409594, + "nutrition": 0.4524590163934426, + "global_facts": 0.29292929292929293, + "machine_learning": 0.23423423423423423, + "security_studies": 0.5204918032786885, + "public_relations": 0.4954128440366973, + "professional_psychology": 0.40098199672667756, + "prehistory": 0.43034055727554177, + "anatomy": 0.48507462686567165, + "human_sexuality": 0.5, + "college_medicine": 0.37790697674418605, + "high_school_government_and_politics": 0.53125, + "college_chemistry": 0.32323232323232326, + "logical_fallacies": 0.5864197530864198, + "high_school_geography": 0.5888324873096447, + "elementary_mathematics": 0.3050397877984085, + "human_aging": 0.4369369369369369, + "college_mathematics": 0.35353535353535354, + "high_school_psychology": 0.5735294117647058, + "formal_logic": 0.32, + "high_school_statistics": 0.33488372093023255, + "international_law": 0.65, + "high_school_mathematics": 0.26394052044609667, + "high_school_computer_science": 0.42424242424242425, + "conceptual_physics": 0.32905982905982906, + "miscellaneous": 0.5332480818414322, + "high_school_chemistry": 0.297029702970297, + "marketing": 0.759656652360515, + "professional_law": 0.3268101761252446, + "management": 0.6372549019607843, + "college_physics": 0.27722772277227725, + "jurisprudence": 0.5887850467289719, + "world_religions": 0.48823529411764705, + "sociology": 0.605, + "us_foreign_policy": 0.5858585858585859, + "high_school_macroeconomics": 0.4473007712082262, + "computer_security": 0.5454545454545454, + "moral_scenarios": 0.27181208053691275, + "moral_disputes": 0.4927536231884058, + "electrical_engineering": 0.4583333333333333, + "astronomy": 0.3841059602649007, + "college_biology": 0.2937062937062937 } }, "prompt_2": { - "accuracy": 0.36589202717196995, + "accuracy": 0.421809081158384, "category_acc": { - "high_school_european_history": 0.40853658536585363, - "business_ethics": 0.42424242424242425, - "clinical_knowledge": 0.3977272727272727, - "medical_genetics": 0.41414141414141414, - "high_school_us_history": 0.39408866995073893, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.4067796610169492, - "virology": 0.36363636363636365, - "high_school_microeconomics": 0.31223628691983124, + "high_school_european_history": 0.6646341463414634, + "business_ethics": 0.5656565656565656, + "clinical_knowledge": 0.4431818181818182, + "medical_genetics": 0.43434343434343436, + "high_school_us_history": 0.46798029556650245, + "high_school_physics": 0.28, + "high_school_world_history": 0.6313559322033898, + "virology": 0.3939393939393939, + "high_school_microeconomics": 0.4345991561181435, "econometrics": 0.2831858407079646, - "college_computer_science": 0.20202020202020202, - "high_school_biology": 0.39158576051779936, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.2846975088967972, - "philosophy": 0.4064516129032258, - "professional_medicine": 0.22878228782287824, - "nutrition": 0.42295081967213116, - "global_facts": 0.30303030303030304, - "machine_learning": 0.3153153153153153, - "security_studies": 0.4426229508196721, - "public_relations": 0.4036697247706422, - "professional_psychology": 0.36824877250409166, - "prehistory": 0.4241486068111455, - "anatomy": 0.44776119402985076, - "human_sexuality": 0.36153846153846153, - "college_medicine": 0.3081395348837209, - "high_school_government_and_politics": 0.4166666666666667, - "college_chemistry": 0.20202020202020202, - "logical_fallacies": 0.3765432098765432, - "high_school_geography": 0.40609137055837563, - "elementary_mathematics": 0.27586206896551724, - "human_aging": 0.42342342342342343, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.43566176470588236, - "formal_logic": 0.224, - "high_school_statistics": 0.24651162790697675, - "international_law": 0.43333333333333335, - "high_school_mathematics": 0.26022304832713755, - "high_school_computer_science": 0.4444444444444444, - "conceptual_physics": 0.3547008547008547, - "miscellaneous": 0.49232736572890023, - "high_school_chemistry": 0.3217821782178218, - "marketing": 0.5793991416309013, - "professional_law": 0.31898238747553814, - "management": 0.3627450980392157, + "college_computer_science": 0.43434343434343436, + "high_school_biology": 0.44660194174757284, + "abstract_algebra": 0.2828282828282828, + "professional_accounting": 0.3309608540925267, + "philosophy": 0.43870967741935485, + "professional_medicine": 0.3837638376383764, + "nutrition": 0.4426229508196721, + "global_facts": 0.3333333333333333, + "machine_learning": 0.23423423423423423, + "security_studies": 0.5163934426229508, + "public_relations": 0.45871559633027525, + "professional_psychology": 0.3911620294599018, + "prehistory": 0.4179566563467492, + "anatomy": 0.4925373134328358, + "human_sexuality": 0.5307692307692308, + "college_medicine": 0.36627906976744184, + "high_school_government_and_politics": 0.5260416666666666, + "college_chemistry": 0.3333333333333333, + "logical_fallacies": 0.5925925925925926, + "high_school_geography": 0.5786802030456852, + "elementary_mathematics": 0.3050397877984085, + "human_aging": 0.42792792792792794, + "college_mathematics": 0.3434343434343434, + "high_school_psychology": 0.5625, + "formal_logic": 0.312, + "high_school_statistics": 0.31627906976744186, + "international_law": 0.65, + "high_school_mathematics": 0.275092936802974, + "high_school_computer_science": 0.45454545454545453, + "conceptual_physics": 0.3418803418803419, + "miscellaneous": 0.5281329923273658, + "high_school_chemistry": 0.26732673267326734, + "marketing": 0.7553648068669528, + "professional_law": 0.32159165035877363, + "management": 0.6176470588235294, "college_physics": 0.2871287128712871, - "jurisprudence": 0.411214953271028, - "world_religions": 0.5058823529411764, - "sociology": 0.47, + "jurisprudence": 0.5420560747663551, + "world_religions": 0.4823529411764706, + "sociology": 0.59, "us_foreign_policy": 0.6060606060606061, - "high_school_macroeconomics": 0.34704370179948585, - "computer_security": 0.47474747474747475, - "moral_scenarios": 0.2539149888143177, - "moral_disputes": 0.39420289855072466, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.33774834437086093, - "college_biology": 0.34265734265734266 + "high_school_macroeconomics": 0.4190231362467866, + "computer_security": 0.5353535353535354, + "moral_scenarios": 0.24272930648769575, + "moral_disputes": 0.5072463768115942, + "electrical_engineering": 0.4583333333333333, + "astronomy": 0.3841059602649007, + "college_biology": 0.3356643356643357 } }, "prompt_3": { - "accuracy": 0.35638183768323206, + "accuracy": 0.4221666070790132, "category_acc": { - "high_school_european_history": 0.43902439024390244, - "business_ethics": 0.3333333333333333, - "clinical_knowledge": 0.32196969696969696, - "medical_genetics": 0.35353535353535354, - "high_school_us_history": 0.4088669950738916, - "high_school_physics": 0.28, - "high_school_world_history": 0.3686440677966102, - "virology": 0.3696969696969697, - "high_school_microeconomics": 0.32489451476793246, + "high_school_european_history": 0.6585365853658537, + "business_ethics": 0.5252525252525253, + "clinical_knowledge": 0.4393939393939394, + "medical_genetics": 0.41414141414141414, + "high_school_us_history": 0.4729064039408867, + "high_school_physics": 0.29333333333333333, + "high_school_world_history": 0.635593220338983, + "virology": 0.3878787878787879, + "high_school_microeconomics": 0.4345991561181435, "econometrics": 0.2831858407079646, - "college_computer_science": 0.2222222222222222, - "high_school_biology": 0.3786407766990291, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.33451957295373663, - "philosophy": 0.38064516129032255, - "professional_medicine": 0.2878228782287823, - "nutrition": 0.3901639344262295, - "global_facts": 0.3333333333333333, - "machine_learning": 0.32432432432432434, - "security_studies": 0.4385245901639344, - "public_relations": 0.3669724770642202, - "professional_psychology": 0.3567921440261866, - "prehistory": 0.4117647058823529, - "anatomy": 0.3880597014925373, - "human_sexuality": 0.36923076923076925, - "college_medicine": 0.3546511627906977, - "high_school_government_and_politics": 0.3854166666666667, - "college_chemistry": 0.24242424242424243, - "logical_fallacies": 0.4012345679012346, - "high_school_geography": 0.3147208121827411, + "college_computer_science": 0.42424242424242425, + "high_school_biology": 0.4401294498381877, + "abstract_algebra": 0.26262626262626265, + "professional_accounting": 0.3416370106761566, + "philosophy": 0.44516129032258067, + "professional_medicine": 0.3874538745387454, + "nutrition": 0.45901639344262296, + "global_facts": 0.29292929292929293, + "machine_learning": 0.23423423423423423, + "security_studies": 0.5122950819672131, + "public_relations": 0.48623853211009177, + "professional_psychology": 0.39279869067103107, + "prehistory": 0.4148606811145511, + "anatomy": 0.5, + "human_sexuality": 0.5230769230769231, + "college_medicine": 0.38953488372093026, + "high_school_government_and_politics": 0.5260416666666666, + "college_chemistry": 0.31313131313131315, + "logical_fallacies": 0.5802469135802469, + "high_school_geography": 0.5634517766497462, "elementary_mathematics": 0.2917771883289125, - "human_aging": 0.4099099099099099, - "college_mathematics": 0.25252525252525254, - "high_school_psychology": 0.40441176470588236, - "formal_logic": 0.216, - "high_school_statistics": 0.24651162790697675, - "international_law": 0.48333333333333334, - "high_school_mathematics": 0.30111524163568776, - "high_school_computer_science": 0.40404040404040403, - "conceptual_physics": 0.38461538461538464, - "miscellaneous": 0.44501278772378516, - "high_school_chemistry": 0.3564356435643564, - "marketing": 0.5021459227467812, - "professional_law": 0.31833007175472927, - "management": 0.38235294117647056, - "college_physics": 0.3069306930693069, - "jurisprudence": 0.411214953271028, - "world_religions": 0.48823529411764705, - "sociology": 0.465, - "us_foreign_policy": 0.5151515151515151, - "high_school_macroeconomics": 0.35475578406169667, - "computer_security": 0.47474747474747475, - "moral_scenarios": 0.23154362416107382, - "moral_disputes": 0.37681159420289856, - "electrical_engineering": 0.4305555555555556, - "astronomy": 0.31788079470198677, - "college_biology": 0.35664335664335667 + "human_aging": 0.44144144144144143, + "college_mathematics": 0.3434343434343434, + "high_school_psychology": 0.5569852941176471, + "formal_logic": 0.312, + "high_school_statistics": 0.3023255813953488, + "international_law": 0.6416666666666667, + "high_school_mathematics": 0.2899628252788104, + "high_school_computer_science": 0.41414141414141414, + "conceptual_physics": 0.34615384615384615, + "miscellaneous": 0.540920716112532, + "high_school_chemistry": 0.26732673267326734, + "marketing": 0.7467811158798283, + "professional_law": 0.31637312459230266, + "management": 0.6078431372549019, + "college_physics": 0.2871287128712871, + "jurisprudence": 0.5607476635514018, + "world_religions": 0.47058823529411764, + "sociology": 0.595, + "us_foreign_policy": 0.5757575757575758, + "high_school_macroeconomics": 0.43444730077120824, + "computer_security": 0.5454545454545454, + "moral_scenarios": 0.25727069351230425, + "moral_disputes": 0.5043478260869565, + "electrical_engineering": 0.4722222222222222, + "astronomy": 0.40397350993377484, + "college_biology": 0.3076923076923077 } }, "prompt_4": { - "accuracy": 0.34637111190561315, + "accuracy": 0.42481229889166966, "category_acc": { - "high_school_european_history": 0.4024390243902439, - "business_ethics": 0.3333333333333333, - "clinical_knowledge": 0.3560606060606061, - "medical_genetics": 0.36363636363636365, - "high_school_us_history": 0.4039408866995074, - "high_school_physics": 0.2733333333333333, - "high_school_world_history": 0.4025423728813559, - "virology": 0.3696969696969697, - "high_school_microeconomics": 0.32489451476793246, - "econometrics": 0.336283185840708, - "college_computer_science": 0.21212121212121213, - "high_school_biology": 0.4077669902912621, - "abstract_algebra": 0.31313131313131315, - "professional_accounting": 0.28113879003558717, - "philosophy": 0.4, - "professional_medicine": 0.24354243542435425, - "nutrition": 0.39672131147540984, - "global_facts": 0.2727272727272727, - "machine_learning": 0.34234234234234234, - "security_studies": 0.47540983606557374, - "public_relations": 0.3577981651376147, - "professional_psychology": 0.353518821603928, - "prehistory": 0.38390092879256965, - "anatomy": 0.3656716417910448, - "human_sexuality": 0.3076923076923077, - "college_medicine": 0.31976744186046513, - "high_school_government_and_politics": 0.4010416666666667, - "college_chemistry": 0.24242424242424243, - "logical_fallacies": 0.37037037037037035, - "high_school_geography": 0.2893401015228426, - "elementary_mathematics": 0.246684350132626, - "human_aging": 0.38738738738738737, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.3915441176470588, - "formal_logic": 0.24, - "high_school_statistics": 0.28837209302325584, - "international_law": 0.4666666666666667, - "high_school_mathematics": 0.25650557620817843, - "high_school_computer_science": 0.42424242424242425, - "conceptual_physics": 0.3333333333333333, - "miscellaneous": 0.40664961636828645, - "high_school_chemistry": 0.3811881188118812, - "marketing": 0.4892703862660944, - "professional_law": 0.32093933463796476, - "management": 0.3627450980392157, - "college_physics": 0.22772277227722773, - "jurisprudence": 0.3925233644859813, - "world_religions": 0.4823529411764706, - "sociology": 0.36, - "us_foreign_policy": 0.48484848484848486, - "high_school_macroeconomics": 0.33161953727506427, - "computer_security": 0.48484848484848486, - "moral_scenarios": 0.24272930648769575, - "moral_disputes": 0.3101449275362319, - "electrical_engineering": 0.4236111111111111, - "astronomy": 0.3509933774834437, - "college_biology": 0.3916083916083916 + "high_school_european_history": 0.6585365853658537, + "business_ethics": 0.5151515151515151, + "clinical_knowledge": 0.4393939393939394, + "medical_genetics": 0.41414141414141414, + "high_school_us_history": 0.46798029556650245, + "high_school_physics": 0.3, + "high_school_world_history": 0.635593220338983, + "virology": 0.4, + "high_school_microeconomics": 0.4388185654008439, + "econometrics": 0.2743362831858407, + "college_computer_science": 0.40404040404040403, + "high_school_biology": 0.4336569579288026, + "abstract_algebra": 0.30303030303030304, + "professional_accounting": 0.3238434163701068, + "philosophy": 0.43548387096774194, + "professional_medicine": 0.3948339483394834, + "nutrition": 0.46885245901639344, + "global_facts": 0.31313131313131315, + "machine_learning": 0.24324324324324326, + "security_studies": 0.5081967213114754, + "public_relations": 0.45871559633027525, + "professional_psychology": 0.3993453355155483, + "prehistory": 0.43653250773993807, + "anatomy": 0.4701492537313433, + "human_sexuality": 0.5384615384615384, + "college_medicine": 0.38372093023255816, + "high_school_government_and_politics": 0.5104166666666666, + "college_chemistry": 0.3333333333333333, + "logical_fallacies": 0.5864197530864198, + "high_school_geography": 0.5685279187817259, + "elementary_mathematics": 0.2917771883289125, + "human_aging": 0.42342342342342343, + "college_mathematics": 0.3434343434343434, + "high_school_psychology": 0.5606617647058824, + "formal_logic": 0.344, + "high_school_statistics": 0.3209302325581395, + "international_law": 0.6333333333333333, + "high_school_mathematics": 0.2936802973977695, + "high_school_computer_science": 0.46464646464646464, + "conceptual_physics": 0.3547008547008547, + "miscellaneous": 0.5319693094629157, + "high_school_chemistry": 0.25742574257425743, + "marketing": 0.759656652360515, + "professional_law": 0.3242009132420091, + "management": 0.6176470588235294, + "college_physics": 0.297029702970297, + "jurisprudence": 0.5700934579439252, + "world_religions": 0.47058823529411764, + "sociology": 0.61, + "us_foreign_policy": 0.5858585858585859, + "high_school_macroeconomics": 0.4370179948586118, + "computer_security": 0.5353535353535354, + "moral_scenarios": 0.2651006711409396, + "moral_disputes": 0.4956521739130435, + "electrical_engineering": 0.4583333333333333, + "astronomy": 0.40397350993377484, + "college_biology": 0.32867132867132864 } }, "prompt_5": { - "accuracy": 0.36245977833392923, + "accuracy": 0.42273864855202004, "category_acc": { - "high_school_european_history": 0.42073170731707316, - "business_ethics": 0.32323232323232326, - "clinical_knowledge": 0.38636363636363635, - "medical_genetics": 0.3838383838383838, - "high_school_us_history": 0.3793103448275862, + "high_school_european_history": 0.6585365853658537, + "business_ethics": 0.5757575757575758, + "clinical_knowledge": 0.45454545454545453, + "medical_genetics": 0.46464646464646464, + "high_school_us_history": 0.45320197044334976, "high_school_physics": 0.2733333333333333, - "high_school_world_history": 0.4449152542372881, - "virology": 0.3515151515151515, - "high_school_microeconomics": 0.34177215189873417, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.21212121212121213, - "high_school_biology": 0.39805825242718446, - "abstract_algebra": 0.3333333333333333, - "professional_accounting": 0.28825622775800713, - "philosophy": 0.3903225806451613, - "professional_medicine": 0.25830258302583026, - "nutrition": 0.42295081967213116, + "high_school_world_history": 0.6271186440677966, + "virology": 0.3878787878787879, + "high_school_microeconomics": 0.4388185654008439, + "econometrics": 0.2743362831858407, + "college_computer_science": 0.4444444444444444, + "high_school_biology": 0.44983818770226536, + "abstract_algebra": 0.29292929292929293, + "professional_accounting": 0.33807829181494664, + "philosophy": 0.43870967741935485, + "professional_medicine": 0.3948339483394834, + "nutrition": 0.4459016393442623, "global_facts": 0.30303030303030304, - "machine_learning": 0.34234234234234234, - "security_studies": 0.45901639344262296, - "public_relations": 0.3944954128440367, - "professional_psychology": 0.35515548281505727, - "prehistory": 0.4117647058823529, - "anatomy": 0.4552238805970149, - "human_sexuality": 0.36923076923076925, - "college_medicine": 0.3023255813953488, - "high_school_government_and_politics": 0.4270833333333333, - "college_chemistry": 0.23232323232323232, - "logical_fallacies": 0.37037037037037035, - "high_school_geography": 0.3248730964467005, - "elementary_mathematics": 0.2891246684350133, - "human_aging": 0.3783783783783784, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.4338235294117647, - "formal_logic": 0.232, - "high_school_statistics": 0.2651162790697674, - "international_law": 0.475, - "high_school_mathematics": 0.275092936802974, + "machine_learning": 0.25225225225225223, + "security_studies": 0.5122950819672131, + "public_relations": 0.45871559633027525, + "professional_psychology": 0.3895253682487725, + "prehistory": 0.4086687306501548, + "anatomy": 0.5149253731343284, + "human_sexuality": 0.5230769230769231, + "college_medicine": 0.3953488372093023, + "high_school_government_and_politics": 0.5208333333333334, + "college_chemistry": 0.32323232323232326, + "logical_fallacies": 0.5802469135802469, + "high_school_geography": 0.5685279187817259, + "elementary_mathematics": 0.30238726790450926, + "human_aging": 0.43243243243243246, + "college_mathematics": 0.3434343434343434, + "high_school_psychology": 0.5625, + "formal_logic": 0.336, + "high_school_statistics": 0.3209302325581395, + "international_law": 0.6166666666666667, + "high_school_mathematics": 0.26765799256505574, "high_school_computer_science": 0.4444444444444444, - "conceptual_physics": 0.358974358974359, - "miscellaneous": 0.46675191815856776, - "high_school_chemistry": 0.38613861386138615, - "marketing": 0.5407725321888412, + "conceptual_physics": 0.36752136752136755, + "miscellaneous": 0.5358056265984654, + "high_school_chemistry": 0.28217821782178215, + "marketing": 0.759656652360515, "professional_law": 0.31833007175472927, - "management": 0.4019607843137255, - "college_physics": 0.27722772277227725, - "jurisprudence": 0.37383177570093457, - "world_religions": 0.5470588235294118, - "sociology": 0.42, - "us_foreign_policy": 0.5252525252525253, - "high_school_macroeconomics": 0.32904884318766064, - "computer_security": 0.494949494949495, - "moral_scenarios": 0.24384787472035793, - "moral_disputes": 0.3652173913043478, - "electrical_engineering": 0.4513888888888889, - "astronomy": 0.3973509933774834, - "college_biology": 0.34965034965034963 + "management": 0.6176470588235294, + "college_physics": 0.2871287128712871, + "jurisprudence": 0.5420560747663551, + "world_religions": 0.4764705882352941, + "sociology": 0.605, + "us_foreign_policy": 0.5656565656565656, + "high_school_macroeconomics": 0.4267352185089974, + "computer_security": 0.5454545454545454, + "moral_scenarios": 0.24049217002237136, + "moral_disputes": 0.5072463768115942, + "electrical_engineering": 0.4722222222222222, + "astronomy": 0.37748344370860926, + "college_biology": 0.3356643356643357 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.29049034175334326 + "accuracy": 0.4078751857355126 }, "prompt_2": { - "accuracy": 0.2704309063893016 + "accuracy": 0.4078751857355126 }, "prompt_3": { - "accuracy": 0.2897473997028232 + "accuracy": 0.4034175334323923 }, "prompt_4": { - "accuracy": 0.2986627043090639 + "accuracy": 0.4063893016344725 }, "prompt_5": { - "accuracy": 0.27340267459138184 + "accuracy": 0.4086181277860327 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.28580323785803236, + "accuracy": 0.3916562889165629, "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.2916666666666667, - "computer_architecture": 0.46153846153846156, - "college_programming": 0.23809523809523808, + "computer_network": 0.3333333333333333, + "operating_system": 0.375, + "computer_architecture": 0.4230769230769231, + "college_programming": 0.2857142857142857, "college_physics": 0.20833333333333334, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.2608695652173913, + "college_chemistry": 0.3448275862068966, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.4782608695652174, "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.19047619047619047, - "metrology_engineer": 0.27586206896551724, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.125, - "high_school_chemistry": 0.16666666666666666, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.5172413793103449, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.3333333333333333, + "high_school_chemistry": 0.20833333333333334, "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.125, + "middle_school_mathematics": 0.20833333333333334, "middle_school_biology": 0.5, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.2, + "middle_school_physics": 0.5833333333333334, + "middle_school_chemistry": 0.36, "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.36666666666666664, - "business_administration": 0.42105263157894735, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.2413793103448276, - "education_science": 0.29411764705882354, - "teacher_qualification": 0.3673469387755102, - "high_school_politics": 0.375, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.14285714285714285, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.4074074074074074, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.17647058823529413, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.125, - "high_school_history": 0.44, - "middle_school_history": 0.18518518518518517, - "civil_servant": 0.25, - "sports_science": 0.2916666666666667, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.3333333333333333, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.3888888888888889, - "tax_accountant": 0.2222222222222222, - "physician": 0.2777777777777778 + "college_economics": 0.4166666666666667, + "business_administration": 0.39473684210526316, + "marxism": 0.4583333333333333, + "mao_zedong_thought": 0.4827586206896552, + "education_science": 0.5588235294117647, + "teacher_qualification": 0.5918367346938775, + "high_school_politics": 0.16666666666666666, + "high_school_geography": 0.5, + "middle_school_politics": 0.7307692307692307, + "middle_school_geography": 0.17647058823529413, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.7083333333333334, + "logic": 0.4444444444444444, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.39473684210526316, + "professional_tour_guide": 0.5, + "legal_professional": 0.39285714285714285, + "high_school_chinese": 0.20833333333333334, + "high_school_history": 0.48, + "middle_school_history": 0.37037037037037035, + "civil_servant": 0.38461538461538464, + "sports_science": 0.4166666666666667, + "plant_protection": 0.4074074074074074, + "basic_medicine": 0.375, + "clinical_medicine": 0.4074074074074074, + "urban_and_rural_planner": 0.47058823529411764, + "accountant": 0.3148148148148148, + "fire_engineer": 0.2777777777777778, + "environmental_impact_assessment_engineer": 0.3611111111111111, + "tax_accountant": 0.2962962962962963, + "physician": 0.3888888888888889 } }, "prompt_2": { - "accuracy": 0.27210460772104605, + "accuracy": 0.3891656288916563, "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.375, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.2857142857142857, + "computer_network": 0.2916666666666667, + "operating_system": 0.3333333333333333, + "computer_architecture": 0.46153846153846156, + "college_programming": 0.3333333333333333, "college_physics": 0.25, - "college_chemistry": 0.13793103448275862, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.17391304347826086, + "college_chemistry": 0.3103448275862069, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.14285714285714285, - "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.08695652173913043, - "high_school_physics": 0.25, + "electrical_engineer": 0.30952380952380953, + "metrology_engineer": 0.41379310344827586, + "high_school_mathematics": 0.2608695652173913, + "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.25, "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.3076923076923077, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.25, - "college_economics": 0.21666666666666667, - "business_administration": 0.2894736842105263, - "marxism": 0.375, - "mao_zedong_thought": 0.27586206896551724, - "education_science": 0.29411764705882354, - "teacher_qualification": 0.30612244897959184, - "high_school_politics": 0.4166666666666667, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.2222222222222222, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.23684210526315788, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.12, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.34615384615384615, - "sports_science": 0.2916666666666667, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.19607843137254902, - "accountant": 0.2777777777777778, + "middle_school_mathematics": 0.20833333333333334, + "middle_school_biology": 0.5, + "middle_school_physics": 0.5833333333333334, + "middle_school_chemistry": 0.36, + "veterinary_medicine": 0.39285714285714285, + "college_economics": 0.4166666666666667, + "business_administration": 0.39473684210526316, + "marxism": 0.5, + "mao_zedong_thought": 0.5517241379310345, + "education_science": 0.5588235294117647, + "teacher_qualification": 0.5714285714285714, + "high_school_politics": 0.25, + "high_school_geography": 0.5, + "middle_school_politics": 0.6153846153846154, + "middle_school_geography": 0.23529411764705882, + "modern_chinese_history": 0.4642857142857143, + "ideological_and_moral_cultivation": 0.6666666666666666, + "logic": 0.48148148148148145, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.35714285714285715, + "art_studies": 0.3684210526315789, + "professional_tour_guide": 0.5, + "legal_professional": 0.39285714285714285, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.48, + "middle_school_history": 0.4074074074074074, + "civil_servant": 0.40384615384615385, + "sports_science": 0.4583333333333333, + "plant_protection": 0.4074074074074074, + "basic_medicine": 0.375, + "clinical_medicine": 0.4074074074074074, + "urban_and_rural_planner": 0.49019607843137253, + "accountant": 0.35185185185185186, "fire_engineer": 0.2222222222222222, "environmental_impact_assessment_engineer": 0.3611111111111111, - "tax_accountant": 0.2222222222222222, - "physician": 0.2777777777777778 + "tax_accountant": 0.25925925925925924, + "physician": 0.3888888888888889 } }, "prompt_3": { - "accuracy": 0.28393524283935245, + "accuracy": 0.3953922789539228, "category_acc": { "computer_network": 0.3333333333333333, - "operating_system": 0.25, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.2857142857142857, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.30434782608695654, + "operating_system": 0.375, + "computer_architecture": 0.38461538461538464, + "college_programming": 0.30952380952380953, + "college_physics": 0.2916666666666667, + "college_chemistry": 0.3448275862068966, + "advanced_mathematics": 0.16666666666666666, + "probability_and_statistics": 0.4782608695652174, "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.08695652173913043, - "high_school_physics": 0.2916666666666667, + "electrical_engineer": 0.3333333333333333, + "metrology_engineer": 0.41379310344827586, + "high_school_mathematics": 0.30434782608695654, + "high_school_physics": 0.20833333333333334, "high_school_chemistry": 0.16666666666666666, - "high_school_biology": 0.2916666666666667, + "high_school_biology": 0.25, "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.46153846153846156, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.12, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.3, - "business_administration": 0.34210526315789475, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.3235294117647059, - "teacher_qualification": 0.30612244897959184, - "high_school_politics": 0.5, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.15384615384615385, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.37037037037037035, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.32142857142857145, + "middle_school_biology": 0.5, + "middle_school_physics": 0.5833333333333334, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.39285714285714285, + "college_economics": 0.4166666666666667, + "business_administration": 0.42105263157894735, + "marxism": 0.5, + "mao_zedong_thought": 0.4827586206896552, + "education_science": 0.5588235294117647, + "teacher_qualification": 0.5918367346938775, + "high_school_politics": 0.3333333333333333, + "high_school_geography": 0.4583333333333333, + "middle_school_politics": 0.7307692307692307, + "middle_school_geography": 0.23529411764705882, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.75, + "logic": 0.48148148148148145, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.39285714285714285, "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.25, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.28, + "professional_tour_guide": 0.5294117647058824, + "legal_professional": 0.39285714285714285, + "high_school_chinese": 0.20833333333333334, + "high_school_history": 0.52, "middle_school_history": 0.4074074074074074, - "civil_servant": 0.28846153846153844, + "civil_servant": 0.4230769230769231, "sports_science": 0.4166666666666667, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.2962962962962963, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.16666666666666666, - "physician": 0.2962962962962963 + "plant_protection": 0.4444444444444444, + "basic_medicine": 0.375, + "clinical_medicine": 0.4074074074074074, + "urban_and_rural_planner": 0.5098039215686274, + "accountant": 0.3148148148148148, + "fire_engineer": 0.2777777777777778, + "environmental_impact_assessment_engineer": 0.3888888888888889, + "tax_accountant": 0.2962962962962963, + "physician": 0.4074074074074074 } }, "prompt_4": { - "accuracy": 0.28953922789539227, + "accuracy": 0.3854296388542964, "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.5, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.2857142857142857, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.17391304347826086, + "computer_network": 0.375, + "operating_system": 0.3333333333333333, + "computer_architecture": 0.4230769230769231, + "college_programming": 0.3333333333333333, + "college_physics": 0.25, + "college_chemistry": 0.3793103448275862, + "advanced_mathematics": 0.16666666666666666, + "probability_and_statistics": 0.43478260869565216, "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.21428571428571427, - "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.17391304347826086, + "electrical_engineer": 0.2857142857142857, + "metrology_engineer": 0.4482758620689655, + "high_school_mathematics": 0.2608695652173913, "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.20833333333333334, + "high_school_chemistry": 0.16666666666666666, "high_school_biology": 0.25, - "middle_school_mathematics": 0.041666666666666664, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.3333333333333333, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.25, - "college_economics": 0.35, - "business_administration": 0.3157894736842105, - "marxism": 0.3333333333333333, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.5, + "middle_school_physics": 0.5833333333333334, + "middle_school_chemistry": 0.32, + "veterinary_medicine": 0.39285714285714285, + "college_economics": 0.43333333333333335, + "business_administration": 0.3684210526315789, + "marxism": 0.5416666666666666, "mao_zedong_thought": 0.4827586206896552, - "education_science": 0.3235294117647059, - "teacher_qualification": 0.3877551020408163, - "high_school_politics": 0.25, - "high_school_geography": 0.25, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.10714285714285714, - "ideological_and_moral_cultivation": 0.20833333333333334, - "logic": 0.4074074074074074, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.14285714285714285, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.2857142857142857, + "education_science": 0.5294117647058824, + "teacher_qualification": 0.5714285714285714, + "high_school_politics": 0.2916666666666667, + "high_school_geography": 0.4583333333333333, + "middle_school_politics": 0.6538461538461539, + "middle_school_geography": 0.17647058823529413, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.6666666666666666, + "logic": 0.48148148148148145, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.35714285714285715, + "art_studies": 0.3684210526315789, + "professional_tour_guide": 0.5, + "legal_professional": 0.35714285714285715, "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.4, - "middle_school_history": 0.25925925925925924, - "civil_servant": 0.28846153846153844, - "sports_science": 0.3333333333333333, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.25925925925925924, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.16666666666666666, - "physician": 0.25925925925925924 + "high_school_history": 0.44, + "middle_school_history": 0.4074074074074074, + "civil_servant": 0.40384615384615385, + "sports_science": 0.375, + "plant_protection": 0.4074074074074074, + "basic_medicine": 0.4166666666666667, + "clinical_medicine": 0.4074074074074074, + "urban_and_rural_planner": 0.5098039215686274, + "accountant": 0.3148148148148148, + "fire_engineer": 0.25, + "environmental_impact_assessment_engineer": 0.3611111111111111, + "tax_accountant": 0.24074074074074073, + "physician": 0.37037037037037035 } }, "prompt_5": { - "accuracy": 0.2714819427148194, + "accuracy": 0.3916562889165629, "category_acc": { - "computer_network": 0.25, - "operating_system": 0.2916666666666667, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.2619047619047619, + "computer_network": 0.2916666666666667, + "operating_system": 0.3333333333333333, + "computer_architecture": 0.38461538461538464, + "college_programming": 0.2857142857142857, "college_physics": 0.2916666666666667, - "college_chemistry": 0.13793103448275862, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.30434782608695654, + "college_chemistry": 0.3448275862068966, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.391304347826087, "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.23809523809523808, + "electrical_engineer": 0.30952380952380953, "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.16666666666666666, - "high_school_biology": 0.2916666666666667, + "high_school_mathematics": 0.2608695652173913, + "high_school_physics": 0.25, + "high_school_chemistry": 0.20833333333333334, + "high_school_biology": 0.20833333333333334, "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.3333333333333333, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.42857142857142855, - "college_economics": 0.3, - "business_administration": 0.34210526315789475, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.3448275862068966, - "education_science": 0.35294117647058826, - "teacher_qualification": 0.24489795918367346, - "high_school_politics": 0.4166666666666667, - "high_school_geography": 0.125, - "middle_school_politics": 0.19230769230769232, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.25, - "logic": 0.2222222222222222, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.2631578947368421, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.44, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.2692307692307692, - "sports_science": 0.3333333333333333, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.25, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.19607843137254902, + "middle_school_biology": 0.5, + "middle_school_physics": 0.5833333333333334, + "middle_school_chemistry": 0.36, + "veterinary_medicine": 0.39285714285714285, + "college_economics": 0.43333333333333335, + "business_administration": 0.42105263157894735, + "marxism": 0.5, + "mao_zedong_thought": 0.5862068965517241, + "education_science": 0.5882352941176471, + "teacher_qualification": 0.5918367346938775, + "high_school_politics": 0.4583333333333333, + "high_school_geography": 0.4583333333333333, + "middle_school_politics": 0.6923076923076923, + "middle_school_geography": 0.23529411764705882, + "modern_chinese_history": 0.42857142857142855, + "ideological_and_moral_cultivation": 0.625, + "logic": 0.48148148148148145, + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.34210526315789475, + "professional_tour_guide": 0.4411764705882353, + "legal_professional": 0.39285714285714285, + "high_school_chinese": 0.25, + "high_school_history": 0.48, + "middle_school_history": 0.4074074074074074, + "civil_servant": 0.38461538461538464, + "sports_science": 0.375, + "plant_protection": 0.4074074074074074, + "basic_medicine": 0.375, + "clinical_medicine": 0.4444444444444444, + "urban_and_rural_planner": 0.47058823529411764, "accountant": 0.3148148148148148, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.16666666666666666, - "physician": 0.2777777777777778 + "fire_engineer": 0.3055555555555556, + "environmental_impact_assessment_engineer": 0.3611111111111111, + "tax_accountant": 0.2962962962962963, + "physician": 0.4074074074074074 } } }, "cmmlu": { "prompt_1": { - "accuracy": 0.24014336917562723 + "accuracy": 0.4121863799283154 }, "prompt_2": { - "accuracy": 0.24014336917562723 + "accuracy": 0.40860215053763443 }, "prompt_3": { - "accuracy": 0.25448028673835127 + "accuracy": 0.3978494623655914 }, "prompt_4": { - "accuracy": 0.25806451612903225 + "accuracy": 0.3978494623655914 }, "prompt_5": { - "accuracy": 0.3046594982078853 + "accuracy": 0.4265232974910394 } }, "cmmlu_full": { "prompt_1": { - "accuracy": 0.2633396649974098, + "accuracy": 0.4134001036090485, "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.30405405405405406, - "ancient_chinese": 0.21341463414634146, - "arts": 0.25625, - "astronomy": 0.26666666666666666, - "business_ethics": 0.2631578947368421, - "chinese_civil_service_exam": 0.2125, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.29906542056074764, - "chinese_history": 0.25077399380804954, - "chinese_literature": 0.28431372549019607, - "chinese_teacher_qualification": 0.3016759776536313, - "clinical_knowledge": 0.21940928270042195, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.2616822429906542, - "college_engineering_hydrology": 0.2358490566037736, - "college_law": 0.19444444444444445, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.3018867924528302, - "college_medicine": 0.2893772893772894, - "computer_science": 0.27450980392156865, - "computer_security": 0.21052631578947367, - "conceptual_physics": 0.272108843537415, - "construction_project_management": 0.35251798561151076, - "economics": 0.2641509433962264, - "education": 0.2883435582822086, - "electrical_engineering": 0.29651162790697677, - "elementary_chinese": 0.1984126984126984, - "elementary_commonsense": 0.26262626262626265, - "elementary_information_and_technology": 0.2184873949579832, - "elementary_mathematics": 0.25217391304347825, - "ethnology": 0.2814814814814815, - "food_science": 0.2937062937062937, - "genetics": 0.26704545454545453, - "global_facts": 0.2953020134228188, - "high_school_biology": 0.24260355029585798, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.21951219512195122, - "high_school_physics": 0.2909090909090909, - "high_school_politics": 0.24475524475524477, - "human_sexuality": 0.2777777777777778, - "international_law": 0.2810810810810811, - "journalism": 0.3023255813953488, - "jurisprudence": 0.23357664233576642, - "legal_and_moral_basis": 0.3177570093457944, - "logical": 0.2682926829268293, - "machine_learning": 0.28688524590163933, - "management": 0.23809523809523808, - "marketing": 0.25555555555555554, - "marxist_theory": 0.24867724867724866, - "modern_chinese": 0.19827586206896552, - "nutrition": 0.2827586206896552, - "philosophy": 0.3619047619047619, - "professional_accounting": 0.29714285714285715, - "professional_law": 0.22274881516587677, - "professional_medicine": 0.26861702127659576, - "professional_psychology": 0.2974137931034483, - "public_relations": 0.25287356321839083, - "security_study": 0.28888888888888886, - "sociology": 0.2610619469026549, - "sports_science": 0.2727272727272727, - "traditional_chinese_medicine": 0.2918918918918919, - "virology": 0.27218934911242604, - "world_history": 0.2732919254658385, - "world_religions": 0.25 - } - }, - "prompt_2": { - "accuracy": 0.26049041616301155, - "category_acc": { - "agronomy": 0.25443786982248523, - "anatomy": 0.30405405405405406, - "ancient_chinese": 0.21951219512195122, - "arts": 0.23125, - "astronomy": 0.24242424242424243, - "business_ethics": 0.3014354066985646, - "chinese_civil_service_exam": 0.28125, - "chinese_driving_rule": 0.33587786259541985, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.2616822429906542, - "chinese_history": 0.25696594427244585, - "chinese_literature": 0.2549019607843137, - "chinese_teacher_qualification": 0.22905027932960895, - "clinical_knowledge": 0.270042194092827, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.16822429906542055, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.26851851851851855, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.2641509433962264, - "college_medicine": 0.2600732600732601, - "computer_science": 0.27450980392156865, - "computer_security": 0.26900584795321636, - "conceptual_physics": 0.29931972789115646, - "construction_project_management": 0.2517985611510791, - "economics": 0.27672955974842767, - "education": 0.25153374233128833, - "electrical_engineering": 0.29651162790697677, - "elementary_chinese": 0.23809523809523808, - "elementary_commonsense": 0.2727272727272727, - "elementary_information_and_technology": 0.24789915966386555, - "elementary_mathematics": 0.22608695652173913, - "ethnology": 0.2222222222222222, - "food_science": 0.3146853146853147, - "genetics": 0.2784090909090909, - "global_facts": 0.30201342281879195, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.15254237288135594, - "high_school_mathematics": 0.24390243902439024, - "high_school_physics": 0.2, - "high_school_politics": 0.2727272727272727, - "human_sexuality": 0.2777777777777778, - "international_law": 0.2810810810810811, - "journalism": 0.27325581395348836, - "jurisprudence": 0.25304136253041365, - "legal_and_moral_basis": 0.2570093457943925, - "logical": 0.23577235772357724, - "machine_learning": 0.30327868852459017, - "management": 0.24761904761904763, - "marketing": 0.2611111111111111, - "marxist_theory": 0.291005291005291, - "modern_chinese": 0.25, - "nutrition": 0.25517241379310346, - "philosophy": 0.2571428571428571, - "professional_accounting": 0.2742857142857143, - "professional_law": 0.24644549763033174, - "professional_medicine": 0.2765957446808511, - "professional_psychology": 0.27155172413793105, - "public_relations": 0.25287356321839083, - "security_study": 0.22962962962962963, - "sociology": 0.22123893805309736, - "sports_science": 0.30303030303030304, - "traditional_chinese_medicine": 0.20540540540540542, - "virology": 0.2603550295857988, - "world_history": 0.3167701863354037, - "world_religions": 0.25625 - } - }, - "prompt_3": { - "accuracy": 0.2654118459678812, - "category_acc": { - "agronomy": 0.26627218934911245, - "anatomy": 0.2635135135135135, + "agronomy": 0.40828402366863903, + "anatomy": 0.32432432432432434, "ancient_chinese": 0.23170731707317074, - "arts": 0.25625, - "astronomy": 0.23030303030303031, - "business_ethics": 0.24880382775119617, - "chinese_civil_service_exam": 0.26875, - "chinese_driving_rule": 0.3435114503816794, - "chinese_food_culture": 0.27205882352941174, - "chinese_foreign_policy": 0.24299065420560748, - "chinese_history": 0.25386996904024767, - "chinese_literature": 0.29901960784313725, - "chinese_teacher_qualification": 0.27932960893854747, - "clinical_knowledge": 0.20675105485232068, + "arts": 0.43125, + "astronomy": 0.2, + "business_ethics": 0.46411483253588515, + "chinese_civil_service_exam": 0.425, + "chinese_driving_rule": 0.6564885496183206, + "chinese_food_culture": 0.36764705882352944, + "chinese_foreign_policy": 0.45794392523364486, + "chinese_history": 0.47678018575851394, + "chinese_literature": 0.30392156862745096, + "chinese_teacher_qualification": 0.5083798882681564, + "clinical_knowledge": 0.3206751054852321, "college_actuarial_science": 0.27358490566037735, - "college_education": 0.24299065420560748, - "college_engineering_hydrology": 0.24528301886792453, - "college_law": 0.25, + "college_education": 0.5700934579439252, + "college_engineering_hydrology": 0.33962264150943394, + "college_law": 0.32407407407407407, "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.2830188679245283, - "college_medicine": 0.28205128205128205, - "computer_science": 0.28431372549019607, - "computer_security": 0.23976608187134502, - "conceptual_physics": 0.272108843537415, - "construction_project_management": 0.26618705035971224, - "economics": 0.29559748427672955, - "education": 0.19631901840490798, - "electrical_engineering": 0.28488372093023256, - "elementary_chinese": 0.25396825396825395, - "elementary_commonsense": 0.23737373737373738, - "elementary_information_and_technology": 0.2647058823529412, - "elementary_mathematics": 0.2391304347826087, - "ethnology": 0.2962962962962963, - "food_science": 0.34965034965034963, - "genetics": 0.20454545454545456, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.28402366863905326, - "high_school_chemistry": 0.2196969696969697, - "high_school_geography": 0.2627118644067797, - "high_school_mathematics": 0.25, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.3146853146853147, - "human_sexuality": 0.2857142857142857, - "international_law": 0.2918918918918919, - "journalism": 0.23837209302325582, - "jurisprudence": 0.22141119221411193, - "legal_and_moral_basis": 0.2757009345794392, - "logical": 0.2682926829268293, - "machine_learning": 0.2786885245901639, - "management": 0.2571428571428571, - "marketing": 0.2222222222222222, - "marxist_theory": 0.328042328042328, - "modern_chinese": 0.20689655172413793, - "nutrition": 0.2896551724137931, - "philosophy": 0.34285714285714286, - "professional_accounting": 0.32571428571428573, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.2925531914893617, - "professional_psychology": 0.28448275862068967, - "public_relations": 0.28735632183908044, - "security_study": 0.22962962962962963, - "sociology": 0.26548672566371684, - "sports_science": 0.26666666666666666, - "traditional_chinese_medicine": 0.2648648648648649, - "virology": 0.25443786982248523, - "world_history": 0.2919254658385093, - "world_religions": 0.31875 + "college_medical_statistics": 0.4056603773584906, + "college_medicine": 0.31868131868131866, + "computer_science": 0.44607843137254904, + "computer_security": 0.4444444444444444, + "conceptual_physics": 0.3197278911564626, + "construction_project_management": 0.381294964028777, + "economics": 0.4716981132075472, + "education": 0.4785276073619632, + "electrical_engineering": 0.38953488372093026, + "elementary_chinese": 0.3134920634920635, + "elementary_commonsense": 0.398989898989899, + "elementary_information_and_technology": 0.6050420168067226, + "elementary_mathematics": 0.30434782608695654, + "ethnology": 0.4074074074074074, + "food_science": 0.46153846153846156, + "genetics": 0.2840909090909091, + "global_facts": 0.35570469798657717, + "high_school_biology": 0.3076923076923077, + "high_school_chemistry": 0.32575757575757575, + "high_school_geography": 0.3813559322033898, + "high_school_mathematics": 0.23780487804878048, + "high_school_physics": 0.33636363636363636, + "high_school_politics": 0.40559440559440557, + "human_sexuality": 0.4444444444444444, + "international_law": 0.43243243243243246, + "journalism": 0.5232558139534884, + "jurisprudence": 0.4768856447688564, + "legal_and_moral_basis": 0.7850467289719626, + "logical": 0.3983739837398374, + "machine_learning": 0.36065573770491804, + "management": 0.5, + "marketing": 0.5444444444444444, + "marxist_theory": 0.5238095238095238, + "modern_chinese": 0.3103448275862069, + "nutrition": 0.45517241379310347, + "philosophy": 0.4095238095238095, + "professional_accounting": 0.45714285714285713, + "professional_law": 0.3459715639810427, + "professional_medicine": 0.2898936170212766, + "professional_psychology": 0.5129310344827587, + "public_relations": 0.5114942528735632, + "security_study": 0.5481481481481482, + "sociology": 0.43805309734513276, + "sports_science": 0.49696969696969695, + "traditional_chinese_medicine": 0.32972972972972975, + "virology": 0.4260355029585799, + "world_history": 0.4409937888198758, + "world_religions": 0.41875 } }, - "prompt_4": { - "accuracy": 0.2769815230530133, + "prompt_2": { + "accuracy": 0.41063719564841994, "category_acc": { - "agronomy": 0.1952662721893491, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.2621951219512195, - "arts": 0.2625, + "agronomy": 0.39644970414201186, + "anatomy": 0.31756756756756754, + "ancient_chinese": 0.24390243902439024, + "arts": 0.45625, "astronomy": 0.21212121212121213, - "business_ethics": 0.3492822966507177, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.3282442748091603, - "chinese_food_culture": 0.21323529411764705, - "chinese_foreign_policy": 0.2336448598130841, - "chinese_history": 0.30030959752321984, - "chinese_literature": 0.28431372549019607, - "chinese_teacher_qualification": 0.20670391061452514, - "clinical_knowledge": 0.24472573839662448, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.308411214953271, - "college_engineering_hydrology": 0.27358490566037735, - "college_law": 0.25, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.3113207547169811, - "college_medicine": 0.23443223443223443, - "computer_science": 0.3088235294117647, - "computer_security": 0.26900584795321636, - "conceptual_physics": 0.2653061224489796, - "construction_project_management": 0.2949640287769784, - "economics": 0.32075471698113206, - "education": 0.32515337423312884, - "electrical_engineering": 0.29651162790697677, - "elementary_chinese": 0.2261904761904762, - "elementary_commonsense": 0.2474747474747475, - "elementary_information_and_technology": 0.3067226890756303, - "elementary_mathematics": 0.26956521739130435, - "ethnology": 0.2740740740740741, - "food_science": 0.2867132867132867, - "genetics": 0.2897727272727273, - "global_facts": 0.2751677852348993, - "high_school_biology": 0.2603550295857988, - "high_school_chemistry": 0.22727272727272727, - "high_school_geography": 0.2627118644067797, - "high_school_mathematics": 0.21951219512195122, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.3006993006993007, - "human_sexuality": 0.25396825396825395, - "international_law": 0.2594594594594595, - "journalism": 0.3081395348837209, - "jurisprudence": 0.26277372262773724, - "legal_and_moral_basis": 0.40654205607476634, - "logical": 0.2682926829268293, - "machine_learning": 0.29508196721311475, - "management": 0.24761904761904763, - "marketing": 0.3388888888888889, - "marxist_theory": 0.30687830687830686, - "modern_chinese": 0.25862068965517243, - "nutrition": 0.2896551724137931, - "philosophy": 0.38095238095238093, - "professional_accounting": 0.2742857142857143, - "professional_law": 0.2843601895734597, - "professional_medicine": 0.2845744680851064, - "professional_psychology": 0.27586206896551724, - "public_relations": 0.3045977011494253, - "security_study": 0.28888888888888886, - "sociology": 0.27876106194690264, - "sports_science": 0.2606060606060606, - "traditional_chinese_medicine": 0.2756756756756757, - "virology": 0.3254437869822485, - "world_history": 0.3167701863354037, - "world_religions": 0.21875 + "business_ethics": 0.4688995215311005, + "chinese_civil_service_exam": 0.425, + "chinese_driving_rule": 0.648854961832061, + "chinese_food_culture": 0.36764705882352944, + "chinese_foreign_policy": 0.45794392523364486, + "chinese_history": 0.47368421052631576, + "chinese_literature": 0.30392156862745096, + "chinese_teacher_qualification": 0.5195530726256983, + "clinical_knowledge": 0.32489451476793246, + "college_actuarial_science": 0.29245283018867924, + "college_education": 0.5607476635514018, + "college_engineering_hydrology": 0.330188679245283, + "college_law": 0.3148148148148148, + "college_mathematics": 0.2571428571428571, + "college_medical_statistics": 0.3867924528301887, + "college_medicine": 0.3076923076923077, + "computer_science": 0.4362745098039216, + "computer_security": 0.43859649122807015, + "conceptual_physics": 0.32653061224489793, + "construction_project_management": 0.3381294964028777, + "economics": 0.46540880503144655, + "education": 0.49693251533742333, + "electrical_engineering": 0.38372093023255816, + "elementary_chinese": 0.2976190476190476, + "elementary_commonsense": 0.3838383838383838, + "elementary_information_and_technology": 0.592436974789916, + "elementary_mathematics": 0.29130434782608694, + "ethnology": 0.4074074074074074, + "food_science": 0.48951048951048953, + "genetics": 0.26136363636363635, + "global_facts": 0.33557046979865773, + "high_school_biology": 0.3254437869822485, + "high_school_chemistry": 0.32575757575757575, + "high_school_geography": 0.3813559322033898, + "high_school_mathematics": 0.21341463414634146, + "high_school_physics": 0.34545454545454546, + "high_school_politics": 0.3916083916083916, + "human_sexuality": 0.4603174603174603, + "international_law": 0.41621621621621624, + "journalism": 0.5348837209302325, + "jurisprudence": 0.4768856447688564, + "legal_and_moral_basis": 0.7523364485981309, + "logical": 0.3821138211382114, + "machine_learning": 0.3442622950819672, + "management": 0.4857142857142857, + "marketing": 0.5388888888888889, + "marxist_theory": 0.5343915343915344, + "modern_chinese": 0.3017241379310345, + "nutrition": 0.4482758620689655, + "philosophy": 0.4095238095238095, + "professional_accounting": 0.44, + "professional_law": 0.36018957345971564, + "professional_medicine": 0.30319148936170215, + "professional_psychology": 0.5129310344827587, + "public_relations": 0.5172413793103449, + "security_study": 0.5407407407407407, + "sociology": 0.43805309734513276, + "sports_science": 0.49696969696969695, + "traditional_chinese_medicine": 0.32432432432432434, + "virology": 0.4319526627218935, + "world_history": 0.4409937888198758, + "world_religions": 0.40625 } }, - "prompt_5": { - "accuracy": 0.27067863926782937, + "prompt_3": { + "accuracy": 0.41158694525988604, "category_acc": { - "agronomy": 0.2485207100591716, - "anatomy": 0.22972972972972974, + "agronomy": 0.42011834319526625, + "anatomy": 0.2972972972972973, "ancient_chinese": 0.2682926829268293, - "arts": 0.21875, - "astronomy": 0.21818181818181817, - "business_ethics": 0.3014354066985646, - "chinese_civil_service_exam": 0.25, - "chinese_driving_rule": 0.3282442748091603, - "chinese_food_culture": 0.21323529411764705, - "chinese_foreign_policy": 0.24299065420560748, - "chinese_history": 0.3281733746130031, - "chinese_literature": 0.28921568627450983, - "chinese_teacher_qualification": 0.21787709497206703, - "clinical_knowledge": 0.21518987341772153, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.2336448598130841, - "college_engineering_hydrology": 0.22641509433962265, - "college_law": 0.25, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.27358490566037735, - "college_medicine": 0.315018315018315, - "computer_science": 0.28921568627450983, - "computer_security": 0.25146198830409355, + "arts": 0.43125, + "astronomy": 0.20606060606060606, + "business_ethics": 0.45933014354066987, + "chinese_civil_service_exam": 0.41875, + "chinese_driving_rule": 0.648854961832061, + "chinese_food_culture": 0.39705882352941174, + "chinese_foreign_policy": 0.4485981308411215, + "chinese_history": 0.47058823529411764, + "chinese_literature": 0.3235294117647059, + "chinese_teacher_qualification": 0.5139664804469274, + "clinical_knowledge": 0.3080168776371308, + "college_actuarial_science": 0.29245283018867924, + "college_education": 0.5607476635514018, + "college_engineering_hydrology": 0.32075471698113206, + "college_law": 0.3333333333333333, + "college_mathematics": 0.22857142857142856, + "college_medical_statistics": 0.3867924528301887, + "college_medicine": 0.31135531135531136, + "computer_science": 0.43137254901960786, + "computer_security": 0.4327485380116959, + "conceptual_physics": 0.3333333333333333, + "construction_project_management": 0.35251798561151076, + "economics": 0.46540880503144655, + "education": 0.4723926380368098, + "electrical_engineering": 0.37790697674418605, + "elementary_chinese": 0.31746031746031744, + "elementary_commonsense": 0.3686868686868687, + "elementary_information_and_technology": 0.6176470588235294, + "elementary_mathematics": 0.32608695652173914, + "ethnology": 0.4222222222222222, + "food_science": 0.46153846153846156, + "genetics": 0.26704545454545453, + "global_facts": 0.37583892617449666, + "high_school_biology": 0.31952662721893493, + "high_school_chemistry": 0.32575757575757575, + "high_school_geography": 0.3898305084745763, + "high_school_mathematics": 0.23170731707317074, + "high_school_physics": 0.35454545454545455, + "high_school_politics": 0.40559440559440557, + "human_sexuality": 0.4444444444444444, + "international_law": 0.41621621621621624, + "journalism": 0.5523255813953488, + "jurisprudence": 0.48418491484184917, + "legal_and_moral_basis": 0.7523364485981309, + "logical": 0.3821138211382114, + "machine_learning": 0.3360655737704918, + "management": 0.47619047619047616, + "marketing": 0.5333333333333333, + "marxist_theory": 0.5238095238095238, + "modern_chinese": 0.31896551724137934, + "nutrition": 0.4689655172413793, + "philosophy": 0.3904761904761905, + "professional_accounting": 0.44571428571428573, + "professional_law": 0.33175355450236965, + "professional_medicine": 0.30319148936170215, + "professional_psychology": 0.5172413793103449, + "public_relations": 0.5057471264367817, + "security_study": 0.562962962962963, + "sociology": 0.4557522123893805, + "sports_science": 0.49696969696969695, + "traditional_chinese_medicine": 0.3027027027027027, + "virology": 0.4319526627218935, + "world_history": 0.40993788819875776, + "world_religions": 0.39375 + } + }, + "prompt_4": { + "accuracy": 0.41201864962873425, + "category_acc": { + "agronomy": 0.41420118343195267, + "anatomy": 0.31756756756756754, + "ancient_chinese": 0.23170731707317074, + "arts": 0.425, + "astronomy": 0.2, + "business_ethics": 0.45933014354066987, + "chinese_civil_service_exam": 0.41875, + "chinese_driving_rule": 0.6259541984732825, + "chinese_food_culture": 0.375, + "chinese_foreign_policy": 0.4766355140186916, + "chinese_history": 0.48606811145510836, + "chinese_literature": 0.29901960784313725, + "chinese_teacher_qualification": 0.5139664804469274, + "clinical_knowledge": 0.31223628691983124, + "college_actuarial_science": 0.29245283018867924, + "college_education": 0.5607476635514018, + "college_engineering_hydrology": 0.33962264150943394, + "college_law": 0.3333333333333333, + "college_mathematics": 0.18095238095238095, + "college_medical_statistics": 0.3867924528301887, + "college_medicine": 0.30036630036630035, + "computer_science": 0.45588235294117646, + "computer_security": 0.4327485380116959, "conceptual_physics": 0.3197278911564626, - "construction_project_management": 0.302158273381295, - "economics": 0.2641509433962264, - "education": 0.2883435582822086, - "electrical_engineering": 0.27906976744186046, - "elementary_chinese": 0.24603174603174602, - "elementary_commonsense": 0.2474747474747475, - "elementary_information_and_technology": 0.226890756302521, - "elementary_mathematics": 0.28695652173913044, - "ethnology": 0.24444444444444444, - "food_science": 0.2937062937062937, - "genetics": 0.2727272727272727, - "global_facts": 0.26174496644295303, + "construction_project_management": 0.34532374100719426, + "economics": 0.4968553459119497, + "education": 0.4785276073619632, + "electrical_engineering": 0.38372093023255816, + "elementary_chinese": 0.2976190476190476, + "elementary_commonsense": 0.3838383838383838, + "elementary_information_and_technology": 0.6050420168067226, + "elementary_mathematics": 0.3, + "ethnology": 0.4148148148148148, + "food_science": 0.46153846153846156, + "genetics": 0.30113636363636365, + "global_facts": 0.348993288590604, "high_school_biology": 0.31952662721893493, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.288135593220339, + "high_school_chemistry": 0.30303030303030304, + "high_school_geography": 0.3983050847457627, "high_school_mathematics": 0.24390243902439024, - "high_school_physics": 0.3, - "high_school_politics": 0.2727272727272727, - "human_sexuality": 0.2698412698412698, - "international_law": 0.31891891891891894, - "journalism": 0.3023255813953488, - "jurisprudence": 0.25547445255474455, - "legal_and_moral_basis": 0.3364485981308411, - "logical": 0.2845528455284553, - "machine_learning": 0.29508196721311475, - "management": 0.23809523809523808, - "marketing": 0.3277777777777778, - "marxist_theory": 0.2857142857142857, - "modern_chinese": 0.23275862068965517, - "nutrition": 0.25517241379310346, + "high_school_physics": 0.35454545454545455, + "high_school_politics": 0.4125874125874126, + "human_sexuality": 0.4603174603174603, + "international_law": 0.42702702702702705, + "journalism": 0.5058139534883721, + "jurisprudence": 0.48418491484184917, + "legal_and_moral_basis": 0.7757009345794392, + "logical": 0.3902439024390244, + "machine_learning": 0.3524590163934426, + "management": 0.5, + "marketing": 0.5444444444444444, + "marxist_theory": 0.5079365079365079, + "modern_chinese": 0.3103448275862069, + "nutrition": 0.4482758620689655, "philosophy": 0.3619047619047619, - "professional_accounting": 0.26285714285714284, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.24202127659574468, - "professional_psychology": 0.27586206896551724, - "public_relations": 0.27011494252873564, - "security_study": 0.23703703703703705, - "sociology": 0.28761061946902655, - "sports_science": 0.2727272727272727, - "traditional_chinese_medicine": 0.2648648648648649, - "virology": 0.2781065088757396, - "world_history": 0.2795031055900621, - "world_religions": 0.26875 + "professional_accounting": 0.4514285714285714, + "professional_law": 0.3459715639810427, + "professional_medicine": 0.30851063829787234, + "professional_psychology": 0.521551724137931, + "public_relations": 0.5287356321839081, + "security_study": 0.5481481481481482, + "sociology": 0.4469026548672566, + "sports_science": 0.5333333333333333, + "traditional_chinese_medicine": 0.32972972972972975, + "virology": 0.40236686390532544, + "world_history": 0.4161490683229814, + "world_religions": 0.41875 } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.21212121212121213 - }, - "prompt_2": { - "accuracy": 0.21212121212121213 - }, - "prompt_3": { - "accuracy": 0.15151515151515152 - }, - "prompt_4": { - "accuracy": 0.18181818181818182 - }, - "prompt_5": { - "accuracy": 0.21212121212121213 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.2409090909090909 - }, - "prompt_2": { - "accuracy": 0.1590909090909091 - }, - "prompt_3": { - "accuracy": 0.17727272727272728 - }, - "prompt_4": { - "accuracy": 0.2431818181818182 - }, - "prompt_5": { - "accuracy": 0.25227272727272726 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3101694915254237 - }, - "prompt_2": { - "accuracy": 0.3176271186440678 - }, - "prompt_3": { - "accuracy": 0.3169491525423729 - }, - "prompt_4": { - "accuracy": 0.29491525423728815 }, "prompt_5": { - "accuracy": 0.31661016949152543 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.3537771129394166 - }, - "prompt_2": { - "accuracy": 0.3661181750186986 - }, - "prompt_3": { - "accuracy": 0.35789080029917725 - }, - "prompt_4": { - "accuracy": 0.3605086013462977 - }, + "accuracy": 0.4110689000172682, + "category_acc": { + "agronomy": 0.3905325443786982, + "anatomy": 0.30405405405405406, + "ancient_chinese": 0.25, + "arts": 0.45, + "astronomy": 0.2, + "business_ethics": 0.47368421052631576, + "chinese_civil_service_exam": 0.41875, + "chinese_driving_rule": 0.6564885496183206, + "chinese_food_culture": 0.3602941176470588, + "chinese_foreign_policy": 0.4672897196261682, + "chinese_history": 0.47987616099071206, + "chinese_literature": 0.3088235294117647, + "chinese_teacher_qualification": 0.4972067039106145, + "clinical_knowledge": 0.31223628691983124, + "college_actuarial_science": 0.27358490566037735, + "college_education": 0.5420560747663551, + "college_engineering_hydrology": 0.3113207547169811, + "college_law": 0.32407407407407407, + "college_mathematics": 0.2571428571428571, + "college_medical_statistics": 0.39622641509433965, + "college_medicine": 0.31868131868131866, + "computer_science": 0.4411764705882353, + "computer_security": 0.42105263157894735, + "conceptual_physics": 0.3401360544217687, + "construction_project_management": 0.3381294964028777, + "economics": 0.48427672955974843, + "education": 0.48466257668711654, + "electrical_engineering": 0.38953488372093026, + "elementary_chinese": 0.30158730158730157, + "elementary_commonsense": 0.3939393939393939, + "elementary_information_and_technology": 0.6008403361344538, + "elementary_mathematics": 0.3130434782608696, + "ethnology": 0.4, + "food_science": 0.4755244755244755, + "genetics": 0.2556818181818182, + "global_facts": 0.33557046979865773, + "high_school_biology": 0.3136094674556213, + "high_school_chemistry": 0.3484848484848485, + "high_school_geography": 0.4067796610169492, + "high_school_mathematics": 0.23780487804878048, + "high_school_physics": 0.33636363636363636, + "high_school_politics": 0.3986013986013986, + "human_sexuality": 0.4603174603174603, + "international_law": 0.42162162162162165, + "journalism": 0.5174418604651163, + "jurisprudence": 0.48905109489051096, + "legal_and_moral_basis": 0.7663551401869159, + "logical": 0.3821138211382114, + "machine_learning": 0.36885245901639346, + "management": 0.49047619047619045, + "marketing": 0.5388888888888889, + "marxist_theory": 0.5291005291005291, + "modern_chinese": 0.3017241379310345, + "nutrition": 0.46206896551724136, + "philosophy": 0.4, + "professional_accounting": 0.44571428571428573, + "professional_law": 0.3412322274881517, + "professional_medicine": 0.30319148936170215, + "professional_psychology": 0.5086206896551724, + "public_relations": 0.5, + "security_study": 0.5185185185185185, + "sociology": 0.4336283185840708, + "sports_science": 0.509090909090909, + "traditional_chinese_medicine": 0.31351351351351353, + "virology": 0.41420118343195267, + "world_history": 0.4409937888198758, + "world_religions": 0.41875 + } + } + }, + "zbench": { + "prompt_1": { + "accuracy": 0.3333333333333333 + }, + "prompt_2": { + "accuracy": 0.36363636363636365 + }, + "prompt_3": { + "accuracy": 0.36363636363636365 + }, + "prompt_4": { + "accuracy": 0.3333333333333333 + }, + "prompt_5": { + "accuracy": 0.36363636363636365 + } + }, + "ind_emotion": { + "prompt_1": { + "accuracy": 0.6227272727272727 + }, + "prompt_2": { + "accuracy": 0.6136363636363636 + }, + "prompt_3": { + "accuracy": 0.5977272727272728 + }, + "prompt_4": { + "accuracy": 0.6136363636363636 + }, + "prompt_5": { + "accuracy": 0.6113636363636363 + } + }, + "ocnli": { + "prompt_1": { + "accuracy": 0.3925423728813559 + }, + "prompt_2": { + "accuracy": 0.3959322033898305 + }, + "prompt_3": { + "accuracy": 0.3752542372881356 + }, + "prompt_4": { + "accuracy": 0.4152542372881356 + }, + "prompt_5": { + "accuracy": 0.4542372881355932 + } + }, + "c3": { + "prompt_1": { + "accuracy": 0.8702318623784593 + }, + "prompt_2": { + "accuracy": 0.8709798055347794 + }, + "prompt_3": { + "accuracy": 0.8739715781600599 + }, + "prompt_4": { + "accuracy": 0.868362004487659 + }, "prompt_5": { - "accuracy": 0.36910994764397903 + "accuracy": 0.8713537771129394 } }, "dream": { "prompt_1": { - "accuracy": 0.49387555120039195 + "accuracy": 0.8907398334149926 }, "prompt_2": { - "accuracy": 0.47084762371386574 + "accuracy": 0.8853503184713376 }, "prompt_3": { - "accuracy": 0.49093581577658013 + "accuracy": 0.8873101420872122 }, "prompt_4": { - "accuracy": 0.4968152866242038 + "accuracy": 0.8863302302792748 }, "prompt_5": { - "accuracy": 0.506124448799608 + "accuracy": 0.8873101420872122 } }, "samsum": { "prompt_1": { - "rouge1": 0.23654306186288257, - "rouge2": 0.08516480930944458, - "rougeL": 0.186716531331191, - "avg_rouge": 0.16947480083450606 + "rouge1": 0.41259329359093655, + "rouge2": 0.2115510244457223, + "rougeL": 0.3530618026494755, + "avg_rouge": 0.3257353735620448 }, "prompt_2": { - "rouge1": 0.2510311908439531, - "rouge2": 0.08936356758500696, - "rougeL": 0.19798966235433332, - "avg_rouge": 0.17946147359443113 + "rouge1": 0.41707374512390494, + "rouge2": 0.2124796660781807, + "rougeL": 0.354080814542538, + "avg_rouge": 0.32787807524820783 }, "prompt_3": { - "rouge1": 0.20657490792902822, - "rouge2": 0.06645521318771101, - "rougeL": 0.16278908395236627, - "avg_rouge": 0.14527306835636852 + "rouge1": 0.41321357066826325, + "rouge2": 0.20913215733645862, + "rougeL": 0.3501808312576957, + "avg_rouge": 0.3241755197541392 }, "prompt_4": { - "rouge1": 0.27247292013114277, - "rouge2": 0.10228961466049344, - "rougeL": 0.2122401956560337, - "avg_rouge": 0.19566757681588998 + "rouge1": 0.41878254040446516, + "rouge2": 0.2122266216884994, + "rougeL": 0.35334307500453, + "avg_rouge": 0.32811741236583153 }, "prompt_5": { - "rouge1": 0.2688485289655838, - "rouge2": 0.10055141866290844, - "rougeL": 0.21335785242188463, - "avg_rouge": 0.19425260001679232 + "rouge1": 0.41256561894917226, + "rouge2": 0.2101843136359623, + "rougeL": 0.34918946838957315, + "avg_rouge": 0.3239798003249026 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.21355592200800247, - "rouge2": 0.05944576461182595, - "rougeL": 0.16163858458066974, - "avg_rouge": 0.14488009040016606 + "rouge1": 0.19650427568779455, + "rouge2": 0.05364004839716688, + "rougeL": 0.14780535686319093, + "avg_rouge": 0.13264989364938415 }, "prompt_2": { - "rouge1": 0.21047761744104962, - "rouge2": 0.05788457200334806, - "rougeL": 0.15820471534384672, - "avg_rouge": 0.14218896826274813 + "rouge1": 0.1973451031462774, + "rouge2": 0.05506989252399569, + "rougeL": 0.14860981077086455, + "avg_rouge": 0.1336749354803792 }, "prompt_3": { - "rouge1": 0.2045322308330721, - "rouge2": 0.05682894819014597, - "rougeL": 0.15480111562583546, - "avg_rouge": 0.13872076488301785 + "rouge1": 0.20097510069358493, + "rouge2": 0.06074357546102678, + "rougeL": 0.15732985995613014, + "avg_rouge": 0.13968284537024728 }, "prompt_4": { - "rouge1": 0.22257402857459527, - "rouge2": 0.06361493370904989, - "rougeL": 0.16567863899457627, - "avg_rouge": 0.15062253375940715 + "rouge1": 0.19718866521664802, + "rouge2": 0.055280092355201836, + "rougeL": 0.14855552896452479, + "avg_rouge": 0.13367476217879154 }, "prompt_5": { - "rouge1": 0.21163598037827852, - "rouge2": 0.056979701942418484, - "rougeL": 0.157701418476116, - "avg_rouge": 0.14210570026560432 + "rouge1": 0.2004555565400454, + "rouge2": 0.05905148166979365, + "rougeL": 0.15259325268376314, + "avg_rouge": 0.13736676363120073 } }, "sst2": { "prompt_1": { - "accuracy": 0.6777522935779816 + "accuracy": 0.948394495412844 }, "prompt_2": { - "accuracy": 0.5458715596330275 + "accuracy": 0.9461009174311926 }, "prompt_3": { - "accuracy": 0.606651376146789 + "accuracy": 0.9461009174311926 }, "prompt_4": { - "accuracy": 0.5275229357798165 + "accuracy": 0.9472477064220184 }, "prompt_5": { - "accuracy": 0.8405963302752294 + "accuracy": 0.9243119266055045 } }, "cola": { "prompt_1": { - "accuracy": 0.6423777564717162 + "accuracy": 0.5656759348034516 }, "prompt_2": { - "accuracy": 0.6855225311601151 + "accuracy": 0.573346116970278 }, "prompt_3": { - "accuracy": 0.6538830297219559 + "accuracy": 0.5695110258868649 }, "prompt_4": { - "accuracy": 0.5110258868648131 + "accuracy": 0.573346116970278 }, "prompt_5": { - "accuracy": 0.6299137104506232 + "accuracy": 0.5043144774688398 } }, "qqp": { "prompt_1": { - "accuracy": 0.4145 + "accuracy": 0.6895 }, "prompt_2": { - "accuracy": 0.4785 + "accuracy": 0.5115 }, "prompt_3": { - "accuracy": 0.5045 + "accuracy": 0.771 }, "prompt_4": { - "accuracy": 0.433 + "accuracy": 0.635 }, "prompt_5": { - "accuracy": 0.4235 + "accuracy": 0.5615 } }, "mnli": { "prompt_1": { - "accuracy": 0.3225 + "accuracy": 0.6 }, "prompt_2": { - "accuracy": 0.327 + "accuracy": 0.6 }, "prompt_3": { - "accuracy": 0.319 + "accuracy": 0.6 }, "prompt_4": { - "accuracy": 0.3325 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.327 + "accuracy": 0.7 } }, "qnli": { "prompt_1": { - "accuracy": 0.511 + "accuracy": 0.5 }, "prompt_2": { - "accuracy": 0.5325 + "accuracy": 0.4 }, "prompt_3": { - "accuracy": 0.5225 + "accuracy": 0.5 }, "prompt_4": { - "accuracy": 0.5285 + "accuracy": 0.7 }, "prompt_5": { - "accuracy": 0.5225 + "accuracy": 0.5 } }, "wnli": { "prompt_1": { - "accuracy": 0.4507042253521127 + "accuracy": 0.5 }, "prompt_2": { - "accuracy": 0.6338028169014085 + "accuracy": 0.7 }, "prompt_3": { - "accuracy": 0.4507042253521127 + "accuracy": 0.4 }, "prompt_4": { - "accuracy": 0.43661971830985913 + "accuracy": 0.3 }, "prompt_5": { - "accuracy": 0.43661971830985913 + "accuracy": 0.6 } }, "rte": { "prompt_1": { - "accuracy": 0.5234657039711191 + "accuracy": 0.6 }, "prompt_2": { - "accuracy": 0.5342960288808665 + "accuracy": 0.6 }, "prompt_3": { - "accuracy": 0.5306859205776173 + "accuracy": 0.7 }, "prompt_4": { - "accuracy": 0.5306859205776173 + "accuracy": 0.7 }, "prompt_5": { - "accuracy": 0.5090252707581228 + "accuracy": 0.8 } }, "mrpc": { "prompt_1": { - "accuracy": 0.4950980392156863 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.4583333333333333 + "accuracy": 0.3 }, "prompt_3": { - "accuracy": 0.45098039215686275 + "accuracy": 0.7 }, "prompt_4": { - "accuracy": 0.46078431372549017 + "accuracy": 0.7 }, "prompt_5": { - "accuracy": 0.5465686274509803 + "accuracy": 0.4 } } }, "five_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.5002100840336134, - "language_acc": { - "English": 0.5252100840336135, - "Chinese": 0.5050420168067227, - "Spanish": 0.4899159663865546, - "Vietnamese": 0.48067226890756304 - }, - "consistency_score_2": 0.6738095238095237, - "consistency_score_3": 0.5390756302521008, - "consistency_score_4": 0.45714285714285713, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.6579831932773109, - "English,Spanish": 0.7159663865546219, - "English,Vietnamese": 0.6705882352941176, - "Chinese,Spanish": 0.653781512605042, - "Chinese,Vietnamese": 0.6672268907563025, - "Spanish,Vietnamese": 0.6773109243697479 - }, - "3_combine": { - "English,Chinese,Spanish": 0.5394957983193277, - "English,Chinese,Vietnamese": 0.526890756302521, - "English,Spanish,Vietnamese": 0.5588235294117647, - "Chinese,Spanish,Vietnamese": 0.5310924369747899 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.45714285714285713 - } - }, - "AC3_2": 0.5741749392378728, - "AC3_3": 0.5189161413004852, - "AC3_4": 0.47770776508380447 - } + "prompt_1": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.3038095238095238, - "language_acc": { - "Spanish": 0.30666666666666664, - "English": 0.32666666666666666, - "Chinese": 0.26, - "Vietnamese": 0.2733333333333333, - "Filipino": 0.3, - "Indonesian": 0.38666666666666666, - "Malay": 0.2733333333333333 - }, - "consistency_score_2": 0.44380952380952393, - "consistency_score_3": 0.2371428571428572, - "consistency_score_4": 0.14095238095238094, - "consistency_score_5": 0.09238095238095238, - "consistency_score_6": 0.06666666666666667, - "consistency_score_7": 0.05333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.44, - "Spanish,Chinese": 0.49333333333333335, - "Spanish,Vietnamese": 0.41333333333333333, - "Spanish,Filipino": 0.36, - "Spanish,Indonesian": 0.4266666666666667, - "Spanish,Malay": 0.44666666666666666, - "English,Chinese": 0.54, - "English,Vietnamese": 0.44666666666666666, - "English,Filipino": 0.41333333333333333, - "English,Indonesian": 0.44666666666666666, - "English,Malay": 0.3933333333333333, - "Chinese,Vietnamese": 0.47333333333333333, - "Chinese,Filipino": 0.5066666666666667, - "Chinese,Indonesian": 0.4666666666666667, - "Chinese,Malay": 0.42, - "Vietnamese,Filipino": 0.38666666666666666, - "Vietnamese,Indonesian": 0.47333333333333333, - "Vietnamese,Malay": 0.43333333333333335, - "Filipino,Indonesian": 0.4533333333333333, - "Filipino,Malay": 0.41333333333333333, - "Indonesian,Malay": 0.47333333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.29333333333333333, - "Spanish,English,Vietnamese": 0.23333333333333334, - "Spanish,English,Filipino": 0.18666666666666668, - "Spanish,English,Indonesian": 0.21333333333333335, - "Spanish,English,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese": 0.28, - "Spanish,Chinese,Filipino": 0.24666666666666667, - "Spanish,Chinese,Indonesian": 0.26666666666666666, - "Spanish,Chinese,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino": 0.18, - "Spanish,Vietnamese,Indonesian": 0.22, - "Spanish,Vietnamese,Malay": 0.2, - "Spanish,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Filipino,Malay": 0.18, - "Spanish,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese": 0.3, - "English,Chinese,Filipino": 0.28, - "English,Chinese,Indonesian": 0.28, - "English,Chinese,Malay": 0.24666666666666667, - "English,Vietnamese,Filipino": 0.21333333333333335, - "English,Vietnamese,Indonesian": 0.23333333333333334, - "English,Vietnamese,Malay": 0.21333333333333335, - "English,Filipino,Indonesian": 0.22666666666666666, - "English,Filipino,Malay": 0.2, - "English,Indonesian,Malay": 0.22, - "Chinese,Vietnamese,Filipino": 0.2733333333333333, - "Chinese,Vietnamese,Indonesian": 0.2733333333333333, - "Chinese,Vietnamese,Malay": 0.23333333333333334, - "Chinese,Filipino,Indonesian": 0.28, - "Chinese,Filipino,Malay": 0.23333333333333334, - "Chinese,Indonesian,Malay": 0.24666666666666667, - "Vietnamese,Filipino,Indonesian": 0.24, - "Vietnamese,Filipino,Malay": 0.20666666666666667, - "Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Filipino,Indonesian,Malay": 0.24 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.18666666666666668, - "Spanish,English,Chinese,Filipino": 0.16, - "Spanish,English,Chinese,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Vietnamese,Malay": 0.12666666666666668, - "Spanish,English,Filipino,Indonesian": 0.12, - "Spanish,English,Filipino,Malay": 0.1, - "Spanish,English,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino": 0.16, - "Spanish,Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Filipino,Malay": 0.12, - "Spanish,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.12, - "English,Chinese,Vietnamese,Filipino": 0.15333333333333332, - "English,Chinese,Vietnamese,Indonesian": 0.16, - "English,Chinese,Vietnamese,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian": 0.17333333333333334, - "English,Chinese,Filipino,Malay": 0.14, - "English,Chinese,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian": 0.14, - "English,Vietnamese,Filipino,Malay": 0.11333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "English,Filipino,Indonesian,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Chinese,Vietnamese,Filipino,Malay": 0.14, - "Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.14 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.08, - "Spanish,English,Chinese,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,English,Vietnamese,Filipino,Malay": 0.06666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.08, - "Spanish,English,Filipino,Indonesian,Malay": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.08, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "English,Chinese,Filipino,Indonesian,Malay": 0.1, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.06, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.08, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.06666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334 - } - }, - "AC3_2": 0.3607012435065001, - "AC3_3": 0.2663682092062926, - "AC3_4": 0.19256449470535283, - "AC3_5": 0.14168040289464226, - "AC3_6": 0.10934018848805299, - "AC3_7": 0.0907377777523712 - } + "prompt_1": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.32224025974025977, - "language_acc": { - "English": 0.3125, - "Filipino": 0.3806818181818182, - "Spanish": 0.32954545454545453, - "Malay": 0.2897727272727273, - "Chinese": 0.3181818181818182, - "Indonesian": 0.3125, - "Vietnamese": 0.3125 - }, - "consistency_score_2": 0.4007034632034632, - "consistency_score_3": 0.19172077922077926, - "consistency_score_4": 0.10633116883116882, - "consistency_score_5": 0.0652056277056277, - "consistency_score_6": 0.04220779220779221, - "consistency_score_7": 0.028409090909090908, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3181818181818182, - "English,Spanish": 0.375, - "English,Malay": 0.375, - "English,Chinese": 0.42613636363636365, - "English,Indonesian": 0.375, - "English,Vietnamese": 0.36363636363636365, - "Filipino,Spanish": 0.4147727272727273, - "Filipino,Malay": 0.4147727272727273, - "Filipino,Chinese": 0.4090909090909091, - "Filipino,Indonesian": 0.5, - "Filipino,Vietnamese": 0.38636363636363635, - "Spanish,Malay": 0.32954545454545453, - "Spanish,Chinese": 0.4090909090909091, - "Spanish,Indonesian": 0.4772727272727273, - "Spanish,Vietnamese": 0.36363636363636365, - "Malay,Chinese": 0.375, - "Malay,Indonesian": 0.42045454545454547, - "Malay,Vietnamese": 0.39204545454545453, - "Chinese,Indonesian": 0.4431818181818182, - "Chinese,Vietnamese": 0.375, - "Indonesian,Vietnamese": 0.4715909090909091 - }, - "3_combine": { - "English,Filipino,Spanish": 0.14204545454545456, - "English,Filipino,Malay": 0.13068181818181818, - "English,Filipino,Chinese": 0.1590909090909091, - "English,Filipino,Indonesian": 0.19318181818181818, - "English,Filipino,Vietnamese": 0.125, - "English,Spanish,Malay": 0.125, - "English,Spanish,Chinese": 0.19886363636363635, - "English,Spanish,Indonesian": 0.19318181818181818, - "English,Spanish,Vietnamese": 0.13636363636363635, - "English,Malay,Chinese": 0.1590909090909091, - "English,Malay,Indonesian": 0.17045454545454544, - "English,Malay,Vietnamese": 0.18181818181818182, - "English,Chinese,Indonesian": 0.2159090909090909, - "English,Chinese,Vietnamese": 0.17613636363636365, - "English,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Malay": 0.17613636363636365, - "Filipino,Spanish,Chinese": 0.2159090909090909, - "Filipino,Spanish,Indonesian": 0.25, - "Filipino,Spanish,Vietnamese": 0.1875, - "Filipino,Malay,Chinese": 0.19886363636363635, - "Filipino,Malay,Indonesian": 0.2556818181818182, - "Filipino,Malay,Vietnamese": 0.17613636363636365, - "Filipino,Chinese,Indonesian": 0.24431818181818182, - "Filipino,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Indonesian,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Chinese": 0.1534090909090909, - "Spanish,Malay,Indonesian": 0.21022727272727273, - "Spanish,Malay,Vietnamese": 0.16477272727272727, - "Spanish,Chinese,Indonesian": 0.2556818181818182, - "Spanish,Chinese,Vietnamese": 0.19886363636363635, - "Spanish,Indonesian,Vietnamese": 0.25, - "Malay,Chinese,Indonesian": 0.23295454545454544, - "Malay,Chinese,Vietnamese": 0.17613636363636365, - "Malay,Indonesian,Vietnamese": 0.2215909090909091, - "Chinese,Indonesian,Vietnamese": 0.2215909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.06818181818181818, - "English,Filipino,Spanish,Chinese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Chinese": 0.06818181818181818, - "English,Filipino,Malay,Indonesian": 0.09659090909090909, - "English,Filipino,Malay,Vietnamese": 0.0625, - "English,Filipino,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Chinese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian": 0.09090909090909091, - "English,Spanish,Malay,Vietnamese": 0.0625, - "English,Spanish,Chinese,Indonesian": 0.13068181818181818, - "English,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "English,Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese": 0.10795454545454546, - "Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.125, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Indonesian": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Indonesian": 0.0625, - "English,Filipino,Malay,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Indonesian": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 - } - }, - "AC3_2": 0.35721394057583195, - "AC3_3": 0.24040792588705062, - "AC3_4": 0.1598995227883493, - "AC3_5": 0.10846355108569626, - "AC3_6": 0.07463916927347766, - "AC3_7": 0.05221485688746601 - } + "prompt_1": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.23300970873786409 - } + "prompt_1": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.22857142857142856 - } + "prompt_1": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.2897196261682243 - } + "prompt_1": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.29, - "category_acc": { - "brand": 0.2, - "demographics": 0.4, - "biology": 0.2, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.5, - "culture": 0.2, - "film": 0.4, - "law": 0.2, - "geography": 0.3 - } - } + "prompt_1": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.07042819290816649 - } + "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.07266952766365038 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.0572216168520532 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.07880065422211031 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.0771333360526057 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.338389731621937 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.32020021451555236, - "category_acc": { - "high_school_european_history": 0.4268292682926829, - "business_ethics": 0.32323232323232326, - "clinical_knowledge": 0.3409090909090909, - "medical_genetics": 0.36363636363636365, - "high_school_us_history": 0.3793103448275862, - "high_school_physics": 0.32, - "high_school_world_history": 0.4533898305084746, - "virology": 0.3212121212121212, - "high_school_microeconomics": 0.2869198312236287, - "econometrics": 0.23893805309734514, - "college_computer_science": 0.21212121212121213, - "high_school_biology": 0.3333333333333333, - "abstract_algebra": 0.2727272727272727, - "professional_accounting": 0.298932384341637, - "philosophy": 0.3419354838709677, - "professional_medicine": 0.2730627306273063, - "nutrition": 0.35737704918032787, - "global_facts": 0.2727272727272727, - "machine_learning": 0.3333333333333333, - "security_studies": 0.32786885245901637, - "public_relations": 0.3302752293577982, - "professional_psychology": 0.33878887070376434, - "prehistory": 0.3498452012383901, - "anatomy": 0.31343283582089554, - "human_sexuality": 0.3769230769230769, - "college_medicine": 0.3023255813953488, - "high_school_government_and_politics": 0.3385416666666667, - "college_chemistry": 0.2828282828282828, - "logical_fallacies": 0.3271604938271605, - "high_school_geography": 0.27411167512690354, - "elementary_mathematics": 0.259946949602122, - "human_aging": 0.36036036036036034, - "college_mathematics": 0.2727272727272727, - "high_school_psychology": 0.34558823529411764, - "formal_logic": 0.288, - "high_school_statistics": 0.20465116279069767, - "international_law": 0.36666666666666664, - "high_school_mathematics": 0.27137546468401486, - "high_school_computer_science": 0.41414141414141414, - "conceptual_physics": 0.36324786324786323, - "miscellaneous": 0.3670076726342711, - "high_school_chemistry": 0.3316831683168317, - "marketing": 0.3605150214592275, - "professional_law": 0.2974559686888454, - "management": 0.30392156862745096, - "college_physics": 0.26732673267326734, - "jurisprudence": 0.32710280373831774, - "world_religions": 0.37058823529411766, - "sociology": 0.33, - "us_foreign_policy": 0.3434343434343434, - "high_school_macroeconomics": 0.3033419023136247, - "computer_security": 0.32323232323232326, - "moral_scenarios": 0.2550335570469799, - "moral_disputes": 0.32463768115942027, - "electrical_engineering": 0.375, - "astronomy": 0.2847682119205298, - "college_biology": 0.34965034965034963 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.28083209509658247 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.2783312577833126, - "category_acc": { - "computer_network": 0.16666666666666666, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.35714285714285715, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.27586206896551724, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.4583333333333333, - "middle_school_mathematics": 0.125, - "middle_school_biology": 0.3076923076923077, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.35, - "business_administration": 0.21052631578947367, - "marxism": 0.25, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.20588235294117646, - "teacher_qualification": 0.3469387755102041, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.11538461538461539, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.14285714285714285, - "ideological_and_moral_cultivation": 0.25, - "logic": 0.2222222222222222, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.25, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.44, - "middle_school_history": 0.14814814814814814, - "civil_servant": 0.3269230769230769, - "sports_science": 0.2916666666666667, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.35185185185185186, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.2222222222222222, - "tax_accountant": 0.25925925925925924, - "physician": 0.2962962962962963 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.31899641577060933 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.2991711276118114, - "category_acc": { - "agronomy": 0.21893491124260356, - "anatomy": 0.3108108108108108, - "ancient_chinese": 0.25, - "arts": 0.36875, - "astronomy": 0.28484848484848485, - "business_ethics": 0.3684210526315789, - "chinese_civil_service_exam": 0.3, - "chinese_driving_rule": 0.35877862595419846, - "chinese_food_culture": 0.2647058823529412, - "chinese_foreign_policy": 0.308411214953271, - "chinese_history": 0.26625386996904027, - "chinese_literature": 0.3137254901960784, - "chinese_teacher_qualification": 0.2681564245810056, - "clinical_knowledge": 0.24050632911392406, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.37383177570093457, - "college_engineering_hydrology": 0.3584905660377358, - "college_law": 0.21296296296296297, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.36792452830188677, - "college_medicine": 0.2564102564102564, - "computer_science": 0.3333333333333333, - "computer_security": 0.2807017543859649, - "conceptual_physics": 0.29931972789115646, - "construction_project_management": 0.35251798561151076, - "economics": 0.32075471698113206, - "education": 0.26380368098159507, - "electrical_engineering": 0.3488372093023256, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.29292929292929293, - "elementary_information_and_technology": 0.3907563025210084, - "elementary_mathematics": 0.24782608695652175, - "ethnology": 0.2814814814814815, - "food_science": 0.38461538461538464, - "genetics": 0.3125, - "global_facts": 0.3221476510067114, - "high_school_biology": 0.28402366863905326, - "high_school_chemistry": 0.21212121212121213, - "high_school_geography": 0.2542372881355932, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.3181818181818182, - "high_school_politics": 0.24475524475524477, - "human_sexuality": 0.3333333333333333, - "international_law": 0.2972972972972973, - "journalism": 0.29651162790697677, - "jurisprudence": 0.2822384428223844, - "legal_and_moral_basis": 0.4252336448598131, - "logical": 0.3333333333333333, - "machine_learning": 0.2786885245901639, - "management": 0.2714285714285714, - "marketing": 0.34444444444444444, - "marxist_theory": 0.3333333333333333, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.3793103448275862, - "philosophy": 0.3142857142857143, - "professional_accounting": 0.32, - "professional_law": 0.24644549763033174, - "professional_medicine": 0.23670212765957446, - "professional_psychology": 0.3146551724137931, - "public_relations": 0.3390804597701149, - "security_study": 0.35555555555555557, - "sociology": 0.28761061946902655, - "sports_science": 0.296969696969697, - "traditional_chinese_medicine": 0.2702702702702703, - "virology": 0.30177514792899407, - "world_history": 0.3167701863354037, - "world_religions": 0.275 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.24242424242424243 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.18863636363636363 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.31796610169491524 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.4050112191473448 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.5600195982361588 - } + "prompt_1": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.16390410600713493, - "rouge2": 0.05650962910733604, - "rougeL": 0.1273026259703825, - "avg_rouge": 0.11590545369495116 - } + "prompt_1": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.1614301845271386, - "rouge2": 0.04841135310290797, - "rougeL": 0.12338839136493196, - "avg_rouge": 0.11107664299832616 - } + "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.5022935779816514 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.4199424736337488 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.5135 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.3455 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.5225 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.5774647887323944 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.5270758122743683 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.5318627450980392 - } + "prompt_1": -1 } } }, - "gemma_2b_it": { + "seallm_7b_v2": { "model_size": "7B", - "model_link": "https://huggingface.co/google/gemma-2b-it", + "model_link": "https://huggingface.co/SeaLLMs/SeaLLM-7B-v2", "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.6342436974789916, + "overall_acc": 0.9228991596638655, "language_acc": { - "English": 0.6714285714285714, - "Chinese": 0.6310924369747899, - "Spanish": 0.6260504201680672, - "Vietnamese": 0.6084033613445378 - }, - "consistency_score_2": 0.7512605042016807, - "consistency_score_3": 0.6470588235294117, - "consistency_score_4": 0.5857142857142857, + "Spanish": 0.9285714285714286, + "English": 0.9436974789915966, + "Chinese": 0.9058823529411765, + "Vietnamese": 0.9134453781512605 + }, + "consistency_score_2": 0.911624649859944, + "consistency_score_3": 0.8710084033613446, + "consistency_score_4": 0.8420168067226891, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.7478991596638656, - "English,Spanish": 0.7966386554621848, - "English,Vietnamese": 0.7470588235294118, - "Chinese,Spanish": 0.7361344537815127, - "Chinese,Vietnamese": 0.7235294117647059, - "Spanish,Vietnamese": 0.7563025210084033 + "Spanish,English": 0.9428571428571428, + "Spanish,Chinese": 0.8991596638655462, + "Spanish,Vietnamese": 0.9126050420168067, + "English,Chinese": 0.9092436974789916, + "English,Vietnamese": 0.9142857142857143, + "Chinese,Vietnamese": 0.8915966386554622 }, "3_combine": { - "English,Chinese,Spanish": 0.6596638655462185, - "English,Chinese,Vietnamese": 0.6327731092436975, - "English,Spanish,Vietnamese": 0.6680672268907563, - "Chinese,Spanish,Vietnamese": 0.6277310924369748 + "Spanish,English,Chinese": 0.8781512605042017, + "Spanish,English,Vietnamese": 0.8873949579831932, + "Spanish,Chinese,Vietnamese": 0.8563025210084033, + "English,Chinese,Vietnamese": 0.8621848739495799 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5857142857142857 + "Spanish,English,Chinese,Vietnamese": 0.8420168067226891 } }, - "AC3_2": 0.687810602584132, - "AC3_3": 0.6405871743180306, - "AC3_4": 0.6090137518258939 + "AC3_2": 0.9172272595983739, + "AC3_3": 0.8962032827621712, + "AC3_4": 0.8806046498567287 }, "prompt_2": { - "overall_acc": 0.6758403361344537, + "overall_acc": 0.915126050420168, "language_acc": { - "English": 0.7084033613445379, - "Chinese": 0.673109243697479, - "Spanish": 0.6563025210084034, - "Vietnamese": 0.6655462184873949 - }, - "consistency_score_2": 0.7474789915966387, - "consistency_score_3": 0.640126050420168, - "consistency_score_4": 0.573109243697479, + "Spanish": 0.9226890756302522, + "English": 0.938655462184874, + "Chinese": 0.9033613445378151, + "Vietnamese": 0.8957983193277311 + }, + "consistency_score_2": 0.8943977591036415, + "consistency_score_3": 0.8453781512605042, + "consistency_score_4": 0.8058823529411765, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.7529411764705882, - "English,Spanish": 0.7722689075630252, - "English,Vietnamese": 0.7655462184873949, - "Chinese,Spanish": 0.7210084033613445, - "Chinese,Vietnamese": 0.7201680672268908, - "Spanish,Vietnamese": 0.7529411764705882 + "Spanish,English": 0.9310924369747899, + "Spanish,Chinese": 0.8899159663865546, + "Spanish,Vietnamese": 0.8789915966386554, + "English,Chinese": 0.9092436974789916, + "English,Vietnamese": 0.8949579831932774, + "Chinese,Vietnamese": 0.8621848739495799 }, "3_combine": { - "English,Chinese,Spanish": 0.6420168067226891, - "English,Chinese,Vietnamese": 0.6403361344537815, - "English,Spanish,Vietnamese": 0.6596638655462185, - "Chinese,Spanish,Vietnamese": 0.6184873949579832 + "Spanish,English,Chinese": 0.8697478991596639, + "Spanish,English,Vietnamese": 0.853781512605042, + "Spanish,Chinese,Vietnamese": 0.8201680672268907, + "English,Chinese,Vietnamese": 0.8378151260504202 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.573109243697479 + "Spanish,English,Chinese,Vietnamese": 0.8058823529411765 } }, - "AC3_2": 0.7098568017114215, - "AC3_3": 0.6574985645099604, - "AC3_4": 0.6202497684868529 + "AC3_2": 0.9046431823003461, + "AC3_3": 0.8788704598285406, + "AC3_4": 0.8570369944354955 }, "prompt_3": { - "overall_acc": 0.6789915966386554, + "overall_acc": 0.9063025210084034, "language_acc": { - "English": 0.7109243697478992, - "Chinese": 0.6764705882352942, - "Spanish": 0.6655462184873949, - "Vietnamese": 0.6630252100840336 - }, - "consistency_score_2": 0.7504201680672269, - "consistency_score_3": 0.6455882352941177, - "consistency_score_4": 0.5823529411764706, + "Spanish": 0.9117647058823529, + "English": 0.938655462184874, + "Chinese": 0.9033613445378151, + "Vietnamese": 0.8714285714285714 + }, + "consistency_score_2": 0.8820728291316526, + "consistency_score_3": 0.8294117647058823, + "consistency_score_4": 0.7899159663865546, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.7546218487394958, - "English,Spanish": 0.7831932773109244, - "English,Vietnamese": 0.761344537815126, - "Chinese,Spanish": 0.726890756302521, - "Chinese,Vietnamese": 0.7260504201680672, - "Spanish,Vietnamese": 0.7504201680672269 + "Spanish,English": 0.9159663865546218, + "Spanish,Chinese": 0.8815126050420168, + "Spanish,Vietnamese": 0.8613445378151261, + "English,Chinese": 0.9033613445378151, + "English,Vietnamese": 0.8764705882352941, + "Chinese,Vietnamese": 0.853781512605042 }, "3_combine": { - "English,Chinese,Spanish": 0.6512605042016807, - "English,Chinese,Vietnamese": 0.6436974789915967, - "English,Spanish,Vietnamese": 0.6630252100840336, - "Chinese,Spanish,Vietnamese": 0.6243697478991597 + "Spanish,English,Chinese": 0.8571428571428571, + "Spanish,English,Vietnamese": 0.83109243697479, + "Spanish,Chinese,Vietnamese": 0.807563025210084, + "English,Chinese,Vietnamese": 0.8218487394957983 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5823529411764706 + "Spanish,English,Chinese,Vietnamese": 0.7899159663865546 } }, - "AC3_2": 0.7129212178199889, - "AC3_3": 0.6618687315822563, - "AC3_4": 0.6269702550743285 + "AC3_2": 0.8940235377855366, + "AC3_3": 0.866154043266771, + "AC3_4": 0.8441162939783743 }, "prompt_4": { - "overall_acc": 0.6348739495798319, + "overall_acc": 0.9163865546218487, "language_acc": { - "English": 0.6655462184873949, - "Chinese": 0.6319327731092437, - "Spanish": 0.6235294117647059, - "Vietnamese": 0.6184873949579832 - }, - "consistency_score_2": 0.7518207282913165, - "consistency_score_3": 0.6460084033613445, - "consistency_score_4": 0.5831932773109244, + "Spanish": 0.9260504201680673, + "English": 0.9361344537815126, + "Chinese": 0.9025210084033614, + "Vietnamese": 0.9008403361344538 + }, + "consistency_score_2": 0.9007002801120448, + "consistency_score_3": 0.8552521008403362, + "consistency_score_4": 0.8243697478991596, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.7546218487394958, - "English,Spanish": 0.7907563025210084, - "English,Vietnamese": 0.7504201680672269, - "Chinese,Spanish": 0.7436974789915967, - "Chinese,Vietnamese": 0.7285714285714285, - "Spanish,Vietnamese": 0.7428571428571429 + "Spanish,English": 0.9294117647058824, + "Spanish,Chinese": 0.8941176470588236, + "Spanish,Vietnamese": 0.8941176470588236, + "English,Chinese": 0.9025210084033614, + "English,Vietnamese": 0.9050420168067227, + "Chinese,Vietnamese": 0.8789915966386554 }, "3_combine": { - "English,Chinese,Spanish": 0.6621848739495798, - "English,Chinese,Vietnamese": 0.6369747899159663, - "English,Spanish,Vietnamese": 0.6588235294117647, - "Chinese,Spanish,Vietnamese": 0.6260504201680672 + "Spanish,English,Chinese": 0.8680672268907563, + "Spanish,English,Vietnamese": 0.8663865546218488, + "Spanish,Chinese,Vietnamese": 0.8378151260504202, + "English,Chinese,Vietnamese": 0.8487394957983193 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5831932773109244 + "Spanish,English,Chinese,Vietnamese": 0.8243697478991596 } }, - "AC3_2": 0.6884159905259617, - "AC3_3": 0.640392781632137, - "AC3_4": 0.6079372486681353 + "AC3_2": 0.9084757102587919, + "AC3_3": 0.8847645354334098, + "AC3_4": 0.8679461356817676 }, "prompt_5": { - "overall_acc": 0.6294117647058823, + "overall_acc": 0.9195378151260504, "language_acc": { - "English": 0.6504201680672269, - "Chinese": 0.634453781512605, - "Spanish": 0.6126050420168068, - "Vietnamese": 0.6201680672268908 + "Spanish": 0.9226890756302522, + "English": 0.9420168067226891, + "Chinese": 0.915126050420168, + "Vietnamese": 0.8983193277310925 }, - "consistency_score_2": 0.7357142857142858, - "consistency_score_3": 0.6239495798319328, - "consistency_score_4": 0.5588235294117647, + "consistency_score_2": 0.9050420168067227, + "consistency_score_3": 0.8621848739495799, + "consistency_score_4": 0.8327731092436975, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.7495798319327731, - "English,Spanish": 0.7663865546218488, - "English,Vietnamese": 0.7252100840336134, - "Chinese,Spanish": 0.7252100840336134, - "Chinese,Vietnamese": 0.719327731092437, - "Spanish,Vietnamese": 0.7285714285714285 + "Spanish,English": 0.9327731092436975, + "Spanish,Chinese": 0.9058823529411765, + "Spanish,Vietnamese": 0.8890756302521008, + "English,Chinese": 0.9159663865546218, + "English,Vietnamese": 0.9, + "Chinese,Vietnamese": 0.8865546218487395 }, "3_combine": { - "English,Chinese,Spanish": 0.6411764705882353, - "English,Chinese,Vietnamese": 0.6210084033613446, - "English,Spanish,Vietnamese": 0.6277310924369748, - "Chinese,Spanish,Vietnamese": 0.6058823529411764 + "Spanish,English,Chinese": 0.8815126050420168, + "Spanish,English,Vietnamese": 0.8638655462184874, + "Spanish,Chinese,Vietnamese": 0.8478991596638655, + "English,Chinese,Vietnamese": 0.8554621848739495 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5588235294117647 + "Spanish,English,Chinese,Vietnamese": 0.8327731092436975 } }, - "AC3_2": 0.678424130451996, - "AC3_3": 0.6266687700393306, - "AC3_4": 0.5920209667527391 + "AC3_2": 0.91223233329785, + "AC3_3": 0.8899382603621049, + "AC3_4": 0.8740074089321783 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.36571428571428566, + "overall_acc": 0.6085714285714285, "language_acc": { - "Spanish": 0.38666666666666666, - "English": 0.4533333333333333, - "Chinese": 0.36666666666666664, - "Vietnamese": 0.38, - "Filipino": 0.29333333333333333, - "Indonesian": 0.32, - "Malay": 0.36 - }, - "consistency_score_2": 0.5457142857142857, - "consistency_score_3": 0.3718095238095238, - "consistency_score_4": 0.27961904761904754, - "consistency_score_5": 0.22095238095238093, - "consistency_score_6": 0.18095238095238095, - "consistency_score_7": 0.15333333333333332, + "Filipino": 0.5733333333333334, + "Vietnamese": 0.56, + "Chinese": 0.5733333333333334, + "Spanish": 0.6533333333333333, + "Malay": 0.54, + "Indonesian": 0.6266666666666667, + "English": 0.7333333333333333 + }, + "consistency_score_2": 0.6314285714285715, + "consistency_score_3": 0.49257142857142855, + "consistency_score_4": 0.4123809523809523, + "consistency_score_5": 0.3574603174603175, + "consistency_score_6": 0.3152380952380952, + "consistency_score_7": 0.28, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.64, - "Spanish,Chinese": 0.5333333333333333, - "Spanish,Vietnamese": 0.5466666666666666, - "Spanish,Filipino": 0.5133333333333333, - "Spanish,Indonesian": 0.5733333333333334, - "Spanish,Malay": 0.54, - "English,Chinese": 0.5733333333333334, - "English,Vietnamese": 0.6, - "English,Filipino": 0.5, - "English,Indonesian": 0.58, - "English,Malay": 0.5533333333333333, - "Chinese,Vietnamese": 0.54, - "Chinese,Filipino": 0.4866666666666667, - "Chinese,Indonesian": 0.5466666666666666, - "Chinese,Malay": 0.56, - "Vietnamese,Filipino": 0.5133333333333333, - "Vietnamese,Indonesian": 0.54, - "Vietnamese,Malay": 0.5533333333333333, - "Filipino,Indonesian": 0.47333333333333333, - "Filipino,Malay": 0.44666666666666666, - "Indonesian,Malay": 0.6466666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.4066666666666667, - "Spanish,English,Vietnamese": 0.44666666666666666, - "Spanish,English,Filipino": 0.36666666666666664, - "Spanish,English,Indonesian": 0.4533333333333333, - "Spanish,English,Malay": 0.43333333333333335, - "Spanish,Chinese,Vietnamese": 0.36666666666666664, - "Spanish,Chinese,Filipino": 0.32, - "Spanish,Chinese,Indonesian": 0.38, - "Spanish,Chinese,Malay": 0.36666666666666664, - "Spanish,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian": 0.38, - "Spanish,Vietnamese,Malay": 0.38666666666666666, - "Spanish,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Filipino,Malay": 0.32, - "Spanish,Indonesian,Malay": 0.41333333333333333, - "English,Chinese,Vietnamese": 0.41333333333333333, - "English,Chinese,Filipino": 0.3333333333333333, - "English,Chinese,Indonesian": 0.41333333333333333, - "English,Chinese,Malay": 0.38, - "English,Vietnamese,Filipino": 0.34, - "English,Vietnamese,Indonesian": 0.4266666666666667, - "English,Vietnamese,Malay": 0.41333333333333333, - "English,Filipino,Indonesian": 0.34, - "English,Filipino,Malay": 0.32, - "English,Indonesian,Malay": 0.44666666666666666, - "Chinese,Vietnamese,Filipino": 0.3333333333333333, - "Chinese,Vietnamese,Indonesian": 0.38, - "Chinese,Vietnamese,Malay": 0.36666666666666664, - "Chinese,Filipino,Indonesian": 0.31333333333333335, - "Chinese,Filipino,Malay": 0.32, - "Chinese,Indonesian,Malay": 0.42, - "Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Vietnamese,Filipino,Malay": 0.30666666666666664, - "Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Filipino,Indonesian,Malay": 0.34 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.31333333333333335, - "Spanish,English,Chinese,Filipino": 0.24666666666666667, - "Spanish,English,Chinese,Indonesian": 0.32, - "Spanish,English,Chinese,Malay": 0.30666666666666664, - "Spanish,English,Vietnamese,Filipino": 0.26, - "Spanish,English,Vietnamese,Indonesian": 0.34, - "Spanish,English,Vietnamese,Malay": 0.3466666666666667, - "Spanish,English,Filipino,Indonesian": 0.2733333333333333, - "Spanish,English,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Indonesian,Malay": 0.36666666666666664, - "Spanish,Chinese,Vietnamese,Filipino": 0.24, - "Spanish,Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.2733333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.25333333333333335, - "Spanish,Chinese,Indonesian,Malay": 0.3, - "Spanish,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,Vietnamese,Filipino,Malay": 0.24, - "Spanish,Vietnamese,Indonesian,Malay": 0.3, - "Spanish,Filipino,Indonesian,Malay": 0.26, - "English,Chinese,Vietnamese,Filipino": 0.26, - "English,Chinese,Vietnamese,Indonesian": 0.32666666666666666, - "English,Chinese,Vietnamese,Malay": 0.3, - "English,Chinese,Filipino,Indonesian": 0.26, - "English,Chinese,Filipino,Malay": 0.24666666666666667, - "English,Chinese,Indonesian,Malay": 0.3333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "English,Vietnamese,Filipino,Malay": 0.25333333333333335, - "English,Vietnamese,Indonesian,Malay": 0.3466666666666667, - "English,Filipino,Indonesian,Malay": 0.28, - "Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.3, - "Chinese,Filipino,Indonesian,Malay": 0.26, - "Vietnamese,Filipino,Indonesian,Malay": 0.22 + "Filipino,Vietnamese": 0.5533333333333333, + "Filipino,Chinese": 0.54, + "Filipino,Spanish": 0.6066666666666667, + "Filipino,Malay": 0.58, + "Filipino,Indonesian": 0.6, + "Filipino,English": 0.5933333333333334, + "Vietnamese,Chinese": 0.6466666666666666, + "Vietnamese,Spanish": 0.66, + "Vietnamese,Malay": 0.5866666666666667, + "Vietnamese,Indonesian": 0.6466666666666666, + "Vietnamese,English": 0.6866666666666666, + "Chinese,Spanish": 0.6533333333333333, + "Chinese,Malay": 0.6, + "Chinese,Indonesian": 0.6666666666666666, + "Chinese,English": 0.6333333333333333, + "Spanish,Malay": 0.68, + "Spanish,Indonesian": 0.6933333333333334, + "Spanish,English": 0.7533333333333333, + "Malay,Indonesian": 0.5933333333333334, + "Malay,English": 0.6066666666666667, + "Indonesian,English": 0.68 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.43333333333333335, + "Filipino,Vietnamese,Spanish": 0.46, + "Filipino,Vietnamese,Malay": 0.43333333333333335, + "Filipino,Vietnamese,Indonesian": 0.46, + "Filipino,Vietnamese,English": 0.4666666666666667, + "Filipino,Chinese,Spanish": 0.4533333333333333, + "Filipino,Chinese,Malay": 0.43333333333333335, + "Filipino,Chinese,Indonesian": 0.4666666666666667, + "Filipino,Chinese,English": 0.43333333333333335, + "Filipino,Spanish,Malay": 0.4866666666666667, + "Filipino,Spanish,Indonesian": 0.49333333333333335, + "Filipino,Spanish,English": 0.5266666666666666, + "Filipino,Malay,Indonesian": 0.44666666666666666, + "Filipino,Malay,English": 0.44666666666666666, + "Filipino,Indonesian,English": 0.4866666666666667, + "Vietnamese,Chinese,Spanish": 0.5133333333333333, + "Vietnamese,Chinese,Malay": 0.44666666666666666, + "Vietnamese,Chinese,Indonesian": 0.5266666666666666, + "Vietnamese,Chinese,English": 0.5266666666666666, + "Vietnamese,Spanish,Malay": 0.49333333333333335, + "Vietnamese,Spanish,Indonesian": 0.5266666666666666, + "Vietnamese,Spanish,English": 0.5866666666666667, + "Vietnamese,Malay,Indonesian": 0.46, + "Vietnamese,Malay,English": 0.47333333333333333, + "Vietnamese,Indonesian,English": 0.54, + "Chinese,Spanish,Malay": 0.5, + "Chinese,Spanish,Indonesian": 0.54, + "Chinese,Spanish,English": 0.5533333333333333, + "Chinese,Malay,Indonesian": 0.47333333333333333, + "Chinese,Malay,English": 0.47333333333333333, + "Chinese,Indonesian,English": 0.5333333333333333, + "Spanish,Malay,Indonesian": 0.5133333333333333, + "Spanish,Malay,English": 0.5666666666666667, + "Spanish,Indonesian,English": 0.5866666666666667, + "Malay,Indonesian,English": 0.48 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.38, + "Filipino,Vietnamese,Chinese,Malay": 0.35333333333333333, + "Filipino,Vietnamese,Chinese,Indonesian": 0.3933333333333333, + "Filipino,Vietnamese,Chinese,English": 0.38666666666666666, + "Filipino,Vietnamese,Spanish,Malay": 0.38666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.4066666666666667, + "Filipino,Vietnamese,Spanish,English": 0.43333333333333335, + "Filipino,Vietnamese,Malay,Indonesian": 0.36666666666666664, + "Filipino,Vietnamese,Malay,English": 0.36666666666666664, + "Filipino,Vietnamese,Indonesian,English": 0.4066666666666667, + "Filipino,Chinese,Spanish,Malay": 0.38666666666666666, + "Filipino,Chinese,Spanish,Indonesian": 0.4066666666666667, + "Filipino,Chinese,Spanish,English": 0.4066666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.36666666666666664, + "Filipino,Chinese,Malay,English": 0.36, + "Filipino,Chinese,Indonesian,English": 0.4, + "Filipino,Spanish,Malay,Indonesian": 0.41333333333333333, + "Filipino,Spanish,Malay,English": 0.44, + "Filipino,Spanish,Indonesian,English": 0.4533333333333333, + "Filipino,Malay,Indonesian,English": 0.38666666666666666, + "Vietnamese,Chinese,Spanish,Malay": 0.4066666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.4533333333333333, + "Vietnamese,Chinese,Spanish,English": 0.47333333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.38666666666666666, + "Vietnamese,Chinese,Malay,English": 0.3933333333333333, + "Vietnamese,Chinese,Indonesian,English": 0.46, + "Vietnamese,Spanish,Malay,Indonesian": 0.42, + "Vietnamese,Spanish,Malay,English": 0.4533333333333333, + "Vietnamese,Spanish,Indonesian,English": 0.47333333333333333, + "Vietnamese,Malay,Indonesian,English": 0.38666666666666666, + "Chinese,Spanish,Malay,Indonesian": 0.4266666666666667, + "Chinese,Spanish,Malay,English": 0.44, + "Chinese,Spanish,Indonesian,English": 0.4866666666666667, + "Chinese,Malay,Indonesian,English": 0.4066666666666667, + "Spanish,Malay,Indonesian,English": 0.4666666666666667 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.26, - "Spanish,English,Chinese,Vietnamese,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - } + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.32666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.35333333333333333, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.36, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.31333333333333335, + "Filipino,Vietnamese,Chinese,Malay,English": 0.31333333333333335, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.35333333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.3466666666666667, + "Filipino,Vietnamese,Spanish,Malay,English": 0.36, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.38, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.32, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.3466666666666667, + "Filipino,Chinese,Spanish,Malay,English": 0.35333333333333333, + "Filipino,Chinese,Spanish,Indonesian,English": 0.38, + "Filipino,Chinese,Malay,Indonesian,English": 0.32666666666666666, + "Filipino,Spanish,Malay,Indonesian,English": 0.38666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.36666666666666664, + "Vietnamese,Chinese,Spanish,Malay,English": 0.38, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.42, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.3466666666666667, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.38, + "Chinese,Spanish,Malay,Indonesian,English": 0.3933333333333333 + }, + "6_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.3, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.3333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.28, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.32, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.34 + }, + "7_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.28 + } + }, + "AC3_2": 0.6197893350388912, + "AC3_3": 0.5444614129537427, + "AC3_4": 0.4916257995254071, + "AC3_5": 0.4503788554061741, + "AC3_6": 0.41533431512440777, + "AC3_7": 0.38353697744879806 + }, + "prompt_2": { + "overall_acc": 0.6057142857142858, + "language_acc": { + "Filipino": 0.49333333333333335, + "Vietnamese": 0.6, + "Chinese": 0.6066666666666667, + "Spanish": 0.6266666666666667, + "Malay": 0.5666666666666667, + "Indonesian": 0.62, + "English": 0.7266666666666667 }, - "AC3_2": 0.43793999099538944, - "AC3_3": 0.3687367177776303, - "AC3_4": 0.3169235958355247, - "AC3_5": 0.27547309828328553, - "AC3_6": 0.2421105026933919, - "AC3_7": 0.21607339445378398 - }, - "prompt_2": { - "overall_acc": 0.35714285714285715, - "language_acc": { - "Spanish": 0.38666666666666666, - "English": 0.43333333333333335, - "Chinese": 0.38, - "Vietnamese": 0.3466666666666667, - "Filipino": 0.31333333333333335, - "Indonesian": 0.32, - "Malay": 0.32 - }, - "consistency_score_2": 0.4904761904761906, - "consistency_score_3": 0.3041904761904762, - "consistency_score_4": 0.20761904761904765, - "consistency_score_5": 0.14825396825396828, - "consistency_score_6": 0.10857142857142857, - "consistency_score_7": 0.08, + "consistency_score_2": 0.5895238095238095, + "consistency_score_3": 0.4453333333333332, + "consistency_score_4": 0.36876190476190474, + "consistency_score_5": 0.3177777777777778, + "consistency_score_6": 0.28095238095238095, + "consistency_score_7": 0.25333333333333335, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5066666666666667, - "Spanish,Chinese": 0.5133333333333333, - "Spanish,Vietnamese": 0.44, - "Spanish,Filipino": 0.4066666666666667, - "Spanish,Indonesian": 0.49333333333333335, - "Spanish,Malay": 0.5333333333333333, - "English,Chinese": 0.62, - "English,Vietnamese": 0.56, - "English,Filipino": 0.5066666666666667, - "English,Indonesian": 0.4266666666666667, - "English,Malay": 0.46, - "Chinese,Vietnamese": 0.5133333333333333, - "Chinese,Filipino": 0.47333333333333333, - "Chinese,Indonesian": 0.48, - "Chinese,Malay": 0.5, - "Vietnamese,Filipino": 0.43333333333333335, - "Vietnamese,Indonesian": 0.46, + "Filipino,Vietnamese": 0.4533333333333333, + "Filipino,Chinese": 0.47333333333333333, + "Filipino,Spanish": 0.56, + "Filipino,Malay": 0.49333333333333335, + "Filipino,Indonesian": 0.5066666666666667, + "Filipino,English": 0.58, + "Vietnamese,Chinese": 0.6266666666666667, + "Vietnamese,Spanish": 0.5666666666666667, "Vietnamese,Malay": 0.5333333333333333, - "Filipino,Indonesian": 0.38, - "Filipino,Malay": 0.44666666666666666, - "Indonesian,Malay": 0.6133333333333333 + "Vietnamese,Indonesian": 0.62, + "Vietnamese,English": 0.6466666666666666, + "Chinese,Spanish": 0.5933333333333334, + "Chinese,Malay": 0.56, + "Chinese,Indonesian": 0.6266666666666667, + "Chinese,English": 0.5933333333333334, + "Spanish,Malay": 0.68, + "Spanish,Indonesian": 0.6333333333333333, + "Spanish,English": 0.76, + "Malay,Indonesian": 0.6066666666666667, + "Malay,English": 0.6133333333333333, + "Indonesian,English": 0.6533333333333333 }, "3_combine": { - "Spanish,English,Chinese": 0.35333333333333333, - "Spanish,English,Vietnamese": 0.3333333333333333, - "Spanish,English,Filipino": 0.26666666666666666, - "Spanish,English,Indonesian": 0.3, - "Spanish,English,Malay": 0.34, - "Spanish,Chinese,Vietnamese": 0.30666666666666664, - "Spanish,Chinese,Filipino": 0.26, - "Spanish,Chinese,Indonesian": 0.32, - "Spanish,Chinese,Malay": 0.3466666666666667, - "Spanish,Vietnamese,Filipino": 0.22666666666666666, - "Spanish,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Malay": 0.32, - "Spanish,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Filipino,Malay": 0.28, - "Spanish,Indonesian,Malay": 0.38, - "English,Chinese,Vietnamese": 0.4, - "English,Chinese,Filipino": 0.34, - "English,Chinese,Indonesian": 0.31333333333333335, - "English,Chinese,Malay": 0.32666666666666666, - "English,Vietnamese,Filipino": 0.30666666666666664, - "English,Vietnamese,Indonesian": 0.3, - "English,Vietnamese,Malay": 0.34, - "English,Filipino,Indonesian": 0.25333333333333335, - "English,Filipino,Malay": 0.28, - "English,Indonesian,Malay": 0.32, - "Chinese,Vietnamese,Filipino": 0.28, - "Chinese,Vietnamese,Indonesian": 0.2733333333333333, - "Chinese,Vietnamese,Malay": 0.32, - "Chinese,Filipino,Indonesian": 0.28, - "Chinese,Filipino,Malay": 0.28, - "Chinese,Indonesian,Malay": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Vietnamese,Filipino,Malay": 0.26, - "Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Filipino,Indonesian,Malay": 0.2866666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.25333333333333335, - "Spanish,English,Chinese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Indonesian": 0.22, - "Spanish,English,Chinese,Malay": 0.24, - "Spanish,English,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.21333333333333335, - "Spanish,English,Vietnamese,Malay": 0.24, - "Spanish,English,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Filipino,Malay": 0.18666666666666668, - "Spanish,English,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino": 0.18, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Malay": 0.22, - "Spanish,Chinese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Filipino,Malay": 0.2, - "Spanish,Chinese,Indonesian,Malay": 0.28, - "Spanish,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.22, - "English,Chinese,Vietnamese,Indonesian": 0.22, - "English,Chinese,Vietnamese,Malay": 0.24, - "English,Chinese,Filipino,Indonesian": 0.20666666666666667, - "English,Chinese,Filipino,Malay": 0.19333333333333333, - "English,Chinese,Indonesian,Malay": 0.24666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "English,Vietnamese,Filipino,Malay": 0.20666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.24, - "English,Filipino,Indonesian,Malay": 0.2, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Chinese,Vietnamese,Filipino,Malay": 0.18, - "Chinese,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 + "Filipino,Vietnamese,Chinese": 0.36, + "Filipino,Vietnamese,Spanish": 0.36, + "Filipino,Vietnamese,Malay": 0.32666666666666666, + "Filipino,Vietnamese,Indonesian": 0.36, + "Filipino,Vietnamese,English": 0.38666666666666666, + "Filipino,Chinese,Spanish": 0.3933333333333333, + "Filipino,Chinese,Malay": 0.36666666666666664, + "Filipino,Chinese,Indonesian": 0.38, + "Filipino,Chinese,English": 0.3933333333333333, + "Filipino,Spanish,Malay": 0.44, + "Filipino,Spanish,Indonesian": 0.42, + "Filipino,Spanish,English": 0.48, + "Filipino,Malay,Indonesian": 0.4066666666666667, + "Filipino,Malay,English": 0.4066666666666667, + "Filipino,Indonesian,English": 0.41333333333333333, + "Vietnamese,Chinese,Spanish": 0.4533333333333333, + "Vietnamese,Chinese,Malay": 0.4266666666666667, + "Vietnamese,Chinese,Indonesian": 0.49333333333333335, + "Vietnamese,Chinese,English": 0.48, + "Vietnamese,Spanish,Malay": 0.44, + "Vietnamese,Spanish,Indonesian": 0.48, + "Vietnamese,Spanish,English": 0.52, + "Vietnamese,Malay,Indonesian": 0.44, + "Vietnamese,Malay,English": 0.4533333333333333, + "Vietnamese,Indonesian,English": 0.5066666666666667, + "Chinese,Spanish,Malay": 0.4533333333333333, + "Chinese,Spanish,Indonesian": 0.4866666666666667, + "Chinese,Spanish,English": 0.5133333333333333, + "Chinese,Malay,Indonesian": 0.48, + "Chinese,Malay,English": 0.44, + "Chinese,Indonesian,English": 0.47333333333333333, + "Spanish,Malay,Indonesian": 0.5333333333333333, + "Spanish,Malay,English": 0.56, + "Spanish,Indonesian,English": 0.56, + "Malay,Indonesian,English": 0.5 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Malay": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Indonesian": 0.31333333333333335, + "Filipino,Vietnamese,Chinese,English": 0.32, + "Filipino,Vietnamese,Spanish,Malay": 0.32, + "Filipino,Vietnamese,Spanish,Indonesian": 0.32666666666666666, + "Filipino,Vietnamese,Spanish,English": 0.3466666666666667, + "Filipino,Vietnamese,Malay,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Malay,English": 0.30666666666666664, + "Filipino,Vietnamese,Indonesian,English": 0.3333333333333333, + "Filipino,Chinese,Spanish,Malay": 0.34, + "Filipino,Chinese,Spanish,Indonesian": 0.34, + "Filipino,Chinese,Spanish,English": 0.36666666666666664, + "Filipino,Chinese,Malay,Indonesian": 0.3333333333333333, + "Filipino,Chinese,Malay,English": 0.31333333333333335, + "Filipino,Chinese,Indonesian,English": 0.32666666666666666, + "Filipino,Spanish,Malay,Indonesian": 0.38666666666666666, + "Filipino,Spanish,Malay,English": 0.4, + "Filipino,Spanish,Indonesian,English": 0.38666666666666666, + "Filipino,Malay,Indonesian,English": 0.35333333333333333, + "Vietnamese,Chinese,Spanish,Malay": 0.37333333333333335, + "Vietnamese,Chinese,Spanish,Indonesian": 0.41333333333333333, + "Vietnamese,Chinese,Spanish,English": 0.41333333333333333, + "Vietnamese,Chinese,Malay,Indonesian": 0.38666666666666666, + "Vietnamese,Chinese,Malay,English": 0.38, + "Vietnamese,Chinese,Indonesian,English": 0.42, + "Vietnamese,Spanish,Malay,Indonesian": 0.4066666666666667, + "Vietnamese,Spanish,Malay,English": 0.41333333333333333, + "Vietnamese,Spanish,Indonesian,English": 0.44666666666666666, + "Vietnamese,Malay,Indonesian,English": 0.4, + "Chinese,Spanish,Malay,Indonesian": 0.4266666666666667, + "Chinese,Spanish,Malay,English": 0.4066666666666667, + "Chinese,Spanish,Indonesian,English": 0.43333333333333335, + "Chinese,Malay,Indonesian,English": 0.4, + "Spanish,Malay,Indonesian,English": 0.47333333333333333 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.14, - "Spanish,English,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,English,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "English,Chinese,Filipino,Indonesian,Malay": 0.16, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.28, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,Malay,English": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.2866666666666667, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.3, + "Filipino,Vietnamese,Spanish,Malay,English": 0.30666666666666664, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.31333333333333335, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.2866666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.31333333333333335, + "Filipino,Chinese,Spanish,Malay,English": 0.31333333333333335, + "Filipino,Chinese,Spanish,Indonesian,English": 0.31333333333333335, + "Filipino,Chinese,Malay,Indonesian,English": 0.2866666666666667, + "Filipino,Spanish,Malay,Indonesian,English": 0.35333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.36, + "Vietnamese,Chinese,Spanish,Malay,English": 0.3466666666666667, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.38, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.35333333333333333, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.38, + "Chinese,Spanish,Malay,Indonesian,English": 0.38 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.12, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.2866666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.2866666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3333333333333333 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.25333333333333335 } }, - "AC3_2": 0.41332263237499334, - "AC3_3": 0.32854674122747657, - "AC3_4": 0.26258732830811815, - "AC3_5": 0.20952979177476555, - "AC3_6": 0.16652059593269275, - "AC3_7": 0.130718954218463 + "AC3_2": 0.5975093909574081, + "AC3_3": 0.5132874229305044, + "AC3_4": 0.45842957663116474, + "AC3_5": 0.41685802676819866, + "AC3_6": 0.3838576031483604, + "AC3_7": 0.3572505542821383 }, "prompt_3": { - "overall_acc": 0.3676190476190476, + "overall_acc": 0.5714285714285714, "language_acc": { - "Spanish": 0.36666666666666664, - "English": 0.4666666666666667, - "Chinese": 0.4, - "Vietnamese": 0.34, - "Filipino": 0.28, - "Indonesian": 0.36, - "Malay": 0.36 + "Filipino": 0.5133333333333333, + "Vietnamese": 0.5133333333333333, + "Chinese": 0.5466666666666666, + "Spanish": 0.62, + "Malay": 0.5266666666666666, + "Indonesian": 0.58, + "English": 0.7 }, - "consistency_score_2": 0.5025396825396827, - "consistency_score_3": 0.31885714285714284, - "consistency_score_4": 0.22266666666666665, - "consistency_score_5": 0.16317460317460317, - "consistency_score_6": 0.12285714285714285, - "consistency_score_7": 0.09333333333333334, + "consistency_score_2": 0.559047619047619, + "consistency_score_3": 0.4116190476190476, + "consistency_score_4": 0.33638095238095234, + "consistency_score_5": 0.28888888888888886, + "consistency_score_6": 0.2561904761904762, + "consistency_score_7": 0.23333333333333334, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6, - "Spanish,Chinese": 0.5466666666666666, - "Spanish,Vietnamese": 0.4266666666666667, - "Spanish,Filipino": 0.43333333333333335, - "Spanish,Indonesian": 0.4866666666666667, - "Spanish,Malay": 0.5666666666666667, - "English,Chinese": 0.5866666666666667, - "English,Vietnamese": 0.5466666666666666, - "English,Filipino": 0.46, - "English,Indonesian": 0.47333333333333333, - "English,Malay": 0.5066666666666667, - "Chinese,Vietnamese": 0.5133333333333333, - "Chinese,Filipino": 0.47333333333333333, - "Chinese,Indonesian": 0.5533333333333333, - "Chinese,Malay": 0.52, - "Vietnamese,Filipino": 0.43333333333333335, - "Vietnamese,Indonesian": 0.46, - "Vietnamese,Malay": 0.5133333333333333, - "Filipino,Indonesian": 0.41333333333333333, - "Filipino,Malay": 0.42, - "Indonesian,Malay": 0.62 + "Filipino,Vietnamese": 0.4666666666666667, + "Filipino,Chinese": 0.47333333333333333, + "Filipino,Spanish": 0.5466666666666666, + "Filipino,Malay": 0.52, + "Filipino,Indonesian": 0.56, + "Filipino,English": 0.5733333333333334, + "Vietnamese,Chinese": 0.58, + "Vietnamese,Spanish": 0.5533333333333333, + "Vietnamese,Malay": 0.58, + "Vietnamese,Indonesian": 0.5266666666666666, + "Vietnamese,English": 0.6, + "Chinese,Spanish": 0.5866666666666667, + "Chinese,Malay": 0.5266666666666666, + "Chinese,Indonesian": 0.5666666666666667, + "Chinese,English": 0.5733333333333334, + "Spanish,Malay": 0.5733333333333334, + "Spanish,Indonesian": 0.5466666666666666, + "Spanish,English": 0.6733333333333333, + "Malay,Indonesian": 0.5466666666666666, + "Malay,English": 0.5333333333333333, + "Indonesian,English": 0.6333333333333333 }, "3_combine": { - "Spanish,English,Chinese": 0.41333333333333333, - "Spanish,English,Vietnamese": 0.35333333333333333, - "Spanish,English,Filipino": 0.30666666666666664, - "Spanish,English,Indonesian": 0.34, - "Spanish,English,Malay": 0.3933333333333333, - "Spanish,Chinese,Vietnamese": 0.30666666666666664, - "Spanish,Chinese,Filipino": 0.2866666666666667, - "Spanish,Chinese,Indonesian": 0.3466666666666667, - "Spanish,Chinese,Malay": 0.36666666666666664, - "Spanish,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,Vietnamese,Indonesian": 0.28, - "Spanish,Vietnamese,Malay": 0.32666666666666666, - "Spanish,Filipino,Indonesian": 0.24666666666666667, - "Spanish,Filipino,Malay": 0.3, - "Spanish,Indonesian,Malay": 0.38666666666666666, - "English,Chinese,Vietnamese": 0.38, - "English,Chinese,Filipino": 0.32, - "English,Chinese,Indonesian": 0.3466666666666667, - "English,Chinese,Malay": 0.3466666666666667, - "English,Vietnamese,Filipino": 0.28, - "English,Vietnamese,Indonesian": 0.30666666666666664, - "English,Vietnamese,Malay": 0.3466666666666667, - "English,Filipino,Indonesian": 0.26, - "English,Filipino,Malay": 0.2866666666666667, - "English,Indonesian,Malay": 0.36, - "Chinese,Vietnamese,Filipino": 0.2733333333333333, - "Chinese,Vietnamese,Indonesian": 0.32, - "Chinese,Vietnamese,Malay": 0.3333333333333333, - "Chinese,Filipino,Indonesian": 0.29333333333333333, - "Chinese,Filipino,Malay": 0.2866666666666667, - "Chinese,Indonesian,Malay": 0.3933333333333333, - "Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Vietnamese,Filipino,Malay": 0.24666666666666667, - "Vietnamese,Indonesian,Malay": 0.36, - "Filipino,Indonesian,Malay": 0.2866666666666667 + "Filipino,Vietnamese,Chinese": 0.35333333333333333, + "Filipino,Vietnamese,Spanish": 0.36666666666666664, + "Filipino,Vietnamese,Malay": 0.38, + "Filipino,Vietnamese,Indonesian": 0.36666666666666664, + "Filipino,Vietnamese,English": 0.38, + "Filipino,Chinese,Spanish": 0.38, + "Filipino,Chinese,Malay": 0.3466666666666667, + "Filipino,Chinese,Indonesian": 0.38, + "Filipino,Chinese,English": 0.3933333333333333, + "Filipino,Spanish,Malay": 0.4066666666666667, + "Filipino,Spanish,Indonesian": 0.4, + "Filipino,Spanish,English": 0.46, + "Filipino,Malay,Indonesian": 0.4, + "Filipino,Malay,English": 0.3933333333333333, + "Filipino,Indonesian,English": 0.43333333333333335, + "Vietnamese,Chinese,Spanish": 0.42, + "Vietnamese,Chinese,Malay": 0.42, + "Vietnamese,Chinese,Indonesian": 0.41333333333333333, + "Vietnamese,Chinese,English": 0.4533333333333333, + "Vietnamese,Spanish,Malay": 0.43333333333333335, + "Vietnamese,Spanish,Indonesian": 0.3933333333333333, + "Vietnamese,Spanish,English": 0.4533333333333333, + "Vietnamese,Malay,Indonesian": 0.3933333333333333, + "Vietnamese,Malay,English": 0.43333333333333335, + "Vietnamese,Indonesian,English": 0.44666666666666666, + "Chinese,Spanish,Malay": 0.4066666666666667, + "Chinese,Spanish,Indonesian": 0.4066666666666667, + "Chinese,Spanish,English": 0.47333333333333333, + "Chinese,Malay,Indonesian": 0.37333333333333335, + "Chinese,Malay,English": 0.4, + "Chinese,Indonesian,English": 0.4533333333333333, + "Spanish,Malay,Indonesian": 0.41333333333333333, + "Spanish,Malay,English": 0.46, + "Spanish,Indonesian,English": 0.4866666666666667, + "Malay,Indonesian,English": 0.43333333333333335 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.25333333333333335, - "Spanish,English,Chinese,Filipino": 0.23333333333333334, - "Spanish,English,Chinese,Indonesian": 0.26666666666666666, - "Spanish,English,Chinese,Malay": 0.28, - "Spanish,English,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Malay": 0.28, - "Spanish,English,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.22, - "Spanish,Chinese,Vietnamese,Malay": 0.23333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.24, - "English,Chinese,Vietnamese,Malay": 0.24666666666666667, - "English,Chinese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Filipino,Malay": 0.21333333333333335, - "English,Chinese,Indonesian,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.22, - "Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 + "Filipino,Vietnamese,Chinese,Spanish": 0.29333333333333333, + "Filipino,Vietnamese,Chinese,Malay": 0.3, + "Filipino,Vietnamese,Chinese,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,English": 0.31333333333333335, + "Filipino,Vietnamese,Spanish,Malay": 0.3333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Spanish,English": 0.32, + "Filipino,Vietnamese,Malay,Indonesian": 0.3, + "Filipino,Vietnamese,Malay,English": 0.31333333333333335, + "Filipino,Vietnamese,Indonesian,English": 0.32, + "Filipino,Chinese,Spanish,Malay": 0.30666666666666664, + "Filipino,Chinese,Spanish,Indonesian": 0.31333333333333335, + "Filipino,Chinese,Spanish,English": 0.3466666666666667, + "Filipino,Chinese,Malay,Indonesian": 0.2866666666666667, + "Filipino,Chinese,Malay,English": 0.3, + "Filipino,Chinese,Indonesian,English": 0.3333333333333333, + "Filipino,Spanish,Malay,Indonesian": 0.34, + "Filipino,Spanish,Malay,English": 0.36666666666666664, + "Filipino,Spanish,Indonesian,English": 0.37333333333333335, + "Filipino,Malay,Indonesian,English": 0.34, + "Vietnamese,Chinese,Spanish,Malay": 0.3466666666666667, + "Vietnamese,Chinese,Spanish,Indonesian": 0.34, + "Vietnamese,Chinese,Spanish,English": 0.38, + "Vietnamese,Chinese,Malay,Indonesian": 0.32, + "Vietnamese,Chinese,Malay,English": 0.36, + "Vietnamese,Chinese,Indonesian,English": 0.38, + "Vietnamese,Spanish,Malay,Indonesian": 0.34, + "Vietnamese,Spanish,Malay,English": 0.38666666666666666, + "Vietnamese,Spanish,Indonesian,English": 0.37333333333333335, + "Vietnamese,Malay,Indonesian,English": 0.34, + "Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, + "Chinese,Spanish,Malay,English": 0.36, + "Chinese,Spanish,Indonesian,English": 0.38, + "Chinese,Malay,Indonesian,English": 0.3333333333333333, + "Spanish,Malay,Indonesian,English": 0.3933333333333333 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,English,Chinese,Vietnamese,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "English,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.2, - "English,Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Malay,English": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.28, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.28, + "Filipino,Vietnamese,Spanish,Malay,English": 0.3, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2866666666666667, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.26666666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, + "Filipino,Chinese,Spanish,Malay,English": 0.2866666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.3, + "Filipino,Chinese,Malay,Indonesian,English": 0.26, + "Filipino,Spanish,Malay,Indonesian,English": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, + "Vietnamese,Chinese,Spanish,Malay,English": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.32666666666666666, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.3, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.32666666666666666, + "Chinese,Spanish,Malay,Indonesian,English": 0.31333333333333335 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.23333333333333334, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.26666666666666666, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2866666666666667 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334 } }, - "AC3_2": 0.4246194471463516, - "AC3_3": 0.34150626283276375, - "AC3_4": 0.2773453801831968, - "AC3_5": 0.22602415124472686, - "AC3_6": 0.18416643546869282, - "AC3_7": 0.14887052338368167 + "AC3_2": 0.5651702972180286, + "AC3_3": 0.47853406034936163, + "AC3_4": 0.4234758107560615, + "AC3_5": 0.3837638375937691, + "AC3_6": 0.35377280942626615, + "AC3_7": 0.3313609467043871 }, "prompt_4": { - "overall_acc": 0.3685714285714286, + "overall_acc": 0.6038095238095238, "language_acc": { - "Spanish": 0.42, - "English": 0.44, - "Chinese": 0.36666666666666664, - "Vietnamese": 0.3933333333333333, - "Filipino": 0.29333333333333333, - "Indonesian": 0.29333333333333333, - "Malay": 0.37333333333333335 - }, - "consistency_score_2": 0.5387301587301587, - "consistency_score_3": 0.3641904761904763, - "consistency_score_4": 0.2718095238095239, - "consistency_score_5": 0.21492063492063493, - "consistency_score_6": 0.17619047619047618, - "consistency_score_7": 0.14666666666666667, + "Filipino": 0.54, + "Vietnamese": 0.56, + "Chinese": 0.5866666666666667, + "Spanish": 0.62, + "Malay": 0.5866666666666667, + "Indonesian": 0.6066666666666667, + "English": 0.7266666666666667 + }, + "consistency_score_2": 0.6155555555555553, + "consistency_score_3": 0.47142857142857153, + "consistency_score_4": 0.3900952380952382, + "consistency_score_5": 0.33492063492063495, + "consistency_score_6": 0.29333333333333333, + "consistency_score_7": 0.26, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6466666666666666, - "Spanish,Chinese": 0.5733333333333334, - "Spanish,Vietnamese": 0.5533333333333333, - "Spanish,Filipino": 0.52, - "Spanish,Indonesian": 0.56, - "Spanish,Malay": 0.58, - "English,Chinese": 0.6, - "English,Vietnamese": 0.56, - "English,Filipino": 0.5, - "English,Indonesian": 0.5333333333333333, - "English,Malay": 0.5533333333333333, - "Chinese,Vietnamese": 0.5333333333333333, - "Chinese,Filipino": 0.49333333333333335, - "Chinese,Indonesian": 0.5266666666666666, - "Chinese,Malay": 0.56, - "Vietnamese,Filipino": 0.44666666666666666, - "Vietnamese,Indonesian": 0.5266666666666666, - "Vietnamese,Malay": 0.58, - "Filipino,Indonesian": 0.42, - "Filipino,Malay": 0.41333333333333333, - "Indonesian,Malay": 0.6333333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.44666666666666666, - "Spanish,English,Vietnamese": 0.44666666666666666, - "Spanish,English,Filipino": 0.37333333333333335, - "Spanish,English,Indonesian": 0.4266666666666667, - "Spanish,English,Malay": 0.44, - "Spanish,Chinese,Vietnamese": 0.38, - "Spanish,Chinese,Filipino": 0.3333333333333333, - "Spanish,Chinese,Indonesian": 0.3933333333333333, - "Spanish,Chinese,Malay": 0.41333333333333333, - "Spanish,Vietnamese,Filipino": 0.31333333333333335, - "Spanish,Vietnamese,Indonesian": 0.38666666666666666, - "Spanish,Vietnamese,Malay": 0.4066666666666667, - "Spanish,Filipino,Indonesian": 0.32, - "Spanish,Filipino,Malay": 0.31333333333333335, - "Spanish,Indonesian,Malay": 0.4266666666666667, - "English,Chinese,Vietnamese": 0.4, - "English,Chinese,Filipino": 0.35333333333333333, - "English,Chinese,Indonesian": 0.3933333333333333, - "English,Chinese,Malay": 0.4, - "English,Vietnamese,Filipino": 0.30666666666666664, - "English,Vietnamese,Indonesian": 0.3933333333333333, - "English,Vietnamese,Malay": 0.3933333333333333, - "English,Filipino,Indonesian": 0.31333333333333335, - "English,Filipino,Malay": 0.29333333333333333, - "English,Indonesian,Malay": 0.4, - "Chinese,Vietnamese,Filipino": 0.2866666666666667, - "Chinese,Vietnamese,Indonesian": 0.35333333333333333, - "Chinese,Vietnamese,Malay": 0.38, - "Chinese,Filipino,Indonesian": 0.3, - "Chinese,Filipino,Malay": 0.2866666666666667, - "Chinese,Indonesian,Malay": 0.4066666666666667, - "Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Vietnamese,Filipino,Malay": 0.2866666666666667, - "Vietnamese,Indonesian,Malay": 0.41333333333333333, - "Filipino,Indonesian,Malay": 0.29333333333333333 + "Filipino,Vietnamese": 0.5333333333333333, + "Filipino,Chinese": 0.5333333333333333, + "Filipino,Spanish": 0.5666666666666667, + "Filipino,Malay": 0.49333333333333335, + "Filipino,Indonesian": 0.5533333333333333, + "Filipino,English": 0.5933333333333334, + "Vietnamese,Chinese": 0.6266666666666667, + "Vietnamese,Spanish": 0.66, + "Vietnamese,Malay": 0.56, + "Vietnamese,Indonesian": 0.64, + "Vietnamese,English": 0.6466666666666666, + "Chinese,Spanish": 0.64, + "Chinese,Malay": 0.6066666666666667, + "Chinese,Indonesian": 0.7, + "Chinese,English": 0.6466666666666666, + "Spanish,Malay": 0.6133333333333333, + "Spanish,Indonesian": 0.6733333333333333, + "Spanish,English": 0.7, + "Malay,Indonesian": 0.6533333333333333, + "Malay,English": 0.5866666666666667, + "Indonesian,English": 0.7 + }, + "3_combine": { + "Filipino,Vietnamese,Chinese": 0.4066666666666667, + "Filipino,Vietnamese,Spanish": 0.4266666666666667, + "Filipino,Vietnamese,Malay": 0.36666666666666664, + "Filipino,Vietnamese,Indonesian": 0.42, + "Filipino,Vietnamese,English": 0.4266666666666667, + "Filipino,Chinese,Spanish": 0.41333333333333333, + "Filipino,Chinese,Malay": 0.38666666666666666, + "Filipino,Chinese,Indonesian": 0.4533333333333333, + "Filipino,Chinese,English": 0.44, + "Filipino,Spanish,Malay": 0.3933333333333333, + "Filipino,Spanish,Indonesian": 0.44, + "Filipino,Spanish,English": 0.4666666666666667, + "Filipino,Malay,Indonesian": 0.4266666666666667, + "Filipino,Malay,English": 0.3933333333333333, + "Filipino,Indonesian,English": 0.46, + "Vietnamese,Chinese,Spanish": 0.5, + "Vietnamese,Chinese,Malay": 0.44, + "Vietnamese,Chinese,Indonesian": 0.5333333333333333, + "Vietnamese,Chinese,English": 0.5066666666666667, + "Vietnamese,Spanish,Malay": 0.46, + "Vietnamese,Spanish,Indonesian": 0.5333333333333333, + "Vietnamese,Spanish,English": 0.5333333333333333, + "Vietnamese,Malay,Indonesian": 0.47333333333333333, + "Vietnamese,Malay,English": 0.44666666666666666, + "Vietnamese,Indonesian,English": 0.5266666666666666, + "Chinese,Spanish,Malay": 0.4866666666666667, + "Chinese,Spanish,Indonesian": 0.5533333333333333, + "Chinese,Spanish,English": 0.5466666666666666, + "Chinese,Malay,Indonesian": 0.5266666666666666, + "Chinese,Malay,English": 0.4533333333333333, + "Chinese,Indonesian,English": 0.54, + "Spanish,Malay,Indonesian": 0.52, + "Spanish,Malay,English": 0.5133333333333333, + "Spanish,Indonesian,English": 0.5733333333333334, + "Malay,Indonesian,English": 0.5133333333333333 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.3333333333333333, - "Spanish,English,Chinese,Filipino": 0.2733333333333333, - "Spanish,English,Chinese,Indonesian": 0.32, - "Spanish,English,Chinese,Malay": 0.3466666666666667, - "Spanish,English,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,English,Vietnamese,Malay": 0.34, - "Spanish,English,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Indonesian,Malay": 0.34, - "Spanish,Chinese,Vietnamese,Filipino": 0.22, - "Spanish,Chinese,Vietnamese,Indonesian": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.32, - "Spanish,Chinese,Filipino,Indonesian": 0.24, - "Spanish,Chinese,Filipino,Malay": 0.23333333333333334, - "Spanish,Chinese,Indonesian,Malay": 0.32666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.31333333333333335, - "English,Chinese,Vietnamese,Malay": 0.30666666666666664, - "English,Chinese,Filipino,Indonesian": 0.24666666666666667, - "English,Chinese,Filipino,Malay": 0.23333333333333334, - "English,Chinese,Indonesian,Malay": 0.32, - "English,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "English,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.22 + "Filipino,Vietnamese,Chinese,Spanish": 0.34, + "Filipino,Vietnamese,Chinese,Malay": 0.32, + "Filipino,Vietnamese,Chinese,Indonesian": 0.37333333333333335, + "Filipino,Vietnamese,Chinese,English": 0.36, + "Filipino,Vietnamese,Spanish,Malay": 0.32, + "Filipino,Vietnamese,Spanish,Indonesian": 0.36666666666666664, + "Filipino,Vietnamese,Spanish,English": 0.36666666666666664, + "Filipino,Vietnamese,Malay,Indonesian": 0.3333333333333333, + "Filipino,Vietnamese,Malay,English": 0.32, + "Filipino,Vietnamese,Indonesian,English": 0.37333333333333335, + "Filipino,Chinese,Spanish,Malay": 0.3333333333333333, + "Filipino,Chinese,Spanish,Indonesian": 0.38, + "Filipino,Chinese,Spanish,English": 0.36666666666666664, + "Filipino,Chinese,Malay,Indonesian": 0.37333333333333335, + "Filipino,Chinese,Malay,English": 0.32, + "Filipino,Chinese,Indonesian,English": 0.38666666666666666, + "Filipino,Spanish,Malay,Indonesian": 0.37333333333333335, + "Filipino,Spanish,Malay,English": 0.3466666666666667, + "Filipino,Spanish,Indonesian,English": 0.3933333333333333, + "Filipino,Malay,Indonesian,English": 0.36, + "Vietnamese,Chinese,Spanish,Malay": 0.3933333333333333, + "Vietnamese,Chinese,Spanish,Indonesian": 0.4666666666666667, + "Vietnamese,Chinese,Spanish,English": 0.44666666666666666, + "Vietnamese,Chinese,Malay,Indonesian": 0.41333333333333333, + "Vietnamese,Chinese,Malay,English": 0.38, + "Vietnamese,Chinese,Indonesian,English": 0.46, + "Vietnamese,Spanish,Malay,Indonesian": 0.4266666666666667, + "Vietnamese,Spanish,Malay,English": 0.42, + "Vietnamese,Spanish,Indonesian,English": 0.48, + "Vietnamese,Malay,Indonesian,English": 0.41333333333333333, + "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, + "Chinese,Spanish,Malay,English": 0.41333333333333333, + "Chinese,Spanish,Indonesian,English": 0.4866666666666667, + "Chinese,Malay,Indonesian,English": 0.43333333333333335, + "Spanish,Malay,Indonesian,English": 0.4666666666666667 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.28, - "Spanish,English,Chinese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.28, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.26, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.26, - "English,Chinese,Filipino,Indonesian,Malay": 0.2, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666 + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.2866666666666667, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.32666666666666666, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.31333333333333335, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Malay,English": 0.28, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.34, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.30666666666666664, + "Filipino,Vietnamese,Spanish,Malay,English": 0.29333333333333333, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.34, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.3, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, + "Filipino,Chinese,Spanish,Malay,English": 0.2866666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.34, + "Filipino,Chinese,Malay,Indonesian,English": 0.31333333333333335, + "Filipino,Spanish,Malay,Indonesian,English": 0.3333333333333333, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.38, + "Vietnamese,Chinese,Spanish,Malay,English": 0.36, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.4266666666666667, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.37333333333333335, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.4, + "Chinese,Spanish,Malay,Indonesian,English": 0.4 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.24, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.28, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.26, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.30666666666666664, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.2733333333333333, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.2866666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.2866666666666667, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.36 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.26 } }, - "AC3_2": 0.4376946915442809, - "AC3_3": 0.36636785616449175, - "AC3_4": 0.31288008833390696, - "AC3_5": 0.27151562252458483, - "AC3_6": 0.23841158836782403, - "AC3_7": 0.20983364136408034 + "AC3_2": 0.609625965410388, + "AC3_3": 0.5294698215374972, + "AC3_4": 0.4739754329771891, + "AC3_5": 0.43085495268933915, + "AC3_6": 0.3948478414280336, + "AC3_7": 0.3634840131883508 }, "prompt_5": { - "overall_acc": 0.36, + "overall_acc": 0.5923809523809523, "language_acc": { - "Spanish": 0.38, - "English": 0.48, - "Chinese": 0.38, - "Vietnamese": 0.36666666666666664, - "Filipino": 0.2866666666666667, - "Indonesian": 0.3333333333333333, - "Malay": 0.29333333333333333 - }, - "consistency_score_2": 0.4961904761904762, - "consistency_score_3": 0.3064761904761905, - "consistency_score_4": 0.21104761904761904, - "consistency_score_5": 0.15682539682539684, - "consistency_score_6": 0.12285714285714286, - "consistency_score_7": 0.1, + "Filipino": 0.56, + "Vietnamese": 0.5866666666666667, + "Chinese": 0.6066666666666667, + "Spanish": 0.56, + "Malay": 0.5266666666666666, + "Indonesian": 0.6066666666666667, + "English": 0.7 + }, + "consistency_score_2": 0.5876190476190475, + "consistency_score_3": 0.4432380952380953, + "consistency_score_4": 0.3651428571428571, + "consistency_score_5": 0.31238095238095237, + "consistency_score_6": 0.27238095238095245, + "consistency_score_7": 0.24, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.5666666666666667, - "Spanish,Chinese": 0.4666666666666667, - "Spanish,Vietnamese": 0.5333333333333333, - "Spanish,Filipino": 0.47333333333333333, - "Spanish,Indonesian": 0.54, - "Spanish,Malay": 0.49333333333333335, - "English,Chinese": 0.5466666666666666, - "English,Vietnamese": 0.5333333333333333, - "English,Filipino": 0.44666666666666666, - "English,Indonesian": 0.47333333333333333, - "English,Malay": 0.48, - "Chinese,Vietnamese": 0.4866666666666667, - "Chinese,Filipino": 0.4866666666666667, - "Chinese,Indonesian": 0.48, - "Chinese,Malay": 0.48, - "Vietnamese,Filipino": 0.43333333333333335, - "Vietnamese,Indonesian": 0.5, - "Vietnamese,Malay": 0.5733333333333334, - "Filipino,Indonesian": 0.41333333333333333, - "Filipino,Malay": 0.4266666666666667, - "Indonesian,Malay": 0.5866666666666667 + "Filipino,Vietnamese": 0.56, + "Filipino,Chinese": 0.5133333333333333, + "Filipino,Spanish": 0.54, + "Filipino,Malay": 0.49333333333333335, + "Filipino,Indonesian": 0.54, + "Filipino,English": 0.6066666666666667, + "Vietnamese,Chinese": 0.6333333333333333, + "Vietnamese,Spanish": 0.6466666666666666, + "Vietnamese,Malay": 0.5466666666666666, + "Vietnamese,Indonesian": 0.6133333333333333, + "Vietnamese,English": 0.6333333333333333, + "Chinese,Spanish": 0.5933333333333334, + "Chinese,Malay": 0.5266666666666666, + "Chinese,Indonesian": 0.6133333333333333, + "Chinese,English": 0.6133333333333333, + "Spanish,Malay": 0.6, + "Spanish,Indonesian": 0.62, + "Spanish,English": 0.66, + "Malay,Indonesian": 0.54, + "Malay,English": 0.5533333333333333, + "Indonesian,English": 0.6933333333333334 }, "3_combine": { - "Spanish,English,Chinese": 0.34, - "Spanish,English,Vietnamese": 0.38, - "Spanish,English,Filipino": 0.29333333333333333, - "Spanish,English,Indonesian": 0.36, - "Spanish,English,Malay": 0.3333333333333333, - "Spanish,Chinese,Vietnamese": 0.30666666666666664, - "Spanish,Chinese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Malay": 0.2866666666666667, - "Spanish,Vietnamese,Filipino": 0.28, - "Spanish,Vietnamese,Indonesian": 0.34, - "Spanish,Vietnamese,Malay": 0.34, - "Spanish,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Filipino,Malay": 0.26, - "Spanish,Indonesian,Malay": 0.36, - "English,Chinese,Vietnamese": 0.36, - "English,Chinese,Filipino": 0.29333333333333333, - "English,Chinese,Indonesian": 0.3333333333333333, - "English,Chinese,Malay": 0.30666666666666664, - "English,Vietnamese,Filipino": 0.26666666666666666, - "English,Vietnamese,Indonesian": 0.3333333333333333, - "English,Vietnamese,Malay": 0.34, - "English,Filipino,Indonesian": 0.26, - "English,Filipino,Malay": 0.25333333333333335, - "English,Indonesian,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Chinese,Vietnamese,Indonesian": 0.30666666666666664, - "Chinese,Vietnamese,Malay": 0.30666666666666664, - "Chinese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Filipino,Malay": 0.26666666666666666, - "Chinese,Indonesian,Malay": 0.3333333333333333, - "Vietnamese,Filipino,Indonesian": 0.22, - "Vietnamese,Filipino,Malay": 0.2733333333333333, - "Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Filipino,Indonesian,Malay": 0.2866666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.24666666666666667, - "Spanish,English,Chinese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,English,Vietnamese,Indonesian": 0.26, - "Spanish,English,Vietnamese,Malay": 0.24666666666666667, - "Spanish,English,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Filipino,Malay": 0.18, - "Spanish,English,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.21333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.2, - "Spanish,Chinese,Filipino,Malay": 0.18, - "Spanish,Chinese,Indonesian,Malay": 0.24666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.26, - "Spanish,Filipino,Indonesian,Malay": 0.22, - "English,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "English,Chinese,Vietnamese,Malay": 0.23333333333333334, - "English,Chinese,Filipino,Indonesian": 0.19333333333333333, - "English,Chinese,Filipino,Malay": 0.19333333333333333, - "English,Chinese,Indonesian,Malay": 0.23333333333333334, - "English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Filipino,Malay": 0.14, - "Spanish,English,Chinese,Indonesian,Malay": 0.2, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.14, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 + "Filipino,Vietnamese,Chinese": 0.41333333333333333, + "Filipino,Vietnamese,Spanish": 0.4266666666666667, + "Filipino,Vietnamese,Malay": 0.38, + "Filipino,Vietnamese,Indonesian": 0.4266666666666667, + "Filipino,Vietnamese,English": 0.44666666666666666, + "Filipino,Chinese,Spanish": 0.41333333333333333, + "Filipino,Chinese,Malay": 0.36, + "Filipino,Chinese,Indonesian": 0.41333333333333333, + "Filipino,Chinese,English": 0.42, + "Filipino,Spanish,Malay": 0.3933333333333333, + "Filipino,Spanish,Indonesian": 0.44, + "Filipino,Spanish,English": 0.4666666666666667, + "Filipino,Malay,Indonesian": 0.38, + "Filipino,Malay,English": 0.38, + "Filipino,Indonesian,English": 0.46, + "Vietnamese,Chinese,Spanish": 0.4866666666666667, + "Vietnamese,Chinese,Malay": 0.42, + "Vietnamese,Chinese,Indonesian": 0.48, + "Vietnamese,Chinese,English": 0.49333333333333335, + "Vietnamese,Spanish,Malay": 0.44666666666666666, + "Vietnamese,Spanish,Indonesian": 0.47333333333333333, + "Vietnamese,Spanish,English": 0.52, + "Vietnamese,Malay,Indonesian": 0.41333333333333333, + "Vietnamese,Malay,English": 0.4266666666666667, + "Vietnamese,Indonesian,English": 0.5, + "Chinese,Spanish,Malay": 0.43333333333333335, + "Chinese,Spanish,Indonesian": 0.47333333333333333, + "Chinese,Spanish,English": 0.49333333333333335, + "Chinese,Malay,Indonesian": 0.41333333333333333, + "Chinese,Malay,English": 0.4266666666666667, + "Chinese,Indonesian,English": 0.5, + "Spanish,Malay,Indonesian": 0.44, + "Spanish,Malay,English": 0.48, + "Spanish,Indonesian,English": 0.5266666666666666, + "Malay,Indonesian,English": 0.44666666666666666 + }, + "4_combine": { + "Filipino,Vietnamese,Chinese,Spanish": 0.35333333333333333, + "Filipino,Vietnamese,Chinese,Malay": 0.3, + "Filipino,Vietnamese,Chinese,Indonesian": 0.36666666666666664, + "Filipino,Vietnamese,Chinese,English": 0.36666666666666664, + "Filipino,Vietnamese,Spanish,Malay": 0.32666666666666666, + "Filipino,Vietnamese,Spanish,Indonesian": 0.36, + "Filipino,Vietnamese,Spanish,English": 0.38666666666666666, + "Filipino,Vietnamese,Malay,Indonesian": 0.32, + "Filipino,Vietnamese,Malay,English": 0.32, + "Filipino,Vietnamese,Indonesian,English": 0.38, + "Filipino,Chinese,Spanish,Malay": 0.32, + "Filipino,Chinese,Spanish,Indonesian": 0.36666666666666664, + "Filipino,Chinese,Spanish,English": 0.36666666666666664, + "Filipino,Chinese,Malay,Indonesian": 0.31333333333333335, + "Filipino,Chinese,Malay,English": 0.30666666666666664, + "Filipino,Chinese,Indonesian,English": 0.37333333333333335, + "Filipino,Spanish,Malay,Indonesian": 0.3466666666666667, + "Filipino,Spanish,Malay,English": 0.35333333333333333, + "Filipino,Spanish,Indonesian,English": 0.3933333333333333, + "Filipino,Malay,Indonesian,English": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Malay": 0.36666666666666664, + "Vietnamese,Chinese,Spanish,Indonesian": 0.4, + "Vietnamese,Chinese,Spanish,English": 0.4266666666666667, + "Vietnamese,Chinese,Malay,Indonesian": 0.36, + "Vietnamese,Chinese,Malay,English": 0.36, + "Vietnamese,Chinese,Indonesian,English": 0.4266666666666667, + "Vietnamese,Spanish,Malay,Indonesian": 0.36666666666666664, + "Vietnamese,Spanish,Malay,English": 0.4, + "Vietnamese,Spanish,Indonesian,English": 0.4266666666666667, + "Vietnamese,Malay,Indonesian,English": 0.36666666666666664, + "Chinese,Spanish,Malay,Indonesian": 0.36666666666666664, + "Chinese,Spanish,Malay,English": 0.38, + "Chinese,Spanish,Indonesian,English": 0.42, + "Chinese,Malay,Indonesian,English": 0.36666666666666664, + "Spanish,Malay,Indonesian,English": 0.4 + }, + "5_combine": { + "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.2733333333333333, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.32, + "Filipino,Vietnamese,Chinese,Spanish,English": 0.32666666666666666, + "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.28, + "Filipino,Vietnamese,Chinese,Malay,English": 0.26666666666666666, + "Filipino,Vietnamese,Chinese,Indonesian,English": 0.34, + "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.29333333333333333, + "Filipino,Vietnamese,Spanish,Malay,English": 0.30666666666666664, + "Filipino,Vietnamese,Spanish,Indonesian,English": 0.3333333333333333, + "Filipino,Vietnamese,Malay,Indonesian,English": 0.2866666666666667, + "Filipino,Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, + "Filipino,Chinese,Spanish,Malay,English": 0.2866666666666667, + "Filipino,Chinese,Spanish,Indonesian,English": 0.3333333333333333, + "Filipino,Chinese,Malay,Indonesian,English": 0.28, + "Filipino,Spanish,Malay,Indonesian,English": 0.31333333333333335, + "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, + "Vietnamese,Chinese,Spanish,Malay,English": 0.3333333333333333, + "Vietnamese,Chinese,Spanish,Indonesian,English": 0.36666666666666664, + "Vietnamese,Chinese,Malay,Indonesian,English": 0.32666666666666666, + "Vietnamese,Spanish,Malay,Indonesian,English": 0.34, + "Chinese,Spanish,Malay,Indonesian,English": 0.3333333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26, + "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.25333333333333335, + "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.3, + "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.25333333333333335, + "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.2733333333333333, + "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666, + "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.3 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 + "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.24 } }, - "AC3_2": 0.4172636262026555, - "AC3_3": 0.33108888248819046, - "AC3_4": 0.2660973982189122, - "AC3_5": 0.21847665843438566, - "AC3_6": 0.1831952662342495, - "AC3_7": 0.15652173909640832 + "AC3_2": 0.5899903915868815, + "AC3_3": 0.5070702505268974, + "AC3_4": 0.4517980050112149, + "AC3_5": 0.4090546365462674, + "AC3_6": 0.37317390387964927, + "AC3_7": 0.34160183062257643 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.34334415584415584, + "overall_acc": 0.4926948051948052, "language_acc": { - "English": 0.39204545454545453, - "Filipino": 0.2897727272727273, - "Spanish": 0.3409090909090909, - "Malay": 0.3977272727272727, - "Chinese": 0.38636363636363635, - "Indonesian": 0.3125, - "Vietnamese": 0.2840909090909091 + "Indonesian": 0.4772727272727273, + "English": 0.5397727272727273, + "Filipino": 0.4602272727272727, + "Spanish": 0.48295454545454547, + "Chinese": 0.48863636363636365, + "Malay": 0.4943181818181818, + "Vietnamese": 0.5056818181818182 }, - "consistency_score_2": 0.5297619047619049, - "consistency_score_3": 0.3464285714285715, - "consistency_score_4": 0.24935064935064932, - "consistency_score_5": 0.1893939393939394, - "consistency_score_6": 0.14853896103896105, - "consistency_score_7": 0.11931818181818182, + "consistency_score_2": 0.6130952380952381, + "consistency_score_3": 0.45925324675324664, + "consistency_score_4": 0.37370129870129876, + "consistency_score_5": 0.31764069264069256, + "consistency_score_6": 0.2767857142857143, + "consistency_score_7": 0.24431818181818182, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4375, - "English,Spanish": 0.6477272727272727, - "English,Malay": 0.5738636363636364, - "English,Chinese": 0.6022727272727273, - "English,Indonesian": 0.6306818181818182, - "English,Vietnamese": 0.5625, - "Filipino,Spanish": 0.42613636363636365, - "Filipino,Malay": 0.4772727272727273, - "Filipino,Chinese": 0.42045454545454547, - "Filipino,Indonesian": 0.48863636363636365, - "Filipino,Vietnamese": 0.4715909090909091, - "Spanish,Malay": 0.5454545454545454, - "Spanish,Chinese": 0.5113636363636364, - "Spanish,Indonesian": 0.5340909090909091, - "Spanish,Vietnamese": 0.5568181818181818, - "Malay,Chinese": 0.4943181818181818, - "Malay,Indonesian": 0.6022727272727273, - "Malay,Vietnamese": 0.5738636363636364, - "Chinese,Indonesian": 0.5170454545454546, - "Chinese,Vietnamese": 0.5170454545454546, - "Indonesian,Vietnamese": 0.5340909090909091 - }, - "3_combine": { - "English,Filipino,Spanish": 0.3068181818181818, - "English,Filipino,Malay": 0.29545454545454547, - "English,Filipino,Chinese": 0.29545454545454547, - "English,Filipino,Indonesian": 0.3125, - "English,Filipino,Vietnamese": 0.30113636363636365, - "English,Spanish,Malay": 0.42613636363636365, + "Indonesian,English": 0.6647727272727273, + "Indonesian,Filipino": 0.6477272727272727, + "Indonesian,Spanish": 0.6420454545454546, + "Indonesian,Chinese": 0.5568181818181818, + "Indonesian,Malay": 0.7272727272727273, + "Indonesian,Vietnamese": 0.6761363636363636, + "English,Filipino": 0.6136363636363636, + "English,Spanish": 0.7272727272727273, + "English,Chinese": 0.5681818181818182, + "English,Malay": 0.6875, + "English,Vietnamese": 0.6420454545454546, + "Filipino,Spanish": 0.5965909090909091, + "Filipino,Chinese": 0.5397727272727273, + "Filipino,Malay": 0.5511363636363636, + "Filipino,Vietnamese": 0.5681818181818182, + "Spanish,Chinese": 0.4943181818181818, + "Spanish,Malay": 0.625, + "Spanish,Vietnamese": 0.6704545454545454, + "Chinese,Malay": 0.5340909090909091, + "Chinese,Vietnamese": 0.5284090909090909, + "Malay,Vietnamese": 0.6136363636363636 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.5, + "Indonesian,English,Spanish": 0.5454545454545454, + "Indonesian,English,Chinese": 0.4431818181818182, + "Indonesian,English,Malay": 0.5681818181818182, + "Indonesian,English,Vietnamese": 0.5284090909090909, + "Indonesian,Filipino,Spanish": 0.4715909090909091, + "Indonesian,Filipino,Chinese": 0.4318181818181818, + "Indonesian,Filipino,Malay": 0.5, + "Indonesian,Filipino,Vietnamese": 0.4659090909090909, + "Indonesian,Spanish,Chinese": 0.3977272727272727, + "Indonesian,Spanish,Malay": 0.5340909090909091, + "Indonesian,Spanish,Vietnamese": 0.5227272727272727, + "Indonesian,Chinese,Malay": 0.44886363636363635, + "Indonesian,Chinese,Vietnamese": 0.42613636363636365, + "Indonesian,Malay,Vietnamese": 0.5454545454545454, + "English,Filipino,Spanish": 0.48863636363636365, + "English,Filipino,Chinese": 0.4147727272727273, + "English,Filipino,Malay": 0.4659090909090909, + "English,Filipino,Vietnamese": 0.4431818181818182, "English,Spanish,Chinese": 0.4318181818181818, - "English,Spanish,Indonesian": 0.4431818181818182, - "English,Spanish,Vietnamese": 0.4375, - "English,Malay,Chinese": 0.38636363636363635, - "English,Malay,Indonesian": 0.42613636363636365, - "English,Malay,Vietnamese": 0.4034090909090909, - "English,Chinese,Indonesian": 0.4147727272727273, - "English,Chinese,Vietnamese": 0.4034090909090909, - "English,Indonesian,Vietnamese": 0.4034090909090909, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.25, - "Filipino,Spanish,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Vietnamese": 0.2897727272727273, - "Filipino,Malay,Chinese": 0.24431818181818182, - "Filipino,Malay,Indonesian": 0.3352272727272727, - "Filipino,Malay,Vietnamese": 0.32386363636363635, - "Filipino,Chinese,Indonesian": 0.2727272727272727, - "Filipino,Chinese,Vietnamese": 0.29545454545454547, - "Filipino,Indonesian,Vietnamese": 0.3068181818181818, - "Spanish,Malay,Chinese": 0.3352272727272727, - "Spanish,Malay,Indonesian": 0.39204545454545453, - "Spanish,Malay,Vietnamese": 0.3806818181818182, - "Spanish,Chinese,Indonesian": 0.32954545454545453, - "Spanish,Chinese,Vietnamese": 0.3409090909090909, - "Spanish,Indonesian,Vietnamese": 0.375, - "Malay,Chinese,Indonesian": 0.3465909090909091, - "Malay,Chinese,Vietnamese": 0.3409090909090909, - "Malay,Indonesian,Vietnamese": 0.3806818181818182, - "Chinese,Indonesian,Vietnamese": 0.3465909090909091 + "English,Spanish,Malay": 0.5511363636363636, + "English,Spanish,Vietnamese": 0.5511363636363636, + "English,Chinese,Malay": 0.4375, + "English,Chinese,Vietnamese": 0.39204545454545453, + "English,Malay,Vietnamese": 0.5056818181818182, + "Filipino,Spanish,Chinese": 0.3693181818181818, + "Filipino,Spanish,Malay": 0.4318181818181818, + "Filipino,Spanish,Vietnamese": 0.45454545454545453, + "Filipino,Chinese,Malay": 0.375, + "Filipino,Chinese,Vietnamese": 0.375, + "Filipino,Malay,Vietnamese": 0.4090909090909091, + "Spanish,Chinese,Malay": 0.38636363636363635, + "Spanish,Chinese,Vietnamese": 0.39204545454545453, + "Spanish,Malay,Vietnamese": 0.48863636363636365, + "Chinese,Malay,Vietnamese": 0.3806818181818182 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.22727272727272727, - "English,Filipino,Spanish,Chinese": 0.2215909090909091, - "English,Filipino,Spanish,Indonesian": 0.22727272727272727, - "English,Filipino,Spanish,Vietnamese": 0.25, - "English,Filipino,Malay,Chinese": 0.20454545454545456, - "English,Filipino,Malay,Indonesian": 0.23863636363636365, - "English,Filipino,Malay,Vietnamese": 0.23295454545454544, - "English,Filipino,Chinese,Indonesian": 0.2215909090909091, - "English,Filipino,Chinese,Vietnamese": 0.23295454545454544, - "English,Filipino,Indonesian,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay,Chinese": 0.29545454545454547, - "English,Spanish,Malay,Indonesian": 0.32954545454545453, - "English,Spanish,Malay,Vietnamese": 0.3352272727272727, - "English,Spanish,Chinese,Indonesian": 0.3068181818181818, - "English,Spanish,Chinese,Vietnamese": 0.3125, - "English,Spanish,Indonesian,Vietnamese": 0.3352272727272727, - "English,Malay,Chinese,Indonesian": 0.3068181818181818, - "English,Malay,Chinese,Vietnamese": 0.2897727272727273, - "English,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "English,Chinese,Indonesian,Vietnamese": 0.3068181818181818, - "Filipino,Spanish,Malay,Chinese": 0.17613636363636365, - "Filipino,Spanish,Malay,Indonesian": 0.21022727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.1875, - "Filipino,Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.1875, - "Filipino,Malay,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.26136363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.2556818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.29545454545454547, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.25 + "Indonesian,English,Filipino,Spanish": 0.42045454545454547, + "Indonesian,English,Filipino,Chinese": 0.3693181818181818, + "Indonesian,English,Filipino,Malay": 0.42613636363636365, + "Indonesian,English,Filipino,Vietnamese": 0.39204545454545453, + "Indonesian,English,Spanish,Chinese": 0.36363636363636365, + "Indonesian,English,Spanish,Malay": 0.48863636363636365, + "Indonesian,English,Spanish,Vietnamese": 0.4602272727272727, + "Indonesian,English,Chinese,Malay": 0.38636363636363635, + "Indonesian,English,Chinese,Vietnamese": 0.3465909090909091, + "Indonesian,English,Malay,Vietnamese": 0.4772727272727273, + "Indonesian,Filipino,Spanish,Chinese": 0.32954545454545453, + "Indonesian,Filipino,Spanish,Malay": 0.4034090909090909, + "Indonesian,Filipino,Spanish,Vietnamese": 0.38636363636363635, + "Indonesian,Filipino,Chinese,Malay": 0.35795454545454547, + "Indonesian,Filipino,Chinese,Vietnamese": 0.32954545454545453, + "Indonesian,Filipino,Malay,Vietnamese": 0.38636363636363635, + "Indonesian,Spanish,Chinese,Malay": 0.3522727272727273, + "Indonesian,Spanish,Chinese,Vietnamese": 0.3409090909090909, + "Indonesian,Spanish,Malay,Vietnamese": 0.45454545454545453, + "Indonesian,Chinese,Malay,Vietnamese": 0.35795454545454547, + "English,Filipino,Spanish,Chinese": 0.3352272727272727, + "English,Filipino,Spanish,Malay": 0.4034090909090909, + "English,Filipino,Spanish,Vietnamese": 0.39204545454545453, + "English,Filipino,Chinese,Malay": 0.3409090909090909, + "English,Filipino,Chinese,Vietnamese": 0.29545454545454547, + "English,Filipino,Malay,Vietnamese": 0.3806818181818182, + "English,Spanish,Chinese,Malay": 0.36363636363636365, + "English,Spanish,Chinese,Vietnamese": 0.3522727272727273, + "English,Spanish,Malay,Vietnamese": 0.44886363636363635, + "English,Chinese,Malay,Vietnamese": 0.32954545454545453, + "Filipino,Spanish,Chinese,Malay": 0.32386363636363635, + "Filipino,Spanish,Chinese,Vietnamese": 0.30113636363636365, + "Filipino,Spanish,Malay,Vietnamese": 0.36363636363636365, + "Filipino,Chinese,Malay,Vietnamese": 0.30113636363636365, + "Spanish,Chinese,Malay,Vietnamese": 0.3181818181818182 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.18181818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, - "English,Filipino,Spanish,Chinese,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese,Indonesian": 0.17045454545454544, - "English,Filipino,Malay,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.23863636363636365, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273 + "Indonesian,English,Filipino,Spanish,Chinese": 0.3068181818181818, + "Indonesian,English,Filipino,Spanish,Malay": 0.375, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.3522727272727273, + "Indonesian,English,Filipino,Chinese,Malay": 0.32386363636363635, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2784090909090909, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.35795454545454547, + "Indonesian,English,Spanish,Chinese,Malay": 0.32954545454545453, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.3125, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.42613636363636365, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.3181818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.3068181818181818, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.3465909090909091, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2897727272727273, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.3068181818181818, + "English,Filipino,Spanish,Chinese,Malay": 0.30113636363636365, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.2727272727272727, + "English,Filipino,Spanish,Malay,Vietnamese": 0.3465909090909091, + "English,Filipino,Chinese,Malay,Vietnamese": 0.2727272727272727, + "English,Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.2840909090909091, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.2556818181818182, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.32954545454545453, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.26136363636363635, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2897727272727273, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182 } }, - "AC3_2": 0.4166519101740687, - "AC3_3": 0.34487946738771746, - "AC3_4": 0.2888943430773179, - "AC3_5": 0.24412484412980684, - "AC3_6": 0.20736627229936222, - "AC3_7": 0.17709330139712967 + "AC3_2": 0.5463403124396696, + "AC3_3": 0.47538663156537675, + "AC3_4": 0.4250265847583037, + "AC3_5": 0.38625956676281836, + "AC3_6": 0.3544492163493555, + "AC3_7": 0.32665448533813757 }, "prompt_2": { - "overall_acc": 0.30275974025974023, + "overall_acc": 0.4675324675324675, "language_acc": { - "English": 0.38636363636363635, - "Filipino": 0.26136363636363635, - "Spanish": 0.25, - "Malay": 0.2897727272727273, - "Chinese": 0.3693181818181818, - "Indonesian": 0.2784090909090909, - "Vietnamese": 0.2840909090909091 + "Indonesian": 0.4772727272727273, + "English": 0.5454545454545454, + "Filipino": 0.42613636363636365, + "Spanish": 0.48295454545454547, + "Chinese": 0.42613636363636365, + "Malay": 0.44886363636363635, + "Vietnamese": 0.4659090909090909 }, - "consistency_score_2": 0.4591450216450217, - "consistency_score_3": 0.27353896103896097, - "consistency_score_4": 0.18457792207792212, - "consistency_score_5": 0.13257575757575757, - "consistency_score_6": 0.09821428571428571, - "consistency_score_7": 0.07386363636363637, + "consistency_score_2": 0.5768398268398267, + "consistency_score_3": 0.4141233766233765, + "consistency_score_4": 0.32678571428571423, + "consistency_score_5": 0.2719155844155844, + "consistency_score_6": 0.23538961038961034, + "consistency_score_7": 0.21022727272727273, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4715909090909091, - "English,Spanish": 0.5227272727272727, - "English,Malay": 0.5056818181818182, - "English,Chinese": 0.6022727272727273, - "English,Indonesian": 0.48863636363636365, - "English,Vietnamese": 0.5227272727272727, - "Filipino,Spanish": 0.3181818181818182, - "Filipino,Malay": 0.42613636363636365, - "Filipino,Chinese": 0.4147727272727273, - "Filipino,Indonesian": 0.45454545454545453, - "Filipino,Vietnamese": 0.4090909090909091, - "Spanish,Malay": 0.4318181818181818, - "Spanish,Chinese": 0.4375, - "Spanish,Indonesian": 0.4318181818181818, - "Spanish,Vietnamese": 0.44886363636363635, - "Malay,Chinese": 0.4147727272727273, - "Malay,Indonesian": 0.5511363636363636, - "Malay,Vietnamese": 0.48863636363636365, - "Chinese,Indonesian": 0.4147727272727273, - "Chinese,Vietnamese": 0.45454545454545453, - "Indonesian,Vietnamese": 0.4318181818181818 - }, - "3_combine": { - "English,Filipino,Spanish": 0.23295454545454544, - "English,Filipino,Malay": 0.2784090909090909, - "English,Filipino,Chinese": 0.3068181818181818, - "English,Filipino,Indonesian": 0.2727272727272727, - "English,Filipino,Vietnamese": 0.2840909090909091, - "English,Spanish,Malay": 0.32386363636363635, - "English,Spanish,Chinese": 0.3409090909090909, - "English,Spanish,Indonesian": 0.3125, - "English,Spanish,Vietnamese": 0.3409090909090909, - "English,Malay,Chinese": 0.32954545454545453, - "English,Malay,Indonesian": 0.35795454545454547, - "English,Malay,Vietnamese": 0.3181818181818182, - "English,Chinese,Indonesian": 0.32954545454545453, - "English,Chinese,Vietnamese": 0.3465909090909091, - "English,Indonesian,Vietnamese": 0.30113636363636365, - "Filipino,Spanish,Malay": 0.20454545454545456, - "Filipino,Spanish,Chinese": 0.18181818181818182, - "Filipino,Spanish,Indonesian": 0.21022727272727273, - "Filipino,Spanish,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese": 0.2215909090909091, - "Filipino,Malay,Indonesian": 0.29545454545454547, - "Filipino,Malay,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian": 0.23295454545454544, - "Filipino,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Chinese": 0.25, - "Spanish,Malay,Indonesian": 0.2897727272727273, - "Spanish,Malay,Vietnamese": 0.2727272727272727, - "Spanish,Chinese,Indonesian": 0.25, - "Spanish,Chinese,Vietnamese": 0.2840909090909091, - "Spanish,Indonesian,Vietnamese": 0.26136363636363635, - "Malay,Chinese,Indonesian": 0.2784090909090909, - "Malay,Chinese,Vietnamese": 0.23863636363636365, - "Malay,Indonesian,Vietnamese": 0.29545454545454547, - "Chinese,Indonesian,Vietnamese": 0.23295454545454544 + "Indonesian,English": 0.6534090909090909, + "Indonesian,Filipino": 0.5568181818181818, + "Indonesian,Spanish": 0.6306818181818182, + "Indonesian,Chinese": 0.5284090909090909, + "Indonesian,Malay": 0.7045454545454546, + "Indonesian,Vietnamese": 0.6079545454545454, + "English,Filipino": 0.5227272727272727, + "English,Spanish": 0.7443181818181818, + "English,Chinese": 0.5795454545454546, + "English,Malay": 0.6477272727272727, + "English,Vietnamese": 0.5795454545454546, + "Filipino,Spanish": 0.5170454545454546, + "Filipino,Chinese": 0.5056818181818182, + "Filipino,Malay": 0.5511363636363636, + "Filipino,Vietnamese": 0.48863636363636365, + "Spanish,Chinese": 0.5397727272727273, + "Spanish,Malay": 0.5965909090909091, + "Spanish,Vietnamese": 0.6022727272727273, + "Chinese,Malay": 0.5056818181818182, + "Chinese,Vietnamese": 0.48863636363636365, + "Malay,Vietnamese": 0.5625 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.42045454545454547, + "Indonesian,English,Spanish": 0.5340909090909091, + "Indonesian,English,Chinese": 0.4431818181818182, + "Indonesian,English,Malay": 0.5284090909090909, + "Indonesian,English,Vietnamese": 0.4602272727272727, + "Indonesian,Filipino,Spanish": 0.3977272727272727, + "Indonesian,Filipino,Chinese": 0.36363636363636365, + "Indonesian,Filipino,Malay": 0.44886363636363635, + "Indonesian,Filipino,Vietnamese": 0.38636363636363635, + "Indonesian,Spanish,Chinese": 0.38636363636363635, + "Indonesian,Spanish,Malay": 0.5, + "Indonesian,Spanish,Vietnamese": 0.45454545454545453, + "Indonesian,Chinese,Malay": 0.4090909090909091, + "Indonesian,Chinese,Vietnamese": 0.36363636363636365, + "Indonesian,Malay,Vietnamese": 0.4659090909090909, + "English,Filipino,Spanish": 0.42613636363636365, + "English,Filipino,Chinese": 0.375, + "English,Filipino,Malay": 0.4090909090909091, + "English,Filipino,Vietnamese": 0.3409090909090909, + "English,Spanish,Chinese": 0.4715909090909091, + "English,Spanish,Malay": 0.5397727272727273, + "English,Spanish,Vietnamese": 0.5, + "English,Chinese,Malay": 0.4090909090909091, + "English,Chinese,Vietnamese": 0.375, + "English,Malay,Vietnamese": 0.42613636363636365, + "Filipino,Spanish,Chinese": 0.3522727272727273, + "Filipino,Spanish,Malay": 0.39204545454545453, + "Filipino,Spanish,Vietnamese": 0.35795454545454547, + "Filipino,Chinese,Malay": 0.35795454545454547, + "Filipino,Chinese,Vietnamese": 0.3181818181818182, + "Filipino,Malay,Vietnamese": 0.35795454545454547, + "Spanish,Chinese,Malay": 0.3806818181818182, + "Spanish,Chinese,Vietnamese": 0.3693181818181818, + "Spanish,Malay,Vietnamese": 0.4318181818181818, + "Chinese,Malay,Vietnamese": 0.3409090909090909 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.18181818181818182, - "English,Filipino,Spanish,Chinese": 0.1534090909090909, - "English,Filipino,Spanish,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Vietnamese": 0.18181818181818182, - "English,Filipino,Malay,Chinese": 0.19886363636363635, - "English,Filipino,Malay,Indonesian": 0.2159090909090909, - "English,Filipino,Malay,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Chinese,Vietnamese": 0.1875, - "English,Filipino,Indonesian,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Chinese": 0.22727272727272727, - "English,Spanish,Malay,Indonesian": 0.2556818181818182, - "English,Spanish,Malay,Vietnamese": 0.24431818181818182, - "English,Spanish,Chinese,Indonesian": 0.2159090909090909, - "English,Spanish,Chinese,Vietnamese": 0.23295454545454544, - "English,Spanish,Indonesian,Vietnamese": 0.2215909090909091, - "English,Malay,Chinese,Indonesian": 0.23863636363636365, - "English,Malay,Chinese,Vietnamese": 0.20454545454545456, - "English,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "English,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese": 0.13068181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "Spanish,Malay,Chinese,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.125, - "English,Filipino,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian": 0.17613636363636365, - "English,Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635 + "Indonesian,English,Filipino,Spanish": 0.3522727272727273, + "Indonesian,English,Filipino,Chinese": 0.3181818181818182, + "Indonesian,English,Filipino,Malay": 0.35795454545454547, + "Indonesian,English,Filipino,Vietnamese": 0.3125, + "Indonesian,English,Spanish,Chinese": 0.3693181818181818, + "Indonesian,English,Spanish,Malay": 0.4602272727272727, + "Indonesian,English,Spanish,Vietnamese": 0.3977272727272727, + "Indonesian,English,Chinese,Malay": 0.35795454545454547, + "Indonesian,English,Chinese,Vietnamese": 0.32386363636363635, + "Indonesian,English,Malay,Vietnamese": 0.3806818181818182, + "Indonesian,Filipino,Spanish,Chinese": 0.29545454545454547, + "Indonesian,Filipino,Spanish,Malay": 0.3522727272727273, + "Indonesian,Filipino,Spanish,Vietnamese": 0.3125, + "Indonesian,Filipino,Chinese,Malay": 0.3125, + "Indonesian,Filipino,Chinese,Vietnamese": 0.2784090909090909, + "Indonesian,Filipino,Malay,Vietnamese": 0.32386363636363635, + "Indonesian,Spanish,Chinese,Malay": 0.3409090909090909, + "Indonesian,Spanish,Chinese,Vietnamese": 0.29545454545454547, + "Indonesian,Spanish,Malay,Vietnamese": 0.3806818181818182, + "Indonesian,Chinese,Malay,Vietnamese": 0.30113636363636365, + "English,Filipino,Spanish,Chinese": 0.32954545454545453, + "English,Filipino,Spanish,Malay": 0.35795454545454547, + "English,Filipino,Spanish,Vietnamese": 0.3068181818181818, + "English,Filipino,Chinese,Malay": 0.3068181818181818, + "English,Filipino,Chinese,Vietnamese": 0.26136363636363635, + "English,Filipino,Malay,Vietnamese": 0.2840909090909091, + "English,Spanish,Chinese,Malay": 0.36363636363636365, + "English,Spanish,Chinese,Vietnamese": 0.3352272727272727, + "English,Spanish,Malay,Vietnamese": 0.3977272727272727, + "English,Chinese,Malay,Vietnamese": 0.2840909090909091, + "Filipino,Spanish,Chinese,Malay": 0.29545454545454547, + "Filipino,Spanish,Chinese,Vietnamese": 0.2556818181818182, + "Filipino,Spanish,Malay,Vietnamese": 0.2897727272727273, + "Filipino,Chinese,Malay,Vietnamese": 0.26136363636363635, + "Spanish,Chinese,Malay,Vietnamese": 0.2840909090909091 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.2840909090909091, + "Indonesian,English,Filipino,Spanish,Malay": 0.3181818181818182, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.2784090909090909, + "Indonesian,English,Filipino,Chinese,Malay": 0.2784090909090909, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.25, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,English,Spanish,Chinese,Malay": 0.32386363636363635, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2840909090909091, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.3522727272727273, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.2727272727272727, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2727272727272727, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.24431818181818182, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453, + "English,Filipino,Spanish,Chinese,Malay": 0.2840909090909091, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.24431818181818182, + "English,Filipino,Spanish,Malay,Vietnamese": 0.26704545454545453, + "English,Filipino,Chinese,Malay,Vietnamese": 0.2215909090909091, + "English,Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.26136363636363635, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.25, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.2159090909090909, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273 } }, - "AC3_2": 0.36490289712595536, - "AC3_3": 0.2874085421121918, - "AC3_4": 0.2293389904095192, - "AC3_5": 0.18440307357761967, - "AC3_6": 0.1483155002521957, - "AC3_7": 0.1187548980875908 + "AC3_2": 0.5164659174527346, + "AC3_3": 0.4392102097367143, + "AC3_4": 0.3846894980684131, + "AC3_5": 0.343849344900539, + "AC3_6": 0.3131279805199235, + "AC3_7": 0.29003810556417425 }, "prompt_3": { - "overall_acc": 0.3116883116883117, + "overall_acc": 0.4521103896103896, "language_acc": { - "English": 0.35795454545454547, - "Filipino": 0.26704545454545453, - "Spanish": 0.32386363636363635, - "Malay": 0.32386363636363635, - "Chinese": 0.3806818181818182, - "Indonesian": 0.2784090909090909, - "Vietnamese": 0.25 + "Indonesian": 0.4375, + "English": 0.5340909090909091, + "Filipino": 0.38636363636363635, + "Spanish": 0.4602272727272727, + "Chinese": 0.4431818181818182, + "Malay": 0.4659090909090909, + "Vietnamese": 0.4375 }, - "consistency_score_2": 0.4824134199134199, - "consistency_score_3": 0.2990259740259741, - "consistency_score_4": 0.2073051948051948, - "consistency_score_5": 0.15232683982683984, - "consistency_score_6": 0.11607142857142858, - "consistency_score_7": 0.09090909090909091, + "consistency_score_2": 0.5446428571428573, + "consistency_score_3": 0.376461038961039, + "consistency_score_4": 0.29188311688311686, + "consistency_score_5": 0.24296536796536794, + "consistency_score_6": 0.2126623376623376, + "consistency_score_7": 0.19318181818181818, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4602272727272727, - "English,Spanish": 0.5852272727272727, - "English,Malay": 0.4943181818181818, - "English,Chinese": 0.5909090909090909, - "English,Indonesian": 0.5340909090909091, - "English,Vietnamese": 0.5340909090909091, - "Filipino,Spanish": 0.38636363636363635, - "Filipino,Malay": 0.4318181818181818, - "Filipino,Chinese": 0.4090909090909091, - "Filipino,Indonesian": 0.5056818181818182, - "Filipino,Vietnamese": 0.44886363636363635, - "Spanish,Malay": 0.44886363636363635, - "Spanish,Chinese": 0.5227272727272727, - "Spanish,Indonesian": 0.45454545454545453, - "Spanish,Vietnamese": 0.45454545454545453, - "Malay,Chinese": 0.4090909090909091, - "Malay,Indonesian": 0.6079545454545454, - "Malay,Vietnamese": 0.4772727272727273, - "Chinese,Indonesian": 0.44886363636363635, - "Chinese,Vietnamese": 0.4715909090909091, - "Indonesian,Vietnamese": 0.45454545454545453 + "Indonesian,English": 0.5852272727272727, + "Indonesian,Filipino": 0.5340909090909091, + "Indonesian,Spanish": 0.5568181818181818, + "Indonesian,Chinese": 0.5170454545454546, + "Indonesian,Malay": 0.6534090909090909, + "Indonesian,Vietnamese": 0.5170454545454546, + "English,Filipino": 0.5113636363636364, + "English,Spanish": 0.6818181818181818, + "English,Chinese": 0.5681818181818182, + "English,Malay": 0.6136363636363636, + "English,Vietnamese": 0.5284090909090909, + "Filipino,Spanish": 0.4715909090909091, + "Filipino,Chinese": 0.4772727272727273, + "Filipino,Malay": 0.5340909090909091, + "Filipino,Vietnamese": 0.48863636363636365, + "Spanish,Chinese": 0.5170454545454546, + "Spanish,Malay": 0.5909090909090909, + "Spanish,Vietnamese": 0.5909090909090909, + "Chinese,Malay": 0.4659090909090909, + "Chinese,Vietnamese": 0.5056818181818182, + "Malay,Vietnamese": 0.5284090909090909 }, "3_combine": { - "English,Filipino,Spanish": 0.2840909090909091, - "English,Filipino,Malay": 0.2840909090909091, - "English,Filipino,Chinese": 0.29545454545454547, - "English,Filipino,Indonesian": 0.2897727272727273, - "English,Filipino,Vietnamese": 0.3068181818181818, - "English,Spanish,Malay": 0.3522727272727273, - "English,Spanish,Chinese": 0.3977272727272727, - "English,Spanish,Indonesian": 0.3522727272727273, - "English,Spanish,Vietnamese": 0.3465909090909091, - "English,Malay,Chinese": 0.32386363636363635, - "English,Malay,Indonesian": 0.3806818181818182, - "English,Malay,Vietnamese": 0.3352272727272727, - "English,Chinese,Indonesian": 0.32954545454545453, - "English,Chinese,Vietnamese": 0.3522727272727273, - "English,Indonesian,Vietnamese": 0.32386363636363635, - "Filipino,Spanish,Malay": 0.25, - "Filipino,Spanish,Chinese": 0.23295454545454544, - "Filipino,Spanish,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Vietnamese": 0.25, - "Filipino,Malay,Chinese": 0.2215909090909091, - "Filipino,Malay,Indonesian": 0.3352272727272727, - "Filipino,Malay,Vietnamese": 0.26136363636363635, - "Filipino,Chinese,Indonesian": 0.24431818181818182, - "Filipino,Chinese,Vietnamese": 0.26136363636363635, - "Filipino,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Chinese": 0.2840909090909091, - "Spanish,Malay,Indonesian": 0.32954545454545453, - "Spanish,Malay,Vietnamese": 0.2897727272727273, - "Spanish,Chinese,Indonesian": 0.2784090909090909, - "Spanish,Chinese,Vietnamese": 0.30113636363636365, - "Spanish,Indonesian,Vietnamese": 0.2784090909090909, - "Malay,Chinese,Indonesian": 0.2840909090909091, - "Malay,Chinese,Vietnamese": 0.26136363636363635, - "Malay,Indonesian,Vietnamese": 0.3409090909090909, - "Chinese,Indonesian,Vietnamese": 0.2556818181818182 + "Indonesian,English,Filipino": 0.375, + "Indonesian,English,Spanish": 0.45454545454545453, + "Indonesian,English,Chinese": 0.4034090909090909, + "Indonesian,English,Malay": 0.4715909090909091, + "Indonesian,English,Vietnamese": 0.3806818181818182, + "Indonesian,Filipino,Spanish": 0.3465909090909091, + "Indonesian,Filipino,Chinese": 0.3352272727272727, + "Indonesian,Filipino,Malay": 0.42613636363636365, + "Indonesian,Filipino,Vietnamese": 0.3352272727272727, + "Indonesian,Spanish,Chinese": 0.3522727272727273, + "Indonesian,Spanish,Malay": 0.4431818181818182, + "Indonesian,Spanish,Vietnamese": 0.38636363636363635, + "Indonesian,Chinese,Malay": 0.36363636363636365, + "Indonesian,Chinese,Vietnamese": 0.3352272727272727, + "Indonesian,Malay,Vietnamese": 0.3977272727272727, + "English,Filipino,Spanish": 0.38636363636363635, + "English,Filipino,Chinese": 0.3465909090909091, + "English,Filipino,Malay": 0.3806818181818182, + "English,Filipino,Vietnamese": 0.3352272727272727, + "English,Spanish,Chinese": 0.42613636363636365, + "English,Spanish,Malay": 0.48863636363636365, + "English,Spanish,Vietnamese": 0.4431818181818182, + "English,Chinese,Malay": 0.3806818181818182, + "English,Chinese,Vietnamese": 0.375, + "English,Malay,Vietnamese": 0.38636363636363635, + "Filipino,Spanish,Chinese": 0.30113636363636365, + "Filipino,Spanish,Malay": 0.36363636363636365, + "Filipino,Spanish,Vietnamese": 0.3409090909090909, + "Filipino,Chinese,Malay": 0.32386363636363635, + "Filipino,Chinese,Vietnamese": 0.30113636363636365, + "Filipino,Malay,Vietnamese": 0.3409090909090909, + "Spanish,Chinese,Malay": 0.3522727272727273, + "Spanish,Chinese,Vietnamese": 0.3693181818181818, + "Spanish,Malay,Vietnamese": 0.4090909090909091, + "Chinese,Malay,Vietnamese": 0.3181818181818182 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.21022727272727273, - "English,Filipino,Spanish,Chinese": 0.19318181818181818, - "English,Filipino,Spanish,Indonesian": 0.20454545454545456, - "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, - "English,Filipino,Malay,Chinese": 0.19318181818181818, - "English,Filipino,Malay,Indonesian": 0.22727272727272727, - "English,Filipino,Malay,Vietnamese": 0.21022727272727273, - "English,Filipino,Chinese,Indonesian": 0.17613636363636365, - "English,Filipino,Chinese,Vietnamese": 0.21022727272727273, - "English,Filipino,Indonesian,Vietnamese": 0.20454545454545456, - "English,Spanish,Malay,Chinese": 0.24431818181818182, - "English,Spanish,Malay,Indonesian": 0.2840909090909091, - "English,Spanish,Malay,Vietnamese": 0.2556818181818182, - "English,Spanish,Chinese,Indonesian": 0.2215909090909091, - "English,Spanish,Chinese,Vietnamese": 0.2556818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.24431818181818182, - "English,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Malay,Chinese,Vietnamese": 0.2215909090909091, - "English,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "English,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay,Chinese": 0.1590909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Chinese,Indonesian": 0.21022727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.1875, - "Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818 + "Indonesian,English,Filipino,Spanish": 0.30113636363636365, + "Indonesian,English,Filipino,Chinese": 0.2840909090909091, + "Indonesian,English,Filipino,Malay": 0.32954545454545453, + "Indonesian,English,Filipino,Vietnamese": 0.2727272727272727, + "Indonesian,English,Spanish,Chinese": 0.3181818181818182, + "Indonesian,English,Spanish,Malay": 0.38636363636363635, + "Indonesian,English,Spanish,Vietnamese": 0.32954545454545453, + "Indonesian,English,Chinese,Malay": 0.32954545454545453, + "Indonesian,English,Chinese,Vietnamese": 0.2897727272727273, + "Indonesian,English,Malay,Vietnamese": 0.32954545454545453, + "Indonesian,Filipino,Spanish,Chinese": 0.25, + "Indonesian,Filipino,Spanish,Malay": 0.3125, + "Indonesian,Filipino,Spanish,Vietnamese": 0.2727272727272727, + "Indonesian,Filipino,Chinese,Malay": 0.2840909090909091, + "Indonesian,Filipino,Chinese,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Malay,Vietnamese": 0.29545454545454547, + "Indonesian,Spanish,Chinese,Malay": 0.29545454545454547, + "Indonesian,Spanish,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,Spanish,Malay,Vietnamese": 0.32954545454545453, + "Indonesian,Chinese,Malay,Vietnamese": 0.26704545454545453, + "English,Filipino,Spanish,Chinese": 0.2784090909090909, + "English,Filipino,Spanish,Malay": 0.3181818181818182, + "English,Filipino,Spanish,Vietnamese": 0.2897727272727273, + "English,Filipino,Chinese,Malay": 0.2727272727272727, + "English,Filipino,Chinese,Vietnamese": 0.24431818181818182, + "English,Filipino,Malay,Vietnamese": 0.2840909090909091, + "English,Spanish,Chinese,Malay": 0.32954545454545453, + "English,Spanish,Chinese,Vietnamese": 0.32386363636363635, + "English,Spanish,Malay,Vietnamese": 0.3465909090909091, + "English,Chinese,Malay,Vietnamese": 0.2727272727272727, + "Filipino,Spanish,Chinese,Malay": 0.2556818181818182, + "Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, + "Filipino,Spanish,Malay,Vietnamese": 0.2840909090909091, + "Filipino,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese,Indonesian": 0.1875, - "English,Spanish,Malay,Chinese,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2159090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.125, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091 + "Indonesian,English,Filipino,Spanish,Chinese": 0.23295454545454544, + "Indonesian,English,Filipino,Spanish,Malay": 0.2784090909090909, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.23863636363636365, + "Indonesian,English,Filipino,Chinese,Malay": 0.2556818181818182, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.21022727272727273, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.26136363636363635, + "Indonesian,English,Spanish,Chinese,Malay": 0.2840909090909091, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2556818181818182, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.29545454545454547, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.25, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.23863636363636365, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.26136363636363635, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2159090909090909, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, + "English,Filipino,Spanish,Chinese,Malay": 0.24431818181818182, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.2215909090909091, + "English,Filipino,Spanish,Malay,Vietnamese": 0.2556818181818182, + "English,Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, + "English,Spanish,Chinese,Malay,Vietnamese": 0.25, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.20454545454545456 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.22727272727272727, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818 } }, - "AC3_2": 0.3786986437276158, - "AC3_3": 0.30522587455388656, - "AC3_4": 0.24899967090977926, - "AC3_5": 0.20464200518099318, - "AC3_6": 0.16915153154083049, - "AC3_7": 0.1407624633081449 + "AC3_2": 0.49408154950836486, + "AC3_3": 0.4108322858071933, + "AC3_4": 0.35474339097107555, + "AC3_5": 0.31607250277284515, + "AC3_6": 0.28926232493309767, + "AC3_7": 0.27069754141030883 }, "prompt_4": { - "overall_acc": 0.3246753246753246, + "overall_acc": 0.4748376623376624, "language_acc": { - "English": 0.3977272727272727, - "Filipino": 0.26136363636363635, - "Spanish": 0.3409090909090909, - "Malay": 0.3409090909090909, - "Chinese": 0.3522727272727273, - "Indonesian": 0.2784090909090909, - "Vietnamese": 0.30113636363636365 + "Indonesian": 0.4659090909090909, + "English": 0.5511363636363636, + "Filipino": 0.42613636363636365, + "Spanish": 0.4431818181818182, + "Chinese": 0.4602272727272727, + "Malay": 0.5056818181818182, + "Vietnamese": 0.4715909090909091 }, - "consistency_score_2": 0.5, - "consistency_score_3": 0.31477272727272726, - "consistency_score_4": 0.21753246753246747, - "consistency_score_5": 0.15692640692640691, - "consistency_score_6": 0.11525974025974027, - "consistency_score_7": 0.08522727272727272, + "consistency_score_2": 0.5817099567099566, + "consistency_score_3": 0.4222402597402597, + "consistency_score_4": 0.335064935064935, + "consistency_score_5": 0.27949134199134196, + "consistency_score_6": 0.24188311688311687, + "consistency_score_7": 0.2159090909090909, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.4431818181818182, - "English,Spanish": 0.5909090909090909, - "English,Malay": 0.5511363636363636, - "English,Chinese": 0.5909090909090909, - "English,Indonesian": 0.5340909090909091, - "English,Vietnamese": 0.5738636363636364, - "Filipino,Spanish": 0.4034090909090909, - "Filipino,Malay": 0.45454545454545453, - "Filipino,Chinese": 0.4147727272727273, - "Filipino,Indonesian": 0.4659090909090909, - "Filipino,Vietnamese": 0.4375, - "Spanish,Malay": 0.5056818181818182, + "Indonesian,English": 0.6193181818181818, + "Indonesian,Filipino": 0.5852272727272727, + "Indonesian,Spanish": 0.5795454545454546, + "Indonesian,Chinese": 0.5056818181818182, + "Indonesian,Malay": 0.6931818181818182, + "Indonesian,Vietnamese": 0.5965909090909091, + "English,Filipino": 0.5170454545454546, + "English,Spanish": 0.7045454545454546, + "English,Chinese": 0.5625, + "English,Malay": 0.6590909090909091, + "English,Vietnamese": 0.6363636363636364, + "Filipino,Spanish": 0.5454545454545454, + "Filipino,Chinese": 0.5454545454545454, + "Filipino,Malay": 0.5568181818181818, + "Filipino,Vietnamese": 0.5397727272727273, "Spanish,Chinese": 0.4943181818181818, - "Spanish,Indonesian": 0.4943181818181818, - "Spanish,Vietnamese": 0.5, - "Malay,Chinese": 0.4431818181818182, - "Malay,Indonesian": 0.5909090909090909, - "Malay,Vietnamese": 0.5227272727272727, - "Chinese,Indonesian": 0.48295454545454547, - "Chinese,Vietnamese": 0.5056818181818182, - "Indonesian,Vietnamese": 0.5 + "Spanish,Malay": 0.5965909090909091, + "Spanish,Vietnamese": 0.6704545454545454, + "Chinese,Malay": 0.4943181818181818, + "Chinese,Vietnamese": 0.5170454545454546, + "Malay,Vietnamese": 0.5965909090909091 }, "3_combine": { - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Malay": 0.2897727272727273, - "English,Filipino,Chinese": 0.2840909090909091, - "English,Filipino,Indonesian": 0.2840909090909091, - "English,Filipino,Vietnamese": 0.2897727272727273, - "English,Spanish,Malay": 0.3806818181818182, - "English,Spanish,Chinese": 0.38636363636363635, - "English,Spanish,Indonesian": 0.375, - "English,Spanish,Vietnamese": 0.39204545454545453, - "English,Malay,Chinese": 0.3465909090909091, - "English,Malay,Indonesian": 0.38636363636363635, - "English,Malay,Vietnamese": 0.3806818181818182, - "English,Chinese,Indonesian": 0.3693181818181818, - "English,Chinese,Vietnamese": 0.3806818181818182, - "English,Indonesian,Vietnamese": 0.3693181818181818, - "Filipino,Spanish,Malay": 0.2556818181818182, - "Filipino,Spanish,Chinese": 0.23295454545454544, - "Filipino,Spanish,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Vietnamese": 0.25, - "Filipino,Malay,Chinese": 0.2159090909090909, - "Filipino,Malay,Indonesian": 0.32954545454545453, - "Filipino,Malay,Vietnamese": 0.26704545454545453, - "Filipino,Chinese,Indonesian": 0.2556818181818182, - "Filipino,Chinese,Vietnamese": 0.25, - "Filipino,Indonesian,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Chinese": 0.2897727272727273, - "Spanish,Malay,Indonesian": 0.3693181818181818, - "Spanish,Malay,Vietnamese": 0.32386363636363635, - "Spanish,Chinese,Indonesian": 0.32386363636363635, - "Spanish,Chinese,Vietnamese": 0.3181818181818182, - "Spanish,Indonesian,Vietnamese": 0.32954545454545453, - "Malay,Chinese,Indonesian": 0.3068181818181818, - "Malay,Chinese,Vietnamese": 0.29545454545454547, - "Malay,Indonesian,Vietnamese": 0.3465909090909091, - "Chinese,Indonesian,Vietnamese": 0.32386363636363635 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.20454545454545456, - "English,Filipino,Spanish,Chinese": 0.1875, - "English,Filipino,Spanish,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Vietnamese": 0.2215909090909091, - "English,Filipino,Malay,Chinese": 0.18181818181818182, - "English,Filipino,Malay,Indonesian": 0.2215909090909091, - "English,Filipino,Malay,Vietnamese": 0.21022727272727273, - "English,Filipino,Chinese,Indonesian": 0.19318181818181818, - "English,Filipino,Chinese,Vietnamese": 0.20454545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.20454545454545456, - "English,Spanish,Malay,Chinese": 0.25, - "English,Spanish,Malay,Indonesian": 0.29545454545454547, - "English,Spanish,Malay,Vietnamese": 0.2727272727272727, - "English,Spanish,Chinese,Indonesian": 0.2784090909090909, - "English,Spanish,Chinese,Vietnamese": 0.26136363636363635, - "English,Spanish,Indonesian,Vietnamese": 0.2784090909090909, - "English,Malay,Chinese,Indonesian": 0.26704545454545453, - "English,Malay,Chinese,Vietnamese": 0.25, - "English,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "English,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Malay,Chinese": 0.1534090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.20454545454545456, - "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.1875, - "Filipino,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.26136363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese,Indonesian": 0.2159090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365 + "Indonesian,English,Filipino": 0.4147727272727273, + "Indonesian,English,Spanish": 0.48863636363636365, + "Indonesian,English,Chinese": 0.39204545454545453, + "Indonesian,English,Malay": 0.5227272727272727, + "Indonesian,English,Vietnamese": 0.4715909090909091, + "Indonesian,Filipino,Spanish": 0.3977272727272727, + "Indonesian,Filipino,Chinese": 0.3806818181818182, + "Indonesian,Filipino,Malay": 0.4602272727272727, + "Indonesian,Filipino,Vietnamese": 0.3977272727272727, + "Indonesian,Spanish,Chinese": 0.3409090909090909, + "Indonesian,Spanish,Malay": 0.4772727272727273, + "Indonesian,Spanish,Vietnamese": 0.4659090909090909, + "Indonesian,Chinese,Malay": 0.4034090909090909, + "Indonesian,Chinese,Vietnamese": 0.35795454545454547, + "Indonesian,Malay,Vietnamese": 0.48863636363636365, + "English,Filipino,Spanish": 0.42613636363636365, + "English,Filipino,Chinese": 0.3806818181818182, + "English,Filipino,Malay": 0.42045454545454547, + "English,Filipino,Vietnamese": 0.4034090909090909, + "English,Spanish,Chinese": 0.42613636363636365, + "English,Spanish,Malay": 0.5113636363636364, + "English,Spanish,Vietnamese": 0.5454545454545454, + "English,Chinese,Malay": 0.42045454545454547, + "English,Chinese,Vietnamese": 0.3977272727272727, + "English,Malay,Vietnamese": 0.48295454545454547, + "Filipino,Spanish,Chinese": 0.36363636363636365, + "Filipino,Spanish,Malay": 0.4034090909090909, + "Filipino,Spanish,Vietnamese": 0.42613636363636365, + "Filipino,Chinese,Malay": 0.35795454545454547, + "Filipino,Chinese,Vietnamese": 0.36363636363636365, + "Filipino,Malay,Vietnamese": 0.3977272727272727, + "Spanish,Chinese,Malay": 0.36363636363636365, + "Spanish,Chinese,Vietnamese": 0.39204545454545453, + "Spanish,Malay,Vietnamese": 0.4715909090909091, + "Chinese,Malay,Vietnamese": 0.36363636363636365 + }, + "4_combine": { + "Indonesian,English,Filipino,Spanish": 0.3409090909090909, + "Indonesian,English,Filipino,Chinese": 0.30113636363636365, + "Indonesian,English,Filipino,Malay": 0.36363636363636365, + "Indonesian,English,Filipino,Vietnamese": 0.32954545454545453, + "Indonesian,English,Spanish,Chinese": 0.32386363636363635, + "Indonesian,English,Spanish,Malay": 0.42045454545454547, + "Indonesian,English,Spanish,Vietnamese": 0.42045454545454547, + "Indonesian,English,Chinese,Malay": 0.3465909090909091, + "Indonesian,English,Chinese,Vietnamese": 0.3125, + "Indonesian,English,Malay,Vietnamese": 0.4147727272727273, + "Indonesian,Filipino,Spanish,Chinese": 0.2727272727272727, + "Indonesian,Filipino,Spanish,Malay": 0.3465909090909091, + "Indonesian,Filipino,Spanish,Vietnamese": 0.32954545454545453, + "Indonesian,Filipino,Chinese,Malay": 0.3125, + "Indonesian,Filipino,Chinese,Vietnamese": 0.2784090909090909, + "Indonesian,Filipino,Malay,Vietnamese": 0.3465909090909091, + "Indonesian,Spanish,Chinese,Malay": 0.3068181818181818, + "Indonesian,Spanish,Chinese,Vietnamese": 0.29545454545454547, + "Indonesian,Spanish,Malay,Vietnamese": 0.3977272727272727, + "Indonesian,Chinese,Malay,Vietnamese": 0.3125, + "English,Filipino,Spanish,Chinese": 0.32386363636363635, + "English,Filipino,Spanish,Malay": 0.3465909090909091, + "English,Filipino,Spanish,Vietnamese": 0.35795454545454547, + "English,Filipino,Chinese,Malay": 0.3181818181818182, + "English,Filipino,Chinese,Vietnamese": 0.29545454545454547, + "English,Filipino,Malay,Vietnamese": 0.3409090909090909, + "English,Spanish,Chinese,Malay": 0.3465909090909091, + "English,Spanish,Chinese,Vietnamese": 0.35795454545454547, + "English,Spanish,Malay,Vietnamese": 0.4318181818181818, + "English,Chinese,Malay,Vietnamese": 0.32386363636363635, + "Filipino,Spanish,Chinese,Malay": 0.29545454545454547, + "Filipino,Spanish,Chinese,Vietnamese": 0.29545454545454547, + "Filipino,Spanish,Malay,Vietnamese": 0.32954545454545453, + "Filipino,Chinese,Malay,Vietnamese": 0.2840909090909091, + "Spanish,Chinese,Malay,Vietnamese": 0.3068181818181818 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.2556818181818182, + "Indonesian,English,Filipino,Spanish,Malay": 0.30113636363636365, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.29545454545454547, + "Indonesian,English,Filipino,Chinese,Malay": 0.2727272727272727, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.24431818181818182, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.30113636363636365, + "Indonesian,English,Spanish,Chinese,Malay": 0.2897727272727273, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2897727272727273, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.3693181818181818, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.2840909090909091, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.2556818181818182, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2897727272727273, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2556818181818182, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.26704545454545453, + "English,Filipino,Spanish,Chinese,Malay": 0.2784090909090909, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.2727272727272727, + "English,Filipino,Spanish,Malay,Vietnamese": 0.30113636363636365, + "English,Filipino,Chinese,Malay,Vietnamese": 0.26136363636363635, + "English,Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.25 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.23863636363636365, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.22727272727272727, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.26704545454545453, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.23295454545454544, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2215909090909091, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2159090909090909 } }, - "AC3_2": 0.3937007873538346, - "AC3_3": 0.31964734930033156, - "AC3_4": 0.2605179251405361, - "AC3_5": 0.21158616659841253, - "AC3_6": 0.17012507783548486, - "AC3_7": 0.13501350131719705 + "AC3_2": 0.522868616609141, + "AC3_3": 0.44699701759051197, + "AC3_4": 0.3928903327095309, + "AC3_5": 0.3518703767071589, + "AC3_6": 0.32050197815827963, + "AC3_7": 0.29684328593074655 }, "prompt_5": { - "overall_acc": 0.3157467532467532, + "overall_acc": 0.47159090909090917, "language_acc": { - "English": 0.38636363636363635, - "Filipino": 0.2556818181818182, - "Spanish": 0.2784090909090909, - "Malay": 0.3465909090909091, - "Chinese": 0.3806818181818182, - "Indonesian": 0.2727272727272727, - "Vietnamese": 0.2897727272727273 + "Indonesian": 0.4318181818181818, + "English": 0.5681818181818182, + "Filipino": 0.4147727272727273, + "Spanish": 0.4375, + "Chinese": 0.5, + "Malay": 0.4772727272727273, + "Vietnamese": 0.4715909090909091 }, - "consistency_score_2": 0.4599567099567099, - "consistency_score_3": 0.2686688311688312, - "consistency_score_4": 0.17224025974025978, - "consistency_score_5": 0.11417748917748917, - "consistency_score_6": 0.07467532467532467, - "consistency_score_7": 0.045454545454545456, + "consistency_score_2": 0.6025432900432899, + "consistency_score_3": 0.4491883116883117, + "consistency_score_4": 0.3680194805194806, + "consistency_score_5": 0.31764069264069267, + "consistency_score_6": 0.2824675324675325, + "consistency_score_7": 0.2556818181818182, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.44886363636363635, - "English,Spanish": 0.5340909090909091, - "English,Malay": 0.5056818181818182, - "English,Chinese": 0.5909090909090909, - "English,Indonesian": 0.5227272727272727, - "English,Vietnamese": 0.4943181818181818, - "Filipino,Spanish": 0.3181818181818182, - "Filipino,Malay": 0.42045454545454547, - "Filipino,Chinese": 0.4318181818181818, - "Filipino,Indonesian": 0.45454545454545453, - "Filipino,Vietnamese": 0.4090909090909091, - "Spanish,Malay": 0.4375, - "Spanish,Chinese": 0.39204545454545453, - "Spanish,Indonesian": 0.4602272727272727, - "Spanish,Vietnamese": 0.4034090909090909, - "Malay,Chinese": 0.4375, - "Malay,Indonesian": 0.5284090909090909, - "Malay,Vietnamese": 0.5113636363636364, - "Chinese,Indonesian": 0.42045454545454547, - "Chinese,Vietnamese": 0.4602272727272727, - "Indonesian,Vietnamese": 0.4772727272727273 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2215909090909091, - "English,Filipino,Malay": 0.2556818181818182, - "English,Filipino,Chinese": 0.30113636363636365, - "English,Filipino,Indonesian": 0.2897727272727273, - "English,Filipino,Vietnamese": 0.25, - "English,Spanish,Malay": 0.3125, - "English,Spanish,Chinese": 0.3125, - "English,Spanish,Indonesian": 0.32954545454545453, - "English,Spanish,Vietnamese": 0.3068181818181818, - "English,Malay,Chinese": 0.32954545454545453, - "English,Malay,Indonesian": 0.35795454545454547, - "English,Malay,Vietnamese": 0.3181818181818182, - "English,Chinese,Indonesian": 0.32386363636363635, - "English,Chinese,Vietnamese": 0.32386363636363635, - "English,Indonesian,Vietnamese": 0.3181818181818182, - "Filipino,Spanish,Malay": 0.1875, - "Filipino,Spanish,Chinese": 0.16477272727272727, - "Filipino,Spanish,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese": 0.20454545454545456, - "Filipino,Malay,Indonesian": 0.30113636363636365, - "Filipino,Malay,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Indonesian": 0.2556818181818182, - "Filipino,Chinese,Vietnamese": 0.22727272727272727, - "Filipino,Indonesian,Vietnamese": 0.2556818181818182, - "Spanish,Malay,Chinese": 0.23863636363636365, - "Spanish,Malay,Indonesian": 0.29545454545454547, - "Spanish,Malay,Vietnamese": 0.25, - "Spanish,Chinese,Indonesian": 0.23295454545454544, - "Spanish,Chinese,Vietnamese": 0.22727272727272727, - "Spanish,Indonesian,Vietnamese": 0.26704545454545453, - "Malay,Chinese,Indonesian": 0.2727272727272727, - "Malay,Chinese,Vietnamese": 0.2727272727272727, - "Malay,Indonesian,Vietnamese": 0.3181818181818182, - "Chinese,Indonesian,Vietnamese": 0.2556818181818182 + "Indonesian,English": 0.6136363636363636, + "Indonesian,Filipino": 0.5965909090909091, + "Indonesian,Spanish": 0.6136363636363636, + "Indonesian,Chinese": 0.5454545454545454, + "Indonesian,Malay": 0.6875, + "Indonesian,Vietnamese": 0.6136363636363636, + "English,Filipino": 0.5625, + "English,Spanish": 0.7102272727272727, + "English,Chinese": 0.625, + "English,Malay": 0.6477272727272727, + "English,Vietnamese": 0.6590909090909091, + "Filipino,Spanish": 0.5681818181818182, + "Filipino,Chinese": 0.5454545454545454, + "Filipino,Malay": 0.5397727272727273, + "Filipino,Vietnamese": 0.5454545454545454, + "Spanish,Chinese": 0.5795454545454546, + "Spanish,Malay": 0.6306818181818182, + "Spanish,Vietnamese": 0.6761363636363636, + "Chinese,Malay": 0.5454545454545454, + "Chinese,Vietnamese": 0.5397727272727273, + "Malay,Vietnamese": 0.6079545454545454 + }, + "3_combine": { + "Indonesian,English,Filipino": 0.44886363636363635, + "Indonesian,English,Spanish": 0.5170454545454546, + "Indonesian,English,Chinese": 0.4318181818181818, + "Indonesian,English,Malay": 0.5056818181818182, + "Indonesian,English,Vietnamese": 0.4772727272727273, + "Indonesian,Filipino,Spanish": 0.4318181818181818, + "Indonesian,Filipino,Chinese": 0.39204545454545453, + "Indonesian,Filipino,Malay": 0.4602272727272727, + "Indonesian,Filipino,Vietnamese": 0.4147727272727273, + "Indonesian,Spanish,Chinese": 0.4147727272727273, + "Indonesian,Spanish,Malay": 0.4943181818181818, + "Indonesian,Spanish,Vietnamese": 0.48863636363636365, + "Indonesian,Chinese,Malay": 0.4375, + "Indonesian,Chinese,Vietnamese": 0.4034090909090909, + "Indonesian,Malay,Vietnamese": 0.5, + "English,Filipino,Spanish": 0.45454545454545453, + "English,Filipino,Chinese": 0.42613636363636365, + "English,Filipino,Malay": 0.42613636363636365, + "English,Filipino,Vietnamese": 0.4375, + "English,Spanish,Chinese": 0.5, + "English,Spanish,Malay": 0.5397727272727273, + "English,Spanish,Vietnamese": 0.5511363636363636, + "English,Chinese,Malay": 0.4431818181818182, + "English,Chinese,Vietnamese": 0.4431818181818182, + "English,Malay,Vietnamese": 0.48295454545454547, + "Filipino,Spanish,Chinese": 0.4034090909090909, + "Filipino,Spanish,Malay": 0.4147727272727273, + "Filipino,Spanish,Vietnamese": 0.4431818181818182, + "Filipino,Chinese,Malay": 0.3977272727272727, + "Filipino,Chinese,Vietnamese": 0.3806818181818182, + "Filipino,Malay,Vietnamese": 0.3977272727272727, + "Spanish,Chinese,Malay": 0.4318181818181818, + "Spanish,Chinese,Vietnamese": 0.4375, + "Spanish,Malay,Vietnamese": 0.5, + "Chinese,Malay,Vietnamese": 0.39204545454545453 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.1534090909090909, - "English,Filipino,Spanish,Chinese": 0.13068181818181818, - "English,Filipino,Spanish,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Chinese": 0.17045454545454544, - "English,Filipino,Malay,Indonesian": 0.20454545454545456, - "English,Filipino,Malay,Vietnamese": 0.17045454545454544, - "English,Filipino,Chinese,Indonesian": 0.19886363636363635, - "English,Filipino,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Indonesian,Vietnamese": 0.1875, - "English,Spanish,Malay,Chinese": 0.20454545454545456, - "English,Spanish,Malay,Indonesian": 0.23295454545454544, - "English,Spanish,Malay,Vietnamese": 0.20454545454545456, - "English,Spanish,Chinese,Indonesian": 0.19318181818181818, - "English,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.21022727272727273, - "English,Malay,Chinese,Indonesian": 0.23295454545454544, - "English,Malay,Chinese,Vietnamese": 0.2215909090909091, - "English,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay,Chinese": 0.10795454545454546, - "Filipino,Spanish,Malay,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.11931818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.17045454545454544, - "Spanish,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.1875, - "Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363 + "Indonesian,English,Filipino,Spanish": 0.3806818181818182, + "Indonesian,English,Filipino,Chinese": 0.3465909090909091, + "Indonesian,English,Filipino,Malay": 0.3806818181818182, + "Indonesian,English,Filipino,Vietnamese": 0.35795454545454547, + "Indonesian,English,Spanish,Chinese": 0.3806818181818182, + "Indonesian,English,Spanish,Malay": 0.4375, + "Indonesian,English,Spanish,Vietnamese": 0.4318181818181818, + "Indonesian,English,Chinese,Malay": 0.3693181818181818, + "Indonesian,English,Chinese,Vietnamese": 0.3522727272727273, + "Indonesian,English,Malay,Vietnamese": 0.4147727272727273, + "Indonesian,Filipino,Spanish,Chinese": 0.32954545454545453, + "Indonesian,Filipino,Spanish,Malay": 0.3693181818181818, + "Indonesian,Filipino,Spanish,Vietnamese": 0.36363636363636365, + "Indonesian,Filipino,Chinese,Malay": 0.3465909090909091, + "Indonesian,Filipino,Chinese,Vietnamese": 0.3068181818181818, + "Indonesian,Filipino,Malay,Vietnamese": 0.3522727272727273, + "Indonesian,Spanish,Chinese,Malay": 0.36363636363636365, + "Indonesian,Spanish,Chinese,Vietnamese": 0.35795454545454547, + "Indonesian,Spanish,Malay,Vietnamese": 0.4147727272727273, + "Indonesian,Chinese,Malay,Vietnamese": 0.3465909090909091, + "English,Filipino,Spanish,Chinese": 0.36363636363636365, + "English,Filipino,Spanish,Malay": 0.3806818181818182, + "English,Filipino,Spanish,Vietnamese": 0.38636363636363635, + "English,Filipino,Chinese,Malay": 0.3409090909090909, + "English,Filipino,Chinese,Vietnamese": 0.32954545454545453, + "English,Filipino,Malay,Vietnamese": 0.36363636363636365, + "English,Spanish,Chinese,Malay": 0.3977272727272727, + "English,Spanish,Chinese,Vietnamese": 0.4034090909090909, + "English,Spanish,Malay,Vietnamese": 0.44886363636363635, + "English,Chinese,Malay,Vietnamese": 0.3522727272727273, + "Filipino,Spanish,Chinese,Malay": 0.3409090909090909, + "Filipino,Spanish,Chinese,Vietnamese": 0.3352272727272727, + "Filipino,Spanish,Malay,Vietnamese": 0.36363636363636365, + "Filipino,Chinese,Malay,Vietnamese": 0.3125, + "Spanish,Chinese,Malay,Vietnamese": 0.35795454545454547 + }, + "5_combine": { + "Indonesian,English,Filipino,Spanish,Chinese": 0.3068181818181818, + "Indonesian,English,Filipino,Spanish,Malay": 0.3352272727272727, + "Indonesian,English,Filipino,Spanish,Vietnamese": 0.32954545454545453, + "Indonesian,English,Filipino,Chinese,Malay": 0.3068181818181818, + "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2840909090909091, + "Indonesian,English,Filipino,Malay,Vietnamese": 0.32386363636363635, + "Indonesian,English,Spanish,Chinese,Malay": 0.3352272727272727, + "Indonesian,English,Spanish,Chinese,Vietnamese": 0.3352272727272727, + "Indonesian,English,Spanish,Malay,Vietnamese": 0.3806818181818182, + "Indonesian,English,Chinese,Malay,Vietnamese": 0.3125, + "Indonesian,Filipino,Spanish,Chinese,Malay": 0.3068181818181818, + "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.2897727272727273, + "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.32386363636363635, + "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2784090909090909, + "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.3181818181818182, + "English,Filipino,Spanish,Chinese,Malay": 0.3181818181818182, + "English,Filipino,Spanish,Chinese,Vietnamese": 0.3125, + "English,Filipino,Spanish,Malay,Vietnamese": 0.3409090909090909, + "English,Filipino,Chinese,Malay,Vietnamese": 0.2897727272727273, + "English,Spanish,Chinese,Malay,Vietnamese": 0.3409090909090909, + "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 + "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.2840909090909091, + "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.2727272727272727, + "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.30113636363636365, + "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.26136363636363635, + "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.30113636363636365, + "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2727272727272727, + "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2840909090909091 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 + "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182 } }, - "AC3_2": 0.3744467948728864, - "AC3_3": 0.2903115980744226, - "AC3_4": 0.2228924185802757, - "AC3_5": 0.1677094145237724, - "AC3_6": 0.12078462075368385, - "AC3_7": 0.07946884573897932 + "AC3_2": 0.529084611812433, + "AC3_3": 0.4601170822682755, + "AC3_4": 0.4134170884693576, + "AC3_5": 0.3796007728505842, + "AC3_6": 0.353312457140737, + "AC3_7": 0.3315873579089522 } }, "sg_eval": { "prompt_1": { - "accuracy": 0.33980582524271846 + "accuracy": 0.5922330097087378 }, "prompt_2": { - "accuracy": 0.39805825242718446 + "accuracy": 0.6116504854368932 }, "prompt_3": { - "accuracy": 0.3883495145631068 + "accuracy": 0.6310679611650486 }, "prompt_4": { - "accuracy": 0.3300970873786408 + "accuracy": 0.6796116504854369 }, "prompt_5": { - "accuracy": 0.36893203883495146 + "accuracy": 0.6310679611650486 } }, "cn_eval": { "prompt_1": { - "accuracy": 0.2857142857142857 + "accuracy": 0.34285714285714286 }, "prompt_2": { "accuracy": 0.3142857142857143 }, "prompt_3": { - "accuracy": 0.3333333333333333 + "accuracy": 0.34285714285714286 }, "prompt_4": { - "accuracy": 0.2761904761904762 + "accuracy": 0.3619047619047619 }, "prompt_5": { - "accuracy": 0.29523809523809524 + "accuracy": 0.3523809523809524 } }, "us_eval": { "prompt_1": { - "accuracy": 0.37383177570093457 + "accuracy": 0.6915887850467289 }, "prompt_2": { - "accuracy": 0.34579439252336447 + "accuracy": 0.7009345794392523 }, "prompt_3": { - "accuracy": 0.32710280373831774 + "accuracy": 0.7476635514018691 }, "prompt_4": { - "accuracy": 0.37383177570093457 + "accuracy": 0.7663551401869159 }, "prompt_5": { - "accuracy": 0.3644859813084112 + "accuracy": 0.719626168224299 } }, "ph_eval": { "prompt_1": { - "accuracy": 0.33, + "accuracy": 0.43, "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.5, - "history": 0.13333333333333333, + "brand": 0.5, + "demographics": 0.0, + "biology": 0.4, + "history": 0.4, "literature": 0.2, - "politics": 0.3, - "culture": 0.3, - "film": 0.3, - "law": 0.2, - "geography": 0.8 + "politics": 0.6, + "culture": 0.5, + "film": 0.5, + "law": 0.3, + "geography": 0.7 } }, "prompt_2": { - "accuracy": 0.27, + "accuracy": 0.43, "category_acc": { - "brand": 0.2, - "demographics": 0.0, + "brand": 0.4, + "demographics": 0.4, "biology": 0.5, - "history": 0.06666666666666667, + "history": 0.13333333333333333, "literature": 0.2, - "politics": 0.2, - "culture": 0.2, - "film": 0.3, - "law": 0.2, - "geography": 0.8 + "politics": 0.9, + "culture": 0.5, + "film": 0.5, + "law": 0.3, + "geography": 0.6 } }, "prompt_3": { - "accuracy": 0.28, + "accuracy": 0.44, "category_acc": { - "brand": 0.2, + "brand": 0.4, "demographics": 0.2, - "biology": 0.6, - "history": 0.06666666666666667, + "biology": 0.5, + "history": 0.26666666666666666, "literature": 0.2, - "politics": 0.3, - "culture": 0.2, - "film": 0.3, - "law": 0.2, + "politics": 0.7, + "culture": 0.6, + "film": 0.4, + "law": 0.5, "geography": 0.6 } }, "prompt_4": { - "accuracy": 0.33, + "accuracy": 0.47, "category_acc": { - "brand": 0.4, + "brand": 0.6, "demographics": 0.2, - "biology": 0.5, - "history": 0.13333333333333333, + "biology": 0.4, + "history": 0.3333333333333333, "literature": 0.3, - "politics": 0.3, - "culture": 0.3, - "film": 0.3, - "law": 0.2, + "politics": 0.8, + "culture": 0.4, + "film": 0.5, + "law": 0.4, "geography": 0.7 } }, "prompt_5": { - "accuracy": 0.3, + "accuracy": 0.42, "category_acc": { - "brand": 0.3, + "brand": 0.4, "demographics": 0.2, - "biology": 0.5, - "history": 0.13333333333333333, - "literature": 0.2, - "politics": 0.3, - "culture": 0.3, - "film": 0.3, - "law": 0.2, + "biology": 0.4, + "history": 0.26666666666666666, + "literature": 0.3, + "politics": 0.7, + "culture": 0.5, + "film": 0.4, + "law": 0.4, "geography": 0.6 } } }, "sing2eng": { "prompt_1": { - "bleu_score": 0.06840406331974451 + "bleu_score": 0.25083986933748753 }, "prompt_2": { - "bleu_score": 0.09065926215718136 + "bleu_score": 0.23347550471383358 }, "prompt_3": { - "bleu_score": 0.10138082117846277 + "bleu_score": 0.22918830819501845 }, "prompt_4": { - "bleu_score": 0.03699501331017958 + "bleu_score": 0.21613737202527156 }, "prompt_5": { - "bleu_score": 0.042702704198049515 + "bleu_score": 0.21084748299774206 } }, "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 + "prompt_1": { + "accuracy": 0.42626343547633355, + "category_acc": { + "History": 0.3413654618473896, + "Geography": 0.35306122448979593, + "Lampungic": 0.36054421768707484, + "Social science": 0.672787979966611, + "Balinese": 0.29936305732484075, + "Makassarese": 0.3817204301075269, + "Banjarese": 0.3541666666666667, + "Chemistry": 0.25401459854014596, + "Biology": 0.357396449704142, + "Science": 0.5964912280701754, + "Christian religion": 0.48258706467661694, + "Art": 0.519134775374376, + "Islam religion": 0.48933143669985774, + "Hindu religion": 0.44, + "Madurese": 0.29152542372881357, + "Sport": 0.40540540540540543, + "Indonesian language": 0.5168119551681195, + "Physics": 0.41414141414141414, + "Minangkabau culture": 0.35175879396984927, + "Dayak language": 0.3119266055045872, + "Sociology": 0.3326612903225806, + "Economy": 0.3319672131147541, + "Sundanese": 0.3197925669835782, + "Javanese": 0.31350806451612906, + "Civic education": 0.4678111587982833 + } + }, + "prompt_2": { + "accuracy": 0.4431537485813472, + "category_acc": { + "History": 0.3634538152610442, + "Geography": 0.35918367346938773, + "Lampungic": 0.40816326530612246, + "Social science": 0.7212020033388982, + "Balinese": 0.3078556263269639, + "Makassarese": 0.3978494623655914, + "Banjarese": 0.3541666666666667, + "Chemistry": 0.29635036496350364, + "Biology": 0.40236686390532544, + "Science": 0.6006191950464397, + "Christian religion": 0.4925373134328358, + "Art": 0.5224625623960066, + "Islam religion": 0.5120910384068279, + "Hindu religion": 0.43333333333333335, + "Madurese": 0.3016949152542373, + "Sport": 0.39864864864864863, + "Indonesian language": 0.5255292652552926, + "Physics": 0.4222222222222222, + "Minangkabau culture": 0.32160804020100503, + "Dayak language": 0.28440366972477066, + "Sociology": 0.37701612903225806, + "Economy": 0.35040983606557374, + "Sundanese": 0.33967156439066554, + "Javanese": 0.3074596774193548, + "Civic education": 0.5150214592274678 + } + }, + "prompt_3": { + "accuracy": 0.4425529074036985, + "category_acc": { + "History": 0.3714859437751004, + "Geography": 0.36122448979591837, + "Lampungic": 0.41496598639455784, + "Social science": 0.6978297161936561, + "Balinese": 0.29936305732484075, + "Makassarese": 0.42473118279569894, + "Banjarese": 0.3680555555555556, + "Chemistry": 0.2686131386861314, + "Biology": 0.3940828402366864, + "Science": 0.6057791537667698, + "Christian religion": 0.48756218905472637, + "Art": 0.5291181364392679, + "Islam religion": 0.5277382645803699, + "Hindu religion": 0.4666666666666667, + "Madurese": 0.2847457627118644, + "Sport": 0.44594594594594594, + "Indonesian language": 0.5214819427148194, + "Physics": 0.42424242424242425, + "Minangkabau culture": 0.3316582914572864, + "Dayak language": 0.29357798165137616, + "Sociology": 0.36088709677419356, + "Economy": 0.3831967213114754, + "Sundanese": 0.35868625756266204, + "Javanese": 0.3034274193548387, + "Civic education": 0.48497854077253216 + } + }, + "prompt_4": { + "accuracy": 0.449295680619534, + "category_acc": { + "History": 0.39357429718875503, + "Geography": 0.39183673469387753, + "Lampungic": 0.35374149659863946, + "Social science": 0.7178631051752922, + "Balinese": 0.3057324840764331, + "Makassarese": 0.3548387096774194, + "Banjarese": 0.3472222222222222, + "Chemistry": 0.24963503649635035, + "Biology": 0.41183431952662725, + "Science": 0.6191950464396285, + "Christian religion": 0.48258706467661694, + "Art": 0.5124792013311148, + "Islam religion": 0.5135135135135135, + "Hindu religion": 0.4866666666666667, + "Madurese": 0.2677966101694915, + "Sport": 0.4391891891891892, + "Indonesian language": 0.5317559153175592, + "Physics": 0.4202020202020202, + "Minangkabau culture": 0.3316582914572864, + "Dayak language": 0.3486238532110092, + "Sociology": 0.4012096774193548, + "Economy": 0.39959016393442626, + "Sundanese": 0.35868625756266204, + "Javanese": 0.3155241935483871, + "Civic education": 0.5092989985693849 + } + }, + "prompt_5": { + "accuracy": 0.43834701916015756, + "category_acc": { + "History": 0.3895582329317269, + "Geography": 0.35918367346938773, + "Lampungic": 0.4013605442176871, + "Social science": 0.67779632721202, + "Balinese": 0.32908704883227174, + "Makassarese": 0.3279569892473118, + "Banjarese": 0.3263888888888889, + "Chemistry": 0.27445255474452557, + "Biology": 0.38698224852071006, + "Science": 0.6160990712074303, + "Christian religion": 0.472636815920398, + "Art": 0.502495840266223, + "Islam religion": 0.5106685633001422, + "Hindu religion": 0.48, + "Madurese": 0.26101694915254237, + "Sport": 0.41216216216216217, + "Indonesian language": 0.5155666251556662, + "Physics": 0.41414141414141414, + "Minangkabau culture": 0.32663316582914576, + "Dayak language": 0.3486238532110092, + "Sociology": 0.3629032258064516, + "Economy": 0.39549180327868855, + "Sundanese": 0.3474503025064823, + "Javanese": 0.29435483870967744, + "Civic education": 0.5135908440629471 + } + } }, "flores_ind2eng": { "prompt_1": { - "bleu_score": 0.12298229499560595 + "bleu_score": 0.33349561081539403 }, "prompt_2": { - "bleu_score": 0.14134858567589187 + "bleu_score": 0.34924350312022184 }, "prompt_3": { - "bleu_score": 0.1476576698323315 + "bleu_score": 0.34642710360033174 }, "prompt_4": { - "bleu_score": 0.11582293658419313 + "bleu_score": 0.351335394546623 }, "prompt_5": { - "bleu_score": 0.07284862501775206 + "bleu_score": 0.33902183478630804 } }, "flores_vie2eng": { "prompt_1": { - "bleu_score": 0.10048619732657606 + "bleu_score": 0.28077818415977956 }, "prompt_2": { - "bleu_score": 0.10927584259296419 + "bleu_score": 0.29263312173317624 }, "prompt_3": { - "bleu_score": 0.11491357800375519 + "bleu_score": 0.29003004114772607 }, "prompt_4": { - "bleu_score": 0.09500480862112987 + "bleu_score": 0.2937442705094875 }, "prompt_5": { - "bleu_score": 0.06169877092755068 + "bleu_score": 0.28097633581136966 } }, "flores_zho2eng": { "prompt_1": { - "bleu_score": 0.09529581280338938 + "bleu_score": 0.20874149964374597 }, "prompt_2": { - "bleu_score": 0.09843116176005445 + "bleu_score": 0.2128290297582051 }, "prompt_3": { - "bleu_score": 0.10307455891305659 + "bleu_score": 0.21013614813602086 }, "prompt_4": { - "bleu_score": 0.08877167685556467 + "bleu_score": 0.2155940961935285 }, "prompt_5": { - "bleu_score": 0.06566543968657373 + "bleu_score": 0.20770689743003645 } }, "flores_zsm2eng": { "prompt_1": { - "bleu_score": 0.10412987965550156 + "bleu_score": 0.355017126213193 }, "prompt_2": { - "bleu_score": 0.11327584521912068 + "bleu_score": 0.3603933291111741 }, "prompt_3": { - "bleu_score": 0.11689632983800974 + "bleu_score": 0.3594937927110909 }, "prompt_4": { - "bleu_score": 0.08946953276015615 + "bleu_score": 0.3585242382452043 }, "prompt_5": { - "bleu_score": 0.04395681863383313 + "bleu_score": 0.3510843144050436 } }, "mmlu": { "prompt_1": { - "accuracy": 0.40373395565927656 + "accuracy": 0.6231038506417736 }, "prompt_2": { - "accuracy": 0.41656942823803966 + "accuracy": 0.6242707117852976 }, "prompt_3": { - "accuracy": 0.411901983663944 + "accuracy": 0.617269544924154 }, "prompt_4": { - "accuracy": 0.4072345390898483 + "accuracy": 0.6231038506417736 }, "prompt_5": { - "accuracy": 0.39556592765460913 + "accuracy": 0.6196032672112018 } }, "mmlu_full": { "prompt_1": { - "accuracy": 0.37690382552735074, + "accuracy": 0.6007150518412585, "category_acc": { - "high_school_european_history": 0.4573170731707317, - "business_ethics": 0.45454545454545453, - "clinical_knowledge": 0.4090909090909091, - "medical_genetics": 0.3939393939393939, - "high_school_us_history": 0.39408866995073893, - "high_school_physics": 0.26666666666666666, - "high_school_world_history": 0.4872881355932203, - "virology": 0.3696969696969697, - "high_school_microeconomics": 0.3291139240506329, - "econometrics": 0.23008849557522124, - "college_computer_science": 0.2828282828282828, - "high_school_biology": 0.42718446601941745, + "high_school_european_history": 0.7439024390243902, + "business_ethics": 0.5656565656565656, + "clinical_knowledge": 0.6704545454545454, + "medical_genetics": 0.6363636363636364, + "high_school_us_history": 0.8029556650246306, + "high_school_physics": 0.36666666666666664, + "high_school_world_history": 0.7966101694915254, + "virology": 0.4727272727272727, + "high_school_microeconomics": 0.6497890295358649, + "econometrics": 0.415929203539823, + "college_computer_science": 0.5151515151515151, + "high_school_biology": 0.7572815533980582, "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.28825622775800713, - "philosophy": 0.36451612903225805, - "professional_medicine": 0.31365313653136534, - "nutrition": 0.46229508196721314, - "global_facts": 0.2828282828282828, - "machine_learning": 0.42342342342342343, - "security_studies": 0.36065573770491804, - "public_relations": 0.43119266055045874, - "professional_psychology": 0.3829787234042553, - "prehistory": 0.4055727554179567, - "anatomy": 0.4253731343283582, - "human_sexuality": 0.4307692307692308, - "college_medicine": 0.3488372093023256, - "high_school_government_and_politics": 0.4322916666666667, - "college_chemistry": 0.1919191919191919, - "logical_fallacies": 0.4074074074074074, - "high_school_geography": 0.467005076142132, - "elementary_mathematics": 0.2625994694960212, - "human_aging": 0.4864864864864865, - "college_mathematics": 0.2727272727272727, - "high_school_psychology": 0.5073529411764706, - "formal_logic": 0.344, - "high_school_statistics": 0.2372093023255814, - "international_law": 0.4666666666666667, - "high_school_mathematics": 0.23048327137546468, - "high_school_computer_science": 0.3939393939393939, - "conceptual_physics": 0.3504273504273504, - "miscellaneous": 0.48081841432225064, - "high_school_chemistry": 0.27722772277227725, - "marketing": 0.592274678111588, - "professional_law": 0.3202870189171559, - "management": 0.43137254901960786, - "college_physics": 0.2376237623762376, - "jurisprudence": 0.40186915887850466, - "world_religions": 0.5117647058823529, - "sociology": 0.495, - "us_foreign_policy": 0.5252525252525253, - "high_school_macroeconomics": 0.37532133676092544, - "computer_security": 0.5151515151515151, - "moral_scenarios": 0.24049217002237136, - "moral_disputes": 0.4666666666666667, - "electrical_engineering": 0.4236111111111111, - "astronomy": 0.31788079470198677, - "college_biology": 0.40559440559440557 - } - }, - "prompt_2": { - "accuracy": 0.3919914193779049, - "category_acc": { - "high_school_european_history": 0.5121951219512195, - "business_ethics": 0.5252525252525253, - "clinical_knowledge": 0.4090909090909091, - "medical_genetics": 0.42424242424242425, - "high_school_us_history": 0.3793103448275862, - "high_school_physics": 0.30666666666666664, - "high_school_world_history": 0.5296610169491526, - "virology": 0.4121212121212121, - "high_school_microeconomics": 0.3291139240506329, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.31313131313131315, - "high_school_biology": 0.45307443365695793, - "abstract_algebra": 0.21212121212121213, - "professional_accounting": 0.2704626334519573, - "philosophy": 0.3935483870967742, - "professional_medicine": 0.3210332103321033, - "nutrition": 0.47540983606557374, - "global_facts": 0.3434343434343434, + "professional_accounting": 0.45195729537366547, + "philosophy": 0.6419354838709678, + "professional_medicine": 0.6568265682656826, + "nutrition": 0.6622950819672131, + "global_facts": 0.26262626262626265, "machine_learning": 0.36936936936936937, - "security_studies": 0.39344262295081966, - "public_relations": 0.4036697247706422, - "professional_psychology": 0.3878887070376432, - "prehistory": 0.42724458204334365, - "anatomy": 0.417910447761194, - "human_sexuality": 0.4307692307692308, - "college_medicine": 0.38372093023255816, - "high_school_government_and_politics": 0.484375, - "college_chemistry": 0.2828282828282828, - "logical_fallacies": 0.4012345679012346, - "high_school_geography": 0.467005076142132, - "elementary_mathematics": 0.2625994694960212, - "human_aging": 0.4594594594594595, - "college_mathematics": 0.3434343434343434, - "high_school_psychology": 0.5606617647058824, - "formal_logic": 0.368, - "high_school_statistics": 0.22790697674418606, - "international_law": 0.5, - "high_school_mathematics": 0.2379182156133829, - "high_school_computer_science": 0.3939393939393939, - "conceptual_physics": 0.38461538461538464, - "miscellaneous": 0.5230179028132992, - "high_school_chemistry": 0.3069306930693069, - "marketing": 0.5708154506437768, - "professional_law": 0.3313763861709067, - "management": 0.4411764705882353, - "college_physics": 0.2079207920792079, - "jurisprudence": 0.4485981308411215, - "world_religions": 0.5235294117647059, - "sociology": 0.525, - "us_foreign_policy": 0.5454545454545454, - "high_school_macroeconomics": 0.390745501285347, - "computer_security": 0.5151515151515151, - "moral_scenarios": 0.2348993288590604, - "moral_disputes": 0.43768115942028984, - "electrical_engineering": 0.4513888888888889, - "astronomy": 0.3509933774834437, - "college_biology": 0.4195804195804196 + "security_studies": 0.7090163934426229, + "public_relations": 0.6146788990825688, + "professional_psychology": 0.6202945990180033, + "prehistory": 0.6873065015479877, + "anatomy": 0.5597014925373134, + "human_sexuality": 0.6461538461538462, + "college_medicine": 0.622093023255814, + "high_school_government_and_politics": 0.8541666666666666, + "college_chemistry": 0.5050505050505051, + "logical_fallacies": 0.7345679012345679, + "high_school_geography": 0.7868020304568528, + "elementary_mathematics": 0.5994694960212201, + "human_aging": 0.6486486486486487, + "college_mathematics": 0.30303030303030304, + "high_school_psychology": 0.8106617647058824, + "formal_logic": 0.432, + "high_school_statistics": 0.42790697674418604, + "international_law": 0.7833333333333333, + "high_school_mathematics": 0.275092936802974, + "high_school_computer_science": 0.6868686868686869, + "conceptual_physics": 0.5299145299145299, + "miscellaneous": 0.8081841432225064, + "high_school_chemistry": 0.42574257425742573, + "marketing": 0.8412017167381974, + "professional_law": 0.4435746901500326, + "management": 0.7745098039215687, + "college_physics": 0.3465346534653465, + "jurisprudence": 0.7383177570093458, + "world_religions": 0.8235294117647058, + "sociology": 0.815, + "us_foreign_policy": 0.8383838383838383, + "high_school_macroeconomics": 0.6143958868894601, + "computer_security": 0.6666666666666666, + "moral_scenarios": 0.3187919463087248, + "moral_disputes": 0.663768115942029, + "electrical_engineering": 0.5416666666666666, + "astronomy": 0.6490066225165563, + "college_biology": 0.7482517482517482 } }, - "prompt_3": { - "accuracy": 0.3948516267429389, + "prompt_2": { + "accuracy": 0.6010010725777619, "category_acc": { - "high_school_european_history": 0.5121951219512195, - "business_ethics": 0.494949494949495, - "clinical_knowledge": 0.42045454545454547, - "medical_genetics": 0.43434343434343436, - "high_school_us_history": 0.37438423645320196, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.5466101694915254, - "virology": 0.4, - "high_school_microeconomics": 0.3291139240506329, - "econometrics": 0.23008849557522124, - "college_computer_science": 0.30303030303030304, - "high_school_biology": 0.4563106796116505, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.29537366548042704, - "philosophy": 0.38387096774193546, - "professional_medicine": 0.31365313653136534, - "nutrition": 0.4786885245901639, - "global_facts": 0.32323232323232326, - "machine_learning": 0.3783783783783784, - "security_studies": 0.4139344262295082, - "public_relations": 0.4036697247706422, - "professional_psychology": 0.38461538461538464, - "prehistory": 0.42105263157894735, - "anatomy": 0.41044776119402987, - "human_sexuality": 0.4307692307692308, - "college_medicine": 0.38372093023255816, - "high_school_government_and_politics": 0.46875, - "college_chemistry": 0.2727272727272727, - "logical_fallacies": 0.3950617283950617, - "high_school_geography": 0.4873096446700508, - "elementary_mathematics": 0.2891246684350133, - "human_aging": 0.46846846846846846, - "college_mathematics": 0.3434343434343434, - "high_school_psychology": 0.5551470588235294, - "formal_logic": 0.36, - "high_school_statistics": 0.25116279069767444, - "international_law": 0.525, - "high_school_mathematics": 0.241635687732342, - "high_school_computer_science": 0.45454545454545453, - "conceptual_physics": 0.39316239316239315, - "miscellaneous": 0.5191815856777494, - "high_school_chemistry": 0.30198019801980197, - "marketing": 0.5836909871244635, - "professional_law": 0.3287671232876712, - "management": 0.46078431372549017, - "college_physics": 0.2079207920792079, - "jurisprudence": 0.4672897196261682, - "world_religions": 0.5176470588235295, - "sociology": 0.515, - "us_foreign_policy": 0.5555555555555556, - "high_school_macroeconomics": 0.41645244215938304, - "computer_security": 0.5050505050505051, - "moral_scenarios": 0.24384787472035793, - "moral_disputes": 0.43768115942028984, - "electrical_engineering": 0.4444444444444444, - "astronomy": 0.3509933774834437, - "college_biology": 0.43356643356643354 + "high_school_european_history": 0.725609756097561, + "business_ethics": 0.5656565656565656, + "clinical_knowledge": 0.6742424242424242, + "medical_genetics": 0.6565656565656566, + "high_school_us_history": 0.8078817733990148, + "high_school_physics": 0.32, + "high_school_world_history": 0.809322033898305, + "virology": 0.4666666666666667, + "high_school_microeconomics": 0.6540084388185654, + "econometrics": 0.4247787610619469, + "college_computer_science": 0.48484848484848486, + "high_school_biology": 0.7637540453074434, + "abstract_algebra": 0.2222222222222222, + "professional_accounting": 0.41637010676156583, + "philosophy": 0.6516129032258065, + "professional_medicine": 0.6014760147601476, + "nutrition": 0.6524590163934426, + "global_facts": 0.31313131313131315, + "machine_learning": 0.43243243243243246, + "security_studies": 0.6844262295081968, + "public_relations": 0.5871559633027523, + "professional_psychology": 0.6121112929623568, + "prehistory": 0.6996904024767802, + "anatomy": 0.5597014925373134, + "human_sexuality": 0.6461538461538462, + "college_medicine": 0.6162790697674418, + "high_school_government_and_politics": 0.8489583333333334, + "college_chemistry": 0.48484848484848486, + "logical_fallacies": 0.7345679012345679, + "high_school_geography": 0.7969543147208121, + "elementary_mathematics": 0.5862068965517241, + "human_aging": 0.6621621621621622, + "college_mathematics": 0.3333333333333333, + "high_school_psychology": 0.8198529411764706, + "formal_logic": 0.4, + "high_school_statistics": 0.4232558139534884, + "international_law": 0.7833333333333333, + "high_school_mathematics": 0.275092936802974, + "high_school_computer_science": 0.6565656565656566, + "conceptual_physics": 0.5256410256410257, + "miscellaneous": 0.8107416879795396, + "high_school_chemistry": 0.44554455445544555, + "marketing": 0.8454935622317596, + "professional_law": 0.44031311154598823, + "management": 0.803921568627451, + "college_physics": 0.37623762376237624, + "jurisprudence": 0.7383177570093458, + "world_religions": 0.8058823529411765, + "sociology": 0.815, + "us_foreign_policy": 0.8080808080808081, + "high_school_macroeconomics": 0.6426735218508998, + "computer_security": 0.7070707070707071, + "moral_scenarios": 0.3434004474272931, + "moral_disputes": 0.663768115942029, + "electrical_engineering": 0.5763888888888888, + "astronomy": 0.6291390728476821, + "college_biology": 0.7132867132867133 + } + }, + "prompt_3": { + "accuracy": 0.6034322488380408, + "category_acc": { + "high_school_european_history": 0.7378048780487805, + "business_ethics": 0.5757575757575758, + "clinical_knowledge": 0.6818181818181818, + "medical_genetics": 0.6363636363636364, + "high_school_us_history": 0.7980295566502463, + "high_school_physics": 0.35333333333333333, + "high_school_world_history": 0.8050847457627118, + "virology": 0.4727272727272727, + "high_school_microeconomics": 0.6455696202531646, + "econometrics": 0.39823008849557523, + "college_computer_science": 0.48484848484848486, + "high_school_biology": 0.7475728155339806, + "abstract_algebra": 0.25252525252525254, + "professional_accounting": 0.4306049822064057, + "philosophy": 0.6709677419354839, + "professional_medicine": 0.6273062730627307, + "nutrition": 0.6622950819672131, + "global_facts": 0.3838383838383838, + "machine_learning": 0.43243243243243246, + "security_studies": 0.7172131147540983, + "public_relations": 0.6146788990825688, + "professional_psychology": 0.6219312602291326, + "prehistory": 0.6811145510835913, + "anatomy": 0.5373134328358209, + "human_sexuality": 0.6538461538461539, + "college_medicine": 0.6104651162790697, + "high_school_government_and_politics": 0.859375, + "college_chemistry": 0.42424242424242425, + "logical_fallacies": 0.7222222222222222, + "high_school_geography": 0.7868020304568528, + "elementary_mathematics": 0.5888594164456233, + "human_aging": 0.6576576576576577, + "college_mathematics": 0.41414141414141414, + "high_school_psychology": 0.8180147058823529, + "formal_logic": 0.4, + "high_school_statistics": 0.4558139534883721, + "international_law": 0.7916666666666666, + "high_school_mathematics": 0.2862453531598513, + "high_school_computer_science": 0.6868686868686869, + "conceptual_physics": 0.5213675213675214, + "miscellaneous": 0.8158567774936062, + "high_school_chemistry": 0.4504950495049505, + "marketing": 0.8412017167381974, + "professional_law": 0.441617742987606, + "management": 0.7941176470588235, + "college_physics": 0.3564356435643564, + "jurisprudence": 0.7663551401869159, + "world_religions": 0.8058823529411765, + "sociology": 0.805, + "us_foreign_policy": 0.8484848484848485, + "high_school_macroeconomics": 0.6401028277634961, + "computer_security": 0.6666666666666666, + "moral_scenarios": 0.32997762863534674, + "moral_disputes": 0.6608695652173913, + "electrical_engineering": 0.5486111111111112, + "astronomy": 0.6357615894039735, + "college_biology": 0.7202797202797203 } }, "prompt_4": { - "accuracy": 0.38584197354308186, + "accuracy": 0.6067929924919556, "category_acc": { - "high_school_european_history": 0.4817073170731707, - "business_ethics": 0.45454545454545453, - "clinical_knowledge": 0.4015151515151515, - "medical_genetics": 0.3838383838383838, - "high_school_us_history": 0.3891625615763547, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.5296610169491526, - "virology": 0.37575757575757573, - "high_school_microeconomics": 0.32489451476793246, - "econometrics": 0.23008849557522124, - "college_computer_science": 0.2828282828282828, - "high_school_biology": 0.4174757281553398, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.30604982206405695, - "philosophy": 0.36774193548387096, - "professional_medicine": 0.30996309963099633, - "nutrition": 0.46229508196721314, - "global_facts": 0.29292929292929293, - "machine_learning": 0.36936936936936937, - "security_studies": 0.4139344262295082, - "public_relations": 0.3944954128440367, - "professional_psychology": 0.38461538461538464, - "prehistory": 0.4086687306501548, - "anatomy": 0.43283582089552236, - "human_sexuality": 0.4076923076923077, - "college_medicine": 0.3488372093023256, - "high_school_government_and_politics": 0.4270833333333333, - "college_chemistry": 0.20202020202020202, - "logical_fallacies": 0.41358024691358025, - "high_school_geography": 0.48223350253807107, - "elementary_mathematics": 0.3050397877984085, - "human_aging": 0.5, - "college_mathematics": 0.32323232323232326, - "high_school_psychology": 0.5055147058823529, - "formal_logic": 0.368, - "high_school_statistics": 0.25116279069767444, - "international_law": 0.49166666666666664, - "high_school_mathematics": 0.24535315985130113, - "high_school_computer_science": 0.41414141414141414, - "conceptual_physics": 0.36324786324786323, - "miscellaneous": 0.5, - "high_school_chemistry": 0.297029702970297, - "marketing": 0.6137339055793991, - "professional_law": 0.33659491193737767, - "management": 0.4215686274509804, - "college_physics": 0.24752475247524752, - "jurisprudence": 0.4766355140186916, - "world_religions": 0.5117647058823529, - "sociology": 0.475, - "us_foreign_policy": 0.5353535353535354, - "high_school_macroeconomics": 0.37532133676092544, - "computer_security": 0.5353535353535354, - "moral_scenarios": 0.24049217002237136, - "moral_disputes": 0.4782608695652174, - "electrical_engineering": 0.4444444444444444, - "astronomy": 0.304635761589404, - "college_biology": 0.40559440559440557 + "high_school_european_history": 0.7317073170731707, + "business_ethics": 0.5858585858585859, + "clinical_knowledge": 0.678030303030303, + "medical_genetics": 0.6363636363636364, + "high_school_us_history": 0.8029556650246306, + "high_school_physics": 0.32666666666666666, + "high_school_world_history": 0.7923728813559322, + "virology": 0.4666666666666667, + "high_school_microeconomics": 0.6666666666666666, + "econometrics": 0.40707964601769914, + "college_computer_science": 0.5454545454545454, + "high_school_biology": 0.7637540453074434, + "abstract_algebra": 0.29292929292929293, + "professional_accounting": 0.45195729537366547, + "philosophy": 0.6645161290322581, + "professional_medicine": 0.6199261992619927, + "nutrition": 0.6655737704918033, + "global_facts": 0.3838383838383838, + "machine_learning": 0.3963963963963964, + "security_studies": 0.6967213114754098, + "public_relations": 0.6330275229357798, + "professional_psychology": 0.6219312602291326, + "prehistory": 0.6965944272445821, + "anatomy": 0.5746268656716418, + "human_sexuality": 0.6923076923076923, + "college_medicine": 0.6162790697674418, + "high_school_government_and_politics": 0.8489583333333334, + "college_chemistry": 0.5151515151515151, + "logical_fallacies": 0.7098765432098766, + "high_school_geography": 0.7969543147208121, + "elementary_mathematics": 0.5888594164456233, + "human_aging": 0.6621621621621622, + "college_mathematics": 0.35353535353535354, + "high_school_psychology": 0.8161764705882353, + "formal_logic": 0.392, + "high_school_statistics": 0.48372093023255813, + "international_law": 0.7833333333333333, + "high_school_mathematics": 0.3048327137546468, + "high_school_computer_science": 0.696969696969697, + "conceptual_physics": 0.5213675213675214, + "miscellaneous": 0.819693094629156, + "high_school_chemistry": 0.400990099009901, + "marketing": 0.8540772532188842, + "professional_law": 0.4461839530332681, + "management": 0.7745098039215687, + "college_physics": 0.36633663366336633, + "jurisprudence": 0.7289719626168224, + "world_religions": 0.7941176470588235, + "sociology": 0.82, + "us_foreign_policy": 0.8181818181818182, + "high_school_macroeconomics": 0.6452442159383034, + "computer_security": 0.6767676767676768, + "moral_scenarios": 0.3288590604026846, + "moral_disputes": 0.6840579710144927, + "electrical_engineering": 0.5486111111111112, + "astronomy": 0.6490066225165563, + "college_biology": 0.7412587412587412 } }, "prompt_5": { - "accuracy": 0.38226671433678944, + "accuracy": 0.5982838755809796, "category_acc": { - "high_school_european_history": 0.4817073170731707, - "business_ethics": 0.46464646464646464, - "clinical_knowledge": 0.4128787878787879, - "medical_genetics": 0.35353535353535354, - "high_school_us_history": 0.39408866995073893, - "high_school_physics": 0.31333333333333335, - "high_school_world_history": 0.5084745762711864, - "virology": 0.3878787878787879, - "high_school_microeconomics": 0.3291139240506329, - "econometrics": 0.24778761061946902, - "college_computer_science": 0.2727272727272727, - "high_school_biology": 0.41423948220064727, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.30604982206405695, - "philosophy": 0.3967741935483871, - "professional_medicine": 0.3062730627306273, - "nutrition": 0.46885245901639344, - "global_facts": 0.32323232323232326, - "machine_learning": 0.36036036036036034, - "security_studies": 0.39344262295081966, - "public_relations": 0.3853211009174312, - "professional_psychology": 0.3911620294599018, - "prehistory": 0.3993808049535604, - "anatomy": 0.4253731343283582, - "human_sexuality": 0.4307692307692308, - "college_medicine": 0.37790697674418605, - "high_school_government_and_politics": 0.4270833333333333, - "college_chemistry": 0.1919191919191919, - "logical_fallacies": 0.4074074074074074, - "high_school_geography": 0.4720812182741117, - "elementary_mathematics": 0.2891246684350133, - "human_aging": 0.481981981981982, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.5036764705882353, - "formal_logic": 0.328, - "high_school_statistics": 0.2558139534883721, - "international_law": 0.49166666666666664, - "high_school_mathematics": 0.24907063197026022, - "high_school_computer_science": 0.3939393939393939, - "conceptual_physics": 0.3547008547008547, - "miscellaneous": 0.4846547314578005, - "high_school_chemistry": 0.3118811881188119, - "marketing": 0.5579399141630901, - "professional_law": 0.33529028049575993, - "management": 0.39215686274509803, - "college_physics": 0.24752475247524752, - "jurisprudence": 0.45794392523364486, - "world_religions": 0.5294117647058824, - "sociology": 0.48, - "us_foreign_policy": 0.5252525252525253, - "high_school_macroeconomics": 0.37789203084832906, - "computer_security": 0.5454545454545454, - "moral_scenarios": 0.22706935123042504, - "moral_disputes": 0.45507246376811594, - "electrical_engineering": 0.4444444444444444, - "astronomy": 0.304635761589404, - "college_biology": 0.3776223776223776 + "high_school_european_history": 0.7073170731707317, + "business_ethics": 0.5454545454545454, + "clinical_knowledge": 0.6553030303030303, + "medical_genetics": 0.6464646464646465, + "high_school_us_history": 0.7635467980295566, + "high_school_physics": 0.36666666666666664, + "high_school_world_history": 0.8050847457627118, + "virology": 0.48484848484848486, + "high_school_microeconomics": 0.6497890295358649, + "econometrics": 0.36283185840707965, + "college_computer_science": 0.5353535353535354, + "high_school_biology": 0.7411003236245954, + "abstract_algebra": 0.2727272727272727, + "professional_accounting": 0.4092526690391459, + "philosophy": 0.6483870967741936, + "professional_medicine": 0.5756457564575646, + "nutrition": 0.6557377049180327, + "global_facts": 0.3939393939393939, + "machine_learning": 0.3963963963963964, + "security_studies": 0.7049180327868853, + "public_relations": 0.5871559633027523, + "professional_psychology": 0.6235679214402619, + "prehistory": 0.6873065015479877, + "anatomy": 0.5373134328358209, + "human_sexuality": 0.6615384615384615, + "college_medicine": 0.6337209302325582, + "high_school_government_and_politics": 0.859375, + "college_chemistry": 0.47474747474747475, + "logical_fallacies": 0.7283950617283951, + "high_school_geography": 0.7918781725888325, + "elementary_mathematics": 0.583554376657825, + "human_aging": 0.6441441441441441, + "college_mathematics": 0.3434343434343434, + "high_school_psychology": 0.8180147058823529, + "formal_logic": 0.376, + "high_school_statistics": 0.4604651162790698, + "international_law": 0.7833333333333333, + "high_school_mathematics": 0.2788104089219331, + "high_school_computer_science": 0.6767676767676768, + "conceptual_physics": 0.5213675213675214, + "miscellaneous": 0.8107416879795396, + "high_school_chemistry": 0.44554455445544555, + "marketing": 0.8326180257510729, + "professional_law": 0.4409654272667971, + "management": 0.7745098039215687, + "college_physics": 0.40594059405940597, + "jurisprudence": 0.7102803738317757, + "world_religions": 0.8352941176470589, + "sociology": 0.825, + "us_foreign_policy": 0.8383838383838383, + "high_school_macroeconomics": 0.6195372750642674, + "computer_security": 0.6666666666666666, + "moral_scenarios": 0.3232662192393736, + "moral_disputes": 0.663768115942029, + "electrical_engineering": 0.5486111111111112, + "astronomy": 0.6490066225165563, + "college_biology": 0.7062937062937062 } } }, "c_eval": { "prompt_1": { - "accuracy": 0.3395245170876672 + "accuracy": 0.42867756315007427 }, "prompt_2": { - "accuracy": 0.3558692421991085 + "accuracy": 0.43684992570579495 }, "prompt_3": { - "accuracy": 0.3536404160475483 + "accuracy": 0.462109955423477 }, "prompt_4": { - "accuracy": 0.32392273402674593 + "accuracy": 0.4182763744427935 }, "prompt_5": { - "accuracy": 0.34175334323922735 + "accuracy": 0.424962852897474 } }, "c_eval_full": { "prompt_1": { - "accuracy": 0.3001245330012453, + "accuracy": 0.4396014943960149, "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.2916666666666667, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.3333333333333333, - "college_physics": 0.2916666666666667, + "computer_network": 0.4583333333333333, + "operating_system": 0.5833333333333334, + "computer_architecture": 0.5, + "college_programming": 0.6666666666666666, + "college_physics": 0.25, "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.3793103448275862, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.21739130434782608, + "discrete_mathematics": 0.38095238095238093, + "electrical_engineer": 0.47619047619047616, + "metrology_engineer": 0.3448275862068966, "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.25, - "high_school_chemistry": 0.041666666666666664, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.24, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.3333333333333333, - "business_administration": 0.2894736842105263, - "marxism": 0.5, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.4411764705882353, - "teacher_qualification": 0.40816326530612246, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.34615384615384615, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.3333333333333333, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.4, - "middle_school_history": 0.2222222222222222, - "civil_servant": 0.28846153846153844, - "sports_science": 0.2916666666666667, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.25, + "high_school_physics": 0.2916666666666667, + "high_school_chemistry": 0.2916666666666667, + "high_school_biology": 0.20833333333333334, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.6923076923076923, + "middle_school_physics": 0.5, + "middle_school_chemistry": 0.48, + "veterinary_medicine": 0.35714285714285715, + "college_economics": 0.48333333333333334, + "business_administration": 0.47368421052631576, + "marxism": 0.4583333333333333, + "mao_zedong_thought": 0.5862068965517241, + "education_science": 0.5, + "teacher_qualification": 0.673469387755102, + "high_school_politics": 0.5, + "high_school_geography": 0.5416666666666666, + "middle_school_politics": 0.7307692307692307, + "middle_school_geography": 0.5294117647058824, + "modern_chinese_history": 0.2857142857142857, + "ideological_and_moral_cultivation": 0.6666666666666666, + "logic": 0.4074074074074074, + "law": 0.3448275862068966, + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.42105263157894735, + "professional_tour_guide": 0.35294117647058826, + "legal_professional": 0.25, + "high_school_chinese": 0.375, + "high_school_history": 0.6, + "middle_school_history": 0.5185185185185185, + "civil_servant": 0.4230769230769231, + "sports_science": 0.4166666666666667, + "plant_protection": 0.5925925925925926, + "basic_medicine": 0.5, "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.24074074074074073, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.2962962962962963, - "physician": 0.4074074074074074 + "urban_and_rural_planner": 0.49019607843137253, + "accountant": 0.3888888888888889, + "fire_engineer": 0.16666666666666666, + "environmental_impact_assessment_engineer": 0.5833333333333334, + "tax_accountant": 0.3148148148148148, + "physician": 0.46296296296296297 } }, "prompt_2": { - "accuracy": 0.32254047322540474, + "accuracy": 0.44956413449564137, "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.25, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.30952380952380953, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.27586206896551724, + "computer_network": 0.4583333333333333, + "operating_system": 0.5833333333333334, + "computer_architecture": 0.46153846153846156, + "college_programming": 0.5238095238095238, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.2413793103448276, "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.2857142857142857, + "probability_and_statistics": 0.2608695652173913, + "discrete_mathematics": 0.5238095238095238, + "electrical_engineer": 0.5476190476190477, "metrology_engineer": 0.3448275862068966, "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.125, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.041666666666666664, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.35, - "business_administration": 0.34210526315789475, - "marxism": 0.5, - "mao_zedong_thought": 0.4482758620689655, - "education_science": 0.5, - "teacher_qualification": 0.42857142857142855, - "high_school_politics": 0.25, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.46153846153846156, + "high_school_physics": 0.25, + "high_school_chemistry": 0.375, + "high_school_biology": 0.4166666666666667, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.7692307692307693, + "middle_school_physics": 0.5416666666666666, + "middle_school_chemistry": 0.48, + "veterinary_medicine": 0.39285714285714285, + "college_economics": 0.4166666666666667, + "business_administration": 0.39473684210526316, + "marxism": 0.4166666666666667, + "mao_zedong_thought": 0.5517241379310345, + "education_science": 0.4411764705882353, + "teacher_qualification": 0.673469387755102, + "high_school_politics": 0.5833333333333334, + "high_school_geography": 0.5416666666666666, + "middle_school_politics": 0.6923076923076923, "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.37037037037037035, + "modern_chinese_history": 0.35714285714285715, + "ideological_and_moral_cultivation": 0.625, + "logic": 0.5185185185185185, "law": 0.2413793103448276, - "chinese_language_and_literature": 0.32142857142857145, + "chinese_language_and_literature": 0.5, "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.44, - "middle_school_history": 0.25925925925925924, - "civil_servant": 0.28846153846153844, - "sports_science": 0.25, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.25, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.35294117647058826, - "accountant": 0.2962962962962963, + "professional_tour_guide": 0.35294117647058826, + "legal_professional": 0.25, + "high_school_chinese": 0.4166666666666667, + "high_school_history": 0.56, + "middle_school_history": 0.5555555555555556, + "civil_servant": 0.4423076923076923, + "sports_science": 0.375, + "plant_protection": 0.5555555555555556, + "basic_medicine": 0.5833333333333334, + "clinical_medicine": 0.4444444444444444, + "urban_and_rural_planner": 0.5686274509803921, + "accountant": 0.3888888888888889, "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.2222222222222222, - "tax_accountant": 0.37037037037037035, - "physician": 0.42592592592592593 + "environmental_impact_assessment_engineer": 0.5833333333333334, + "tax_accountant": 0.48148148148148145, + "physician": 0.4444444444444444 } }, "prompt_3": { - "accuracy": 0.3275217932752179, + "accuracy": 0.4396014943960149, "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.25, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.30952380952380953, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.125, + "computer_network": 0.4166666666666667, + "operating_system": 0.625, + "computer_architecture": 0.6538461538461539, + "college_programming": 0.5714285714285714, + "college_physics": 0.375, + "college_chemistry": 0.27586206896551724, + "advanced_mathematics": 0.2916666666666667, + "probability_and_statistics": 0.13043478260869565, + "discrete_mathematics": 0.23809523809523808, + "electrical_engineer": 0.47619047619047616, + "metrology_engineer": 0.3103448275862069, + "high_school_mathematics": 0.13043478260869565, + "high_school_physics": 0.375, + "high_school_chemistry": 0.3333333333333333, "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.125, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.24, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.35, - "business_administration": 0.34210526315789475, - "marxism": 0.5, - "mao_zedong_thought": 0.41379310344827586, + "middle_school_mathematics": 0.2916666666666667, + "middle_school_biology": 0.6153846153846154, + "middle_school_physics": 0.4166666666666667, + "middle_school_chemistry": 0.44, + "veterinary_medicine": 0.35714285714285715, + "college_economics": 0.4166666666666667, + "business_administration": 0.47368421052631576, + "marxism": 0.4583333333333333, + "mao_zedong_thought": 0.6206896551724138, "education_science": 0.5, - "teacher_qualification": 0.4489795918367347, - "high_school_politics": 0.25, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.5, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.20833333333333334, - "logic": 0.37037037037037035, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.25, - "high_school_chinese": 0.25, - "high_school_history": 0.4, - "middle_school_history": 0.25925925925925924, - "civil_servant": 0.28846153846153844, - "sports_science": 0.25, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.2962962962962963, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.19444444444444445, + "teacher_qualification": 0.7142857142857143, + "high_school_politics": 0.4583333333333333, + "high_school_geography": 0.5416666666666666, + "middle_school_politics": 0.6538461538461539, + "middle_school_geography": 0.47058823529411764, + "modern_chinese_history": 0.39285714285714285, + "ideological_and_moral_cultivation": 0.6666666666666666, + "logic": 0.3333333333333333, + "law": 0.41379310344827586, + "chinese_language_and_literature": 0.39285714285714285, + "art_studies": 0.34210526315789475, + "professional_tour_guide": 0.38235294117647056, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.375, + "high_school_history": 0.6, + "middle_school_history": 0.4444444444444444, + "civil_servant": 0.5, + "sports_science": 0.3333333333333333, + "plant_protection": 0.5925925925925926, + "basic_medicine": 0.5416666666666666, + "clinical_medicine": 0.37037037037037035, + "urban_and_rural_planner": 0.5098039215686274, + "accountant": 0.3148148148148148, + "fire_engineer": 0.3333333333333333, + "environmental_impact_assessment_engineer": 0.5833333333333334, "tax_accountant": 0.4074074074074074, - "physician": 0.4074074074074074 + "physician": 0.4444444444444444 } }, "prompt_4": { - "accuracy": 0.29763387297633875, + "accuracy": 0.4202988792029888, "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.25, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.3333333333333333, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.14285714285714285, + "computer_network": 0.5416666666666666, + "operating_system": 0.5833333333333334, + "computer_architecture": 0.5, + "college_programming": 0.47619047619047616, + "college_physics": 0.125, + "college_chemistry": 0.3103448275862069, + "advanced_mathematics": 0.25, + "probability_and_statistics": 0.30434782608695654, + "discrete_mathematics": 0.5238095238095238, "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.125, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.08333333333333333, - "middle_school_biology": 0.38461538461538464, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.2, + "metrology_engineer": 0.3793103448275862, + "high_school_mathematics": 0.08695652173913043, + "high_school_physics": 0.4583333333333333, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.20833333333333334, + "middle_school_mathematics": 0.25, + "middle_school_biology": 0.6923076923076923, + "middle_school_physics": 0.5, + "middle_school_chemistry": 0.4, "veterinary_medicine": 0.25, - "college_economics": 0.26666666666666666, - "business_administration": 0.34210526315789475, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.4411764705882353, - "teacher_qualification": 0.42857142857142855, - "high_school_politics": 0.25, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.375, + "college_economics": 0.4, + "business_administration": 0.42105263157894735, + "marxism": 0.5, + "mao_zedong_thought": 0.5172413793103449, + "education_science": 0.5, + "teacher_qualification": 0.6530612244897959, + "high_school_politics": 0.4583333333333333, + "high_school_geography": 0.5833333333333334, + "middle_school_politics": 0.5769230769230769, + "middle_school_geography": 0.35294117647058826, + "modern_chinese_history": 0.39285714285714285, + "ideological_and_moral_cultivation": 0.5, "logic": 0.4074074074074074, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.4, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.28846153846153844, - "sports_science": 0.2916666666666667, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.25, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.2037037037037037, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.2777777777777778, - "physician": 0.4074074074074074 + "law": 0.3103448275862069, + "chinese_language_and_literature": 0.35714285714285715, + "art_studies": 0.42105263157894735, + "professional_tour_guide": 0.35294117647058826, + "legal_professional": 0.32142857142857145, + "high_school_chinese": 0.2916666666666667, + "high_school_history": 0.52, + "middle_school_history": 0.5925925925925926, + "civil_servant": 0.40384615384615385, + "sports_science": 0.375, + "plant_protection": 0.5185185185185185, + "basic_medicine": 0.6666666666666666, + "clinical_medicine": 0.48148148148148145, + "urban_and_rural_planner": 0.5294117647058824, + "accountant": 0.3333333333333333, + "fire_engineer": 0.19444444444444445, + "environmental_impact_assessment_engineer": 0.4722222222222222, + "tax_accountant": 0.3888888888888889, + "physician": 0.4444444444444444 } }, "prompt_5": { - "accuracy": 0.3038605230386052, + "accuracy": 0.42092154420921546, "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.2916666666666667, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.3333333333333333, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.125, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.125, - "middle_school_biology": 0.3076923076923077, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.24, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.2833333333333333, - "business_administration": 0.3157894736842105, - "marxism": 0.5, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.5, - "teacher_qualification": 0.40816326530612246, - "high_school_politics": 0.25, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.25, - "ideological_and_moral_cultivation": 0.375, - "logic": 0.3333333333333333, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.25, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.4, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.25, - "sports_science": 0.2916666666666667, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.2777777777777778, + "computer_network": 0.5, + "operating_system": 0.625, + "computer_architecture": 0.46153846153846156, + "college_programming": 0.5714285714285714, + "college_physics": 0.20833333333333334, + "college_chemistry": 0.2413793103448276, + "advanced_mathematics": 0.20833333333333334, + "probability_and_statistics": 0.13043478260869565, + "discrete_mathematics": 0.2857142857142857, + "electrical_engineer": 0.42857142857142855, + "metrology_engineer": 0.5172413793103449, + "high_school_mathematics": 0.34782608695652173, + "high_school_physics": 0.3333333333333333, + "high_school_chemistry": 0.3333333333333333, + "high_school_biology": 0.375, + "middle_school_mathematics": 0.375, + "middle_school_biology": 0.5769230769230769, + "middle_school_physics": 0.4166666666666667, + "middle_school_chemistry": 0.36, + "veterinary_medicine": 0.35714285714285715, + "college_economics": 0.38333333333333336, + "business_administration": 0.39473684210526316, + "marxism": 0.375, + "mao_zedong_thought": 0.6206896551724138, + "education_science": 0.47058823529411764, + "teacher_qualification": 0.673469387755102, + "high_school_politics": 0.5833333333333334, + "high_school_geography": 0.5, + "middle_school_politics": 0.6923076923076923, + "middle_school_geography": 0.47058823529411764, + "modern_chinese_history": 0.35714285714285715, + "ideological_and_moral_cultivation": 0.7083333333333334, + "logic": 0.4074074074074074, + "law": 0.4827586206896552, + "chinese_language_and_literature": 0.42857142857142855, + "art_studies": 0.3157894736842105, + "professional_tour_guide": 0.4411764705882353, + "legal_professional": 0.2857142857142857, + "high_school_chinese": 0.375, + "high_school_history": 0.36, + "middle_school_history": 0.5185185185185185, + "civil_servant": 0.38461538461538464, + "sports_science": 0.375, + "plant_protection": 0.6296296296296297, + "basic_medicine": 0.4583333333333333, + "clinical_medicine": 0.48148148148148145, + "urban_and_rural_planner": 0.39215686274509803, + "accountant": 0.3148148148148148, "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.2222222222222222, - "tax_accountant": 0.3148148148148148, - "physician": 0.37037037037037035 + "environmental_impact_assessment_engineer": 0.4444444444444444, + "tax_accountant": 0.3333333333333333, + "physician": 0.42592592592592593 } } }, "cmmlu": { "prompt_1": { - "accuracy": 0.3118279569892473 + "accuracy": 0.4731182795698925 }, "prompt_2": { - "accuracy": 0.3010752688172043 + "accuracy": 0.5053763440860215 }, "prompt_3": { - "accuracy": 0.3225806451612903 + "accuracy": 0.4982078853046595 }, "prompt_4": { - "accuracy": 0.2974910394265233 + "accuracy": 0.4659498207885305 }, "prompt_5": { - "accuracy": 0.3046594982078853 + "accuracy": 0.46236559139784944 } }, "cmmlu_full": { "prompt_1": { - "accuracy": 0.3238646175099292, + "accuracy": 0.442756000690727, "category_acc": { - "agronomy": 0.30177514792899407, - "anatomy": 0.28378378378378377, - "ancient_chinese": 0.2804878048780488, - "arts": 0.30625, - "astronomy": 0.3090909090909091, - "business_ethics": 0.3684210526315789, - "chinese_civil_service_exam": 0.3375, - "chinese_driving_rule": 0.4351145038167939, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.308411214953271, - "chinese_history": 0.30959752321981426, - "chinese_literature": 0.30392156862745096, - "chinese_teacher_qualification": 0.329608938547486, - "clinical_knowledge": 0.3037974683544304, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.40186915887850466, - "college_engineering_hydrology": 0.3490566037735849, - "college_law": 0.26851851851851855, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.3113207547169811, - "college_medicine": 0.28205128205128205, - "computer_science": 0.36764705882352944, - "computer_security": 0.3567251461988304, - "conceptual_physics": 0.2925170068027211, - "construction_project_management": 0.3669064748201439, - "economics": 0.33962264150943394, - "education": 0.34355828220858897, - "electrical_engineering": 0.3488372093023256, + "agronomy": 0.4319526627218935, + "anatomy": 0.2972972972972973, + "ancient_chinese": 0.2926829268292683, + "arts": 0.5, + "astronomy": 0.3939393939393939, + "business_ethics": 0.5119617224880383, + "chinese_civil_service_exam": 0.3625, + "chinese_driving_rule": 0.5725190839694656, + "chinese_food_culture": 0.33088235294117646, + "chinese_foreign_policy": 0.4766355140186916, + "chinese_history": 0.43653250773993807, + "chinese_literature": 0.3137254901960784, + "chinese_teacher_qualification": 0.553072625698324, + "clinical_knowledge": 0.33755274261603374, + "college_actuarial_science": 0.2358490566037736, + "college_education": 0.5420560747663551, + "college_engineering_hydrology": 0.5094339622641509, + "college_law": 0.25925925925925924, + "college_mathematics": 0.2571428571428571, + "college_medical_statistics": 0.4056603773584906, + "college_medicine": 0.34798534798534797, + "computer_science": 0.5784313725490197, + "computer_security": 0.5730994152046783, + "conceptual_physics": 0.4421768707482993, + "construction_project_management": 0.4028776978417266, + "economics": 0.5660377358490566, + "education": 0.5030674846625767, + "electrical_engineering": 0.4476744186046512, "elementary_chinese": 0.2857142857142857, - "elementary_commonsense": 0.31313131313131315, - "elementary_information_and_technology": 0.42857142857142855, - "elementary_mathematics": 0.25217391304347825, - "ethnology": 0.28888888888888886, - "food_science": 0.35664335664335667, - "genetics": 0.3068181818181818, - "global_facts": 0.28187919463087246, - "high_school_biology": 0.30177514792899407, + "elementary_commonsense": 0.45454545454545453, + "elementary_information_and_technology": 0.6974789915966386, + "elementary_mathematics": 0.391304347826087, + "ethnology": 0.4, + "food_science": 0.4755244755244755, + "genetics": 0.39204545454545453, + "global_facts": 0.5369127516778524, + "high_school_biology": 0.3431952662721893, "high_school_chemistry": 0.25, - "high_school_geography": 0.2627118644067797, - "high_school_mathematics": 0.25609756097560976, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.34265734265734266, - "human_sexuality": 0.3333333333333333, - "international_law": 0.3027027027027027, - "journalism": 0.3546511627906977, - "jurisprudence": 0.35036496350364965, - "legal_and_moral_basis": 0.46261682242990654, - "logical": 0.2845528455284553, - "machine_learning": 0.22950819672131148, - "management": 0.38095238095238093, - "marketing": 0.40555555555555556, - "marxist_theory": 0.3492063492063492, - "modern_chinese": 0.19827586206896552, - "nutrition": 0.36551724137931035, - "philosophy": 0.3523809523809524, - "professional_accounting": 0.3314285714285714, - "professional_law": 0.3222748815165877, - "professional_medicine": 0.2632978723404255, - "professional_psychology": 0.35344827586206895, - "public_relations": 0.3735632183908046, - "security_study": 0.34814814814814815, - "sociology": 0.3584070796460177, - "sports_science": 0.3515151515151515, - "traditional_chinese_medicine": 0.2918918918918919, - "virology": 0.33136094674556216, - "world_history": 0.35403726708074534, - "world_religions": 0.31875 + "high_school_geography": 0.3728813559322034, + "high_school_mathematics": 0.2926829268292683, + "high_school_physics": 0.38181818181818183, + "high_school_politics": 0.34965034965034963, + "human_sexuality": 0.5873015873015873, + "international_law": 0.3621621621621622, + "journalism": 0.5232558139534884, + "jurisprudence": 0.46958637469586373, + "legal_and_moral_basis": 0.705607476635514, + "logical": 0.42276422764227645, + "machine_learning": 0.4426229508196721, + "management": 0.5857142857142857, + "marketing": 0.5833333333333334, + "marxist_theory": 0.5185185185185185, + "modern_chinese": 0.3448275862068966, + "nutrition": 0.4896551724137931, + "philosophy": 0.41904761904761906, + "professional_accounting": 0.4742857142857143, + "professional_law": 0.36018957345971564, + "professional_medicine": 0.324468085106383, + "professional_psychology": 0.46551724137931033, + "public_relations": 0.47701149425287354, + "security_study": 0.5333333333333333, + "sociology": 0.5309734513274337, + "sports_science": 0.48484848484848486, + "traditional_chinese_medicine": 0.372972972972973, + "virology": 0.48520710059171596, + "world_history": 0.45962732919254656, + "world_religions": 0.54375 } }, "prompt_2": { - "accuracy": 0.3350889310999827, + "accuracy": 0.44620963564151267, "category_acc": { - "agronomy": 0.33136094674556216, - "anatomy": 0.27702702702702703, + "agronomy": 0.4556213017751479, + "anatomy": 0.30405405405405406, "ancient_chinese": 0.27439024390243905, - "arts": 0.3375, - "astronomy": 0.3151515151515151, - "business_ethics": 0.36363636363636365, + "arts": 0.50625, + "astronomy": 0.3393939393939394, + "business_ethics": 0.5454545454545454, "chinese_civil_service_exam": 0.33125, - "chinese_driving_rule": 0.5038167938931297, - "chinese_food_culture": 0.22058823529411764, - "chinese_foreign_policy": 0.35514018691588783, - "chinese_history": 0.33436532507739936, - "chinese_literature": 0.3088235294117647, - "chinese_teacher_qualification": 0.3854748603351955, - "clinical_knowledge": 0.2742616033755274, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.38317757009345793, - "college_engineering_hydrology": 0.3867924528301887, - "college_law": 0.25925925925925924, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.3584905660377358, - "college_medicine": 0.315018315018315, - "computer_science": 0.3284313725490196, - "computer_security": 0.32748538011695905, - "conceptual_physics": 0.3197278911564626, - "construction_project_management": 0.37410071942446044, - "economics": 0.34591194968553457, - "education": 0.3987730061349693, - "electrical_engineering": 0.36046511627906974, - "elementary_chinese": 0.27380952380952384, - "elementary_commonsense": 0.3484848484848485, - "elementary_information_and_technology": 0.4579831932773109, - "elementary_mathematics": 0.25217391304347825, - "ethnology": 0.2814814814814815, - "food_science": 0.38461538461538464, - "genetics": 0.3181818181818182, - "global_facts": 0.2483221476510067, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.3135593220338983, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.2909090909090909, - "high_school_politics": 0.32867132867132864, - "human_sexuality": 0.3968253968253968, - "international_law": 0.2756756756756757, - "journalism": 0.37209302325581395, - "jurisprudence": 0.36253041362530414, - "legal_and_moral_basis": 0.5373831775700935, - "logical": 0.3008130081300813, - "machine_learning": 0.21311475409836064, - "management": 0.35714285714285715, - "marketing": 0.4, - "marxist_theory": 0.36507936507936506, - "modern_chinese": 0.22413793103448276, - "nutrition": 0.3724137931034483, - "philosophy": 0.38095238095238093, - "professional_accounting": 0.3314285714285714, - "professional_law": 0.33175355450236965, - "professional_medicine": 0.2553191489361702, - "professional_psychology": 0.34051724137931033, - "public_relations": 0.3850574712643678, - "security_study": 0.35555555555555557, - "sociology": 0.36283185840707965, - "sports_science": 0.41818181818181815, - "traditional_chinese_medicine": 0.3027027027027027, - "virology": 0.378698224852071, - "world_history": 0.36645962732919257, - "world_religions": 0.34375 + "chinese_driving_rule": 0.6183206106870229, + "chinese_food_culture": 0.3161764705882353, + "chinese_foreign_policy": 0.4485981308411215, + "chinese_history": 0.44891640866873067, + "chinese_literature": 0.3333333333333333, + "chinese_teacher_qualification": 0.5307262569832403, + "clinical_knowledge": 0.350210970464135, + "college_actuarial_science": 0.25471698113207547, + "college_education": 0.5887850467289719, + "college_engineering_hydrology": 0.5, + "college_law": 0.32407407407407407, + "college_mathematics": 0.26666666666666666, + "college_medical_statistics": 0.29245283018867924, + "college_medicine": 0.3772893772893773, + "computer_science": 0.5735294117647058, + "computer_security": 0.6081871345029239, + "conceptual_physics": 0.38095238095238093, + "construction_project_management": 0.4028776978417266, + "economics": 0.559748427672956, + "education": 0.50920245398773, + "electrical_engineering": 0.45930232558139533, + "elementary_chinese": 0.29365079365079366, + "elementary_commonsense": 0.4494949494949495, + "elementary_information_and_technology": 0.6974789915966386, + "elementary_mathematics": 0.3652173913043478, + "ethnology": 0.4074074074074074, + "food_science": 0.4755244755244755, + "genetics": 0.3977272727272727, + "global_facts": 0.5302013422818792, + "high_school_biology": 0.3136094674556213, + "high_school_chemistry": 0.2878787878787879, + "high_school_geography": 0.3305084745762712, + "high_school_mathematics": 0.2621951219512195, + "high_school_physics": 0.43636363636363634, + "high_school_politics": 0.42657342657342656, + "human_sexuality": 0.5634920634920635, + "international_law": 0.3891891891891892, + "journalism": 0.4883720930232558, + "jurisprudence": 0.46958637469586373, + "legal_and_moral_basis": 0.7523364485981309, + "logical": 0.4634146341463415, + "machine_learning": 0.45901639344262296, + "management": 0.5619047619047619, + "marketing": 0.6055555555555555, + "marxist_theory": 0.5079365079365079, + "modern_chinese": 0.35344827586206895, + "nutrition": 0.503448275862069, + "philosophy": 0.45714285714285713, + "professional_accounting": 0.5085714285714286, + "professional_law": 0.3696682464454976, + "professional_medicine": 0.31648936170212766, + "professional_psychology": 0.4698275862068966, + "public_relations": 0.4942528735632184, + "security_study": 0.5259259259259259, + "sociology": 0.4778761061946903, + "sports_science": 0.48484848484848486, + "traditional_chinese_medicine": 0.3783783783783784, + "virology": 0.5266272189349113, + "world_history": 0.5031055900621118, + "world_religions": 0.525 } }, "prompt_3": { - "accuracy": 0.33517527197375235, - "category_acc": { - "agronomy": 0.33136094674556216, - "anatomy": 0.28378378378378377, - "ancient_chinese": 0.2682926829268293, - "arts": 0.3375, - "astronomy": 0.3090909090909091, - "business_ethics": 0.37799043062200954, - "chinese_civil_service_exam": 0.3125, - "chinese_driving_rule": 0.48854961832061067, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.35514018691588783, - "chinese_history": 0.32507739938080493, - "chinese_literature": 0.3088235294117647, - "chinese_teacher_qualification": 0.3854748603351955, - "clinical_knowledge": 0.27848101265822783, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.40186915887850466, - "college_engineering_hydrology": 0.37735849056603776, - "college_law": 0.28703703703703703, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.3490566037735849, - "college_medicine": 0.30036630036630035, - "computer_science": 0.3333333333333333, - "computer_security": 0.3333333333333333, - "conceptual_physics": 0.3197278911564626, - "construction_project_management": 0.38848920863309355, - "economics": 0.34591194968553457, - "education": 0.4049079754601227, - "electrical_engineering": 0.37209302325581395, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.3282828282828283, - "elementary_information_and_technology": 0.47478991596638653, - "elementary_mathematics": 0.2391304347826087, - "ethnology": 0.25925925925925924, - "food_science": 0.34965034965034963, - "genetics": 0.32954545454545453, - "global_facts": 0.2550335570469799, - "high_school_biology": 0.28402366863905326, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.3050847457627119, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.2909090909090909, - "high_school_politics": 0.34965034965034963, - "human_sexuality": 0.3888888888888889, - "international_law": 0.2864864864864865, - "journalism": 0.36046511627906974, - "jurisprudence": 0.3746958637469586, - "legal_and_moral_basis": 0.5373831775700935, - "logical": 0.2926829268292683, - "machine_learning": 0.20491803278688525, - "management": 0.35714285714285715, - "marketing": 0.37222222222222223, - "marxist_theory": 0.36507936507936506, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.36551724137931035, - "philosophy": 0.3238095238095238, - "professional_accounting": 0.3485714285714286, - "professional_law": 0.33175355450236965, - "professional_medicine": 0.2632978723404255, - "professional_psychology": 0.34051724137931033, - "public_relations": 0.41954022988505746, - "security_study": 0.32592592592592595, - "sociology": 0.37168141592920356, - "sports_science": 0.4121212121212121, - "traditional_chinese_medicine": 0.31891891891891894, - "virology": 0.3727810650887574, - "world_history": 0.36645962732919257, - "world_religions": 0.33125 - } - }, - "prompt_4": { - "accuracy": 0.31453980314280783, + "accuracy": 0.439129683992402, "category_acc": { - "agronomy": 0.3076923076923077, + "agronomy": 0.4319526627218935, "anatomy": 0.3108108108108108, - "ancient_chinese": 0.31097560975609756, - "arts": 0.31875, - "astronomy": 0.32727272727272727, - "business_ethics": 0.33014354066985646, - "chinese_civil_service_exam": 0.325, - "chinese_driving_rule": 0.37404580152671757, - "chinese_food_culture": 0.23529411764705882, - "chinese_foreign_policy": 0.3177570093457944, - "chinese_history": 0.3126934984520124, - "chinese_literature": 0.3088235294117647, - "chinese_teacher_qualification": 0.3575418994413408, - "clinical_knowledge": 0.27848101265822783, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.40186915887850466, - "college_engineering_hydrology": 0.3113207547169811, - "college_law": 0.25925925925925924, - "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.3018867924528302, - "college_medicine": 0.25274725274725274, - "computer_science": 0.31862745098039214, - "computer_security": 0.3333333333333333, - "conceptual_physics": 0.29931972789115646, - "construction_project_management": 0.31654676258992803, - "economics": 0.3333333333333333, - "education": 0.37423312883435583, - "electrical_engineering": 0.3430232558139535, - "elementary_chinese": 0.25793650793650796, - "elementary_commonsense": 0.29797979797979796, - "elementary_information_and_technology": 0.3865546218487395, - "elementary_mathematics": 0.26521739130434785, - "ethnology": 0.32592592592592595, - "food_science": 0.3006993006993007, - "genetics": 0.29545454545454547, - "global_facts": 0.2483221476510067, - "high_school_biology": 0.26627218934911245, + "ancient_chinese": 0.2682926829268293, + "arts": 0.4875, + "astronomy": 0.34545454545454546, + "business_ethics": 0.5550239234449761, + "chinese_civil_service_exam": 0.36875, + "chinese_driving_rule": 0.5725190839694656, + "chinese_food_culture": 0.3235294117647059, + "chinese_foreign_policy": 0.5420560747663551, + "chinese_history": 0.43962848297213625, + "chinese_literature": 0.30392156862745096, + "chinese_teacher_qualification": 0.4972067039106145, + "clinical_knowledge": 0.34177215189873417, + "college_actuarial_science": 0.29245283018867924, + "college_education": 0.5794392523364486, + "college_engineering_hydrology": 0.49056603773584906, + "college_law": 0.37037037037037035, + "college_mathematics": 0.22857142857142856, + "college_medical_statistics": 0.3113207547169811, + "college_medicine": 0.3772893772893773, + "computer_science": 0.5784313725490197, + "computer_security": 0.5789473684210527, + "conceptual_physics": 0.3877551020408163, + "construction_project_management": 0.43884892086330934, + "economics": 0.5534591194968553, + "education": 0.48466257668711654, + "electrical_engineering": 0.45930232558139533, + "elementary_chinese": 0.2976190476190476, + "elementary_commonsense": 0.42424242424242425, + "elementary_information_and_technology": 0.6974789915966386, + "elementary_mathematics": 0.3521739130434783, + "ethnology": 0.4222222222222222, + "food_science": 0.44755244755244755, + "genetics": 0.39204545454545453, + "global_facts": 0.5100671140939598, + "high_school_biology": 0.35502958579881655, "high_school_chemistry": 0.21212121212121213, - "high_school_geography": 0.2966101694915254, - "high_school_mathematics": 0.2682926829268293, - "high_school_physics": 0.3, - "high_school_politics": 0.3146853146853147, - "human_sexuality": 0.3253968253968254, - "international_law": 0.31891891891891894, - "journalism": 0.3023255813953488, - "jurisprudence": 0.340632603406326, - "legal_and_moral_basis": 0.4485981308411215, - "logical": 0.3089430894308943, - "machine_learning": 0.1721311475409836, - "management": 0.35714285714285715, - "marketing": 0.37777777777777777, - "marxist_theory": 0.3386243386243386, - "modern_chinese": 0.25, - "nutrition": 0.31724137931034485, - "philosophy": 0.38095238095238093, - "professional_accounting": 0.32, - "professional_law": 0.2985781990521327, - "professional_medicine": 0.26861702127659576, - "professional_psychology": 0.36637931034482757, - "public_relations": 0.3793103448275862, - "security_study": 0.32592592592592595, - "sociology": 0.336283185840708, - "sports_science": 0.3151515151515151, - "traditional_chinese_medicine": 0.2918918918918919, - "virology": 0.3136094674556213, - "world_history": 0.3416149068322981, - "world_religions": 0.3375 + "high_school_geography": 0.3983050847457627, + "high_school_mathematics": 0.25609756097560976, + "high_school_physics": 0.3181818181818182, + "high_school_politics": 0.38461538461538464, + "human_sexuality": 0.5396825396825397, + "international_law": 0.3945945945945946, + "journalism": 0.4883720930232558, + "jurisprudence": 0.4720194647201946, + "legal_and_moral_basis": 0.6682242990654206, + "logical": 0.45528455284552843, + "machine_learning": 0.4426229508196721, + "management": 0.5761904761904761, + "marketing": 0.5722222222222222, + "marxist_theory": 0.5238095238095238, + "modern_chinese": 0.29310344827586204, + "nutrition": 0.5103448275862069, + "philosophy": 0.5238095238095238, + "professional_accounting": 0.45714285714285713, + "professional_law": 0.35071090047393366, + "professional_medicine": 0.3058510638297872, + "professional_psychology": 0.46120689655172414, + "public_relations": 0.5172413793103449, + "security_study": 0.5185185185185185, + "sociology": 0.5176991150442478, + "sports_science": 0.4121212121212121, + "traditional_chinese_medicine": 0.33513513513513515, + "virology": 0.5088757396449705, + "world_history": 0.546583850931677, + "world_religions": 0.5125 + } + }, + "prompt_4": { + "accuracy": 0.4350716629252288, + "category_acc": { + "agronomy": 0.4260355029585799, + "anatomy": 0.30405405405405406, + "ancient_chinese": 0.2926829268292683, + "arts": 0.4625, + "astronomy": 0.296969696969697, + "business_ethics": 0.5215311004784688, + "chinese_civil_service_exam": 0.35625, + "chinese_driving_rule": 0.5648854961832062, + "chinese_food_culture": 0.35294117647058826, + "chinese_foreign_policy": 0.514018691588785, + "chinese_history": 0.42724458204334365, + "chinese_literature": 0.3382352941176471, + "chinese_teacher_qualification": 0.5083798882681564, + "clinical_knowledge": 0.3080168776371308, + "college_actuarial_science": 0.20754716981132076, + "college_education": 0.5046728971962616, + "college_engineering_hydrology": 0.5, + "college_law": 0.3611111111111111, + "college_mathematics": 0.2857142857142857, + "college_medical_statistics": 0.330188679245283, + "college_medicine": 0.3516483516483517, + "computer_science": 0.5343137254901961, + "computer_security": 0.5614035087719298, + "conceptual_physics": 0.41496598639455784, + "construction_project_management": 0.45323741007194246, + "economics": 0.5094339622641509, + "education": 0.4662576687116564, + "electrical_engineering": 0.43023255813953487, + "elementary_chinese": 0.2976190476190476, + "elementary_commonsense": 0.4292929292929293, + "elementary_information_and_technology": 0.680672268907563, + "elementary_mathematics": 0.39565217391304347, + "ethnology": 0.4, + "food_science": 0.5104895104895105, + "genetics": 0.38636363636363635, + "global_facts": 0.5033557046979866, + "high_school_biology": 0.33136094674556216, + "high_school_chemistry": 0.24242424242424243, + "high_school_geography": 0.3898305084745763, + "high_school_mathematics": 0.3170731707317073, + "high_school_physics": 0.3181818181818182, + "high_school_politics": 0.40559440559440557, + "human_sexuality": 0.5714285714285714, + "international_law": 0.3783783783783784, + "journalism": 0.5, + "jurisprudence": 0.49148418491484186, + "legal_and_moral_basis": 0.719626168224299, + "logical": 0.44715447154471544, + "machine_learning": 0.4672131147540984, + "management": 0.5619047619047619, + "marketing": 0.5944444444444444, + "marxist_theory": 0.5343915343915344, + "modern_chinese": 0.27586206896551724, + "nutrition": 0.4689655172413793, + "philosophy": 0.44761904761904764, + "professional_accounting": 0.5028571428571429, + "professional_law": 0.3459715639810427, + "professional_medicine": 0.31648936170212766, + "professional_psychology": 0.4482758620689655, + "public_relations": 0.4827586206896552, + "security_study": 0.45185185185185184, + "sociology": 0.4690265486725664, + "sports_science": 0.4484848484848485, + "traditional_chinese_medicine": 0.34594594594594597, + "virology": 0.4911242603550296, + "world_history": 0.4906832298136646, + "world_religions": 0.5125 } }, "prompt_5": { - "accuracy": 0.32844068381972025, + "accuracy": 0.4328268002072181, "category_acc": { - "agronomy": 0.3076923076923077, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.27439024390243905, - "arts": 0.3125, - "astronomy": 0.3393939393939394, - "business_ethics": 0.3684210526315789, - "chinese_civil_service_exam": 0.2875, - "chinese_driving_rule": 0.4122137404580153, - "chinese_food_culture": 0.2647058823529412, - "chinese_foreign_policy": 0.3177570093457944, - "chinese_history": 0.3188854489164087, - "chinese_literature": 0.31862745098039214, - "chinese_teacher_qualification": 0.35195530726256985, - "clinical_knowledge": 0.28270042194092826, + "agronomy": 0.40236686390532544, + "anatomy": 0.30405405405405406, + "ancient_chinese": 0.2682926829268293, + "arts": 0.53125, + "astronomy": 0.30303030303030304, + "business_ethics": 0.5358851674641149, + "chinese_civil_service_exam": 0.33125, + "chinese_driving_rule": 0.5801526717557252, + "chinese_food_culture": 0.27941176470588236, + "chinese_foreign_policy": 0.4392523364485981, + "chinese_history": 0.4674922600619195, + "chinese_literature": 0.28431372549019607, + "chinese_teacher_qualification": 0.5139664804469274, + "clinical_knowledge": 0.3333333333333333, "college_actuarial_science": 0.22641509433962265, - "college_education": 0.38317757009345793, - "college_engineering_hydrology": 0.3584905660377358, - "college_law": 0.26851851851851855, + "college_education": 0.5887850467289719, + "college_engineering_hydrology": 0.42452830188679247, + "college_law": 0.35185185185185186, "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.37735849056603776, - "college_medicine": 0.315018315018315, - "computer_science": 0.35294117647058826, - "computer_security": 0.3684210526315789, - "conceptual_physics": 0.2925170068027211, - "construction_project_management": 0.37410071942446044, - "economics": 0.33962264150943394, - "education": 0.37423312883435583, - "electrical_engineering": 0.32558139534883723, - "elementary_chinese": 0.2976190476190476, - "elementary_commonsense": 0.30303030303030304, - "elementary_information_and_technology": 0.42016806722689076, - "elementary_mathematics": 0.26521739130434785, - "ethnology": 0.3037037037037037, - "food_science": 0.3356643356643357, - "genetics": 0.3181818181818182, - "global_facts": 0.2550335570469799, - "high_school_biology": 0.3254437869822485, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.2966101694915254, - "high_school_mathematics": 0.2865853658536585, - "high_school_physics": 0.3, - "high_school_politics": 0.34265734265734266, - "human_sexuality": 0.35714285714285715, - "international_law": 0.3027027027027027, - "journalism": 0.3546511627906977, - "jurisprudence": 0.35279805352798055, - "legal_and_moral_basis": 0.4672897196261682, - "logical": 0.3008130081300813, - "machine_learning": 0.2540983606557377, - "management": 0.3619047619047619, - "marketing": 0.4166666666666667, - "marxist_theory": 0.3492063492063492, - "modern_chinese": 0.21551724137931033, - "nutrition": 0.35172413793103446, - "philosophy": 0.34285714285714286, - "professional_accounting": 0.33714285714285713, - "professional_law": 0.3127962085308057, - "professional_medicine": 0.2712765957446808, - "professional_psychology": 0.34051724137931033, - "public_relations": 0.3793103448275862, - "security_study": 0.34814814814814815, - "sociology": 0.35398230088495575, - "sports_science": 0.3939393939393939, - "traditional_chinese_medicine": 0.2972972972972973, - "virology": 0.3431952662721893, - "world_history": 0.36645962732919257, - "world_religions": 0.3375 + "college_medical_statistics": 0.3490566037735849, + "college_medicine": 0.38095238095238093, + "computer_science": 0.5686274509803921, + "computer_security": 0.5555555555555556, + "conceptual_physics": 0.3469387755102041, + "construction_project_management": 0.3669064748201439, + "economics": 0.5220125786163522, + "education": 0.50920245398773, + "electrical_engineering": 0.45930232558139533, + "elementary_chinese": 0.26587301587301587, + "elementary_commonsense": 0.4090909090909091, + "elementary_information_and_technology": 0.6932773109243697, + "elementary_mathematics": 0.32608695652173914, + "ethnology": 0.4222222222222222, + "food_science": 0.4125874125874126, + "genetics": 0.4034090909090909, + "global_facts": 0.47651006711409394, + "high_school_biology": 0.3136094674556213, + "high_school_chemistry": 0.3181818181818182, + "high_school_geography": 0.4067796610169492, + "high_school_mathematics": 0.3048780487804878, + "high_school_physics": 0.43636363636363634, + "high_school_politics": 0.38461538461538464, + "human_sexuality": 0.5714285714285714, + "international_law": 0.3567567567567568, + "journalism": 0.4476744186046512, + "jurisprudence": 0.45498783454987834, + "legal_and_moral_basis": 0.705607476635514, + "logical": 0.42276422764227645, + "machine_learning": 0.45901639344262296, + "management": 0.5952380952380952, + "marketing": 0.6111111111111112, + "marxist_theory": 0.5185185185185185, + "modern_chinese": 0.3017241379310345, + "nutrition": 0.5103448275862069, + "philosophy": 0.44761904761904764, + "professional_accounting": 0.44, + "professional_law": 0.33175355450236965, + "professional_medicine": 0.27393617021276595, + "professional_psychology": 0.4956896551724138, + "public_relations": 0.47126436781609193, + "security_study": 0.5703703703703704, + "sociology": 0.5088495575221239, + "sports_science": 0.509090909090909, + "traditional_chinese_medicine": 0.32972972972972975, + "virology": 0.4970414201183432, + "world_history": 0.4782608695652174, + "world_religions": 0.5125 } } }, "zbench": { "prompt_1": { - "accuracy": 0.21212121212121213 + "accuracy": 0.3333333333333333 }, "prompt_2": { - "accuracy": 0.24242424242424243 + "accuracy": 0.3333333333333333 }, "prompt_3": { - "accuracy": 0.2727272727272727 + "accuracy": 0.3333333333333333 }, "prompt_4": { - "accuracy": 0.12121212121212122 + "accuracy": 0.3939393939393939 }, "prompt_5": { - "accuracy": 0.21212121212121213 + "accuracy": 0.36363636363636365 } }, "ind_emotion": { "prompt_1": { - "accuracy": 0.19318181818181818 + "accuracy": 0.6227272727272727 }, "prompt_2": { - "accuracy": 0.3477272727272727 + "accuracy": 0.6431818181818182 }, "prompt_3": { - "accuracy": 0.32954545454545453 + "accuracy": 0.6090909090909091 }, "prompt_4": { - "accuracy": 0.31363636363636366 + "accuracy": 0.5863636363636363 }, "prompt_5": { - "accuracy": 0.29545454545454547 + "accuracy": 0.6272727272727273 } }, "ocnli": { "prompt_1": { - "accuracy": 0.3142372881355932 + "accuracy": 0.5227118644067796 }, "prompt_2": { - "accuracy": 0.3284745762711864 + "accuracy": 0.49254237288135594 }, "prompt_3": { - "accuracy": 0.32440677966101694 + "accuracy": 0.5033898305084745 }, "prompt_4": { - "accuracy": 0.33016949152542374 + "accuracy": 0.5271186440677966 }, "prompt_5": { - "accuracy": 0.336271186440678 + "accuracy": 0.5393220338983051 } }, "c3": { "prompt_1": { - "accuracy": 0.5287958115183246 + "accuracy": 0.8163799551234107 }, "prompt_2": { - "accuracy": 0.5755422587883321 + "accuracy": 0.8156320119670905 }, "prompt_3": { - "accuracy": 0.5766641735228123 + "accuracy": 0.8148840688107704 }, "prompt_4": { - "accuracy": 0.5373971578160059 + "accuracy": 0.8156320119670905 }, "prompt_5": { - "accuracy": 0.5347793567688855 + "accuracy": 0.8178758414360509 } }, "dream": { "prompt_1": { - "accuracy": 0.636452719255267 + "accuracy": 0.8721215090641842 }, "prompt_2": { - "accuracy": 0.6359627633512984 + "accuracy": 0.8652621264086232 }, "prompt_3": { - "accuracy": 0.6295933365997061 + "accuracy": 0.8931896129348359 }, "prompt_4": { - "accuracy": 0.6114649681528662 + "accuracy": 0.8775110240078393 }, "prompt_5": { - "accuracy": 0.6393924546790789 + "accuracy": 0.9039686428221461 } }, "samsum": { "prompt_1": { - "rouge1": 0.3114676254323794, - "rouge2": 0.09828457453388299, - "rougeL": 0.22791145657880788, - "avg_rouge": 0.2125545521816901 + "rouge1": 0.3379569725563289, + "rouge2": 0.1297505746252268, + "rougeL": 0.2583327984045818, + "avg_rouge": 0.2420134485287125 }, "prompt_2": { - "rouge1": 0.3147789520616352, - "rouge2": 0.10216816870583775, - "rougeL": 0.2287159994553074, - "avg_rouge": 0.21522104007426013 + "rouge1": 0.36693774963873615, + "rouge2": 0.13995152326152985, + "rougeL": 0.28220133592060764, + "avg_rouge": 0.26303020294029117 }, "prompt_3": { - "rouge1": 0.32008112809488953, - "rouge2": 0.10125139172804017, - "rougeL": 0.230546722031336, - "avg_rouge": 0.21729308061808858 + "rouge1": 0.35187138311903504, + "rouge2": 0.13105884717466826, + "rougeL": 0.2673882223081265, + "avg_rouge": 0.2501061508672766 }, "prompt_4": { - "rouge1": 0.3148972898941042, - "rouge2": 0.10516840419821284, - "rougeL": 0.23173161184029895, - "avg_rouge": 0.21726576864420533 + "rouge1": 0.35465837446263077, + "rouge2": 0.13333498040065456, + "rougeL": 0.27015665755068613, + "avg_rouge": 0.25271667080465715 }, "prompt_5": { - "rouge1": 0.3246043884384595, - "rouge2": 0.10637349123994702, - "rougeL": 0.23880705409546057, - "avg_rouge": 0.22326164459128903 + "rouge1": 0.3742821190677485, + "rouge2": 0.14040447358984648, + "rougeL": 0.28825299811660415, + "avg_rouge": 0.2676465302580664 } }, "dialogsum": { "prompt_1": { - "rouge1": 0.22778557503334432, - "rouge2": 0.05722609557281417, - "rougeL": 0.16993454716995432, - "avg_rouge": 0.15164873925870428 + "rouge1": 0.23089767016259732, + "rouge2": 0.06172943469639488, + "rougeL": 0.17016908958572435, + "avg_rouge": 0.15426539814823884 }, "prompt_2": { - "rouge1": 0.23906369567621766, - "rouge2": 0.05753514696815149, - "rougeL": 0.18111443503121383, - "avg_rouge": 0.15923775922519431 + "rouge1": 0.22861164883981736, + "rouge2": 0.059782881376602634, + "rougeL": 0.16816208890349965, + "avg_rouge": 0.15218553970663987 }, "prompt_3": { - "rouge1": 0.23404311352793605, - "rouge2": 0.0553907932779348, - "rougeL": 0.1757440121930614, - "avg_rouge": 0.1550593063329774 + "rouge1": 0.2294992806454931, + "rouge2": 0.060170947657632076, + "rougeL": 0.16849272526047757, + "avg_rouge": 0.1527209845212009 }, "prompt_4": { - "rouge1": 0.18719575606074948, - "rouge2": 0.042201569749929634, - "rougeL": 0.14550994993185537, - "avg_rouge": 0.12496909191417815 + "rouge1": 0.23145317379810715, + "rouge2": 0.06210396398371093, + "rougeL": 0.17045338878061678, + "avg_rouge": 0.1546701755208116 }, "prompt_5": { - "rouge1": 0.23108739724727537, - "rouge2": 0.0509668914533323, - "rougeL": 0.1768690339312652, - "avg_rouge": 0.15297444087729095 + "rouge1": 0.22469808863669913, + "rouge2": 0.056208955341898646, + "rougeL": 0.1678153274684139, + "avg_rouge": 0.14957412381567056 } }, "sst2": { "prompt_1": { - "accuracy": 0.8130733944954128 + "accuracy": 0.9403669724770642 }, "prompt_2": { - "accuracy": 0.8188073394495413 + "accuracy": 0.9380733944954128 }, "prompt_3": { - "accuracy": 0.7855504587155964 + "accuracy": 0.9461009174311926 }, "prompt_4": { - "accuracy": 0.7878440366972477 + "accuracy": 0.9346330275229358 }, "prompt_5": { - "accuracy": 0.8451834862385321 + "accuracy": 0.908256880733945 } }, "cola": { "prompt_1": { - "accuracy": 0.47746883988494726 + "accuracy": 0.7689357622243528 }, "prompt_2": { - "accuracy": 0.5302013422818792 + "accuracy": 0.7545541706615532 }, "prompt_3": { - "accuracy": 0.6270373921380633 + "accuracy": 0.7900287631831256 }, "prompt_4": { - "accuracy": 0.6423777564717162 + "accuracy": 0.8015340364333653 }, "prompt_5": { - "accuracy": 0.5982742090124641 + "accuracy": 0.7727708533077661 } }, "qqp": { "prompt_1": { - "accuracy": 0.392 + "accuracy": 0.7925 }, "prompt_2": { - "accuracy": 0.46 + "accuracy": 0.792 }, "prompt_3": { - "accuracy": 0.406 + "accuracy": 0.7845 }, "prompt_4": { - "accuracy": 0.393 + "accuracy": 0.779 }, "prompt_5": { - "accuracy": 0.4385 + "accuracy": 0.78 } }, "mnli": { "prompt_1": { - "accuracy": 0.3685 + "accuracy": 0.7 }, "prompt_2": { - "accuracy": 0.366 + "accuracy": 0.7 }, "prompt_3": { - "accuracy": 0.38 + "accuracy": 0.7 }, "prompt_4": { - "accuracy": 0.3775 + "accuracy": 0.7 }, "prompt_5": { - "accuracy": 0.35 + "accuracy": 0.7 } }, "qnli": { "prompt_1": { - "accuracy": 0.5685 + "accuracy": 1.0 }, "prompt_2": { - "accuracy": 0.59 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.5445 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.5065 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.509 + "accuracy": 0.9 } }, "wnli": { "prompt_1": { - "accuracy": 0.5211267605633803 + "accuracy": 0.8 }, "prompt_2": { - "accuracy": 0.4507042253521127 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.4647887323943662 + "accuracy": 0.8 }, "prompt_4": { - "accuracy": 0.43661971830985913 + "accuracy": 0.6 }, "prompt_5": { - "accuracy": 0.4647887323943662 + "accuracy": 0.8 } }, "rte": { "prompt_1": { - "accuracy": 0.5090252707581228 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.51985559566787 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.5306859205776173 + "accuracy": 1.0 }, "prompt_4": { - "accuracy": 0.5270758122743683 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.5487364620938628 + "accuracy": 0.8 } }, "mrpc": { "prompt_1": { - "accuracy": 0.49019607843137253 + "accuracy": 0.9 }, "prompt_2": { - "accuracy": 0.46078431372549017 + "accuracy": 0.9 }, "prompt_3": { - "accuracy": 0.47549019607843135 + "accuracy": 0.9 }, "prompt_4": { - "accuracy": 0.4681372549019608 + "accuracy": 0.9 }, "prompt_5": { - "accuracy": 0.4681372549019608 + "accuracy": 0.9 } } }, "five_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.5186974789915966, - "language_acc": { - "English": 0.5327731092436975, - "Chinese": 0.5, - "Spanish": 0.5403361344537815, - "Vietnamese": 0.5016806722689076 - }, - "consistency_score_2": 0.560924369747899, - "consistency_score_3": 0.4023109243697479, - "consistency_score_4": 0.3126050420168067, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.553781512605042, - "English,Spanish": 0.619327731092437, - "English,Vietnamese": 0.5436974789915966, - "Chinese,Spanish": 0.5756302521008403, - "Chinese,Vietnamese": 0.5218487394957984, - "Spanish,Vietnamese": 0.5512605042016807 - }, - "3_combine": { - "English,Chinese,Spanish": 0.42521008403361343, - "English,Chinese,Vietnamese": 0.380672268907563, - "English,Spanish,Vietnamese": 0.4176470588235294, - "Chinese,Spanish,Vietnamese": 0.38571428571428573 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3126050420168067 - } - }, - "AC3_2": 0.538985121143913, - "AC3_3": 0.4531503978836942, - "AC3_4": 0.39010454821665447 - } + "prompt_1": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.3142857142857142, - "language_acc": { - "Spanish": 0.30666666666666664, - "English": 0.3466666666666667, - "Chinese": 0.32666666666666666, - "Vietnamese": 0.24, - "Filipino": 0.26666666666666666, - "Indonesian": 0.37333333333333335, - "Malay": 0.34 - }, - "consistency_score_2": 0.3422222222222222, - "consistency_score_3": 0.14780952380952384, - "consistency_score_4": 0.07276190476190475, - "consistency_score_5": 0.039999999999999994, - "consistency_score_6": 0.025714285714285714, - "consistency_score_7": 0.02, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.4666666666666667, - "Spanish,Chinese": 0.38666666666666666, - "Spanish,Vietnamese": 0.3933333333333333, - "Spanish,Filipino": 0.32, - "Spanish,Indonesian": 0.2866666666666667, - "Spanish,Malay": 0.4066666666666667, - "English,Chinese": 0.41333333333333333, - "English,Vietnamese": 0.32, - "English,Filipino": 0.36, - "English,Indonesian": 0.3466666666666667, - "English,Malay": 0.38666666666666666, - "Chinese,Vietnamese": 0.3466666666666667, - "Chinese,Filipino": 0.32666666666666666, - "Chinese,Indonesian": 0.30666666666666664, - "Chinese,Malay": 0.29333333333333333, - "Vietnamese,Filipino": 0.3, - "Vietnamese,Indonesian": 0.30666666666666664, - "Vietnamese,Malay": 0.3, - "Filipino,Indonesian": 0.2866666666666667, - "Filipino,Malay": 0.2866666666666667, - "Indonesian,Malay": 0.3466666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.25333333333333335, - "Spanish,English,Vietnamese": 0.19333333333333333, - "Spanish,English,Filipino": 0.18666666666666668, - "Spanish,English,Indonesian": 0.17333333333333334, - "Spanish,English,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese": 0.16666666666666666, - "Spanish,Chinese,Filipino": 0.14666666666666667, - "Spanish,Chinese,Indonesian": 0.12, - "Spanish,Chinese,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino": 0.12666666666666668, - "Spanish,Vietnamese,Indonesian": 0.10666666666666667, - "Spanish,Vietnamese,Malay": 0.16666666666666666, - "Spanish,Filipino,Indonesian": 0.10666666666666667, - "Spanish,Filipino,Malay": 0.14, - "Spanish,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese": 0.15333333333333332, - "English,Chinese,Filipino": 0.17333333333333334, - "English,Chinese,Indonesian": 0.17333333333333334, - "English,Chinese,Malay": 0.2, - "English,Vietnamese,Filipino": 0.12666666666666668, - "English,Vietnamese,Indonesian": 0.12666666666666668, - "English,Vietnamese,Malay": 0.13333333333333333, - "English,Filipino,Indonesian": 0.13333333333333333, - "English,Filipino,Malay": 0.14666666666666667, - "English,Indonesian,Malay": 0.17333333333333334, - "Chinese,Vietnamese,Filipino": 0.12666666666666668, - "Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "Chinese,Vietnamese,Malay": 0.13333333333333333, - "Chinese,Filipino,Indonesian": 0.14, - "Chinese,Filipino,Malay": 0.10666666666666667, - "Chinese,Indonesian,Malay": 0.12666666666666668, - "Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Vietnamese,Filipino,Malay": 0.11333333333333333, - "Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.10666666666666667, - "Spanish,English,Chinese,Filipino": 0.12666666666666668, - "Spanish,English,Chinese,Indonesian": 0.1, - "Spanish,English,Chinese,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino": 0.08, - "Spanish,English,Vietnamese,Indonesian": 0.07333333333333333, - "Spanish,English,Vietnamese,Malay": 0.08666666666666667, - "Spanish,English,Filipino,Indonesian": 0.06, - "Spanish,English,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino": 0.06666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.04666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.09333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.05333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.08, - "Spanish,Chinese,Indonesian,Malay": 0.07333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.04, - "Spanish,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.04666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.04666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.07333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.06666666666666667, - "English,Chinese,Vietnamese,Malay": 0.07333333333333333, - "English,Chinese,Filipino,Indonesian": 0.08, - "English,Chinese,Filipino,Malay": 0.08, - "English,Chinese,Indonesian,Malay": 0.08666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.06, - "English,Vietnamese,Filipino,Malay": 0.06, - "English,Vietnamese,Indonesian,Malay": 0.05333333333333334, - "English,Filipino,Indonesian,Malay": 0.06666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.05333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.06666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.05333333333333334, - "Chinese,Filipino,Indonesian,Malay": 0.04, - "Vietnamese,Filipino,Indonesian,Malay": 0.04 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.06, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.04, - "Spanish,English,Chinese,Vietnamese,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.05333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.06, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.02666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.04666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.03333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.02666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.02, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.05333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.03333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.02666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.02666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.04666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.02666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.03333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.03333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.02 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.02, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.02666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.02666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.02, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.02, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.02 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.02 - } - }, - "AC3_2": 0.32765957441817556, - "AC3_3": 0.20105994578148068, - "AC3_4": 0.11816647915957099, - "AC3_5": 0.07096774191545265, - "AC3_6": 0.04753901559226044, - "AC3_7": 0.037606837595587694 - } + "prompt_1": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.30032467532467527, - "language_acc": { - "English": 0.375, - "Filipino": 0.2556818181818182, - "Spanish": 0.3068181818181818, - "Malay": 0.25, - "Chinese": 0.3181818181818182, - "Indonesian": 0.2784090909090909, - "Vietnamese": 0.3181818181818182 - }, - "consistency_score_2": 0.3709415584415584, - "consistency_score_3": 0.18035714285714288, - "consistency_score_4": 0.10633116883116883, - "consistency_score_5": 0.0698051948051948, - "consistency_score_6": 0.047889610389610385, - "consistency_score_7": 0.03409090909090909, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3522727272727273, - "English,Spanish": 0.4431818181818182, - "English,Malay": 0.36363636363636365, - "English,Chinese": 0.375, - "English,Indonesian": 0.4090909090909091, - "English,Vietnamese": 0.3806818181818182, - "Filipino,Spanish": 0.3125, - "Filipino,Malay": 0.38636363636363635, - "Filipino,Chinese": 0.3693181818181818, - "Filipino,Indonesian": 0.3352272727272727, - "Filipino,Vietnamese": 0.3352272727272727, - "Spanish,Malay": 0.3806818181818182, - "Spanish,Chinese": 0.4147727272727273, - "Spanish,Indonesian": 0.39204545454545453, - "Spanish,Vietnamese": 0.35795454545454547, - "Malay,Chinese": 0.32954545454545453, - "Malay,Indonesian": 0.42613636363636365, - "Malay,Vietnamese": 0.30113636363636365, - "Chinese,Indonesian": 0.39204545454545453, - "Chinese,Vietnamese": 0.3806818181818182, - "Indonesian,Vietnamese": 0.3522727272727273 - }, - "3_combine": { - "English,Filipino,Spanish": 0.17613636363636365, - "English,Filipino,Malay": 0.17613636363636365, - "English,Filipino,Chinese": 0.17045454545454544, - "English,Filipino,Indonesian": 0.16477272727272727, - "English,Filipino,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay": 0.21022727272727273, - "English,Spanish,Chinese": 0.20454545454545456, - "English,Spanish,Indonesian": 0.22727272727272727, - "English,Spanish,Vietnamese": 0.1875, - "English,Malay,Chinese": 0.1590909090909091, - "English,Malay,Indonesian": 0.2159090909090909, - "English,Malay,Vietnamese": 0.1534090909090909, - "English,Chinese,Indonesian": 0.19318181818181818, - "English,Chinese,Vietnamese": 0.18181818181818182, - "English,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Malay": 0.17613636363636365, - "Filipino,Spanish,Chinese": 0.19318181818181818, - "Filipino,Spanish,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Chinese": 0.1590909090909091, - "Filipino,Malay,Indonesian": 0.19318181818181818, - "Filipino,Malay,Vietnamese": 0.13636363636363635, - "Filipino,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Chinese,Vietnamese": 0.17045454545454544, - "Filipino,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Malay,Chinese": 0.20454545454545456, - "Spanish,Malay,Indonesian": 0.23295454545454544, - "Spanish,Malay,Vietnamese": 0.17045454545454544, - "Spanish,Chinese,Indonesian": 0.21022727272727273, - "Spanish,Chinese,Vietnamese": 0.20454545454545456, - "Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "Malay,Chinese,Indonesian": 0.19318181818181818, - "Malay,Chinese,Vietnamese": 0.13068181818181818, - "Malay,Indonesian,Vietnamese": 0.17613636363636365, - "Chinese,Indonesian,Vietnamese": 0.17045454545454544 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.11363636363636363, - "English,Filipino,Spanish,Chinese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese": 0.09659090909090909, - "English,Filipino,Malay,Indonesian": 0.125, - "English,Filipino,Malay,Vietnamese": 0.09090909090909091, - "English,Filipino,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese": 0.10795454545454546, - "English,Spanish,Malay,Indonesian": 0.14772727272727273, - "English,Spanish,Malay,Vietnamese": 0.11931818181818182, - "English,Spanish,Chinese,Indonesian": 0.125, - "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "English,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese": 0.11931818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.125, - "Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Malay,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Malay,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0625, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - } - }, - "AC3_2": 0.3319186858595657, - "AC3_3": 0.2253702899130701, - "AC3_4": 0.15705601779583958, - "AC3_5": 0.11328035996028073, - "AC3_6": 0.08260678712851871, - "AC3_7": 0.061231244465361725 - } + "prompt_1": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.3106796116504854 - } + "prompt_1": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.23809523809523808 - } + "prompt_1": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.308411214953271 - } + "prompt_1": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.3, - "category_acc": { - "brand": 0.3, - "demographics": 0.6, - "biology": 0.1, - "history": 0.3333333333333333, - "literature": 0.1, - "politics": 0.6, - "culture": 0.1, - "film": 0.5, - "law": 0.3, - "geography": 0.2 - } - } + "prompt_1": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.007187159214661441 - } + "prompt_1": -1 }, "indommlu": { "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.006999970820512095 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.00674481031148272 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.011346370672295878 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.006775845043378504 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.29871645274212366 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.3205577404361816, - "category_acc": { - "high_school_european_history": 0.3902439024390244, - "business_ethics": 0.4444444444444444, - "clinical_knowledge": 0.35984848484848486, - "medical_genetics": 0.25252525252525254, - "high_school_us_history": 0.3645320197044335, - "high_school_physics": 0.2, - "high_school_world_history": 0.4025423728813559, - "virology": 0.3151515151515151, - "high_school_microeconomics": 0.26582278481012656, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.2727272727272727, - "high_school_biology": 0.3365695792880259, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.3202846975088968, - "philosophy": 0.3161290322580645, - "professional_medicine": 0.22878228782287824, - "nutrition": 0.39672131147540984, - "global_facts": 0.2727272727272727, - "machine_learning": 0.35135135135135137, - "security_studies": 0.38524590163934425, - "public_relations": 0.3761467889908257, - "professional_psychology": 0.31751227495908346, - "prehistory": 0.28792569659442724, - "anatomy": 0.3283582089552239, - "human_sexuality": 0.35384615384615387, - "college_medicine": 0.3430232558139535, - "high_school_government_and_politics": 0.359375, - "college_chemistry": 0.24242424242424243, - "logical_fallacies": 0.2777777777777778, - "high_school_geography": 0.3147208121827411, - "elementary_mathematics": 0.2572944297082228, - "human_aging": 0.3738738738738739, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.36580882352941174, - "formal_logic": 0.288, - "high_school_statistics": 0.24651162790697675, - "international_law": 0.375, - "high_school_mathematics": 0.26394052044609667, - "high_school_computer_science": 0.29292929292929293, - "conceptual_physics": 0.28205128205128205, - "miscellaneous": 0.3618925831202046, - "high_school_chemistry": 0.25742574257425743, - "marketing": 0.4248927038626609, - "professional_law": 0.31702544031311153, - "management": 0.3333333333333333, - "college_physics": 0.2376237623762376, - "jurisprudence": 0.3177570093457944, - "world_religions": 0.36470588235294116, - "sociology": 0.375, - "us_foreign_policy": 0.47474747474747475, - "high_school_macroeconomics": 0.29562982005141386, - "computer_security": 0.29292929292929293, - "moral_scenarios": 0.24608501118568232, - "moral_disputes": 0.4, - "electrical_engineering": 0.4236111111111111, - "astronomy": 0.26490066225165565, - "college_biology": 0.32167832167832167 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.2919762258543834 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.2658779576587796, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.25, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.23809523809523808, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.16666666666666666, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.25, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.125, - "middle_school_mathematics": 0.125, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.25, - "college_economics": 0.21666666666666667, - "business_administration": 0.34210526315789475, - "marxism": 0.375, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.35294117647058826, - "teacher_qualification": 0.30612244897959184, - "high_school_politics": 0.5, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.25925925925925924, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.17647058823529413, - "legal_professional": 0.25, - "high_school_chinese": 0.25, - "high_school_history": 0.32, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.21153846153846154, - "sports_science": 0.16666666666666666, - "plant_protection": 0.37037037037037035, - "basic_medicine": 0.20833333333333334, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.24074074074074073, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.16666666666666666, - "tax_accountant": 0.2222222222222222, - "physician": 0.35185185185185186 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.26881720430107525 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.26774304955966155, - "category_acc": { - "agronomy": 0.31952662721893493, - "anatomy": 0.25, - "ancient_chinese": 0.23170731707317074, - "arts": 0.2375, - "astronomy": 0.30303030303030304, - "business_ethics": 0.2727272727272727, - "chinese_civil_service_exam": 0.29375, - "chinese_driving_rule": 0.3435114503816794, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.2336448598130841, - "chinese_history": 0.23529411764705882, - "chinese_literature": 0.3088235294117647, - "chinese_teacher_qualification": 0.3128491620111732, - "clinical_knowledge": 0.28270042194092826, - "college_actuarial_science": 0.22641509433962265, - "college_education": 0.2616822429906542, - "college_engineering_hydrology": 0.32075471698113206, - "college_law": 0.19444444444444445, - "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.16981132075471697, - "college_medicine": 0.23809523809523808, - "computer_science": 0.28921568627450983, - "computer_security": 0.32748538011695905, - "conceptual_physics": 0.30612244897959184, - "construction_project_management": 0.2014388489208633, - "economics": 0.29559748427672955, - "education": 0.2822085889570552, - "electrical_engineering": 0.29651162790697677, - "elementary_chinese": 0.27380952380952384, - "elementary_commonsense": 0.25757575757575757, - "elementary_information_and_technology": 0.31932773109243695, - "elementary_mathematics": 0.2956521739130435, - "ethnology": 0.26666666666666666, - "food_science": 0.23776223776223776, - "genetics": 0.2727272727272727, - "global_facts": 0.2684563758389262, - "high_school_biology": 0.28402366863905326, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.2288135593220339, - "high_school_mathematics": 0.23170731707317074, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.3006993006993007, - "human_sexuality": 0.2777777777777778, - "international_law": 0.22702702702702704, - "journalism": 0.3023255813953488, - "jurisprudence": 0.291970802919708, - "legal_and_moral_basis": 0.3411214953271028, - "logical": 0.21138211382113822, - "machine_learning": 0.22131147540983606, - "management": 0.2523809523809524, - "marketing": 0.24444444444444444, - "marxist_theory": 0.2698412698412698, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.2896551724137931, - "philosophy": 0.23809523809523808, - "professional_accounting": 0.22857142857142856, - "professional_law": 0.24170616113744076, - "professional_medicine": 0.2765957446808511, - "professional_psychology": 0.2801724137931034, - "public_relations": 0.28160919540229884, - "security_study": 0.25925925925925924, - "sociology": 0.23008849557522124, - "sports_science": 0.19393939393939394, - "traditional_chinese_medicine": 0.23243243243243245, - "virology": 0.28402366863905326, - "world_history": 0.2546583850931677, - "world_religions": 0.26875 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.3333333333333333 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.15227272727272728 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.32677966101694916 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.4012715033657442 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.5071043606075453 - } + "prompt_1": -1 }, "samsum": { - "prompt_1": { - "rouge1": 0.0998170669258938, - "rouge2": 0.006253668575785717, - "rougeL": 0.07726068103530165, - "avg_rouge": 0.061110472178993715 - } + "prompt_1": -1 }, "dialogsum": { - "prompt_1": { - "rouge1": 0.10985081119178293, - "rouge2": 0.007904839134367962, - "rougeL": 0.08495481492281567, - "avg_rouge": 0.06757015508298886 - } + "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.5676605504587156 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.5081495685522531 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.4845 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.3235 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.5015 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.49295774647887325 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.5451263537906137 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.5220588235294118 - } + "prompt_1": -1 } } }, - "gemma_7b": { - "model_size": "7B", - "model_link": "https://huggingface.co/google/gemma-7b", + "gpt_35_turbo_1106": { + "model_size": "NA", + "model_link": "https://openai.com/blog/chatgpt", "zero_shot": { "cross_xquad": { "prompt_1": { - "overall_acc": 0.8472689075630252, + "overall_acc": 0.915126050420168, "language_acc": { - "English": 0.8873949579831932, - "Chinese": 0.7899159663865546, - "Spanish": 0.8663865546218488, - "Vietnamese": 0.8453781512605042 - }, - "consistency_score_2": 0.8180672268907562, - "consistency_score_3": 0.7384453781512605, - "consistency_score_4": 0.6773109243697479, + "Spanish": 0.9201680672268907, + "English": 0.9436974789915966, + "Chinese": 0.9042016806722689, + "Vietnamese": 0.892436974789916 + }, + "consistency_score_2": 0.900280112044818, + "consistency_score_3": 0.8569327731092438, + "consistency_score_4": 0.826890756302521, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.8016806722689076, - "English,Spanish": 0.8697478991596639, - "English,Vietnamese": 0.8495798319327731, - "Chinese,Spanish": 0.7840336134453781, - "Chinese,Vietnamese": 0.7663865546218488, - "Spanish,Vietnamese": 0.8369747899159664 + "Spanish,English": 0.9285714285714286, + "Spanish,Chinese": 0.8949579831932774, + "Spanish,Vietnamese": 0.892436974789916, + "English,Chinese": 0.9050420168067227, + "English,Vietnamese": 0.9033613445378151, + "Chinese,Vietnamese": 0.8773109243697479 }, "3_combine": { - "English,Chinese,Spanish": 0.7369747899159664, - "English,Chinese,Vietnamese": 0.7201680672268908, - "English,Spanish,Vietnamese": 0.7890756302521008, - "Chinese,Spanish,Vietnamese": 0.7075630252100841 + "Spanish,English,Chinese": 0.8697478991596639, + "Spanish,English,Vietnamese": 0.8672268907563025, + "Spanish,Chinese,Vietnamese": 0.8411764705882353, + "English,Chinese,Vietnamese": 0.8495798319327731 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.6773109243697479 + "Spanish,English,Chinese,Vietnamese": 0.826890756302521 } }, - "AC3_2": 0.8324120413403068, - "AC3_3": 0.7891230020922688, - "AC3_4": 0.7528165792487318 + "AC3_2": 0.9076423779936448, + "AC3_3": 0.8850738967286825, + "AC3_4": 0.8687737902187682 }, "prompt_2": { - "overall_acc": 0.8834033613445378, + "overall_acc": 0.9155462184873949, "language_acc": { - "English": 0.9126050420168067, - "Chinese": 0.8369747899159664, - "Spanish": 0.8983193277310925, - "Vietnamese": 0.8857142857142857 + "Spanish": 0.9193277310924369, + "English": 0.9436974789915966, + "Chinese": 0.9058823529411765, + "Vietnamese": 0.8932773109243698 }, - "consistency_score_2": 0.8648459383753501, - "consistency_score_3": 0.8035714285714286, - "consistency_score_4": 0.7571428571428571, + "consistency_score_2": 0.9009803921568628, + "consistency_score_3": 0.8571428571428572, + "consistency_score_4": 0.826890756302521, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.8420168067226891, - "English,Spanish": 0.9235294117647059, - "English,Vietnamese": 0.8991596638655462, - "Chinese,Spanish": 0.8218487394957983, - "Chinese,Vietnamese": 0.8151260504201681, - "Spanish,Vietnamese": 0.8873949579831932 + "Spanish,English": 0.926890756302521, + "Spanish,Chinese": 0.8966386554621849, + "Spanish,Vietnamese": 0.892436974789916, + "English,Chinese": 0.9067226890756303, + "English,Vietnamese": 0.9033613445378151, + "Chinese,Vietnamese": 0.8798319327731092 }, "3_combine": { - "English,Chinese,Spanish": 0.7983193277310925, - "English,Chinese,Vietnamese": 0.7840336134453781, - "English,Spanish,Vietnamese": 0.8596638655462185, - "Chinese,Spanish,Vietnamese": 0.7722689075630252 + "Spanish,English,Chinese": 0.8697478991596639, + "Spanish,English,Vietnamese": 0.865546218487395, + "Spanish,Chinese,Vietnamese": 0.8420168067226891, + "English,Chinese,Vietnamese": 0.8512605042016806 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7571428571428571 + "Spanish,English,Chinese,Vietnamese": 0.826890756302521 } }, - "AC3_2": 0.8740261575790896, - "AC3_3": 0.8415984699732395, - "AC3_4": 0.8154144484803805 + "AC3_2": 0.9082049072020327, + "AC3_3": 0.8853824534766179, + "AC3_4": 0.8689630855460749 }, "prompt_3": { - "overall_acc": 0.8689075630252101, + "overall_acc": 0.9065126050420168, "language_acc": { - "English": 0.8983193277310925, - "Chinese": 0.8294117647058824, - "Spanish": 0.880672268907563, - "Vietnamese": 0.8672268907563025 + "Spanish": 0.9159663865546218, + "English": 0.9277310924369748, + "Chinese": 0.9008403361344538, + "Vietnamese": 0.8815126050420168 }, - "consistency_score_2": 0.8532212885154061, - "consistency_score_3": 0.7871848739495798, - "consistency_score_4": 0.7403361344537815, + "consistency_score_2": 0.8900560224089636, + "consistency_score_3": 0.8407563025210084, + "consistency_score_4": 0.8050420168067227, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.8361344537815126, - "English,Spanish": 0.9067226890756303, - "English,Vietnamese": 0.8823529411764706, - "Chinese,Spanish": 0.819327731092437, - "Chinese,Vietnamese": 0.8117647058823529, - "Spanish,Vietnamese": 0.8630252100840337 + "Spanish,English": 0.9184873949579831, + "Spanish,Chinese": 0.892436974789916, + "Spanish,Vietnamese": 0.873109243697479, + "English,Chinese": 0.8974789915966387, + "English,Vietnamese": 0.8890756302521008, + "Chinese,Vietnamese": 0.8697478991596639 }, "3_combine": { - "English,Chinese,Spanish": 0.7865546218487395, - "English,Chinese,Vietnamese": 0.7722689075630252, - "English,Spanish,Vietnamese": 0.8319327731092437, - "Chinese,Spanish,Vietnamese": 0.7579831932773109 + "Spanish,English,Chinese": 0.8571428571428571, + "Spanish,English,Vietnamese": 0.846218487394958, + "Spanish,Chinese,Vietnamese": 0.8252100840336134, + "English,Chinese,Vietnamese": 0.8344537815126051 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7403361344537815 + "Spanish,English,Chinese,Vietnamese": 0.8050420168067227 } }, - "AC3_2": 0.8609929853028961, - "AC3_3": 0.826029846114136, - "AC3_4": 0.799485705459677 + "AC3_2": 0.8982089424628893, + "AC3_3": 0.8723971252063294, + "AC3_4": 0.8527694371714862 }, "prompt_4": { - "overall_acc": 0.8535714285714286, + "overall_acc": 0.9142857142857143, "language_acc": { - "English": 0.8941176470588236, - "Chinese": 0.8025210084033614, - "Spanish": 0.8697478991596639, - "Vietnamese": 0.8478991596638655 - }, - "consistency_score_2": 0.8274509803921567, - "consistency_score_3": 0.7512605042016807, - "consistency_score_4": 0.6974789915966386, + "Spanish": 0.9319327731092437, + "English": 0.938655462184874, + "Chinese": 0.9016806722689076, + "Vietnamese": 0.8848739495798319 + }, + "consistency_score_2": 0.8978991596638656, + "consistency_score_3": 0.8525210084033614, + "consistency_score_4": 0.8184873949579832, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.807563025210084, - "English,Spanish": 0.8865546218487395, - "English,Vietnamese": 0.8571428571428571, - "Chinese,Spanish": 0.7941176470588235, - "Chinese,Vietnamese": 0.7840336134453781, - "Spanish,Vietnamese": 0.8352941176470589 + "Spanish,English": 0.938655462184874, + "Spanish,Chinese": 0.8983193277310925, + "Spanish,Vietnamese": 0.8949579831932774, + "English,Chinese": 0.8983193277310925, + "English,Vietnamese": 0.8865546218487395, + "Chinese,Vietnamese": 0.8705882352941177 }, "3_combine": { - "English,Chinese,Spanish": 0.7546218487394958, - "English,Chinese,Vietnamese": 0.7327731092436974, - "English,Spanish,Vietnamese": 0.7983193277310925, - "Chinese,Spanish,Vietnamese": 0.719327731092437 + "Spanish,English,Chinese": 0.8705882352941177, + "Spanish,English,Vietnamese": 0.8630252100840337, + "Spanish,Chinese,Vietnamese": 0.8411764705882353, + "English,Chinese,Vietnamese": 0.8352941176470589 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.6974789915966386 + "Spanish,English,Chinese,Vietnamese": 0.8184873949579832 } }, - "AC3_2": 0.8403082690606141, - "AC3_3": 0.7991547135439795, - "AC3_4": 0.7676709041512765 + "AC3_2": 0.9060183497893602, + "AC3_3": 0.8823237641763225, + "AC3_4": 0.863738395405328 }, "prompt_5": { - "overall_acc": 0.8605042016806723, + "overall_acc": 0.9058823529411765, "language_acc": { - "English": 0.8991596638655462, - "Chinese": 0.8302521008403362, - "Spanish": 0.8756302521008403, - "Vietnamese": 0.8369747899159664 + "Spanish": 0.9168067226890756, + "English": 0.926890756302521, + "Chinese": 0.9025210084033614, + "Vietnamese": 0.8773109243697479 }, - "consistency_score_2": 0.8400560224089636, - "consistency_score_3": 0.7695378151260504, - "consistency_score_4": 0.7184873949579832, + "consistency_score_2": 0.8901960784313726, + "consistency_score_3": 0.8418067226890756, + "consistency_score_4": 0.8067226890756303, "detailed_consistency_score": { "2_combine": { - "English,Chinese": 0.8394957983193277, - "English,Spanish": 0.9050420168067227, - "English,Vietnamese": 0.846218487394958, - "Chinese,Spanish": 0.8235294117647058, - "Chinese,Vietnamese": 0.7899159663865546, - "Spanish,Vietnamese": 0.8361344537815126 + "Spanish,English": 0.9193277310924369, + "Spanish,Chinese": 0.8957983193277311, + "Spanish,Vietnamese": 0.8697478991596639, + "English,Chinese": 0.9016806722689076, + "English,Vietnamese": 0.8899159663865546, + "Chinese,Vietnamese": 0.8647058823529412 }, "3_combine": { - "English,Chinese,Spanish": 0.7924369747899159, - "English,Chinese,Vietnamese": 0.7470588235294118, - "English,Spanish,Vietnamese": 0.8008403361344538, - "Chinese,Spanish,Vietnamese": 0.7378151260504202 + "Spanish,English,Chinese": 0.8630252100840337, + "Spanish,English,Vietnamese": 0.8453781512605042, + "Spanish,Chinese,Vietnamese": 0.8235294117647058, + "English,Chinese,Vietnamese": 0.8352941176470589 }, "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7184873949579832 + "Spanish,English,Chinese,Vietnamese": 0.8067226890756303 } }, - "AC3_2": 0.8501571736743808, - "AC3_3": 0.8124827659654418, - "AC3_4": 0.7831091989253643 + "AC3_2": 0.8979707166209516, + "AC3_3": 0.8726699334120924, + "AC3_4": 0.8534318535552013 } }, "cross_mmlu": { "prompt_1": { - "overall_acc": 0.5133333333333333, + "overall_acc": 0.661904761904762, "language_acc": { - "Spanish": 0.5666666666666667, - "English": 0.64, - "Chinese": 0.4266666666666667, - "Vietnamese": 0.47333333333333333, - "Filipino": 0.5933333333333334, - "Indonesian": 0.49333333333333335, - "Malay": 0.4 + "Vietnamese": 0.6133333333333333, + "Malay": 0.6133333333333333, + "Filipino": 0.6266666666666667, + "Indonesian": 0.66, + "Chinese": 0.6533333333333333, + "Spanish": 0.72, + "English": 0.7466666666666667 }, - "consistency_score_2": 0.5603174603174603, - "consistency_score_3": 0.38400000000000006, - "consistency_score_4": 0.2834285714285714, - "consistency_score_5": 0.21777777777777776, - "consistency_score_6": 0.17047619047619048, - "consistency_score_7": 0.13333333333333333, + "consistency_score_2": 0.6663492063492062, + "consistency_score_3": 0.5316190476190475, + "consistency_score_4": 0.449904761904762, + "consistency_score_5": 0.3923809523809524, + "consistency_score_6": 0.3485714285714286, + "consistency_score_7": 0.31333333333333335, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6866666666666666, - "Spanish,Chinese": 0.5666666666666667, - "Spanish,Vietnamese": 0.5533333333333333, - "Spanish,Filipino": 0.7133333333333334, - "Spanish,Indonesian": 0.6266666666666667, - "Spanish,Malay": 0.5066666666666667, - "English,Chinese": 0.6, - "English,Vietnamese": 0.5, - "English,Filipino": 0.6866666666666666, - "English,Indonesian": 0.6066666666666667, - "English,Malay": 0.5066666666666667, - "Chinese,Vietnamese": 0.44, - "Chinese,Filipino": 0.5533333333333333, - "Chinese,Indonesian": 0.5466666666666666, - "Chinese,Malay": 0.49333333333333335, - "Vietnamese,Filipino": 0.52, - "Vietnamese,Indonesian": 0.44666666666666666, - "Vietnamese,Malay": 0.5333333333333333, - "Filipino,Indonesian": 0.6333333333333333, - "Filipino,Malay": 0.5266666666666666, - "Indonesian,Malay": 0.52 + "Vietnamese,Malay": 0.5666666666666667, + "Vietnamese,Filipino": 0.6133333333333333, + "Vietnamese,Indonesian": 0.6466666666666666, + "Vietnamese,Chinese": 0.5933333333333334, + "Vietnamese,Spanish": 0.6666666666666666, + "Vietnamese,English": 0.64, + "Malay,Filipino": 0.62, + "Malay,Indonesian": 0.6866666666666666, + "Malay,Chinese": 0.64, + "Malay,Spanish": 0.6933333333333334, + "Malay,English": 0.6733333333333333, + "Filipino,Indonesian": 0.6733333333333333, + "Filipino,Chinese": 0.6133333333333333, + "Filipino,Spanish": 0.72, + "Filipino,English": 0.6666666666666666, + "Indonesian,Chinese": 0.6666666666666666, + "Indonesian,Spanish": 0.76, + "Indonesian,English": 0.7333333333333333, + "Chinese,Spanish": 0.6666666666666666, + "Chinese,English": 0.6533333333333333, + "Spanish,English": 0.8 }, "3_combine": { - "Spanish,English,Chinese": 0.44, - "Spanish,English,Vietnamese": 0.4066666666666667, - "Spanish,English,Filipino": 0.5666666666666667, - "Spanish,English,Indonesian": 0.4866666666666667, - "Spanish,English,Malay": 0.38666666666666666, - "Spanish,Chinese,Vietnamese": 0.32666666666666666, - "Spanish,Chinese,Filipino": 0.4533333333333333, - "Spanish,Chinese,Indonesian": 0.4066666666666667, - "Spanish,Chinese,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Filipino": 0.4266666666666667, - "Spanish,Vietnamese,Indonesian": 0.36, - "Spanish,Vietnamese,Malay": 0.35333333333333333, - "Spanish,Filipino,Indonesian": 0.5133333333333333, - "Spanish,Filipino,Malay": 0.3933333333333333, - "Spanish,Indonesian,Malay": 0.36, - "English,Chinese,Vietnamese": 0.31333333333333335, - "English,Chinese,Filipino": 0.44666666666666666, - "English,Chinese,Indonesian": 0.42, - "English,Chinese,Malay": 0.3466666666666667, - "English,Vietnamese,Filipino": 0.4, - "English,Vietnamese,Indonesian": 0.3333333333333333, - "English,Vietnamese,Malay": 0.34, - "English,Filipino,Indonesian": 0.4866666666666667, - "English,Filipino,Malay": 0.4066666666666667, - "English,Indonesian,Malay": 0.36666666666666664, - "Chinese,Vietnamese,Filipino": 0.32, - "Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Malay": 0.29333333333333333, - "Chinese,Filipino,Indonesian": 0.4, - "Chinese,Filipino,Malay": 0.3466666666666667, - "Chinese,Indonesian,Malay": 0.3333333333333333, - "Vietnamese,Filipino,Indonesian": 0.34, - "Vietnamese,Filipino,Malay": 0.36, - "Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Filipino,Indonesian,Malay": 0.37333333333333335 + "Vietnamese,Malay,Filipino": 0.4533333333333333, + "Vietnamese,Malay,Indonesian": 0.4866666666666667, + "Vietnamese,Malay,Chinese": 0.4533333333333333, + "Vietnamese,Malay,Spanish": 0.49333333333333335, + "Vietnamese,Malay,English": 0.4866666666666667, + "Vietnamese,Filipino,Indonesian": 0.5133333333333333, + "Vietnamese,Filipino,Chinese": 0.4533333333333333, + "Vietnamese,Filipino,Spanish": 0.54, + "Vietnamese,Filipino,English": 0.5133333333333333, + "Vietnamese,Indonesian,Chinese": 0.5, + "Vietnamese,Indonesian,Spanish": 0.5733333333333334, + "Vietnamese,Indonesian,English": 0.56, + "Vietnamese,Chinese,Spanish": 0.5, + "Vietnamese,Chinese,English": 0.4866666666666667, + "Vietnamese,Spanish,English": 0.58, + "Malay,Filipino,Indonesian": 0.5266666666666666, + "Malay,Filipino,Chinese": 0.48, + "Malay,Filipino,Spanish": 0.54, + "Malay,Filipino,English": 0.52, + "Malay,Indonesian,Chinese": 0.5266666666666666, + "Malay,Indonesian,Spanish": 0.5933333333333334, + "Malay,Indonesian,English": 0.5733333333333334, + "Malay,Chinese,Spanish": 0.5133333333333333, + "Malay,Chinese,English": 0.5133333333333333, + "Malay,Spanish,English": 0.6066666666666667, + "Filipino,Indonesian,Chinese": 0.49333333333333335, + "Filipino,Indonesian,Spanish": 0.5866666666666667, + "Filipino,Indonesian,English": 0.5533333333333333, + "Filipino,Chinese,Spanish": 0.5133333333333333, + "Filipino,Chinese,English": 0.5, + "Filipino,Spanish,English": 0.6066666666666667, + "Indonesian,Chinese,Spanish": 0.5666666666666667, + "Indonesian,Chinese,English": 0.5466666666666666, + "Indonesian,Spanish,English": 0.6733333333333333, + "Chinese,Spanish,English": 0.58 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.26666666666666666, - "Spanish,English,Chinese,Filipino": 0.37333333333333335, - "Spanish,English,Chinese,Indonesian": 0.34, - "Spanish,English,Chinese,Malay": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino": 0.3466666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.3, - "Spanish,English,Vietnamese,Malay": 0.28, - "Spanish,English,Filipino,Indonesian": 0.41333333333333333, - "Spanish,English,Filipino,Malay": 0.34, - "Spanish,English,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Filipino": 0.28, - "Spanish,Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Malay": 0.22, - "Spanish,Chinese,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.2866666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.3, - "English,Chinese,Vietnamese,Filipino": 0.26, - "English,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "English,Chinese,Vietnamese,Malay": 0.22, - "English,Chinese,Filipino,Indonesian": 0.3466666666666667, - "English,Chinese,Filipino,Malay": 0.29333333333333333, - "English,Chinese,Indonesian,Malay": 0.26, - "English,Vietnamese,Filipino,Indonesian": 0.28, - "English,Vietnamese,Filipino,Malay": 0.29333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.24, - "English,Filipino,Indonesian,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.22, - "Chinese,Vietnamese,Filipino,Malay": 0.24, - "Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667 + "Vietnamese,Malay,Filipino,Indonesian": 0.41333333333333333, + "Vietnamese,Malay,Filipino,Chinese": 0.38, + "Vietnamese,Malay,Filipino,Spanish": 0.4266666666666667, + "Vietnamese,Malay,Filipino,English": 0.4066666666666667, + "Vietnamese,Malay,Indonesian,Chinese": 0.4066666666666667, + "Vietnamese,Malay,Indonesian,Spanish": 0.4533333333333333, + "Vietnamese,Malay,Indonesian,English": 0.4533333333333333, + "Vietnamese,Malay,Chinese,Spanish": 0.4066666666666667, + "Vietnamese,Malay,Chinese,English": 0.4, + "Vietnamese,Malay,Spanish,English": 0.46, + "Vietnamese,Filipino,Indonesian,Chinese": 0.4, + "Vietnamese,Filipino,Indonesian,Spanish": 0.48, + "Vietnamese,Filipino,Indonesian,English": 0.46, + "Vietnamese,Filipino,Chinese,Spanish": 0.41333333333333333, + "Vietnamese,Filipino,Chinese,English": 0.4, + "Vietnamese,Filipino,Spanish,English": 0.48, + "Vietnamese,Indonesian,Chinese,Spanish": 0.46, + "Vietnamese,Indonesian,Chinese,English": 0.4533333333333333, + "Vietnamese,Indonesian,Spanish,English": 0.5266666666666666, + "Vietnamese,Chinese,Spanish,English": 0.4533333333333333, + "Malay,Filipino,Indonesian,Chinese": 0.41333333333333333, + "Malay,Filipino,Indonesian,Spanish": 0.4866666666666667, + "Malay,Filipino,Indonesian,English": 0.4666666666666667, + "Malay,Filipino,Chinese,Spanish": 0.43333333333333335, + "Malay,Filipino,Chinese,English": 0.42, + "Malay,Filipino,Spanish,English": 0.48, + "Malay,Indonesian,Chinese,Spanish": 0.46, + "Malay,Indonesian,Chinese,English": 0.46, + "Malay,Indonesian,Spanish,English": 0.54, + "Malay,Chinese,Spanish,English": 0.4666666666666667, + "Filipino,Indonesian,Chinese,Spanish": 0.44666666666666666, + "Filipino,Indonesian,Chinese,English": 0.43333333333333335, + "Filipino,Indonesian,Spanish,English": 0.5266666666666666, + "Filipino,Chinese,Spanish,English": 0.4666666666666667, + "Indonesian,Chinese,Spanish,English": 0.5133333333333333 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.20666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.18666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian": 0.3, - "Spanish,English,Chinese,Filipino,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.22, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,English,Filipino,Indonesian,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.22, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "English,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666 + "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.34, + "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.4, + "Vietnamese,Malay,Filipino,Indonesian,English": 0.38666666666666666, + "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.36, + "Vietnamese,Malay,Filipino,Chinese,English": 0.34, + "Vietnamese,Malay,Filipino,Spanish,English": 0.3933333333333333, + "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.38, + "Vietnamese,Malay,Indonesian,Chinese,English": 0.38, + "Vietnamese,Malay,Indonesian,Spanish,English": 0.4266666666666667, + "Vietnamese,Malay,Chinese,Spanish,English": 0.38, + "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.38, + "Vietnamese,Filipino,Indonesian,Chinese,English": 0.36666666666666664, + "Vietnamese,Filipino,Indonesian,Spanish,English": 0.44, + "Vietnamese,Filipino,Chinese,Spanish,English": 0.38, + "Vietnamese,Indonesian,Chinese,Spanish,English": 0.4266666666666667, + "Malay,Filipino,Indonesian,Chinese,Spanish": 0.38666666666666666, + "Malay,Filipino,Indonesian,Chinese,English": 0.38, + "Malay,Filipino,Indonesian,Spanish,English": 0.44666666666666666, + "Malay,Filipino,Chinese,Spanish,English": 0.4, + "Malay,Indonesian,Chinese,Spanish,English": 0.43333333333333335, + "Filipino,Indonesian,Chinese,Spanish,English": 0.41333333333333333 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.2, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 + "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.3333333333333333, + "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.32, + "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.37333333333333335, + "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.3333333333333333, + "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.36, + "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.35333333333333333, + "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.36666666666666664 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 + "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.31333333333333335 } }, - "AC3_2": 0.5357973585150474, - "AC3_3": 0.43934621094658105, - "AC3_4": 0.3652115705972421, - "AC3_5": 0.30581560279505055, - "AC3_6": 0.2559517176970173, - "AC3_7": 0.21168384876451626 + "AC3_2": 0.6641195483429715, + "AC3_3": 0.5896508746637111, + "AC3_4": 0.5356926666021684, + "AC3_5": 0.49269153004317184, + "AC3_6": 0.45665813917656617, + "AC3_7": 0.4253255207897209 }, "prompt_2": { - "overall_acc": 0.5314285714285715, + "overall_acc": 0.6885714285714286, "language_acc": { - "Spanish": 0.5933333333333334, - "English": 0.64, - "Chinese": 0.4533333333333333, - "Vietnamese": 0.5466666666666666, - "Filipino": 0.5866666666666667, - "Indonesian": 0.4666666666666667, - "Malay": 0.43333333333333335 + "Vietnamese": 0.6333333333333333, + "Malay": 0.6533333333333333, + "Filipino": 0.6666666666666666, + "Indonesian": 0.6933333333333334, + "Chinese": 0.6533333333333333, + "Spanish": 0.7333333333333333, + "English": 0.7866666666666666 }, - "consistency_score_2": 0.540952380952381, - "consistency_score_3": 0.3641904761904761, - "consistency_score_4": 0.2668571428571429, - "consistency_score_5": 0.20285714285714285, - "consistency_score_6": 0.1580952380952381, - "consistency_score_7": 0.12666666666666668, + "consistency_score_2": 0.6926984126984127, + "consistency_score_3": 0.5647619047619048, + "consistency_score_4": 0.4864761904761906, + "consistency_score_5": 0.43142857142857144, + "consistency_score_6": 0.3904761904761905, + "consistency_score_7": 0.36, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6333333333333333, - "Spanish,Chinese": 0.46, - "Spanish,Vietnamese": 0.58, - "Spanish,Filipino": 0.6066666666666667, - "Spanish,Indonesian": 0.56, - "Spanish,Malay": 0.48, - "English,Chinese": 0.5, - "English,Vietnamese": 0.5866666666666667, - "English,Filipino": 0.64, - "English,Indonesian": 0.56, - "English,Malay": 0.48, - "Chinese,Vietnamese": 0.5066666666666667, - "Chinese,Filipino": 0.5, - "Chinese,Indonesian": 0.5066666666666667, - "Chinese,Malay": 0.5066666666666667, - "Vietnamese,Filipino": 0.6066666666666667, - "Vietnamese,Indonesian": 0.5466666666666666, - "Vietnamese,Malay": 0.5066666666666667, - "Filipino,Indonesian": 0.6133333333333333, - "Filipino,Malay": 0.4533333333333333, - "Indonesian,Malay": 0.5266666666666666 + "Vietnamese,Malay": 0.62, + "Vietnamese,Filipino": 0.66, + "Vietnamese,Indonesian": 0.7066666666666667, + "Vietnamese,Chinese": 0.6533333333333333, + "Vietnamese,Spanish": 0.6666666666666666, + "Vietnamese,English": 0.68, + "Malay,Filipino": 0.6466666666666666, + "Malay,Indonesian": 0.6866666666666666, + "Malay,Chinese": 0.66, + "Malay,Spanish": 0.68, + "Malay,English": 0.6733333333333333, + "Filipino,Indonesian": 0.72, + "Filipino,Chinese": 0.6066666666666667, + "Filipino,Spanish": 0.7066666666666667, + "Filipino,English": 0.7333333333333333, + "Indonesian,Chinese": 0.6933333333333334, + "Indonesian,Spanish": 0.7666666666666667, + "Indonesian,English": 0.7933333333333333, + "Chinese,Spanish": 0.6533333333333333, + "Chinese,English": 0.7, + "Spanish,English": 0.84 }, "3_combine": { - "Spanish,English,Chinese": 0.32666666666666666, - "Spanish,English,Vietnamese": 0.44666666666666666, - "Spanish,English,Filipino": 0.4866666666666667, - "Spanish,English,Indonesian": 0.4266666666666667, - "Spanish,English,Malay": 0.34, - "Spanish,Chinese,Vietnamese": 0.34, - "Spanish,Chinese,Filipino": 0.36, - "Spanish,Chinese,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Malay": 0.28, - "Spanish,Vietnamese,Filipino": 0.43333333333333335, - "Spanish,Vietnamese,Indonesian": 0.4066666666666667, - "Spanish,Vietnamese,Malay": 0.35333333333333333, - "Spanish,Filipino,Indonesian": 0.42, - "Spanish,Filipino,Malay": 0.34, - "Spanish,Indonesian,Malay": 0.3333333333333333, - "English,Chinese,Vietnamese": 0.3466666666666667, - "English,Chinese,Filipino": 0.38, - "English,Chinese,Indonesian": 0.3333333333333333, - "English,Chinese,Malay": 0.3, - "English,Vietnamese,Filipino": 0.46, - "English,Vietnamese,Indonesian": 0.41333333333333333, - "English,Vietnamese,Malay": 0.35333333333333333, - "English,Filipino,Indonesian": 0.44666666666666666, - "English,Filipino,Malay": 0.32666666666666666, - "English,Indonesian,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Filipino": 0.3933333333333333, - "Chinese,Vietnamese,Indonesian": 0.3333333333333333, - "Chinese,Vietnamese,Malay": 0.32666666666666666, - "Chinese,Filipino,Indonesian": 0.36666666666666664, - "Chinese,Filipino,Malay": 0.29333333333333333, - "Chinese,Indonesian,Malay": 0.30666666666666664, - "Vietnamese,Filipino,Indonesian": 0.43333333333333335, - "Vietnamese,Filipino,Malay": 0.3466666666666667, - "Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Filipino,Indonesian,Malay": 0.32 + "Vietnamese,Malay,Filipino": 0.49333333333333335, + "Vietnamese,Malay,Indonesian": 0.5333333333333333, + "Vietnamese,Malay,Chinese": 0.5, + "Vietnamese,Malay,Spanish": 0.5066666666666667, + "Vietnamese,Malay,English": 0.52, + "Vietnamese,Filipino,Indonesian": 0.5666666666666667, + "Vietnamese,Filipino,Chinese": 0.4866666666666667, + "Vietnamese,Filipino,Spanish": 0.54, + "Vietnamese,Filipino,English": 0.5666666666666667, + "Vietnamese,Indonesian,Chinese": 0.5466666666666666, + "Vietnamese,Indonesian,Spanish": 0.58, + "Vietnamese,Indonesian,English": 0.6133333333333333, + "Vietnamese,Chinese,Spanish": 0.5066666666666667, + "Vietnamese,Chinese,English": 0.54, + "Vietnamese,Spanish,English": 0.6066666666666667, + "Malay,Filipino,Indonesian": 0.56, + "Malay,Filipino,Chinese": 0.49333333333333335, + "Malay,Filipino,Spanish": 0.5533333333333333, + "Malay,Filipino,English": 0.56, + "Malay,Indonesian,Chinese": 0.54, + "Malay,Indonesian,Spanish": 0.5933333333333334, + "Malay,Indonesian,English": 0.6066666666666667, + "Malay,Chinese,Spanish": 0.5333333333333333, + "Malay,Chinese,English": 0.5533333333333333, + "Malay,Spanish,English": 0.6133333333333333, + "Filipino,Indonesian,Chinese": 0.5333333333333333, + "Filipino,Indonesian,Spanish": 0.6333333333333333, + "Filipino,Indonesian,English": 0.6533333333333333, + "Filipino,Chinese,Spanish": 0.52, + "Filipino,Chinese,English": 0.5533333333333333, + "Filipino,Spanish,English": 0.66, + "Indonesian,Chinese,Spanish": 0.5733333333333334, + "Indonesian,Chinese,English": 0.6133333333333333, + "Indonesian,Spanish,English": 0.7066666666666667, + "Chinese,Spanish,English": 0.6066666666666667 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.26666666666666666, - "Spanish,English,Chinese,Filipino": 0.2866666666666667, - "Spanish,English,Chinese,Indonesian": 0.25333333333333335, - "Spanish,English,Chinese,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.36666666666666664, - "Spanish,English,Vietnamese,Indonesian": 0.3466666666666667, - "Spanish,English,Vietnamese,Malay": 0.28, - "Spanish,English,Filipino,Indonesian": 0.36666666666666664, - "Spanish,English,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.23333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.22, - "Spanish,Chinese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino": 0.29333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.26, - "English,Chinese,Vietnamese,Malay": 0.23333333333333334, - "English,Chinese,Filipino,Indonesian": 0.3, - "English,Chinese,Filipino,Malay": 0.22, - "English,Chinese,Indonesian,Malay": 0.20666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "English,Vietnamese,Filipino,Malay": 0.2733333333333333, - "English,Vietnamese,Indonesian,Malay": 0.26, - "English,Filipino,Indonesian,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Chinese,Filipino,Indonesian,Malay": 0.22, - "Vietnamese,Filipino,Indonesian,Malay": 0.24 + "Vietnamese,Malay,Filipino,Indonesian": 0.46, + "Vietnamese,Malay,Filipino,Chinese": 0.4066666666666667, + "Vietnamese,Malay,Filipino,Spanish": 0.44, + "Vietnamese,Malay,Filipino,English": 0.46, + "Vietnamese,Malay,Indonesian,Chinese": 0.44, + "Vietnamese,Malay,Indonesian,Spanish": 0.48, + "Vietnamese,Malay,Indonesian,English": 0.4866666666666667, + "Vietnamese,Malay,Chinese,Spanish": 0.4266666666666667, + "Vietnamese,Malay,Chinese,English": 0.44666666666666666, + "Vietnamese,Malay,Spanish,English": 0.47333333333333333, + "Vietnamese,Filipino,Indonesian,Chinese": 0.44, + "Vietnamese,Filipino,Indonesian,Spanish": 0.5066666666666667, + "Vietnamese,Filipino,Indonesian,English": 0.52, + "Vietnamese,Filipino,Chinese,Spanish": 0.4266666666666667, + "Vietnamese,Filipino,Chinese,English": 0.46, + "Vietnamese,Filipino,Spanish,English": 0.52, + "Vietnamese,Indonesian,Chinese,Spanish": 0.4666666666666667, + "Vietnamese,Indonesian,Chinese,English": 0.5, + "Vietnamese,Indonesian,Spanish,English": 0.5533333333333333, + "Vietnamese,Chinese,Spanish,English": 0.4866666666666667, + "Malay,Filipino,Indonesian,Chinese": 0.44666666666666666, + "Malay,Filipino,Indonesian,Spanish": 0.5133333333333333, + "Malay,Filipino,Indonesian,English": 0.5266666666666666, + "Malay,Filipino,Chinese,Spanish": 0.44666666666666666, + "Malay,Filipino,Chinese,English": 0.47333333333333333, + "Malay,Filipino,Spanish,English": 0.5133333333333333, + "Malay,Indonesian,Chinese,Spanish": 0.4866666666666667, + "Malay,Indonesian,Chinese,English": 0.5066666666666667, + "Malay,Indonesian,Spanish,English": 0.56, + "Malay,Chinese,Spanish,English": 0.5133333333333333, + "Filipino,Indonesian,Chinese,Spanish": 0.47333333333333333, + "Filipino,Indonesian,Chinese,English": 0.5, + "Filipino,Indonesian,Spanish,English": 0.6066666666666667, + "Filipino,Chinese,Spanish,English": 0.5066666666666667, + "Indonesian,Chinese,Spanish,English": 0.5533333333333333 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.21333333333333335, - "Spanish,English,Chinese,Vietnamese,Malay": 0.18, - "Spanish,English,Chinese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.18, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - } - }, - "AC3_2": 0.5361481856896639, - "AC3_3": 0.43219541831475, - "AC3_4": 0.3553001328934204, - "AC3_5": 0.2936297942901949, - "AC3_6": 0.24369376476339485, - "AC3_7": 0.20457308245806055 - }, + "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.38, + "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.4266666666666667, + "Vietnamese,Malay,Filipino,Indonesian,English": 0.43333333333333335, + "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.37333333333333335, + "Vietnamese,Malay,Filipino,Chinese,English": 0.4, + "Vietnamese,Malay,Filipino,Spanish,English": 0.42, + "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.4066666666666667, + "Vietnamese,Malay,Indonesian,Chinese,English": 0.42, + "Vietnamese,Malay,Indonesian,Spanish,English": 0.4533333333333333, + "Vietnamese,Malay,Chinese,Spanish,English": 0.41333333333333333, + "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.4, + "Vietnamese,Filipino,Indonesian,Chinese,English": 0.42, + "Vietnamese,Filipino,Indonesian,Spanish,English": 0.4866666666666667, + "Vietnamese,Filipino,Chinese,Spanish,English": 0.4266666666666667, + "Vietnamese,Indonesian,Chinese,Spanish,English": 0.46, + "Malay,Filipino,Indonesian,Chinese,Spanish": 0.42, + "Malay,Filipino,Indonesian,Chinese,English": 0.44, + "Malay,Filipino,Indonesian,Spanish,English": 0.49333333333333335, + "Malay,Filipino,Chinese,Spanish,English": 0.44, + "Malay,Indonesian,Chinese,Spanish,English": 0.48, + "Filipino,Indonesian,Chinese,Spanish,English": 0.4666666666666667 + }, + "6_combine": { + "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.36, + "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.37333333333333335, + "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.4066666666666667, + "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.37333333333333335, + "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.4, + "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.4, + "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.42 + }, + "7_combine": { + "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.36 + } + }, + "AC3_2": 0.6906287552443499, + "AC3_3": 0.6205514545748909, + "AC3_4": 0.570144732832474, + "AC3_5": 0.5304810495153178, + "AC3_6": 0.49834825364187546, + "AC3_7": 0.4728065394644463 + }, "prompt_3": { - "overall_acc": 0.5076190476190476, + "overall_acc": 0.7009523809523809, "language_acc": { - "Spanish": 0.5733333333333334, - "English": 0.5933333333333334, - "Chinese": 0.46, - "Vietnamese": 0.4666666666666667, - "Filipino": 0.5733333333333334, - "Indonesian": 0.4866666666666667, - "Malay": 0.4 + "Vietnamese": 0.6266666666666667, + "Malay": 0.68, + "Filipino": 0.6933333333333334, + "Indonesian": 0.6933333333333334, + "Chinese": 0.6533333333333333, + "Spanish": 0.7333333333333333, + "English": 0.8266666666666667 }, - "consistency_score_2": 0.5298412698412699, - "consistency_score_3": 0.3455238095238095, - "consistency_score_4": 0.2504761904761904, - "consistency_score_5": 0.19428571428571428, - "consistency_score_6": 0.1580952380952381, - "consistency_score_7": 0.13333333333333333, + "consistency_score_2": 0.6974603174603174, + "consistency_score_3": 0.5767619047619048, + "consistency_score_4": 0.5036190476190475, + "consistency_score_5": 0.45269841269841266, + "consistency_score_6": 0.41523809523809524, + "consistency_score_7": 0.38666666666666666, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.6066666666666667, - "Spanish,Chinese": 0.54, - "Spanish,Vietnamese": 0.46, - "Spanish,Filipino": 0.6266666666666667, - "Spanish,Indonesian": 0.5333333333333333, - "Spanish,Malay": 0.4866666666666667, - "English,Chinese": 0.5333333333333333, - "English,Vietnamese": 0.44, - "English,Filipino": 0.6266666666666667, - "English,Indonesian": 0.5, - "English,Malay": 0.48, - "Chinese,Vietnamese": 0.5333333333333333, - "Chinese,Filipino": 0.5, - "Chinese,Indonesian": 0.48, - "Chinese,Malay": 0.4866666666666667, - "Vietnamese,Filipino": 0.5266666666666666, - "Vietnamese,Indonesian": 0.5533333333333333, - "Vietnamese,Malay": 0.5266666666666666, - "Filipino,Indonesian": 0.5866666666666667, - "Filipino,Malay": 0.49333333333333335, - "Indonesian,Malay": 0.6066666666666667 + "Vietnamese,Malay": 0.6466666666666666, + "Vietnamese,Filipino": 0.68, + "Vietnamese,Indonesian": 0.64, + "Vietnamese,Chinese": 0.62, + "Vietnamese,Spanish": 0.6533333333333333, + "Vietnamese,English": 0.64, + "Malay,Filipino": 0.7, + "Malay,Indonesian": 0.7466666666666667, + "Malay,Chinese": 0.6466666666666666, + "Malay,Spanish": 0.7666666666666667, + "Malay,English": 0.74, + "Filipino,Indonesian": 0.7, + "Filipino,Chinese": 0.6466666666666666, + "Filipino,Spanish": 0.7266666666666667, + "Filipino,English": 0.7466666666666667, + "Indonesian,Chinese": 0.6466666666666666, + "Indonesian,Spanish": 0.74, + "Indonesian,English": 0.76, + "Chinese,Spanish": 0.68, + "Chinese,English": 0.6866666666666666, + "Spanish,English": 0.8333333333333334 }, "3_combine": { - "Spanish,English,Chinese": 0.38666666666666666, - "Spanish,English,Vietnamese": 0.30666666666666664, - "Spanish,English,Filipino": 0.44666666666666666, - "Spanish,English,Indonesian": 0.36, - "Spanish,English,Malay": 0.34, - "Spanish,Chinese,Vietnamese": 0.32666666666666666, - "Spanish,Chinese,Filipino": 0.38, - "Spanish,Chinese,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino": 0.36666666666666664, - "Spanish,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,Vietnamese,Malay": 0.30666666666666664, - "Spanish,Filipino,Indonesian": 0.4, - "Spanish,Filipino,Malay": 0.36666666666666664, - "Spanish,Indonesian,Malay": 0.35333333333333333, - "English,Chinese,Vietnamese": 0.32666666666666666, - "English,Chinese,Filipino": 0.36666666666666664, - "English,Chinese,Indonesian": 0.31333333333333335, - "English,Chinese,Malay": 0.30666666666666664, - "English,Vietnamese,Filipino": 0.3333333333333333, - "English,Vietnamese,Indonesian": 0.3, - "English,Vietnamese,Malay": 0.28, - "English,Filipino,Indonesian": 0.38666666666666666, - "English,Filipino,Malay": 0.35333333333333333, - "English,Indonesian,Malay": 0.34, - "Chinese,Vietnamese,Filipino": 0.36, - "Chinese,Vietnamese,Indonesian": 0.35333333333333333, - "Chinese,Vietnamese,Malay": 0.3333333333333333, - "Chinese,Filipino,Indonesian": 0.3466666666666667, - "Chinese,Filipino,Malay": 0.31333333333333335, - "Chinese,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian": 0.38, - "Vietnamese,Filipino,Malay": 0.32666666666666666, - "Vietnamese,Indonesian,Malay": 0.38, - "Filipino,Indonesian,Malay": 0.38 + "Vietnamese,Malay,Filipino": 0.5466666666666666, + "Vietnamese,Malay,Indonesian": 0.5533333333333333, + "Vietnamese,Malay,Chinese": 0.5133333333333333, + "Vietnamese,Malay,Spanish": 0.56, + "Vietnamese,Malay,English": 0.5466666666666666, + "Vietnamese,Filipino,Indonesian": 0.5533333333333333, + "Vietnamese,Filipino,Chinese": 0.5133333333333333, + "Vietnamese,Filipino,Spanish": 0.5666666666666667, + "Vietnamese,Filipino,English": 0.5666666666666667, + "Vietnamese,Indonesian,Chinese": 0.49333333333333335, + "Vietnamese,Indonesian,Spanish": 0.5466666666666666, + "Vietnamese,Indonesian,English": 0.56, + "Vietnamese,Chinese,Spanish": 0.52, + "Vietnamese,Chinese,English": 0.52, + "Vietnamese,Spanish,English": 0.5933333333333334, + "Malay,Filipino,Indonesian": 0.6, + "Malay,Filipino,Chinese": 0.5266666666666666, + "Malay,Filipino,Spanish": 0.62, + "Malay,Filipino,English": 0.62, + "Malay,Indonesian,Chinese": 0.5466666666666666, + "Malay,Indonesian,Spanish": 0.64, + "Malay,Indonesian,English": 0.6466666666666666, + "Malay,Chinese,Spanish": 0.5733333333333334, + "Malay,Chinese,English": 0.5666666666666667, + "Malay,Spanish,English": 0.6866666666666666, + "Filipino,Indonesian,Chinese": 0.5266666666666666, + "Filipino,Indonesian,Spanish": 0.62, + "Filipino,Indonesian,English": 0.6333333333333333, + "Filipino,Chinese,Spanish": 0.5666666666666667, + "Filipino,Chinese,English": 0.56, + "Filipino,Spanish,English": 0.6666666666666666, + "Indonesian,Chinese,Spanish": 0.56, + "Indonesian,Chinese,English": 0.5666666666666667, + "Indonesian,Spanish,English": 0.6866666666666666, + "Chinese,Spanish,English": 0.62 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.26, - "Spanish,English,Chinese,Filipino": 0.2866666666666667, - "Spanish,English,Chinese,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Malay": 0.24, - "Spanish,English,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.21333333333333335, - "Spanish,English,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Filipino,Indonesian": 0.29333333333333333, - "Spanish,English,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.23333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.26, - "Spanish,Chinese,Filipino,Malay": 0.26, - "Spanish,Chinese,Indonesian,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.24, - "Spanish,Filipino,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.2733333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "English,Chinese,Vietnamese,Malay": 0.21333333333333335, - "English,Chinese,Filipino,Indonesian": 0.25333333333333335, - "English,Chinese,Filipino,Malay": 0.25333333333333335, - "English,Chinese,Indonesian,Malay": 0.21333333333333335, - "English,Vietnamese,Filipino,Indonesian": 0.24, - "English,Vietnamese,Filipino,Malay": 0.23333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "English,Filipino,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 + "Vietnamese,Malay,Filipino,Indonesian": 0.49333333333333335, + "Vietnamese,Malay,Filipino,Chinese": 0.44666666666666666, + "Vietnamese,Malay,Filipino,Spanish": 0.5066666666666667, + "Vietnamese,Malay,Filipino,English": 0.5, + "Vietnamese,Malay,Indonesian,Chinese": 0.44, + "Vietnamese,Malay,Indonesian,Spanish": 0.5, + "Vietnamese,Malay,Indonesian,English": 0.49333333333333335, + "Vietnamese,Malay,Chinese,Spanish": 0.47333333333333333, + "Vietnamese,Malay,Chinese,English": 0.4666666666666667, + "Vietnamese,Malay,Spanish,English": 0.5266666666666666, + "Vietnamese,Filipino,Indonesian,Chinese": 0.44666666666666666, + "Vietnamese,Filipino,Indonesian,Spanish": 0.5, + "Vietnamese,Filipino,Indonesian,English": 0.5133333333333333, + "Vietnamese,Filipino,Chinese,Spanish": 0.47333333333333333, + "Vietnamese,Filipino,Chinese,English": 0.47333333333333333, + "Vietnamese,Filipino,Spanish,English": 0.54, + "Vietnamese,Indonesian,Chinese,Spanish": 0.4533333333333333, + "Vietnamese,Indonesian,Chinese,English": 0.46, + "Vietnamese,Indonesian,Spanish,English": 0.5266666666666666, + "Vietnamese,Chinese,Spanish,English": 0.49333333333333335, + "Malay,Filipino,Indonesian,Chinese": 0.4666666666666667, + "Malay,Filipino,Indonesian,Spanish": 0.54, + "Malay,Filipino,Indonesian,English": 0.5533333333333333, + "Malay,Filipino,Chinese,Spanish": 0.5, + "Malay,Filipino,Chinese,English": 0.4866666666666667, + "Malay,Filipino,Spanish,English": 0.58, + "Malay,Indonesian,Chinese,Spanish": 0.5066666666666667, + "Malay,Indonesian,Chinese,English": 0.5066666666666667, + "Malay,Indonesian,Spanish,English": 0.6066666666666667, + "Malay,Chinese,Spanish,English": 0.54, + "Filipino,Indonesian,Chinese,Spanish": 0.4866666666666667, + "Filipino,Indonesian,Chinese,English": 0.4866666666666667, + "Filipino,Indonesian,Spanish,English": 0.5866666666666667, + "Filipino,Chinese,Spanish,English": 0.5266666666666666, + "Indonesian,Chinese,Spanish,English": 0.5266666666666666 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.22, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,English,Chinese,Vietnamese,Malay": 0.18666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian": 0.2, - "Spanish,English,Chinese,Filipino,Malay": 0.21333333333333335, - "Spanish,English,Chinese,Indonesian,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.2, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 + "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.4066666666666667, + "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.46, + "Vietnamese,Malay,Filipino,Indonesian,English": 0.46, + "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.43333333333333335, + "Vietnamese,Malay,Filipino,Chinese,English": 0.4266666666666667, + "Vietnamese,Malay,Filipino,Spanish,English": 0.4866666666666667, + "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.4266666666666667, + "Vietnamese,Malay,Indonesian,Chinese,English": 0.42, + "Vietnamese,Malay,Indonesian,Spanish,English": 0.48, + "Vietnamese,Malay,Chinese,Spanish,English": 0.4533333333333333, + "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.42, + "Vietnamese,Filipino,Indonesian,Chinese,English": 0.4266666666666667, + "Vietnamese,Filipino,Indonesian,Spanish,English": 0.4866666666666667, + "Vietnamese,Filipino,Chinese,Spanish,English": 0.4533333333333333, + "Vietnamese,Indonesian,Chinese,Spanish,English": 0.44, + "Malay,Filipino,Indonesian,Chinese,Spanish": 0.44666666666666666, + "Malay,Filipino,Indonesian,Chinese,English": 0.44, + "Malay,Filipino,Indonesian,Spanish,English": 0.52, + "Malay,Filipino,Chinese,Spanish,English": 0.47333333333333333, + "Malay,Indonesian,Chinese,Spanish,English": 0.4866666666666667, + "Filipino,Indonesian,Chinese,Spanish,English": 0.46 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 + "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.4, + "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.3933333333333333, + "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.44666666666666666, + "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.42, + "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.41333333333333333, + "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.4066666666666667, + "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.4266666666666667 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 + "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.38666666666666666 } }, - "AC3_2": 0.5184921605841666, - "AC3_3": 0.4111725618544405, - "AC3_4": 0.3354367073020077, - "AC3_5": 0.2810157006771571, - "AC3_6": 0.24110089239514768, - "AC3_7": 0.21119366019488556 + "AC3_2": 0.6992019890316715, + "AC3_3": 0.6328216485860745, + "AC3_4": 0.5861221047072226, + "AC3_5": 0.5501145267757028, + "AC3_6": 0.5215277100134094, + "AC3_7": 0.4984004669711045 }, "prompt_4": { - "overall_acc": 0.539047619047619, + "overall_acc": 0.6647619047619048, "language_acc": { - "Spanish": 0.6, - "English": 0.6333333333333333, - "Chinese": 0.5266666666666666, - "Vietnamese": 0.5, - "Filipino": 0.5933333333333334, - "Indonesian": 0.5, - "Malay": 0.42 + "Vietnamese": 0.5733333333333334, + "Malay": 0.6333333333333333, + "Filipino": 0.6266666666666667, + "Indonesian": 0.6933333333333334, + "Chinese": 0.6266666666666667, + "Spanish": 0.72, + "English": 0.78 }, - "consistency_score_2": 0.5730158730158731, - "consistency_score_3": 0.3965714285714285, - "consistency_score_4": 0.2971428571428571, - "consistency_score_5": 0.23365079365079366, - "consistency_score_6": 0.1904761904761905, - "consistency_score_7": 0.16, + "consistency_score_2": 0.6571428571428571, + "consistency_score_3": 0.520952380952381, + "consistency_score_4": 0.44552380952380943, + "consistency_score_5": 0.39587301587301593, + "consistency_score_6": 0.35999999999999993, + "consistency_score_7": 0.3333333333333333, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.7066666666666667, - "Spanish,Chinese": 0.62, - "Spanish,Vietnamese": 0.5133333333333333, - "Spanish,Filipino": 0.64, - "Spanish,Indonesian": 0.6466666666666666, - "Spanish,Malay": 0.52, - "English,Chinese": 0.5866666666666667, - "English,Vietnamese": 0.4666666666666667, - "English,Filipino": 0.6466666666666666, - "English,Indonesian": 0.62, - "English,Malay": 0.5266666666666666, - "Chinese,Vietnamese": 0.48, - "Chinese,Filipino": 0.6533333333333333, - "Chinese,Indonesian": 0.62, - "Chinese,Malay": 0.5133333333333333, - "Vietnamese,Filipino": 0.5733333333333334, - "Vietnamese,Indonesian": 0.4866666666666667, - "Vietnamese,Malay": 0.5266666666666666, - "Filipino,Indonesian": 0.6866666666666666, - "Filipino,Malay": 0.5, - "Indonesian,Malay": 0.5 + "Vietnamese,Malay": 0.58, + "Vietnamese,Filipino": 0.6333333333333333, + "Vietnamese,Indonesian": 0.6266666666666667, + "Vietnamese,Chinese": 0.6533333333333333, + "Vietnamese,Spanish": 0.6133333333333333, + "Vietnamese,English": 0.6, + "Malay,Filipino": 0.62, + "Malay,Indonesian": 0.6666666666666666, + "Malay,Chinese": 0.5666666666666667, + "Malay,Spanish": 0.68, + "Malay,English": 0.6266666666666667, + "Filipino,Indonesian": 0.7, + "Filipino,Chinese": 0.6, + "Filipino,Spanish": 0.68, + "Filipino,English": 0.68, + "Indonesian,Chinese": 0.64, + "Indonesian,Spanish": 0.72, + "Indonesian,English": 0.7466666666666667, + "Chinese,Spanish": 0.6333333333333333, + "Chinese,English": 0.7, + "Spanish,English": 0.8333333333333334 }, "3_combine": { - "Spanish,English,Chinese": 0.49333333333333335, - "Spanish,English,Vietnamese": 0.38, - "Spanish,English,Filipino": 0.52, - "Spanish,English,Indonesian": 0.5066666666666667, - "Spanish,English,Malay": 0.41333333333333333, - "Spanish,Chinese,Vietnamese": 0.35333333333333333, - "Spanish,Chinese,Filipino": 0.49333333333333335, - "Spanish,Chinese,Indonesian": 0.47333333333333333, - "Spanish,Chinese,Malay": 0.36, - "Spanish,Vietnamese,Filipino": 0.4, - "Spanish,Vietnamese,Indonesian": 0.36666666666666664, - "Spanish,Vietnamese,Malay": 0.32666666666666666, - "Spanish,Filipino,Indonesian": 0.5266666666666666, - "Spanish,Filipino,Malay": 0.35333333333333333, - "Spanish,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese": 0.32, - "English,Chinese,Filipino": 0.48, - "English,Chinese,Indonesian": 0.4533333333333333, - "English,Chinese,Malay": 0.35333333333333333, - "English,Vietnamese,Filipino": 0.37333333333333335, - "English,Vietnamese,Indonesian": 0.32666666666666666, - "English,Vietnamese,Malay": 0.32, - "English,Filipino,Indonesian": 0.5, - "English,Filipino,Malay": 0.36666666666666664, - "English,Indonesian,Malay": 0.34, - "Chinese,Vietnamese,Filipino": 0.3933333333333333, - "Chinese,Vietnamese,Indonesian": 0.34, - "Chinese,Vietnamese,Malay": 0.31333333333333335, - "Chinese,Filipino,Indonesian": 0.52, - "Chinese,Filipino,Malay": 0.37333333333333335, - "Chinese,Indonesian,Malay": 0.35333333333333333, - "Vietnamese,Filipino,Indonesian": 0.4066666666666667, - "Vietnamese,Filipino,Malay": 0.3466666666666667, - "Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Filipino,Indonesian,Malay": 0.37333333333333335 + "Vietnamese,Malay,Filipino": 0.46, + "Vietnamese,Malay,Indonesian": 0.47333333333333333, + "Vietnamese,Malay,Chinese": 0.4533333333333333, + "Vietnamese,Malay,Spanish": 0.49333333333333335, + "Vietnamese,Malay,English": 0.4533333333333333, + "Vietnamese,Filipino,Indonesian": 0.5133333333333333, + "Vietnamese,Filipino,Chinese": 0.48, + "Vietnamese,Filipino,Spanish": 0.5, + "Vietnamese,Filipino,English": 0.5, + "Vietnamese,Indonesian,Chinese": 0.5133333333333333, + "Vietnamese,Indonesian,Spanish": 0.5066666666666667, + "Vietnamese,Indonesian,English": 0.5333333333333333, + "Vietnamese,Chinese,Spanish": 0.4866666666666667, + "Vietnamese,Chinese,English": 0.5133333333333333, + "Vietnamese,Spanish,English": 0.54, + "Malay,Filipino,Indonesian": 0.5333333333333333, + "Malay,Filipino,Chinese": 0.44, + "Malay,Filipino,Spanish": 0.5133333333333333, + "Malay,Filipino,English": 0.5066666666666667, + "Malay,Indonesian,Chinese": 0.4666666666666667, + "Malay,Indonesian,Spanish": 0.5733333333333334, + "Malay,Indonesian,English": 0.56, + "Malay,Chinese,Spanish": 0.4866666666666667, + "Malay,Chinese,English": 0.4866666666666667, + "Malay,Spanish,English": 0.5933333333333334, + "Filipino,Indonesian,Chinese": 0.5066666666666667, + "Filipino,Indonesian,Spanish": 0.5733333333333334, + "Filipino,Indonesian,English": 0.5866666666666667, + "Filipino,Chinese,Spanish": 0.5, + "Filipino,Chinese,English": 0.52, + "Filipino,Spanish,English": 0.6133333333333333, + "Indonesian,Chinese,Spanish": 0.5266666666666666, + "Indonesian,Chinese,English": 0.56, + "Indonesian,Spanish,English": 0.6666666666666666, + "Chinese,Spanish,English": 0.6 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.2866666666666667, - "Spanish,English,Chinese,Filipino": 0.4066666666666667, - "Spanish,English,Chinese,Indonesian": 0.38666666666666666, - "Spanish,English,Chinese,Malay": 0.32, - "Spanish,English,Vietnamese,Filipino": 0.32, - "Spanish,English,Vietnamese,Indonesian": 0.28, - "Spanish,English,Vietnamese,Malay": 0.2733333333333333, - "Spanish,English,Filipino,Indonesian": 0.42, - "Spanish,English,Filipino,Malay": 0.31333333333333335, - "Spanish,English,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian": 0.28, - "Spanish,Chinese,Vietnamese,Malay": 0.24, - "Spanish,Chinese,Filipino,Indonesian": 0.41333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.29333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.26, - "Spanish,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino": 0.2866666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "English,Chinese,Vietnamese,Malay": 0.23333333333333334, - "English,Chinese,Filipino,Indonesian": 0.4, - "English,Chinese,Filipino,Malay": 0.3, - "English,Chinese,Indonesian,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.28, - "English,Vietnamese,Filipino,Malay": 0.26, - "English,Vietnamese,Indonesian,Malay": 0.24, - "English,Filipino,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Chinese,Vietnamese,Filipino,Malay": 0.26, - "Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.26 + "Vietnamese,Malay,Filipino,Indonesian": 0.41333333333333333, + "Vietnamese,Malay,Filipino,Chinese": 0.38666666666666666, + "Vietnamese,Malay,Filipino,Spanish": 0.4266666666666667, + "Vietnamese,Malay,Filipino,English": 0.4, + "Vietnamese,Malay,Indonesian,Chinese": 0.3933333333333333, + "Vietnamese,Malay,Indonesian,Spanish": 0.44, + "Vietnamese,Malay,Indonesian,English": 0.4266666666666667, + "Vietnamese,Malay,Chinese,Spanish": 0.4066666666666667, + "Vietnamese,Malay,Chinese,English": 0.4, + "Vietnamese,Malay,Spanish,English": 0.44666666666666666, + "Vietnamese,Filipino,Indonesian,Chinese": 0.42, + "Vietnamese,Filipino,Indonesian,Spanish": 0.4533333333333333, + "Vietnamese,Filipino,Indonesian,English": 0.46, + "Vietnamese,Filipino,Chinese,Spanish": 0.41333333333333333, + "Vietnamese,Filipino,Chinese,English": 0.43333333333333335, + "Vietnamese,Filipino,Spanish,English": 0.4666666666666667, + "Vietnamese,Indonesian,Chinese,Spanish": 0.42, + "Vietnamese,Indonesian,Chinese,English": 0.46, + "Vietnamese,Indonesian,Spanish,English": 0.48, + "Vietnamese,Chinese,Spanish,English": 0.4666666666666667, + "Malay,Filipino,Indonesian,Chinese": 0.3933333333333333, + "Malay,Filipino,Indonesian,Spanish": 0.48, + "Malay,Filipino,Indonesian,English": 0.47333333333333333, + "Malay,Filipino,Chinese,Spanish": 0.41333333333333333, + "Malay,Filipino,Chinese,English": 0.4, + "Malay,Filipino,Spanish,English": 0.4866666666666667, + "Malay,Indonesian,Chinese,Spanish": 0.44, + "Malay,Indonesian,Chinese,English": 0.44, + "Malay,Indonesian,Spanish,English": 0.54, + "Malay,Chinese,Spanish,English": 0.47333333333333333, + "Filipino,Indonesian,Chinese,Spanish": 0.4533333333333333, + "Filipino,Indonesian,Chinese,English": 0.47333333333333333, + "Filipino,Indonesian,Spanish,English": 0.5333333333333333, + "Filipino,Chinese,Spanish,English": 0.47333333333333333, + "Indonesian,Chinese,Spanish,English": 0.5066666666666667 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.26, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.22, - "Spanish,English,Chinese,Filipino,Indonesian": 0.34, - "Spanish,English,Chinese,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.24, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.24, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 + "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.3466666666666667, + "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.4, + "Vietnamese,Malay,Filipino,Indonesian,English": 0.38, + "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.36666666666666664, + "Vietnamese,Malay,Filipino,Chinese,English": 0.35333333333333333, + "Vietnamese,Malay,Filipino,Spanish,English": 0.4, + "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.37333333333333335, + "Vietnamese,Malay,Indonesian,Chinese,English": 0.37333333333333335, + "Vietnamese,Malay,Indonesian,Spanish,English": 0.42, + "Vietnamese,Malay,Chinese,Spanish,English": 0.3933333333333333, + "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.37333333333333335, + "Vietnamese,Filipino,Indonesian,Chinese,English": 0.4, + "Vietnamese,Filipino,Indonesian,Spanish,English": 0.4266666666666667, + "Vietnamese,Filipino,Chinese,Spanish,English": 0.4, + "Vietnamese,Indonesian,Chinese,Spanish,English": 0.41333333333333333, + "Malay,Filipino,Indonesian,Chinese,Spanish": 0.38666666666666666, + "Malay,Filipino,Indonesian,Chinese,English": 0.38, + "Malay,Filipino,Indonesian,Spanish,English": 0.46, + "Malay,Filipino,Chinese,Spanish,English": 0.4, + "Malay,Indonesian,Chinese,Spanish,English": 0.43333333333333335, + "Filipino,Indonesian,Chinese,Spanish,English": 0.43333333333333335 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 + "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.34, + "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.3333333333333333, + "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.38, + "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.35333333333333333, + "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.36666666666666664, + "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.36666666666666664, + "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.38 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16 + "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.3333333333333333 } }, - "AC3_2": 0.5555129616291515, - "AC3_3": 0.4569613773359895, - "AC3_4": 0.3831044581383693, - "AC3_5": 0.32599757400797286, - "AC3_6": 0.2814870072969863, - "AC3_7": 0.2467574931527118 + "AC3_2": 0.6609304239928177, + "AC3_3": 0.5841361636531642, + "AC3_4": 0.5334973735110832, + "AC3_5": 0.4962335199999633, + "AC3_6": 0.4670631969804445, + "AC3_7": 0.4440203561896099 }, "prompt_5": { - "overall_acc": 0.5752380952380952, + "overall_acc": 0.699047619047619, "language_acc": { - "Spanish": 0.5866666666666667, - "English": 0.64, - "Chinese": 0.5666666666666667, - "Vietnamese": 0.5133333333333333, - "Filipino": 0.58, - "Indonesian": 0.5733333333333334, - "Malay": 0.5666666666666667 + "Vietnamese": 0.62, + "Malay": 0.68, + "Filipino": 0.6733333333333333, + "Indonesian": 0.7, + "Chinese": 0.6933333333333334, + "Spanish": 0.7266666666666667, + "English": 0.8 }, - "consistency_score_2": 0.6476190476190475, - "consistency_score_3": 0.4988571428571429, - "consistency_score_4": 0.40685714285714286, - "consistency_score_5": 0.3409523809523809, - "consistency_score_6": 0.2914285714285714, - "consistency_score_7": 0.25333333333333335, + "consistency_score_2": 0.7066666666666666, + "consistency_score_3": 0.5843809523809523, + "consistency_score_4": 0.5060952380952382, + "consistency_score_5": 0.448888888888889, + "consistency_score_6": 0.4038095238095238, + "consistency_score_7": 0.36666666666666664, "detailed_consistency_score": { "2_combine": { - "Spanish,English": 0.78, - "Spanish,Chinese": 0.6466666666666666, - "Spanish,Vietnamese": 0.56, - "Spanish,Filipino": 0.6533333333333333, - "Spanish,Indonesian": 0.68, - "Spanish,Malay": 0.7133333333333334, - "English,Chinese": 0.6533333333333333, - "English,Vietnamese": 0.56, - "English,Filipino": 0.6533333333333333, - "English,Indonesian": 0.68, - "English,Malay": 0.6666666666666666, - "Chinese,Vietnamese": 0.5733333333333334, - "Chinese,Filipino": 0.6733333333333333, - "Chinese,Indonesian": 0.6533333333333333, - "Chinese,Malay": 0.6333333333333333, - "Vietnamese,Filipino": 0.6, - "Vietnamese,Indonesian": 0.5933333333333334, - "Vietnamese,Malay": 0.5533333333333333, + "Vietnamese,Malay": 0.6533333333333333, + "Vietnamese,Filipino": 0.6533333333333333, + "Vietnamese,Indonesian": 0.66, + "Vietnamese,Chinese": 0.64, + "Vietnamese,Spanish": 0.6333333333333333, + "Vietnamese,English": 0.6533333333333333, + "Malay,Filipino": 0.6866666666666666, + "Malay,Indonesian": 0.7466666666666667, + "Malay,Chinese": 0.6733333333333333, + "Malay,Spanish": 0.72, + "Malay,English": 0.74, "Filipino,Indonesian": 0.72, - "Filipino,Malay": 0.66, - "Indonesian,Malay": 0.6933333333333334 + "Filipino,Chinese": 0.68, + "Filipino,Spanish": 0.7466666666666667, + "Filipino,English": 0.7333333333333333, + "Indonesian,Chinese": 0.6933333333333334, + "Indonesian,Spanish": 0.7466666666666667, + "Indonesian,English": 0.76, + "Chinese,Spanish": 0.7066666666666667, + "Chinese,English": 0.7666666666666667, + "Spanish,English": 0.8266666666666667 }, "3_combine": { - "Spanish,English,Chinese": 0.5533333333333333, - "Spanish,English,Vietnamese": 0.46, - "Spanish,English,Filipino": 0.5666666666666667, - "Spanish,English,Indonesian": 0.5733333333333334, - "Spanish,English,Malay": 0.5866666666666667, - "Spanish,Chinese,Vietnamese": 0.44, - "Spanish,Chinese,Filipino": 0.5066666666666667, - "Spanish,Chinese,Indonesian": 0.5133333333333333, - "Spanish,Chinese,Malay": 0.5133333333333333, - "Spanish,Vietnamese,Filipino": 0.44666666666666666, - "Spanish,Vietnamese,Indonesian": 0.44666666666666666, - "Spanish,Vietnamese,Malay": 0.4533333333333333, - "Spanish,Filipino,Indonesian": 0.5533333333333333, - "Spanish,Filipino,Malay": 0.5266666666666666, - "Spanish,Indonesian,Malay": 0.5533333333333333, - "English,Chinese,Vietnamese": 0.4266666666666667, - "English,Chinese,Filipino": 0.5133333333333333, - "English,Chinese,Indonesian": 0.52, - "English,Chinese,Malay": 0.49333333333333335, - "English,Vietnamese,Filipino": 0.44666666666666666, - "English,Vietnamese,Indonesian": 0.44666666666666666, - "English,Vietnamese,Malay": 0.44, - "English,Filipino,Indonesian": 0.5466666666666666, - "English,Filipino,Malay": 0.5133333333333333, - "English,Indonesian,Malay": 0.5333333333333333, - "Chinese,Vietnamese,Filipino": 0.48, - "Chinese,Vietnamese,Indonesian": 0.44666666666666666, - "Chinese,Vietnamese,Malay": 0.44, - "Chinese,Filipino,Indonesian": 0.5666666666666667, - "Chinese,Filipino,Malay": 0.52, - "Chinese,Indonesian,Malay": 0.52, - "Vietnamese,Filipino,Indonesian": 0.4866666666666667, - "Vietnamese,Filipino,Malay": 0.44666666666666666, - "Vietnamese,Indonesian,Malay": 0.44, - "Filipino,Indonesian,Malay": 0.54 + "Vietnamese,Malay,Filipino": 0.52, + "Vietnamese,Malay,Indonesian": 0.5466666666666666, + "Vietnamese,Malay,Chinese": 0.5066666666666667, + "Vietnamese,Malay,Spanish": 0.5466666666666666, + "Vietnamese,Malay,English": 0.56, + "Vietnamese,Filipino,Indonesian": 0.54, + "Vietnamese,Filipino,Chinese": 0.5133333333333333, + "Vietnamese,Filipino,Spanish": 0.5466666666666666, + "Vietnamese,Filipino,English": 0.5533333333333333, + "Vietnamese,Indonesian,Chinese": 0.5133333333333333, + "Vietnamese,Indonesian,Spanish": 0.5466666666666666, + "Vietnamese,Indonesian,English": 0.5666666666666667, + "Vietnamese,Chinese,Spanish": 0.5133333333333333, + "Vietnamese,Chinese,English": 0.5533333333333333, + "Vietnamese,Spanish,English": 0.5866666666666667, + "Malay,Filipino,Indonesian": 0.6066666666666667, + "Malay,Filipino,Chinese": 0.54, + "Malay,Filipino,Spanish": 0.6066666666666667, + "Malay,Filipino,English": 0.6066666666666667, + "Malay,Indonesian,Chinese": 0.58, + "Malay,Indonesian,Spanish": 0.6333333333333333, + "Malay,Indonesian,English": 0.6533333333333333, + "Malay,Chinese,Spanish": 0.5666666666666667, + "Malay,Chinese,English": 0.6066666666666667, + "Malay,Spanish,English": 0.66, + "Filipino,Indonesian,Chinese": 0.5666666666666667, + "Filipino,Indonesian,Spanish": 0.6333333333333333, + "Filipino,Indonesian,English": 0.64, + "Filipino,Chinese,Spanish": 0.5933333333333334, + "Filipino,Chinese,English": 0.6133333333333333, + "Filipino,Spanish,English": 0.68, + "Indonesian,Chinese,Spanish": 0.5866666666666667, + "Indonesian,Chinese,English": 0.6266666666666667, + "Indonesian,Spanish,English": 0.68, + "Chinese,Spanish,English": 0.66 }, "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.37333333333333335, - "Spanish,English,Chinese,Filipino": 0.44, - "Spanish,English,Chinese,Indonesian": 0.44666666666666666, - "Spanish,English,Chinese,Malay": 0.4533333333333333, - "Spanish,English,Vietnamese,Filipino": 0.38666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.38, - "Spanish,English,Vietnamese,Malay": 0.38, - "Spanish,English,Filipino,Indonesian": 0.47333333333333333, - "Spanish,English,Filipino,Malay": 0.4666666666666667, - "Spanish,English,Indonesian,Malay": 0.4866666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.38, - "Spanish,Chinese,Vietnamese,Indonesian": 0.36, - "Spanish,Chinese,Vietnamese,Malay": 0.36666666666666664, - "Spanish,Chinese,Filipino,Indonesian": 0.44, - "Spanish,Chinese,Filipino,Malay": 0.4266666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.44666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.3933333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.36666666666666664, - "Spanish,Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Spanish,Filipino,Indonesian,Malay": 0.46, - "English,Chinese,Vietnamese,Filipino": 0.38666666666666666, - "English,Chinese,Vietnamese,Indonesian": 0.36, - "English,Chinese,Vietnamese,Malay": 0.35333333333333333, - "English,Chinese,Filipino,Indonesian": 0.44666666666666666, - "English,Chinese,Filipino,Malay": 0.41333333333333333, - "English,Chinese,Indonesian,Malay": 0.4266666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "English,Vietnamese,Filipino,Malay": 0.36666666666666664, - "English,Vietnamese,Indonesian,Malay": 0.36, - "English,Filipino,Indonesian,Malay": 0.44, - "Chinese,Vietnamese,Filipino,Indonesian": 0.41333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.3933333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Chinese,Filipino,Indonesian,Malay": 0.44666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.38666666666666666 + "Vietnamese,Malay,Filipino,Indonesian": 0.4666666666666667, + "Vietnamese,Malay,Filipino,Chinese": 0.43333333333333335, + "Vietnamese,Malay,Filipino,Spanish": 0.47333333333333333, + "Vietnamese,Malay,Filipino,English": 0.48, + "Vietnamese,Malay,Indonesian,Chinese": 0.44666666666666666, + "Vietnamese,Malay,Indonesian,Spanish": 0.4866666666666667, + "Vietnamese,Malay,Indonesian,English": 0.5, + "Vietnamese,Malay,Chinese,Spanish": 0.4533333333333333, + "Vietnamese,Malay,Chinese,English": 0.48, + "Vietnamese,Malay,Spanish,English": 0.5133333333333333, + "Vietnamese,Filipino,Indonesian,Chinese": 0.44, + "Vietnamese,Filipino,Indonesian,Spanish": 0.4866666666666667, + "Vietnamese,Filipino,Indonesian,English": 0.5, + "Vietnamese,Filipino,Chinese,Spanish": 0.46, + "Vietnamese,Filipino,Chinese,English": 0.4866666666666667, + "Vietnamese,Filipino,Spanish,English": 0.52, + "Vietnamese,Indonesian,Chinese,Spanish": 0.44666666666666666, + "Vietnamese,Indonesian,Chinese,English": 0.4866666666666667, + "Vietnamese,Indonesian,Spanish,English": 0.52, + "Vietnamese,Chinese,Spanish,English": 0.5, + "Malay,Filipino,Indonesian,Chinese": 0.4866666666666667, + "Malay,Filipino,Indonesian,Spanish": 0.5533333333333333, + "Malay,Filipino,Indonesian,English": 0.5533333333333333, + "Malay,Filipino,Chinese,Spanish": 0.5, + "Malay,Filipino,Chinese,English": 0.52, + "Malay,Filipino,Spanish,English": 0.5666666666666667, + "Malay,Indonesian,Chinese,Spanish": 0.5133333333333333, + "Malay,Indonesian,Chinese,English": 0.5466666666666666, + "Malay,Indonesian,Spanish,English": 0.5933333333333334, + "Malay,Chinese,Spanish,English": 0.54, + "Filipino,Indonesian,Chinese,Spanish": 0.5133333333333333, + "Filipino,Indonesian,Chinese,English": 0.5333333333333333, + "Filipino,Indonesian,Spanish,English": 0.6, + "Filipino,Chinese,Spanish,English": 0.56, + "Indonesian,Chinese,Spanish,English": 0.5533333333333333 }, "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,English,Chinese,Vietnamese,Malay": 0.32, - "Spanish,English,Chinese,Filipino,Indonesian": 0.38, - "Spanish,English,Chinese,Filipino,Malay": 0.38, - "Spanish,English,Chinese,Indonesian,Malay": 0.4, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.34, - "Spanish,English,Vietnamese,Filipino,Malay": 0.32666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.32, - "Spanish,English,Filipino,Indonesian,Malay": 0.4066666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.32, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.38, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.32, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.3, - "English,Chinese,Filipino,Indonesian,Malay": 0.36666666666666664, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.32, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.34 + "Vietnamese,Malay,Filipino,Indonesian,Chinese": 0.38666666666666666, + "Vietnamese,Malay,Filipino,Indonesian,Spanish": 0.43333333333333335, + "Vietnamese,Malay,Filipino,Indonesian,English": 0.44, + "Vietnamese,Malay,Filipino,Chinese,Spanish": 0.4066666666666667, + "Vietnamese,Malay,Filipino,Chinese,English": 0.4266666666666667, + "Vietnamese,Malay,Filipino,Spanish,English": 0.4533333333333333, + "Vietnamese,Malay,Indonesian,Chinese,Spanish": 0.4066666666666667, + "Vietnamese,Malay,Indonesian,Chinese,English": 0.43333333333333335, + "Vietnamese,Malay,Indonesian,Spanish,English": 0.4666666666666667, + "Vietnamese,Malay,Chinese,Spanish,English": 0.44, + "Vietnamese,Filipino,Indonesian,Chinese,Spanish": 0.4066666666666667, + "Vietnamese,Filipino,Indonesian,Chinese,English": 0.43333333333333335, + "Vietnamese,Filipino,Indonesian,Spanish,English": 0.47333333333333333, + "Vietnamese,Filipino,Chinese,Spanish,English": 0.4533333333333333, + "Vietnamese,Indonesian,Chinese,Spanish,English": 0.44, + "Malay,Filipino,Indonesian,Chinese,Spanish": 0.46, + "Malay,Filipino,Indonesian,Chinese,English": 0.47333333333333333, + "Malay,Filipino,Indonesian,Spanish,English": 0.5266666666666666, + "Malay,Filipino,Chinese,Spanish,English": 0.48, + "Malay,Indonesian,Chinese,Spanish,English": 0.49333333333333335, + "Filipino,Indonesian,Chinese,Spanish,English": 0.49333333333333335 }, "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.2866666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.34, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.28, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.28 + "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish": 0.36666666666666664, + "Vietnamese,Malay,Filipino,Indonesian,Chinese,English": 0.38666666666666666, + "Vietnamese,Malay,Filipino,Indonesian,Spanish,English": 0.42, + "Vietnamese,Malay,Filipino,Chinese,Spanish,English": 0.4, + "Vietnamese,Malay,Indonesian,Chinese,Spanish,English": 0.4, + "Vietnamese,Filipino,Indonesian,Chinese,Spanish,English": 0.4066666666666667, + "Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.44666666666666666 }, "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 + "Vietnamese,Malay,Filipino,Indonesian,Chinese,Spanish,English": 0.36666666666666664 } }, - "AC3_2": 0.609286455965603, - "AC3_3": 0.5343318217018299, - "AC3_4": 0.47661309980739347, - "AC3_5": 0.42813978809305764, - "AC3_6": 0.3868634222473556, - "AC3_7": 0.3517547892295775 + "AC3_2": 0.7028364949816184, + "AC3_3": 0.6365918952422053, + "AC3_4": 0.5871248691814143, + "AC3_5": 0.5467109144066519, + "AC3_6": 0.5119105189107344, + "AC3_7": 0.48102472441122346 } }, "cross_logiqa": { "prompt_1": { - "overall_acc": 0.46022727272727276, + "overall_acc": 0.47970779220779225, "language_acc": { - "English": 0.4943181818181818, - "Filipino": 0.4715909090909091, - "Spanish": 0.48295454545454547, - "Malay": 0.42045454545454547, - "Chinese": 0.4431818181818182, - "Indonesian": 0.4318181818181818, - "Vietnamese": 0.4772727272727273 + "English": 0.5625, + "Filipino": 0.3806818181818182, + "Vietnamese": 0.4375, + "Chinese": 0.5568181818181818, + "Indonesian": 0.4375, + "Malay": 0.4431818181818182, + "Spanish": 0.5397727272727273 }, - "consistency_score_2": 0.569805194805195, - "consistency_score_3": 0.4011363636363635, - "consistency_score_4": 0.3087662337662337, - "consistency_score_5": 0.24972943722943727, - "consistency_score_6": 0.2077922077922078, + "consistency_score_2": 0.549512987012987, + "consistency_score_3": 0.37905844155844165, + "consistency_score_4": 0.29107142857142865, + "consistency_score_5": 0.2372835497835498, + "consistency_score_6": 0.2012987012987013, "consistency_score_7": 0.17613636363636365, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.625, - "English,Spanish": 0.625, - "English,Malay": 0.5965909090909091, - "English,Chinese": 0.5284090909090909, - "English,Indonesian": 0.5795454545454546, - "English,Vietnamese": 0.5852272727272727, - "Filipino,Spanish": 0.625, - "Filipino,Malay": 0.6306818181818182, - "Filipino,Chinese": 0.5284090909090909, - "Filipino,Indonesian": 0.6590909090909091, - "Filipino,Vietnamese": 0.6079545454545454, - "Spanish,Malay": 0.5795454545454546, - "Spanish,Chinese": 0.5113636363636364, - "Spanish,Indonesian": 0.5625, - "Spanish,Vietnamese": 0.5795454545454546, - "Malay,Chinese": 0.4943181818181818, - "Malay,Indonesian": 0.5909090909090909, - "Malay,Vietnamese": 0.5284090909090909, - "Chinese,Indonesian": 0.4772727272727273, - "Chinese,Vietnamese": 0.45454545454545453, - "Indonesian,Vietnamese": 0.5965909090909091 + "English,Filipino": 0.4772727272727273, + "English,Vietnamese": 0.5738636363636364, + "English,Chinese": 0.5795454545454546, + "English,Indonesian": 0.6079545454545454, + "English,Malay": 0.6193181818181818, + "English,Spanish": 0.7443181818181818, + "Filipino,Vietnamese": 0.4602272727272727, + "Filipino,Chinese": 0.44886363636363635, + "Filipino,Indonesian": 0.44886363636363635, + "Filipino,Malay": 0.48863636363636365, + "Filipino,Spanish": 0.4602272727272727, + "Vietnamese,Chinese": 0.5170454545454546, + "Vietnamese,Indonesian": 0.5397727272727273, + "Vietnamese,Malay": 0.5170454545454546, + "Vietnamese,Spanish": 0.5965909090909091, + "Chinese,Indonesian": 0.5511363636363636, + "Chinese,Malay": 0.5170454545454546, + "Chinese,Spanish": 0.6022727272727273, + "Indonesian,Malay": 0.5909090909090909, + "Indonesian,Spanish": 0.625, + "Malay,Spanish": 0.5738636363636364 }, "3_combine": { - "English,Filipino,Spanish": 0.4715909090909091, - "English,Filipino,Malay": 0.48295454545454547, - "English,Filipino,Chinese": 0.3806818181818182, - "English,Filipino,Indonesian": 0.4659090909090909, - "English,Filipino,Vietnamese": 0.4431818181818182, - "English,Spanish,Malay": 0.4431818181818182, - "English,Spanish,Chinese": 0.3806818181818182, - "English,Spanish,Indonesian": 0.4318181818181818, - "English,Spanish,Vietnamese": 0.4375, - "English,Malay,Chinese": 0.36363636363636365, - "English,Malay,Indonesian": 0.4431818181818182, - "English,Malay,Vietnamese": 0.3977272727272727, - "English,Chinese,Indonesian": 0.3465909090909091, - "English,Chinese,Vietnamese": 0.3409090909090909, - "English,Indonesian,Vietnamese": 0.42613636363636365, - "Filipino,Spanish,Malay": 0.45454545454545453, - "Filipino,Spanish,Chinese": 0.38636363636363635, - "Filipino,Spanish,Indonesian": 0.4659090909090909, - "Filipino,Spanish,Vietnamese": 0.4431818181818182, - "Filipino,Malay,Chinese": 0.375, - "Filipino,Malay,Indonesian": 0.48295454545454547, - "Filipino,Malay,Vietnamese": 0.42045454545454547, - "Filipino,Chinese,Indonesian": 0.36363636363636365, - "Filipino,Chinese,Vietnamese": 0.3465909090909091, - "Filipino,Indonesian,Vietnamese": 0.4772727272727273, - "Spanish,Malay,Chinese": 0.3465909090909091, - "Spanish,Malay,Indonesian": 0.42045454545454547, - "Spanish,Malay,Vietnamese": 0.38636363636363635, - "Spanish,Chinese,Indonesian": 0.32954545454545453, - "Spanish,Chinese,Vietnamese": 0.3125, - "Spanish,Indonesian,Vietnamese": 0.42045454545454547, - "Malay,Chinese,Indonesian": 0.3352272727272727, - "Malay,Chinese,Vietnamese": 0.30113636363636365, - "Malay,Indonesian,Vietnamese": 0.4034090909090909, - "Chinese,Indonesian,Vietnamese": 0.3125 + "English,Filipino,Vietnamese": 0.3068181818181818, + "English,Filipino,Chinese": 0.3125, + "English,Filipino,Indonesian": 0.3409090909090909, + "English,Filipino,Malay": 0.35795454545454547, + "English,Filipino,Spanish": 0.3693181818181818, + "English,Vietnamese,Chinese": 0.375, + "English,Vietnamese,Indonesian": 0.42045454545454547, + "English,Vietnamese,Malay": 0.4090909090909091, + "English,Vietnamese,Spanish": 0.48295454545454547, + "English,Chinese,Indonesian": 0.42613636363636365, + "English,Chinese,Malay": 0.4034090909090909, + "English,Chinese,Spanish": 0.48863636363636365, + "English,Indonesian,Malay": 0.4659090909090909, + "English,Indonesian,Spanish": 0.5170454545454546, + "English,Malay,Spanish": 0.5, + "Filipino,Vietnamese,Chinese": 0.2840909090909091, + "Filipino,Vietnamese,Indonesian": 0.29545454545454547, + "Filipino,Vietnamese,Malay": 0.30113636363636365, + "Filipino,Vietnamese,Spanish": 0.3125, + "Filipino,Chinese,Indonesian": 0.30113636363636365, + "Filipino,Chinese,Malay": 0.29545454545454547, + "Filipino,Chinese,Spanish": 0.3181818181818182, + "Filipino,Indonesian,Malay": 0.32954545454545453, + "Filipino,Indonesian,Spanish": 0.3181818181818182, + "Filipino,Malay,Spanish": 0.32386363636363635, + "Vietnamese,Chinese,Indonesian": 0.375, + "Vietnamese,Chinese,Malay": 0.32954545454545453, + "Vietnamese,Chinese,Spanish": 0.4090909090909091, + "Vietnamese,Indonesian,Malay": 0.3806818181818182, + "Vietnamese,Indonesian,Spanish": 0.4431818181818182, + "Vietnamese,Malay,Spanish": 0.38636363636363635, + "Chinese,Indonesian,Malay": 0.3977272727272727, + "Chinese,Indonesian,Spanish": 0.4431818181818182, + "Chinese,Malay,Spanish": 0.38636363636363635, + "Indonesian,Malay,Spanish": 0.4602272727272727 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.375, - "English,Filipino,Spanish,Chinese": 0.3125, - "English,Filipino,Spanish,Indonesian": 0.375, - "English,Filipino,Spanish,Vietnamese": 0.35795454545454547, - "English,Filipino,Malay,Chinese": 0.3125, - "English,Filipino,Malay,Indonesian": 0.39204545454545453, - "English,Filipino,Malay,Vietnamese": 0.3465909090909091, - "English,Filipino,Chinese,Indonesian": 0.30113636363636365, - "English,Filipino,Chinese,Vietnamese": 0.2784090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.35795454545454547, - "English,Spanish,Malay,Chinese": 0.2897727272727273, - "English,Spanish,Malay,Indonesian": 0.35795454545454547, - "English,Spanish,Malay,Vietnamese": 0.3181818181818182, - "English,Spanish,Chinese,Indonesian": 0.2784090909090909, - "English,Spanish,Chinese,Vietnamese": 0.26704545454545453, - "English,Spanish,Indonesian,Vietnamese": 0.3409090909090909, - "English,Malay,Chinese,Indonesian": 0.2897727272727273, - "English,Malay,Chinese,Vietnamese": 0.25, - "English,Malay,Indonesian,Vietnamese": 0.3352272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Malay,Chinese": 0.30113636363636365, - "Filipino,Spanish,Malay,Indonesian": 0.36363636363636365, - "Filipino,Spanish,Malay,Vietnamese": 0.32386363636363635, - "Filipino,Spanish,Chinese,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Indonesian,Vietnamese": 0.3693181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.29545454545454547, - "Filipino,Malay,Chinese,Vietnamese": 0.26136363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.3522727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Chinese,Indonesian": 0.26136363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Indonesian,Vietnamese": 0.32954545454545453, - "Spanish,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182 + "English,Filipino,Vietnamese,Chinese": 0.23295454545454544, + "English,Filipino,Vietnamese,Indonesian": 0.2556818181818182, + "English,Filipino,Vietnamese,Malay": 0.2556818181818182, + "English,Filipino,Vietnamese,Spanish": 0.2727272727272727, + "English,Filipino,Chinese,Indonesian": 0.25, + "English,Filipino,Chinese,Malay": 0.25, + "English,Filipino,Chinese,Spanish": 0.2727272727272727, + "English,Filipino,Indonesian,Malay": 0.29545454545454547, + "English,Filipino,Indonesian,Spanish": 0.2840909090909091, + "English,Filipino,Malay,Spanish": 0.29545454545454547, + "English,Vietnamese,Chinese,Indonesian": 0.3125, + "English,Vietnamese,Chinese,Malay": 0.2840909090909091, + "English,Vietnamese,Chinese,Spanish": 0.3409090909090909, + "English,Vietnamese,Indonesian,Malay": 0.32954545454545453, + "English,Vietnamese,Indonesian,Spanish": 0.375, + "English,Vietnamese,Malay,Spanish": 0.3465909090909091, + "English,Chinese,Indonesian,Malay": 0.3465909090909091, + "English,Chinese,Indonesian,Spanish": 0.38636363636363635, + "English,Chinese,Malay,Spanish": 0.35795454545454547, + "English,Indonesian,Malay,Spanish": 0.4090909090909091, + "Filipino,Vietnamese,Chinese,Indonesian": 0.22727272727272727, + "Filipino,Vietnamese,Chinese,Malay": 0.2215909090909091, + "Filipino,Vietnamese,Chinese,Spanish": 0.25, + "Filipino,Vietnamese,Indonesian,Malay": 0.25, + "Filipino,Vietnamese,Indonesian,Spanish": 0.24431818181818182, + "Filipino,Vietnamese,Malay,Spanish": 0.23863636363636365, + "Filipino,Chinese,Indonesian,Malay": 0.2556818181818182, + "Filipino,Chinese,Indonesian,Spanish": 0.24431818181818182, + "Filipino,Chinese,Malay,Spanish": 0.24431818181818182, + "Filipino,Indonesian,Malay,Spanish": 0.2727272727272727, + "Vietnamese,Chinese,Indonesian,Malay": 0.2840909090909091, + "Vietnamese,Chinese,Indonesian,Spanish": 0.3409090909090909, + "Vietnamese,Chinese,Malay,Spanish": 0.2897727272727273, + "Vietnamese,Indonesian,Malay,Spanish": 0.32954545454545453, + "Chinese,Indonesian,Malay,Spanish": 0.3409090909090909 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.26136363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.3181818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.2784090909090909, - "English,Filipino,Spanish,Chinese,Indonesian": 0.25, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.30113636363636365, - "English,Filipino,Malay,Chinese,Indonesian": 0.26704545454545453, - "English,Filipino,Malay,Chinese,Vietnamese": 0.22727272727272727, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.20454545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.23863636363636365, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635 + "English,Filipino,Vietnamese,Chinese,Indonesian": 0.20454545454545456, + "English,Filipino,Vietnamese,Chinese,Malay": 0.19886363636363635, + "English,Filipino,Vietnamese,Chinese,Spanish": 0.2215909090909091, + "English,Filipino,Vietnamese,Indonesian,Malay": 0.23295454545454544, + "English,Filipino,Vietnamese,Indonesian,Spanish": 0.22727272727272727, + "English,Filipino,Vietnamese,Malay,Spanish": 0.2215909090909091, + "English,Filipino,Chinese,Indonesian,Malay": 0.22727272727272727, + "English,Filipino,Chinese,Indonesian,Spanish": 0.22727272727272727, + "English,Filipino,Chinese,Malay,Spanish": 0.22727272727272727, + "English,Filipino,Indonesian,Malay,Spanish": 0.2556818181818182, + "English,Vietnamese,Chinese,Indonesian,Malay": 0.2556818181818182, + "English,Vietnamese,Chinese,Indonesian,Spanish": 0.2897727272727273, + "English,Vietnamese,Chinese,Malay,Spanish": 0.26136363636363635, + "English,Vietnamese,Indonesian,Malay,Spanish": 0.29545454545454547, + "English,Chinese,Indonesian,Malay,Spanish": 0.3181818181818182, + "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.20454545454545456, + "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.21022727272727273, + "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.20454545454545456, + "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.2159090909090909, + "Filipino,Chinese,Indonesian,Malay,Spanish": 0.2215909090909091, + "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.26136363636363635 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.2215909090909091, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1875, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2556818181818182, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1875 + "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.1875, + "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.19318181818181818, + "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.1875, + "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.20454545454545456, + "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.21022727272727273, + "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.23863636363636365, + "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.1875 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365 + "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.17613636363636365 } }, - "AC3_2": 0.5091876208403145, - "AC3_3": 0.42865495317642854, - "AC3_4": 0.36958086240803956, - "AC3_5": 0.32377269258188995, - "AC3_6": 0.2863139290414214, - "AC3_7": 0.25476866879113375 + "AC3_2": 0.5122431786957747, + "AC3_3": 0.42348495072769815, + "AC3_4": 0.3623066854747182, + "AC3_5": 0.3175122518547604, + "AC3_6": 0.2835936411205318, + "AC3_7": 0.2576648289436108 }, "prompt_2": { - "overall_acc": 0.4853896103896104, + "overall_acc": 0.4878246753246754, "language_acc": { - "English": 0.5795454545454546, - "Filipino": 0.4772727272727273, - "Spanish": 0.4715909090909091, - "Malay": 0.48295454545454547, - "Chinese": 0.4090909090909091, - "Indonesian": 0.48295454545454547, - "Vietnamese": 0.4943181818181818 - }, - "consistency_score_2": 0.5614177489177488, - "consistency_score_3": 0.38847402597402597, - "consistency_score_4": 0.28993506493506493, - "consistency_score_5": 0.22537878787878793, - "consistency_score_6": 0.18181818181818182, - "consistency_score_7": 0.1534090909090909, + "English": 0.5909090909090909, + "Filipino": 0.4147727272727273, + "Vietnamese": 0.4318181818181818, + "Chinese": 0.5625, + "Indonesian": 0.4772727272727273, + "Malay": 0.4318181818181818, + "Spanish": 0.5056818181818182 + }, + "consistency_score_2": 0.5744047619047619, + "consistency_score_3": 0.40649350649350646, + "consistency_score_4": 0.3180194805194805, + "consistency_score_5": 0.26379870129870125, + "consistency_score_6": 0.22646103896103895, + "consistency_score_7": 0.19886363636363635, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.6363636363636364, - "English,Spanish": 0.6136363636363636, - "English,Malay": 0.6306818181818182, - "English,Chinese": 0.4602272727272727, - "English,Indonesian": 0.6420454545454546, - "English,Vietnamese": 0.6022727272727273, - "Filipino,Spanish": 0.6306818181818182, - "Filipino,Malay": 0.6079545454545454, - "Filipino,Chinese": 0.4375, - "Filipino,Indonesian": 0.6193181818181818, - "Filipino,Vietnamese": 0.5625, - "Spanish,Malay": 0.6079545454545454, - "Spanish,Chinese": 0.4659090909090909, - "Spanish,Indonesian": 0.5681818181818182, - "Spanish,Vietnamese": 0.5795454545454546, - "Malay,Chinese": 0.4147727272727273, - "Malay,Indonesian": 0.6647727272727273, - "Malay,Vietnamese": 0.5340909090909091, - "Chinese,Indonesian": 0.45454545454545453, - "Chinese,Vietnamese": 0.44886363636363635, - "Indonesian,Vietnamese": 0.6079545454545454 + "English,Filipino": 0.5284090909090909, + "English,Vietnamese": 0.5454545454545454, + "English,Chinese": 0.5795454545454546, + "English,Indonesian": 0.6818181818181818, + "English,Malay": 0.5795454545454546, + "English,Spanish": 0.6875, + "Filipino,Vietnamese": 0.5170454545454546, + "Filipino,Chinese": 0.5227272727272727, + "Filipino,Indonesian": 0.5568181818181818, + "Filipino,Malay": 0.5965909090909091, + "Filipino,Spanish": 0.5511363636363636, + "Vietnamese,Chinese": 0.5170454545454546, + "Vietnamese,Indonesian": 0.5625, + "Vietnamese,Malay": 0.5284090909090909, + "Vietnamese,Spanish": 0.5681818181818182, + "Chinese,Indonesian": 0.5795454545454546, + "Chinese,Malay": 0.5681818181818182, + "Chinese,Spanish": 0.5852272727272727, + "Indonesian,Malay": 0.6420454545454546, + "Indonesian,Spanish": 0.6136363636363636, + "Malay,Spanish": 0.5511363636363636 }, "3_combine": { - "English,Filipino,Spanish": 0.4715909090909091, - "English,Filipino,Malay": 0.4772727272727273, - "English,Filipino,Chinese": 0.32386363636363635, - "English,Filipino,Indonesian": 0.4772727272727273, - "English,Filipino,Vietnamese": 0.42613636363636365, - "English,Spanish,Malay": 0.4602272727272727, - "English,Spanish,Chinese": 0.3409090909090909, - "English,Spanish,Indonesian": 0.44886363636363635, - "English,Spanish,Vietnamese": 0.4318181818181818, - "English,Malay,Chinese": 0.3125, - "English,Malay,Indonesian": 0.4943181818181818, - "English,Malay,Vietnamese": 0.42613636363636365, - "English,Chinese,Indonesian": 0.32954545454545453, - "English,Chinese,Vietnamese": 0.3068181818181818, - "English,Indonesian,Vietnamese": 0.45454545454545453, - "Filipino,Spanish,Malay": 0.44886363636363635, - "Filipino,Spanish,Chinese": 0.3352272727272727, - "Filipino,Spanish,Indonesian": 0.4431818181818182, - "Filipino,Spanish,Vietnamese": 0.42613636363636365, - "Filipino,Malay,Chinese": 0.2897727272727273, - "Filipino,Malay,Indonesian": 0.4659090909090909, - "Filipino,Malay,Vietnamese": 0.3977272727272727, - "Filipino,Chinese,Indonesian": 0.3068181818181818, - "Filipino,Chinese,Vietnamese": 0.2840909090909091, - "Filipino,Indonesian,Vietnamese": 0.4318181818181818, - "Spanish,Malay,Chinese": 0.3125, - "Spanish,Malay,Indonesian": 0.45454545454545453, - "Spanish,Malay,Vietnamese": 0.4090909090909091, - "Spanish,Chinese,Indonesian": 0.3125, - "Spanish,Chinese,Vietnamese": 0.3181818181818182, - "Spanish,Indonesian,Vietnamese": 0.42045454545454547, - "Malay,Chinese,Indonesian": 0.3181818181818182, - "Malay,Chinese,Vietnamese": 0.2784090909090909, - "Malay,Indonesian,Vietnamese": 0.4375, - "Chinese,Indonesian,Vietnamese": 0.32386363636363635 + "English,Filipino,Vietnamese": 0.3522727272727273, + "English,Filipino,Chinese": 0.36363636363636365, + "English,Filipino,Indonesian": 0.42613636363636365, + "English,Filipino,Malay": 0.4034090909090909, + "English,Filipino,Spanish": 0.42045454545454547, + "English,Vietnamese,Chinese": 0.375, + "English,Vietnamese,Indonesian": 0.44886363636363635, + "English,Vietnamese,Malay": 0.39204545454545453, + "English,Vietnamese,Spanish": 0.42613636363636365, + "English,Chinese,Indonesian": 0.44886363636363635, + "English,Chinese,Malay": 0.3977272727272727, + "English,Chinese,Spanish": 0.4602272727272727, + "English,Indonesian,Malay": 0.4772727272727273, + "English,Indonesian,Spanish": 0.5227272727272727, + "English,Malay,Spanish": 0.4318181818181818, + "Filipino,Vietnamese,Chinese": 0.32954545454545453, + "Filipino,Vietnamese,Indonesian": 0.3806818181818182, + "Filipino,Vietnamese,Malay": 0.375, + "Filipino,Vietnamese,Spanish": 0.375, + "Filipino,Chinese,Indonesian": 0.3806818181818182, + "Filipino,Chinese,Malay": 0.39204545454545453, + "Filipino,Chinese,Spanish": 0.3806818181818182, + "Filipino,Indonesian,Malay": 0.4431818181818182, + "Filipino,Indonesian,Spanish": 0.3977272727272727, + "Filipino,Malay,Spanish": 0.4034090909090909, + "Vietnamese,Chinese,Indonesian": 0.38636363636363635, + "Vietnamese,Chinese,Malay": 0.35795454545454547, + "Vietnamese,Chinese,Spanish": 0.38636363636363635, + "Vietnamese,Indonesian,Malay": 0.4090909090909091, + "Vietnamese,Indonesian,Spanish": 0.42045454545454547, + "Vietnamese,Malay,Spanish": 0.39204545454545453, + "Chinese,Indonesian,Malay": 0.4147727272727273, + "Chinese,Indonesian,Spanish": 0.42613636363636365, + "Chinese,Malay,Spanish": 0.39204545454545453, + "Indonesian,Malay,Spanish": 0.4375 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.375, - "English,Filipino,Spanish,Chinese": 0.2727272727272727, - "English,Filipino,Spanish,Indonesian": 0.36363636363636365, - "English,Filipino,Spanish,Vietnamese": 0.3409090909090909, - "English,Filipino,Malay,Chinese": 0.25, - "English,Filipino,Malay,Indonesian": 0.38636363636363635, - "English,Filipino,Malay,Vietnamese": 0.3352272727272727, - "English,Filipino,Chinese,Indonesian": 0.25, - "English,Filipino,Chinese,Vietnamese": 0.23295454545454544, - "English,Filipino,Indonesian,Vietnamese": 0.3409090909090909, - "English,Spanish,Malay,Chinese": 0.26136363636363635, - "English,Spanish,Malay,Indonesian": 0.375, - "English,Spanish,Malay,Vietnamese": 0.3352272727272727, - "English,Spanish,Chinese,Indonesian": 0.26136363636363635, - "English,Spanish,Chinese,Vietnamese": 0.26136363636363635, - "English,Spanish,Indonesian,Vietnamese": 0.3352272727272727, - "English,Malay,Chinese,Indonesian": 0.2556818181818182, - "English,Malay,Chinese,Vietnamese": 0.23863636363636365, - "English,Malay,Indonesian,Vietnamese": 0.35795454545454547, - "English,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Malay,Chinese": 0.24431818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.35795454545454547, - "Filipino,Spanish,Malay,Vietnamese": 0.32386363636363635, - "Filipino,Spanish,Chinese,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.3352272727272727, - "Filipino,Malay,Chinese,Indonesian": 0.23295454545454544, - "Filipino,Malay,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.3409090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Chinese,Indonesian": 0.23863636363636365, - "Spanish,Malay,Chinese,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Indonesian,Vietnamese": 0.3352272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365 + "English,Filipino,Vietnamese,Chinese": 0.2556818181818182, + "English,Filipino,Vietnamese,Indonesian": 0.3181818181818182, + "English,Filipino,Vietnamese,Malay": 0.29545454545454547, + "English,Filipino,Vietnamese,Spanish": 0.30113636363636365, + "English,Filipino,Chinese,Indonesian": 0.3068181818181818, + "English,Filipino,Chinese,Malay": 0.29545454545454547, + "English,Filipino,Chinese,Spanish": 0.30113636363636365, + "English,Filipino,Indonesian,Malay": 0.3465909090909091, + "English,Filipino,Indonesian,Spanish": 0.3522727272727273, + "English,Filipino,Malay,Spanish": 0.3409090909090909, + "English,Vietnamese,Chinese,Indonesian": 0.32386363636363635, + "English,Vietnamese,Chinese,Malay": 0.2897727272727273, + "English,Vietnamese,Chinese,Spanish": 0.32954545454545453, + "English,Vietnamese,Indonesian,Malay": 0.3352272727272727, + "English,Vietnamese,Indonesian,Spanish": 0.36363636363636365, + "English,Vietnamese,Malay,Spanish": 0.3181818181818182, + "English,Chinese,Indonesian,Malay": 0.32954545454545453, + "English,Chinese,Indonesian,Spanish": 0.3806818181818182, + "English,Chinese,Malay,Spanish": 0.32386363636363635, + "English,Indonesian,Malay,Spanish": 0.38636363636363635, + "Filipino,Vietnamese,Chinese,Indonesian": 0.26704545454545453, + "Filipino,Vietnamese,Chinese,Malay": 0.2784090909090909, + "Filipino,Vietnamese,Chinese,Spanish": 0.2840909090909091, + "Filipino,Vietnamese,Indonesian,Malay": 0.3125, + "Filipino,Vietnamese,Indonesian,Spanish": 0.3181818181818182, + "Filipino,Vietnamese,Malay,Spanish": 0.3125, + "Filipino,Chinese,Indonesian,Malay": 0.3125, + "Filipino,Chinese,Indonesian,Spanish": 0.30113636363636365, + "Filipino,Chinese,Malay,Spanish": 0.3125, + "Filipino,Indonesian,Malay,Spanish": 0.3465909090909091, + "Vietnamese,Chinese,Indonesian,Malay": 0.30113636363636365, + "Vietnamese,Chinese,Indonesian,Spanish": 0.32954545454545453, + "Vietnamese,Chinese,Malay,Spanish": 0.30113636363636365, + "Vietnamese,Indonesian,Malay,Spanish": 0.32954545454545453, + "Chinese,Indonesian,Malay,Spanish": 0.32954545454545453 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.2215909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.3068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.2784090909090909, - "English,Filipino,Spanish,Chinese,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.2727272727272727, - "English,Filipino,Malay,Chinese,Indonesian": 0.20454545454545456, - "English,Filipino,Malay,Chinese,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.1875, - "English,Spanish,Malay,Chinese,Indonesian": 0.21022727272727273, - "English,Spanish,Malay,Chinese,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1875, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635 + "English,Filipino,Vietnamese,Chinese,Indonesian": 0.22727272727272727, + "English,Filipino,Vietnamese,Chinese,Malay": 0.22727272727272727, + "English,Filipino,Vietnamese,Chinese,Spanish": 0.23863636363636365, + "English,Filipino,Vietnamese,Indonesian,Malay": 0.26136363636363635, + "English,Filipino,Vietnamese,Indonesian,Spanish": 0.2784090909090909, + "English,Filipino,Vietnamese,Malay,Spanish": 0.26704545454545453, + "English,Filipino,Chinese,Indonesian,Malay": 0.2556818181818182, + "English,Filipino,Chinese,Indonesian,Spanish": 0.26704545454545453, + "English,Filipino,Chinese,Malay,Spanish": 0.2556818181818182, + "English,Filipino,Indonesian,Malay,Spanish": 0.3068181818181818, + "English,Vietnamese,Chinese,Indonesian,Malay": 0.2556818181818182, + "English,Vietnamese,Chinese,Indonesian,Spanish": 0.29545454545454547, + "English,Vietnamese,Chinese,Malay,Spanish": 0.26136363636363635, + "English,Vietnamese,Indonesian,Malay,Spanish": 0.2897727272727273, + "English,Chinese,Indonesian,Malay,Spanish": 0.29545454545454547, + "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.23295454545454544, + "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.25, + "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2556818181818182, + "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.2784090909090909, + "Filipino,Chinese,Indonesian,Malay,Spanish": 0.26704545454545453, + "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.2727272727272727 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.18181818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727 + "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.19886363636363635, + "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.2215909090909091, + "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2159090909090909, + "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.24431818181818182, + "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.23295454545454544, + "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.24431818181818182, + "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.22727272727272727 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 + "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.19886363636363635 } }, - "AC3_2": 0.5206427714976617, - "AC3_3": 0.43155762122586583, - "AC3_4": 0.36302589784088446, - "AC3_5": 0.30782607186727007, - "AC3_6": 0.2645432426057831, - "AC3_7": 0.23313503519510861 + "AC3_2": 0.5275862382153784, + "AC3_3": 0.44346087742835405, + "AC3_4": 0.38503164336923823, + "AC3_5": 0.3424255279901211, + "AC3_6": 0.30932519181029544, + "AC3_7": 0.28254620670718816 }, - "prompt_3": { - "overall_acc": 0.450487012987013, + "prompt_3": -1, + "prompt_4": { + "overall_acc": 0.5137987012987013, "language_acc": { - "English": 0.5056818181818182, - "Filipino": 0.44886363636363635, - "Spanish": 0.4602272727272727, - "Malay": 0.4715909090909091, - "Chinese": 0.39204545454545453, - "Indonesian": 0.4431818181818182, - "Vietnamese": 0.4318181818181818 + "English": 0.5681818181818182, + "Filipino": 0.4772727272727273, + "Vietnamese": 0.4318181818181818, + "Chinese": 0.5397727272727273, + "Indonesian": 0.5284090909090909, + "Malay": 0.5056818181818182, + "Spanish": 0.5454545454545454 }, - "consistency_score_2": 0.5378787878787878, - "consistency_score_3": 0.35308441558441567, - "consistency_score_4": 0.2498376623376624, - "consistency_score_5": 0.18425324675324675, - "consistency_score_6": 0.1396103896103896, - "consistency_score_7": 0.10795454545454546, + "consistency_score_2": 0.5868506493506492, + "consistency_score_3": 0.4237012987012987, + "consistency_score_4": 0.34139610389610386, + "consistency_score_5": 0.2924783549783549, + "consistency_score_6": 0.25892857142857145, + "consistency_score_7": 0.23295454545454544, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.6193181818181818, - "English,Spanish": 0.6306818181818182, - "English,Malay": 0.6079545454545454, - "English,Chinese": 0.4772727272727273, - "English,Indonesian": 0.5852272727272727, - "English,Vietnamese": 0.5397727272727273, - "Filipino,Spanish": 0.6704545454545454, - "Filipino,Malay": 0.5397727272727273, - "Filipino,Chinese": 0.4431818181818182, - "Filipino,Indonesian": 0.5681818181818182, + "English,Filipino": 0.5625, + "English,Vietnamese": 0.5852272727272727, + "English,Chinese": 0.5681818181818182, + "English,Indonesian": 0.6136363636363636, + "English,Malay": 0.6022727272727273, + "English,Spanish": 0.7443181818181818, "Filipino,Vietnamese": 0.5625, - "Spanish,Malay": 0.5454545454545454, - "Spanish,Chinese": 0.4715909090909091, - "Spanish,Indonesian": 0.5284090909090909, - "Spanish,Vietnamese": 0.5397727272727273, - "Malay,Chinese": 0.42613636363636365, - "Malay,Indonesian": 0.6193181818181818, - "Malay,Vietnamese": 0.5, - "Chinese,Indonesian": 0.42613636363636365, - "Chinese,Vietnamese": 0.45454545454545453, - "Indonesian,Vietnamese": 0.5397727272727273 + "Filipino,Chinese": 0.5454545454545454, + "Filipino,Indonesian": 0.5454545454545454, + "Filipino,Malay": 0.5852272727272727, + "Filipino,Spanish": 0.5625, + "Vietnamese,Chinese": 0.5113636363636364, + "Vietnamese,Indonesian": 0.5056818181818182, + "Vietnamese,Malay": 0.5454545454545454, + "Vietnamese,Spanish": 0.6079545454545454, + "Chinese,Indonesian": 0.5454545454545454, + "Chinese,Malay": 0.5795454545454546, + "Chinese,Spanish": 0.5511363636363636, + "Indonesian,Malay": 0.6931818181818182, + "Indonesian,Spanish": 0.6306818181818182, + "Malay,Spanish": 0.6761363636363636 }, "3_combine": { - "English,Filipino,Spanish": 0.4943181818181818, - "English,Filipino,Malay": 0.4147727272727273, - "English,Filipino,Chinese": 0.3181818181818182, - "English,Filipino,Indonesian": 0.4090909090909091, "English,Filipino,Vietnamese": 0.3977272727272727, - "English,Spanish,Malay": 0.42045454545454547, - "English,Spanish,Chinese": 0.3409090909090909, - "English,Spanish,Indonesian": 0.4147727272727273, - "English,Spanish,Vietnamese": 0.4034090909090909, - "English,Malay,Chinese": 0.29545454545454547, - "English,Malay,Indonesian": 0.42045454545454547, - "English,Malay,Vietnamese": 0.3693181818181818, - "English,Chinese,Indonesian": 0.2897727272727273, - "English,Chinese,Vietnamese": 0.29545454545454547, - "English,Indonesian,Vietnamese": 0.3693181818181818, - "Filipino,Spanish,Malay": 0.4090909090909091, - "Filipino,Spanish,Chinese": 0.3522727272727273, - "Filipino,Spanish,Indonesian": 0.4147727272727273, - "Filipino,Spanish,Vietnamese": 0.4318181818181818, - "Filipino,Malay,Chinese": 0.26136363636363635, - "Filipino,Malay,Indonesian": 0.38636363636363635, - "Filipino,Malay,Vietnamese": 0.35795454545454547, - "Filipino,Chinese,Indonesian": 0.26704545454545453, - "Filipino,Chinese,Vietnamese": 0.30113636363636365, - "Filipino,Indonesian,Vietnamese": 0.39204545454545453, - "Spanish,Malay,Chinese": 0.2840909090909091, - "Spanish,Malay,Indonesian": 0.39204545454545453, - "Spanish,Malay,Vietnamese": 0.3409090909090909, - "Spanish,Chinese,Indonesian": 0.26704545454545453, - "Spanish,Chinese,Vietnamese": 0.3125, - "Spanish,Indonesian,Vietnamese": 0.375, - "Malay,Chinese,Indonesian": 0.2784090909090909, - "Malay,Chinese,Vietnamese": 0.24431818181818182, - "Malay,Indonesian,Vietnamese": 0.3693181818181818, - "Chinese,Indonesian,Vietnamese": 0.26704545454545453 + "English,Filipino,Chinese": 0.38636363636363635, + "English,Filipino,Indonesian": 0.4090909090909091, + "English,Filipino,Malay": 0.42613636363636365, + "English,Filipino,Spanish": 0.4659090909090909, + "English,Vietnamese,Chinese": 0.3693181818181818, + "English,Vietnamese,Indonesian": 0.4090909090909091, + "English,Vietnamese,Malay": 0.4147727272727273, + "English,Vietnamese,Spanish": 0.4943181818181818, + "English,Chinese,Indonesian": 0.3977272727272727, + "English,Chinese,Malay": 0.42045454545454547, + "English,Chinese,Spanish": 0.4772727272727273, + "English,Indonesian,Malay": 0.4943181818181818, + "English,Indonesian,Spanish": 0.5170454545454546, + "English,Malay,Spanish": 0.5397727272727273, + "Filipino,Vietnamese,Chinese": 0.3465909090909091, + "Filipino,Vietnamese,Indonesian": 0.3806818181818182, + "Filipino,Vietnamese,Malay": 0.3977272727272727, + "Filipino,Vietnamese,Spanish": 0.4147727272727273, + "Filipino,Chinese,Indonesian": 0.375, + "Filipino,Chinese,Malay": 0.4090909090909091, + "Filipino,Chinese,Spanish": 0.3806818181818182, + "Filipino,Indonesian,Malay": 0.4602272727272727, + "Filipino,Indonesian,Spanish": 0.4318181818181818, + "Filipino,Malay,Spanish": 0.4602272727272727, + "Vietnamese,Chinese,Indonesian": 0.32386363636363635, + "Vietnamese,Chinese,Malay": 0.36363636363636365, + "Vietnamese,Chinese,Spanish": 0.3806818181818182, + "Vietnamese,Indonesian,Malay": 0.4090909090909091, + "Vietnamese,Indonesian,Spanish": 0.4090909090909091, + "Vietnamese,Malay,Spanish": 0.45454545454545453, + "Chinese,Indonesian,Malay": 0.4375, + "Chinese,Indonesian,Spanish": 0.4034090909090909, + "Chinese,Malay,Spanish": 0.4375, + "Indonesian,Malay,Spanish": 0.5340909090909091 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.3465909090909091, - "English,Filipino,Spanish,Chinese": 0.2784090909090909, - "English,Filipino,Spanish,Indonesian": 0.3352272727272727, - "English,Filipino,Spanish,Vietnamese": 0.3352272727272727, - "English,Filipino,Malay,Chinese": 0.2215909090909091, - "English,Filipino,Malay,Indonesian": 0.30113636363636365, - "English,Filipino,Malay,Vietnamese": 0.2897727272727273, - "English,Filipino,Chinese,Indonesian": 0.2159090909090909, - "English,Filipino,Chinese,Vietnamese": 0.22727272727272727, - "English,Filipino,Indonesian,Vietnamese": 0.2897727272727273, - "English,Spanish,Malay,Chinese": 0.22727272727272727, - "English,Spanish,Malay,Indonesian": 0.3181818181818182, - "English,Spanish,Malay,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese,Indonesian": 0.2159090909090909, - "English,Spanish,Chinese,Vietnamese": 0.23863636363636365, - "English,Spanish,Indonesian,Vietnamese": 0.2897727272727273, - "English,Malay,Chinese,Indonesian": 0.19886363636363635, - "English,Malay,Chinese,Vietnamese": 0.18181818181818182, - "English,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Malay,Chinese": 0.2215909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.3068181818181818, - "Filipino,Spanish,Malay,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.2556818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.3125, - "Filipino,Malay,Chinese,Indonesian": 0.1875, - "Filipino,Malay,Chinese,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "Spanish,Malay,Chinese,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365 + "English,Filipino,Vietnamese,Chinese": 0.2784090909090909, + "English,Filipino,Vietnamese,Indonesian": 0.32954545454545453, + "English,Filipino,Vietnamese,Malay": 0.3409090909090909, + "English,Filipino,Vietnamese,Spanish": 0.35795454545454547, + "English,Filipino,Chinese,Indonesian": 0.3125, + "English,Filipino,Chinese,Malay": 0.32954545454545453, + "English,Filipino,Chinese,Spanish": 0.3465909090909091, + "English,Filipino,Indonesian,Malay": 0.3693181818181818, + "English,Filipino,Indonesian,Spanish": 0.3806818181818182, + "English,Filipino,Malay,Spanish": 0.3977272727272727, + "English,Vietnamese,Chinese,Indonesian": 0.2784090909090909, + "English,Vietnamese,Chinese,Malay": 0.29545454545454547, + "English,Vietnamese,Chinese,Spanish": 0.32954545454545453, + "English,Vietnamese,Indonesian,Malay": 0.3465909090909091, + "English,Vietnamese,Indonesian,Spanish": 0.36363636363636365, + "English,Vietnamese,Malay,Spanish": 0.39204545454545453, + "English,Chinese,Indonesian,Malay": 0.36363636363636365, + "English,Chinese,Indonesian,Spanish": 0.36363636363636365, + "English,Chinese,Malay,Spanish": 0.39204545454545453, + "English,Indonesian,Malay,Spanish": 0.44886363636363635, + "Filipino,Vietnamese,Chinese,Indonesian": 0.2784090909090909, + "Filipino,Vietnamese,Chinese,Malay": 0.3068181818181818, + "Filipino,Vietnamese,Chinese,Spanish": 0.2840909090909091, + "Filipino,Vietnamese,Indonesian,Malay": 0.3409090909090909, + "Filipino,Vietnamese,Indonesian,Spanish": 0.32954545454545453, + "Filipino,Vietnamese,Malay,Spanish": 0.36363636363636365, + "Filipino,Chinese,Indonesian,Malay": 0.32954545454545453, + "Filipino,Chinese,Indonesian,Spanish": 0.3125, + "Filipino,Chinese,Malay,Spanish": 0.3352272727272727, + "Filipino,Indonesian,Malay,Spanish": 0.4090909090909091, + "Vietnamese,Chinese,Indonesian,Malay": 0.29545454545454547, + "Vietnamese,Chinese,Indonesian,Spanish": 0.2897727272727273, + "Vietnamese,Chinese,Malay,Spanish": 0.3181818181818182, + "Vietnamese,Indonesian,Malay,Spanish": 0.36363636363636365, + "Chinese,Indonesian,Malay,Spanish": 0.375 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.19886363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.26136363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.25, - "English,Filipino,Spanish,Chinese,Indonesian": 0.1875, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.25, - "English,Filipino,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.2159090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 + "English,Filipino,Vietnamese,Chinese,Indonesian": 0.25, + "English,Filipino,Vietnamese,Chinese,Malay": 0.26136363636363635, + "English,Filipino,Vietnamese,Chinese,Spanish": 0.26136363636363635, + "English,Filipino,Vietnamese,Indonesian,Malay": 0.30113636363636365, + "English,Filipino,Vietnamese,Indonesian,Spanish": 0.3068181818181818, + "English,Filipino,Vietnamese,Malay,Spanish": 0.32954545454545453, + "English,Filipino,Chinese,Indonesian,Malay": 0.29545454545454547, + "English,Filipino,Chinese,Indonesian,Spanish": 0.30113636363636365, + "English,Filipino,Chinese,Malay,Spanish": 0.3125, + "English,Filipino,Indonesian,Malay,Spanish": 0.35795454545454547, + "English,Vietnamese,Chinese,Indonesian,Malay": 0.26136363636363635, + "English,Vietnamese,Chinese,Indonesian,Spanish": 0.26136363636363635, + "English,Vietnamese,Chinese,Malay,Spanish": 0.2840909090909091, + "English,Vietnamese,Indonesian,Malay,Spanish": 0.32954545454545453, + "English,Chinese,Indonesian,Malay,Spanish": 0.3465909090909091, + "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.26704545454545453, + "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.25, + "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2727272727272727, + "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.3181818181818182, + "Filipino,Chinese,Indonesian,Malay,Spanish": 0.30113636363636365, + "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.2727272727272727 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125 + "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.23863636363636365, + "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.23863636363636365, + "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.25, + "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.29545454545454547, + "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.2897727272727273, + "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.2556818181818182, + "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.24431818181818182 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 + "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.23295454545454544 } }, - "AC3_2": 0.49031928920206447, - "AC3_3": 0.39588252651507944, - "AC3_4": 0.32141841116736963, - "AC3_5": 0.2615359384448628, - "AC3_6": 0.2131602922163289, - "AC3_7": 0.17417085092018547 + "AC3_2": 0.5479003849634371, + "AC3_3": 0.46442064424123597, + "AC3_4": 0.4102196919912001, + "AC3_5": 0.3727626819026909, + "AC3_6": 0.3443314825484767, + "AC3_7": 0.32056571141952267 }, - "prompt_4": { - "overall_acc": 0.47402597402597413, + "prompt_5": { + "overall_acc": 0.49025974025974023, "language_acc": { - "English": 0.4943181818181818, - "Filipino": 0.4602272727272727, - "Spanish": 0.48863636363636365, - "Malay": 0.4318181818181818, - "Chinese": 0.48295454545454547, - "Indonesian": 0.45454545454545453, - "Vietnamese": 0.5056818181818182 + "English": 0.5454545454545454, + "Filipino": 0.4090909090909091, + "Vietnamese": 0.48295454545454547, + "Chinese": 0.5397727272727273, + "Indonesian": 0.42613636363636365, + "Malay": 0.4715909090909091, + "Spanish": 0.5568181818181818 }, - "consistency_score_2": 0.5533008658008658, - "consistency_score_3": 0.3902597402597403, - "consistency_score_4": 0.30600649350649345, - "consistency_score_5": 0.2532467532467533, - "consistency_score_6": 0.21590909090909088, - "consistency_score_7": 0.1875, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.6022727272727273, - "English,Spanish": 0.6022727272727273, - "English,Malay": 0.5681818181818182, - "English,Chinese": 0.44886363636363635, - "English,Indonesian": 0.5795454545454546, - "English,Vietnamese": 0.5454545454545454, - "Filipino,Spanish": 0.6193181818181818, - "Filipino,Malay": 0.6193181818181818, - "Filipino,Chinese": 0.4431818181818182, - "Filipino,Indonesian": 0.625, - "Filipino,Vietnamese": 0.5852272727272727, - "Spanish,Malay": 0.6193181818181818, - "Spanish,Chinese": 0.4943181818181818, - "Spanish,Indonesian": 0.5568181818181818, - "Spanish,Vietnamese": 0.5852272727272727, - "Malay,Chinese": 0.48863636363636365, - "Malay,Indonesian": 0.6420454545454546, - "Malay,Vietnamese": 0.5511363636363636, - "Chinese,Indonesian": 0.4375, - "Chinese,Vietnamese": 0.4375, - "Indonesian,Vietnamese": 0.5681818181818182 - }, - "3_combine": { - "English,Filipino,Spanish": 0.4602272727272727, - "English,Filipino,Malay": 0.4375, - "English,Filipino,Chinese": 0.30113636363636365, - "English,Filipino,Indonesian": 0.44886363636363635, - "English,Filipino,Vietnamese": 0.4147727272727273, - "English,Spanish,Malay": 0.44886363636363635, - "English,Spanish,Chinese": 0.3465909090909091, - "English,Spanish,Indonesian": 0.42613636363636365, - "English,Spanish,Vietnamese": 0.42613636363636365, - "English,Malay,Chinese": 0.3181818181818182, - "English,Malay,Indonesian": 0.44886363636363635, - "English,Malay,Vietnamese": 0.39204545454545453, - "English,Chinese,Indonesian": 0.29545454545454547, - "English,Chinese,Vietnamese": 0.2897727272727273, - "English,Indonesian,Vietnamese": 0.3977272727272727, - "Filipino,Spanish,Malay": 0.4772727272727273, - "Filipino,Spanish,Chinese": 0.3522727272727273, - "Filipino,Spanish,Indonesian": 0.44886363636363635, - "Filipino,Spanish,Vietnamese": 0.4431818181818182, - "Filipino,Malay,Chinese": 0.3409090909090909, - "Filipino,Malay,Indonesian": 0.48295454545454547, - "Filipino,Malay,Vietnamese": 0.4318181818181818, - "Filipino,Chinese,Indonesian": 0.3125, - "Filipino,Chinese,Vietnamese": 0.3068181818181818, - "Filipino,Indonesian,Vietnamese": 0.4375, - "Spanish,Malay,Chinese": 0.3693181818181818, - "Spanish,Malay,Indonesian": 0.4602272727272727, - "Spanish,Malay,Vietnamese": 0.44886363636363635, - "Spanish,Chinese,Indonesian": 0.32954545454545453, - "Spanish,Chinese,Vietnamese": 0.3409090909090909, - "Spanish,Indonesian,Vietnamese": 0.42613636363636365, - "Malay,Chinese,Indonesian": 0.35795454545454547, - "Malay,Chinese,Vietnamese": 0.3125, - "Malay,Indonesian,Vietnamese": 0.42613636363636365, - "Chinese,Indonesian,Vietnamese": 0.30113636363636365 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.375, - "English,Filipino,Spanish,Chinese": 0.2784090909090909, - "English,Filipino,Spanish,Indonesian": 0.36363636363636365, - "English,Filipino,Spanish,Vietnamese": 0.3522727272727273, - "English,Filipino,Malay,Chinese": 0.2727272727272727, - "English,Filipino,Malay,Indonesian": 0.3693181818181818, - "English,Filipino,Malay,Vietnamese": 0.3409090909090909, - "English,Filipino,Chinese,Indonesian": 0.24431818181818182, - "English,Filipino,Chinese,Vietnamese": 0.23863636363636365, - "English,Filipino,Indonesian,Vietnamese": 0.3522727272727273, - "English,Spanish,Malay,Chinese": 0.2840909090909091, - "English,Spanish,Malay,Indonesian": 0.36363636363636365, - "English,Spanish,Malay,Vietnamese": 0.3465909090909091, - "English,Spanish,Chinese,Indonesian": 0.26136363636363635, - "English,Spanish,Chinese,Vietnamese": 0.26704545454545453, - "English,Spanish,Indonesian,Vietnamese": 0.3352272727272727, - "English,Malay,Chinese,Indonesian": 0.26136363636363635, - "English,Malay,Chinese,Vietnamese": 0.24431818181818182, - "English,Malay,Indonesian,Vietnamese": 0.3352272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Malay,Chinese": 0.30113636363636365, - "Filipino,Spanish,Malay,Indonesian": 0.38636363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.3806818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Chinese,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Indonesian,Vietnamese": 0.3693181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.2784090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.26136363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.36363636363636365, - "Filipino,Chinese,Indonesian,Vietnamese": 0.25, - "Spanish,Malay,Chinese,Indonesian": 0.29545454545454547, - "Spanish,Malay,Chinese,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.3693181818181818, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.2556818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.3181818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.3125, - "English,Filipino,Spanish,Chinese,Indonesian": 0.22727272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.22727272727272727, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.3068181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.22727272727272727, - "English,Filipino,Malay,Chinese,Vietnamese": 0.2215909090909091, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "English,Spanish,Malay,Chinese,Indonesian": 0.23863636363636365, - "English,Spanish,Malay,Chinese,Vietnamese": 0.23295454545454544, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.25, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.25, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.32954545454545453, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.2159090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1875 - } - }, - "AC3_2": 0.5106047494278415, - "AC3_3": 0.4280835617680661, - "AC3_4": 0.3719204832151548, - "AC3_5": 0.330125231865554, - "AC3_6": 0.2966844919356078, - "AC3_7": 0.2687116564010978 - }, - "prompt_5": { - "overall_acc": 0.4724025974025974, - "language_acc": { - "English": 0.5, - "Filipino": 0.4659090909090909, - "Spanish": 0.48295454545454547, - "Malay": 0.45454545454545453, - "Chinese": 0.48863636363636365, - "Indonesian": 0.4318181818181818, - "Vietnamese": 0.48295454545454547 - }, - "consistency_score_2": 0.5625, - "consistency_score_3": 0.3969155844155845, - "consistency_score_4": 0.3084415584415585, - "consistency_score_5": 0.25270562770562766, - "consistency_score_6": 0.21428571428571427, - "consistency_score_7": 0.1875, + "consistency_score_2": 0.5622294372294371, + "consistency_score_3": 0.39285714285714296, + "consistency_score_4": 0.30909090909090897, + "consistency_score_5": 0.26136363636363635, + "consistency_score_6": 0.23133116883116883, + "consistency_score_7": 0.21022727272727273, "detailed_consistency_score": { "2_combine": { - "English,Filipino": 0.6306818181818182, - "English,Spanish": 0.6477272727272727, - "English,Malay": 0.6306818181818182, - "English,Chinese": 0.5284090909090909, + "English,Filipino": 0.48863636363636365, + "English,Vietnamese": 0.5852272727272727, + "English,Chinese": 0.5625, "English,Indonesian": 0.5738636363636364, - "English,Vietnamese": 0.5795454545454546, - "Filipino,Spanish": 0.6022727272727273, - "Filipino,Malay": 0.5909090909090909, - "Filipino,Chinese": 0.4715909090909091, - "Filipino,Indonesian": 0.5965909090909091, - "Filipino,Vietnamese": 0.5681818181818182, - "Spanish,Malay": 0.6136363636363636, - "Spanish,Chinese": 0.5170454545454546, - "Spanish,Indonesian": 0.5454545454545454, - "Spanish,Vietnamese": 0.5909090909090909, - "Malay,Chinese": 0.4602272727272727, - "Malay,Indonesian": 0.6193181818181818, - "Malay,Vietnamese": 0.5397727272727273, - "Chinese,Indonesian": 0.44886363636363635, - "Chinese,Vietnamese": 0.4772727272727273, - "Indonesian,Vietnamese": 0.5795454545454546 + "English,Malay": 0.6193181818181818, + "English,Spanish": 0.7045454545454546, + "Filipino,Vietnamese": 0.5340909090909091, + "Filipino,Chinese": 0.5113636363636364, + "Filipino,Indonesian": 0.5340909090909091, + "Filipino,Malay": 0.5568181818181818, + "Filipino,Spanish": 0.5056818181818182, + "Vietnamese,Chinese": 0.5113636363636364, + "Vietnamese,Indonesian": 0.48863636363636365, + "Vietnamese,Malay": 0.5909090909090909, + "Vietnamese,Spanish": 0.5738636363636364, + "Chinese,Indonesian": 0.5397727272727273, + "Chinese,Malay": 0.5511363636363636, + "Chinese,Spanish": 0.5511363636363636, + "Indonesian,Malay": 0.625, + "Indonesian,Spanish": 0.6022727272727273, + "Malay,Spanish": 0.5965909090909091 }, "3_combine": { - "English,Filipino,Spanish": 0.48295454545454547, - "English,Filipino,Malay": 0.4659090909090909, - "English,Filipino,Chinese": 0.375, - "English,Filipino,Indonesian": 0.4375, - "English,Filipino,Vietnamese": 0.4318181818181818, - "English,Spanish,Malay": 0.4943181818181818, - "English,Spanish,Chinese": 0.39204545454545453, - "English,Spanish,Indonesian": 0.4318181818181818, - "English,Spanish,Vietnamese": 0.45454545454545453, - "English,Malay,Chinese": 0.36363636363636365, - "English,Malay,Indonesian": 0.44886363636363635, - "English,Malay,Vietnamese": 0.4318181818181818, - "English,Chinese,Indonesian": 0.32386363636363635, - "English,Chinese,Vietnamese": 0.3522727272727273, - "English,Indonesian,Vietnamese": 0.4147727272727273, - "Filipino,Spanish,Malay": 0.4375, - "Filipino,Spanish,Chinese": 0.35795454545454547, - "Filipino,Spanish,Indonesian": 0.42045454545454547, - "Filipino,Spanish,Vietnamese": 0.42613636363636365, - "Filipino,Malay,Chinese": 0.3465909090909091, - "Filipino,Malay,Indonesian": 0.4431818181818182, - "Filipino,Malay,Vietnamese": 0.3977272727272727, - "Filipino,Chinese,Indonesian": 0.3352272727272727, - "Filipino,Chinese,Vietnamese": 0.3352272727272727, - "Filipino,Indonesian,Vietnamese": 0.4147727272727273, - "Spanish,Malay,Chinese": 0.36363636363636365, - "Spanish,Malay,Indonesian": 0.42613636363636365, - "Spanish,Malay,Vietnamese": 0.4318181818181818, - "Spanish,Chinese,Indonesian": 0.3181818181818182, - "Spanish,Chinese,Vietnamese": 0.3465909090909091, - "Spanish,Indonesian,Vietnamese": 0.4034090909090909, - "Malay,Chinese,Indonesian": 0.3352272727272727, - "Malay,Chinese,Vietnamese": 0.3181818181818182, - "Malay,Indonesian,Vietnamese": 0.4147727272727273, - "Chinese,Indonesian,Vietnamese": 0.3181818181818182 + "English,Filipino,Vietnamese": 0.35795454545454547, + "English,Filipino,Chinese": 0.3409090909090909, + "English,Filipino,Indonesian": 0.3465909090909091, + "English,Filipino,Malay": 0.3693181818181818, + "English,Filipino,Spanish": 0.38636363636363635, + "English,Vietnamese,Chinese": 0.3693181818181818, + "English,Vietnamese,Indonesian": 0.3806818181818182, + "English,Vietnamese,Malay": 0.4375, + "English,Vietnamese,Spanish": 0.4659090909090909, + "English,Chinese,Indonesian": 0.39204545454545453, + "English,Chinese,Malay": 0.3977272727272727, + "English,Chinese,Spanish": 0.44886363636363635, + "English,Indonesian,Malay": 0.44886363636363635, + "English,Indonesian,Spanish": 0.4715909090909091, + "English,Malay,Spanish": 0.48863636363636365, + "Filipino,Vietnamese,Chinese": 0.3465909090909091, + "Filipino,Vietnamese,Indonesian": 0.3465909090909091, + "Filipino,Vietnamese,Malay": 0.39204545454545453, + "Filipino,Vietnamese,Spanish": 0.36363636363636365, + "Filipino,Chinese,Indonesian": 0.3465909090909091, + "Filipino,Chinese,Malay": 0.375, + "Filipino,Chinese,Spanish": 0.3352272727272727, + "Filipino,Indonesian,Malay": 0.4034090909090909, + "Filipino,Indonesian,Spanish": 0.375, + "Filipino,Malay,Spanish": 0.3693181818181818, + "Vietnamese,Chinese,Indonesian": 0.3352272727272727, + "Vietnamese,Chinese,Malay": 0.36363636363636365, + "Vietnamese,Chinese,Spanish": 0.38636363636363635, + "Vietnamese,Indonesian,Malay": 0.3977272727272727, + "Vietnamese,Indonesian,Spanish": 0.3977272727272727, + "Vietnamese,Malay,Spanish": 0.42045454545454547, + "Chinese,Indonesian,Malay": 0.4147727272727273, + "Chinese,Indonesian,Spanish": 0.42045454545454547, + "Chinese,Malay,Spanish": 0.39204545454545453, + "Indonesian,Malay,Spanish": 0.4659090909090909 }, "4_combine": { - "English,Filipino,Spanish,Malay": 0.39204545454545453, - "English,Filipino,Spanish,Chinese": 0.3068181818181818, - "English,Filipino,Spanish,Indonesian": 0.36363636363636365, - "English,Filipino,Spanish,Vietnamese": 0.3693181818181818, - "English,Filipino,Malay,Chinese": 0.30113636363636365, - "English,Filipino,Malay,Indonesian": 0.3522727272727273, - "English,Filipino,Malay,Vietnamese": 0.3522727272727273, - "English,Filipino,Chinese,Indonesian": 0.26704545454545453, - "English,Filipino,Chinese,Vietnamese": 0.2784090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.3352272727272727, - "English,Spanish,Malay,Chinese": 0.3181818181818182, - "English,Spanish,Malay,Indonesian": 0.36363636363636365, - "English,Spanish,Malay,Vietnamese": 0.375, - "English,Spanish,Chinese,Indonesian": 0.2840909090909091, - "English,Spanish,Chinese,Vietnamese": 0.30113636363636365, - "English,Spanish,Indonesian,Vietnamese": 0.3409090909090909, - "English,Malay,Chinese,Indonesian": 0.2727272727272727, - "English,Malay,Chinese,Vietnamese": 0.2784090909090909, - "English,Malay,Indonesian,Vietnamese": 0.3409090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.25, - "Filipino,Spanish,Malay,Chinese": 0.2897727272727273, - "Filipino,Spanish,Malay,Indonesian": 0.3409090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.3352272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.26136363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.2784090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.32954545454545453, - "Filipino,Malay,Chinese,Indonesian": 0.2784090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.2727272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.32386363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.2727272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.32954545454545453, - "Spanish,Chinese,Indonesian,Vietnamese": 0.25, - "Malay,Chinese,Indonesian,Vietnamese": 0.25 + "English,Filipino,Vietnamese,Chinese": 0.26704545454545453, + "English,Filipino,Vietnamese,Indonesian": 0.2840909090909091, + "English,Filipino,Vietnamese,Malay": 0.30113636363636365, + "English,Filipino,Vietnamese,Spanish": 0.3125, + "English,Filipino,Chinese,Indonesian": 0.2727272727272727, + "English,Filipino,Chinese,Malay": 0.2784090909090909, + "English,Filipino,Chinese,Spanish": 0.2840909090909091, + "English,Filipino,Indonesian,Malay": 0.30113636363636365, + "English,Filipino,Indonesian,Spanish": 0.3068181818181818, + "English,Filipino,Malay,Spanish": 0.3068181818181818, + "English,Vietnamese,Chinese,Indonesian": 0.2840909090909091, + "English,Vietnamese,Chinese,Malay": 0.2897727272727273, + "English,Vietnamese,Chinese,Spanish": 0.3352272727272727, + "English,Vietnamese,Indonesian,Malay": 0.32954545454545453, + "English,Vietnamese,Indonesian,Spanish": 0.3409090909090909, + "English,Vietnamese,Malay,Spanish": 0.35795454545454547, + "English,Chinese,Indonesian,Malay": 0.3352272727272727, + "English,Chinese,Indonesian,Spanish": 0.36363636363636365, + "English,Chinese,Malay,Spanish": 0.3465909090909091, + "English,Indonesian,Malay,Spanish": 0.39204545454545453, + "Filipino,Vietnamese,Chinese,Indonesian": 0.2727272727272727, + "Filipino,Vietnamese,Chinese,Malay": 0.2840909090909091, + "Filipino,Vietnamese,Chinese,Spanish": 0.2727272727272727, + "Filipino,Vietnamese,Indonesian,Malay": 0.30113636363636365, + "Filipino,Vietnamese,Indonesian,Spanish": 0.30113636363636365, + "Filipino,Vietnamese,Malay,Spanish": 0.3068181818181818, + "Filipino,Chinese,Indonesian,Malay": 0.30113636363636365, + "Filipino,Chinese,Indonesian,Spanish": 0.29545454545454547, + "Filipino,Chinese,Malay,Spanish": 0.2727272727272727, + "Filipino,Indonesian,Malay,Spanish": 0.3125, + "Vietnamese,Chinese,Indonesian,Malay": 0.29545454545454547, + "Vietnamese,Chinese,Indonesian,Spanish": 0.3125, + "Vietnamese,Chinese,Malay,Spanish": 0.3068181818181818, + "Vietnamese,Indonesian,Malay,Spanish": 0.3409090909090909, + "Chinese,Indonesian,Malay,Spanish": 0.3522727272727273 }, "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.2727272727272727, - "English,Filipino,Spanish,Malay,Indonesian": 0.3068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.3125, - "English,Filipino,Spanish,Chinese,Indonesian": 0.23863636363636365, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.2556818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.29545454545454547, - "English,Filipino,Malay,Chinese,Indonesian": 0.23295454545454544, - "English,Filipino,Malay,Chinese,Vietnamese": 0.24431818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.25, - "English,Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.29545454545454547, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.23295454545454544, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909 + "English,Filipino,Vietnamese,Chinese,Indonesian": 0.23863636363636365, + "English,Filipino,Vietnamese,Chinese,Malay": 0.23295454545454544, + "English,Filipino,Vietnamese,Chinese,Spanish": 0.24431818181818182, + "English,Filipino,Vietnamese,Indonesian,Malay": 0.2556818181818182, + "English,Filipino,Vietnamese,Indonesian,Spanish": 0.26704545454545453, + "English,Filipino,Vietnamese,Malay,Spanish": 0.26704545454545453, + "English,Filipino,Chinese,Indonesian,Malay": 0.24431818181818182, + "English,Filipino,Chinese,Indonesian,Spanish": 0.26136363636363635, + "English,Filipino,Chinese,Malay,Spanish": 0.24431818181818182, + "English,Filipino,Indonesian,Malay,Spanish": 0.2727272727272727, + "English,Vietnamese,Chinese,Indonesian,Malay": 0.2556818181818182, + "English,Vietnamese,Chinese,Indonesian,Spanish": 0.2784090909090909, + "English,Vietnamese,Chinese,Malay,Spanish": 0.26704545454545453, + "English,Vietnamese,Indonesian,Malay,Spanish": 0.29545454545454547, + "English,Chinese,Indonesian,Malay,Spanish": 0.32386363636363635, + "Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.24431818181818182, + "Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.2556818181818182, + "Filipino,Vietnamese,Chinese,Malay,Spanish": 0.23863636363636365, + "Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.26704545454545453, + "Filipino,Chinese,Indonesian,Malay,Spanish": 0.2556818181818182, + "Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.2784090909090909 }, "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.23295454545454544, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2556818181818182, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818 + "English,Filipino,Vietnamese,Chinese,Indonesian,Malay": 0.2159090909090909, + "English,Filipino,Vietnamese,Chinese,Indonesian,Spanish": 0.23295454545454544, + "English,Filipino,Vietnamese,Chinese,Malay,Spanish": 0.2159090909090909, + "English,Filipino,Vietnamese,Indonesian,Malay,Spanish": 0.23863636363636365, + "English,Filipino,Chinese,Indonesian,Malay,Spanish": 0.23863636363636365, + "English,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.25, + "Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.22727272727272727 }, "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1875 + "English,Filipino,Vietnamese,Chinese,Indonesian,Malay,Spanish": 0.21022727272727273 } }, - "AC3_2": 0.5135294117150848, - "AC3_3": 0.43138164351689456, - "AC3_4": 0.3732078731600778, - "AC3_5": 0.32927166112923617, - "AC3_6": 0.2948328267047849, - "AC3_7": 0.26845018446116475 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.5825242718446602 - }, - "prompt_2": { - "accuracy": 0.5631067961165048 - }, - "prompt_3": { - "accuracy": 0.6019417475728155 - }, - "prompt_4": { - "accuracy": 0.3106796116504854 - }, - "prompt_5": { - "accuracy": 0.5728155339805825 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.4 - }, - "prompt_2": { - "accuracy": 0.3523809523809524 - }, - "prompt_3": { - "accuracy": 0.4380952380952381 - }, - "prompt_4": { - "accuracy": 0.3333333333333333 - }, - "prompt_5": { - "accuracy": 0.34285714285714286 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.6635514018691588 - }, - "prompt_2": { - "accuracy": 0.7102803738317757 - }, - "prompt_3": { - "accuracy": 0.7102803738317757 - }, - "prompt_4": { - "accuracy": 0.4766355140186916 - }, - "prompt_5": { - "accuracy": 0.6635514018691588 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.52, - "category_acc": { - "brand": 0.5, - "demographics": 0.2, - "biology": 0.4, - "history": 0.6, - "literature": 0.4, - "politics": 0.7, - "culture": 0.6, - "film": 0.5, - "law": 0.5, - "geography": 0.6 - } - }, - "prompt_2": { - "accuracy": 0.51, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.3, - "history": 0.6, - "literature": 0.4, - "politics": 0.7, - "culture": 0.5, - "film": 0.7, - "law": 0.6, - "geography": 0.5 - } - }, - "prompt_3": { - "accuracy": 0.52, - "category_acc": { - "brand": 0.5, - "demographics": 0.2, - "biology": 0.3, - "history": 0.4666666666666667, - "literature": 0.5, - "politics": 0.8, - "culture": 0.6, - "film": 0.6, - "law": 0.5, - "geography": 0.6 - } - }, - "prompt_4": { - "accuracy": 0.36, - "category_acc": { - "brand": 0.5, - "demographics": 0.4, - "biology": 0.1, - "history": 0.3333333333333333, - "literature": 0.2, - "politics": 0.8, - "culture": 0.2, - "film": 0.4, - "law": 0.4, - "geography": 0.3 - } - }, - "prompt_5": { - "accuracy": 0.55, - "category_acc": { - "brand": 0.5, - "demographics": 0.2, - "biology": 0.4, - "history": 0.5333333333333333, - "literature": 0.5, - "politics": 0.8, - "culture": 0.7, - "film": 0.6, - "law": 0.6, - "geography": 0.5 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.06768576871680226 - }, - "prompt_2": { - "bleu_score": 0.07537211725219173 - }, - "prompt_3": { - "bleu_score": 0.067893201379615 - }, - "prompt_4": { - "bleu_score": 0.06860409223920591 - }, - "prompt_5": { - "bleu_score": 0.054699323963346584 - } - }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.09694513383692231 - }, - "prompt_2": { - "bleu_score": 0.09667804069703814 - }, - "prompt_3": { - "bleu_score": 0.09176921113104079 - }, - "prompt_4": { - "bleu_score": 0.091977688283435 - }, - "prompt_5": { - "bleu_score": 0.17916920218674579 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.07515663977438763 - }, - "prompt_2": { - "bleu_score": 0.12170650854729724 - }, - "prompt_3": { - "bleu_score": 0.1115212793762873 - }, - "prompt_4": { - "bleu_score": 0.07062748836809796 - }, - "prompt_5": { - "bleu_score": 0.09654427568024729 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.06658954469519791 - }, - "prompt_2": { - "bleu_score": 0.08090750076122914 - }, - "prompt_3": { - "bleu_score": 0.08082539857207657 - }, - "prompt_4": { - "bleu_score": 0.06491223088446361 - }, - "prompt_5": { - "bleu_score": 0.08128298950369667 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.10384187879690358 - }, - "prompt_2": { - "bleu_score": 0.13249387087809075 - }, - "prompt_3": { - "bleu_score": 0.1333205871967059 - }, - "prompt_4": { - "bleu_score": 0.09048762545192097 - }, - "prompt_5": { - "bleu_score": 0.16840186415610886 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.6056009334889149 - }, - "prompt_2": { - "accuracy": 0.5892648774795799 - }, - "prompt_3": { - "accuracy": 0.603267211201867 - }, - "prompt_4": { - "accuracy": 0.5682613768961493 - }, - "prompt_5": { - "accuracy": 0.6056009334889149 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.5874865927779764, - "category_acc": { - "high_school_european_history": 0.7621951219512195, - "business_ethics": 0.6363636363636364, - "clinical_knowledge": 0.6553030303030303, - "medical_genetics": 0.6565656565656566, - "high_school_us_history": 0.7733990147783252, - "high_school_physics": 0.3466666666666667, - "high_school_world_history": 0.7966101694915254, - "virology": 0.5151515151515151, - "high_school_microeconomics": 0.6329113924050633, - "econometrics": 0.4336283185840708, - "college_computer_science": 0.43434343434343436, - "high_school_biology": 0.7669902912621359, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.4875444839857651, - "philosophy": 0.6483870967741936, - "professional_medicine": 0.6236162361623616, - "nutrition": 0.6885245901639344, - "global_facts": 0.35353535353535354, - "machine_learning": 0.4774774774774775, - "security_studies": 0.6762295081967213, - "public_relations": 0.5321100917431193, - "professional_psychology": 0.630114566284779, - "prehistory": 0.7306501547987616, - "anatomy": 0.5895522388059702, - "human_sexuality": 0.6846153846153846, - "college_medicine": 0.6162790697674418, - "high_school_government_and_politics": 0.8125, - "college_chemistry": 0.41414141414141414, - "logical_fallacies": 0.6481481481481481, - "high_school_geography": 0.7918781725888325, - "elementary_mathematics": 0.41379310344827586, - "human_aging": 0.6351351351351351, - "college_mathematics": 0.3434343434343434, - "high_school_psychology": 0.8051470588235294, - "formal_logic": 0.368, - "high_school_statistics": 0.5162790697674419, - "international_law": 0.675, - "high_school_mathematics": 0.30855018587360594, - "high_school_computer_science": 0.6262626262626263, - "conceptual_physics": 0.5384615384615384, - "miscellaneous": 0.7391304347826086, - "high_school_chemistry": 0.5346534653465347, - "marketing": 0.8068669527896996, - "professional_law": 0.46966731898238745, - "management": 0.7058823529411765, - "college_physics": 0.31683168316831684, - "jurisprudence": 0.719626168224299, - "world_religions": 0.8058823529411765, - "sociology": 0.735, - "us_foreign_policy": 0.7777777777777778, - "high_school_macroeconomics": 0.5604113110539846, - "computer_security": 0.5858585858585859, - "moral_scenarios": 0.24049217002237136, - "moral_disputes": 0.6347826086956522, - "electrical_engineering": 0.5486111111111112, - "astronomy": 0.7086092715231788, - "college_biology": 0.7622377622377622 - } - }, - "prompt_2": { - "accuracy": 0.5941365749016804, - "category_acc": { - "high_school_european_history": 0.7621951219512195, - "business_ethics": 0.6060606060606061, - "clinical_knowledge": 0.6628787878787878, - "medical_genetics": 0.6868686868686869, - "high_school_us_history": 0.7832512315270936, - "high_school_physics": 0.35333333333333333, - "high_school_world_history": 0.8135593220338984, - "virology": 0.5212121212121212, - "high_school_microeconomics": 0.6075949367088608, - "econometrics": 0.3893805309734513, - "college_computer_science": 0.4444444444444444, - "high_school_biology": 0.7475728155339806, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.501779359430605, - "philosophy": 0.6806451612903226, - "professional_medicine": 0.6457564575645757, - "nutrition": 0.7049180327868853, - "global_facts": 0.3838383838383838, - "machine_learning": 0.4594594594594595, - "security_studies": 0.6762295081967213, - "public_relations": 0.6055045871559633, - "professional_psychology": 0.646481178396072, - "prehistory": 0.7244582043343654, - "anatomy": 0.6119402985074627, - "human_sexuality": 0.7615384615384615, - "college_medicine": 0.5930232558139535, - "high_school_government_and_politics": 0.8177083333333334, - "college_chemistry": 0.42424242424242425, - "logical_fallacies": 0.7098765432098766, - "high_school_geography": 0.751269035532995, - "elementary_mathematics": 0.4297082228116711, - "human_aging": 0.6531531531531531, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.8180147058823529, - "formal_logic": 0.4, - "high_school_statistics": 0.5069767441860465, - "international_law": 0.7166666666666667, - "high_school_mathematics": 0.3271375464684015, - "high_school_computer_science": 0.6262626262626263, - "conceptual_physics": 0.5256410256410257, - "miscellaneous": 0.7480818414322251, - "high_school_chemistry": 0.5198019801980198, - "marketing": 0.7939914163090128, - "professional_law": 0.4683626875407697, - "management": 0.7549019607843137, - "college_physics": 0.32673267326732675, - "jurisprudence": 0.6915887850467289, - "world_religions": 0.7647058823529411, - "sociology": 0.765, - "us_foreign_policy": 0.7878787878787878, - "high_school_macroeconomics": 0.596401028277635, - "computer_security": 0.6666666666666666, - "moral_scenarios": 0.24161073825503357, - "moral_disputes": 0.6202898550724638, - "electrical_engineering": 0.5347222222222222, - "astronomy": 0.7019867549668874, - "college_biology": 0.7552447552447552 - } - }, - "prompt_3": { - "accuracy": 0.6023596710761531, - "category_acc": { - "high_school_european_history": 0.7865853658536586, - "business_ethics": 0.6161616161616161, - "clinical_knowledge": 0.6628787878787878, - "medical_genetics": 0.7070707070707071, - "high_school_us_history": 0.7684729064039408, - "high_school_physics": 0.3333333333333333, - "high_school_world_history": 0.809322033898305, - "virology": 0.5454545454545454, - "high_school_microeconomics": 0.6329113924050633, - "econometrics": 0.415929203539823, - "college_computer_science": 0.48484848484848486, - "high_school_biology": 0.7605177993527508, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.505338078291815, - "philosophy": 0.6483870967741936, - "professional_medicine": 0.6457564575645757, - "nutrition": 0.6885245901639344, - "global_facts": 0.40404040404040403, - "machine_learning": 0.40540540540540543, - "security_studies": 0.680327868852459, - "public_relations": 0.6146788990825688, - "professional_psychology": 0.6563011456628478, - "prehistory": 0.718266253869969, - "anatomy": 0.5895522388059702, - "human_sexuality": 0.7153846153846154, - "college_medicine": 0.6162790697674418, - "high_school_government_and_politics": 0.8177083333333334, - "college_chemistry": 0.42424242424242425, - "logical_fallacies": 0.7283950617283951, - "high_school_geography": 0.7918781725888325, - "elementary_mathematics": 0.44297082228116713, - "human_aging": 0.6936936936936937, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.8106617647058824, - "formal_logic": 0.384, - "high_school_statistics": 0.5581395348837209, - "international_law": 0.7166666666666667, - "high_school_mathematics": 0.34944237918215615, - "high_school_computer_science": 0.6363636363636364, - "conceptual_physics": 0.5470085470085471, - "miscellaneous": 0.7621483375959079, - "high_school_chemistry": 0.5445544554455446, - "marketing": 0.8540772532188842, - "professional_law": 0.4781474233529028, - "management": 0.7941176470588235, - "college_physics": 0.3564356435643564, - "jurisprudence": 0.719626168224299, - "world_religions": 0.7823529411764706, - "sociology": 0.76, - "us_foreign_policy": 0.8787878787878788, - "high_school_macroeconomics": 0.5912596401028277, - "computer_security": 0.6767676767676768, - "moral_scenarios": 0.24272930648769575, - "moral_disputes": 0.6202898550724638, - "electrical_engineering": 0.5763888888888888, - "astronomy": 0.7019867549668874, - "college_biology": 0.7552447552447552 - } - }, - "prompt_4": { - "accuracy": 0.5788344654987486, - "category_acc": { - "high_school_european_history": 0.7865853658536586, - "business_ethics": 0.5757575757575758, - "clinical_knowledge": 0.6022727272727273, - "medical_genetics": 0.7070707070707071, - "high_school_us_history": 0.7832512315270936, - "high_school_physics": 0.34, - "high_school_world_history": 0.8135593220338984, - "virology": 0.5333333333333333, - "high_school_microeconomics": 0.6286919831223629, - "econometrics": 0.4424778761061947, - "college_computer_science": 0.43434343434343436, - "high_school_biology": 0.7572815533980582, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.47330960854092524, - "philosophy": 0.6709677419354839, - "professional_medicine": 0.6125461254612546, - "nutrition": 0.639344262295082, - "global_facts": 0.3333333333333333, - "machine_learning": 0.43243243243243246, - "security_studies": 0.6475409836065574, - "public_relations": 0.5779816513761468, - "professional_psychology": 0.6235679214402619, - "prehistory": 0.7306501547987616, - "anatomy": 0.5746268656716418, - "human_sexuality": 0.6615384615384615, - "college_medicine": 0.6046511627906976, - "high_school_government_and_politics": 0.78125, - "college_chemistry": 0.43434343434343436, - "logical_fallacies": 0.6604938271604939, - "high_school_geography": 0.7106598984771574, - "elementary_mathematics": 0.4005305039787798, - "human_aging": 0.6486486486486487, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.7849264705882353, - "formal_logic": 0.408, - "high_school_statistics": 0.4790697674418605, - "international_law": 0.7, - "high_school_mathematics": 0.32342007434944237, - "high_school_computer_science": 0.5959595959595959, - "conceptual_physics": 0.5811965811965812, - "miscellaneous": 0.6508951406649617, - "high_school_chemistry": 0.5247524752475248, - "marketing": 0.8154506437768241, - "professional_law": 0.4683626875407697, - "management": 0.6764705882352942, - "college_physics": 0.3564356435643564, - "jurisprudence": 0.7663551401869159, - "world_religions": 0.7647058823529411, - "sociology": 0.73, - "us_foreign_policy": 0.7878787878787878, - "high_school_macroeconomics": 0.583547557840617, - "computer_security": 0.6262626262626263, - "moral_scenarios": 0.24496644295302014, - "moral_disputes": 0.6115942028985507, - "electrical_engineering": 0.6041666666666666, - "astronomy": 0.6821192052980133, - "college_biology": 0.7552447552447552 - } - }, - "prompt_5": { - "accuracy": 0.5907758312477654, - "category_acc": { - "high_school_european_history": 0.7439024390243902, - "business_ethics": 0.5757575757575758, - "clinical_knowledge": 0.6590909090909091, - "medical_genetics": 0.696969696969697, - "high_school_us_history": 0.7684729064039408, - "high_school_physics": 0.35333333333333333, - "high_school_world_history": 0.7966101694915254, - "virology": 0.509090909090909, - "high_school_microeconomics": 0.6329113924050633, - "econometrics": 0.4778761061946903, - "college_computer_science": 0.46464646464646464, - "high_school_biology": 0.7443365695792881, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.505338078291815, - "philosophy": 0.6612903225806451, - "professional_medicine": 0.6051660516605166, - "nutrition": 0.7049180327868853, - "global_facts": 0.36363636363636365, - "machine_learning": 0.42342342342342343, - "security_studies": 0.6475409836065574, - "public_relations": 0.6055045871559633, - "professional_psychology": 0.6415711947626841, - "prehistory": 0.7213622291021672, - "anatomy": 0.5671641791044776, - "human_sexuality": 0.7076923076923077, - "college_medicine": 0.5930232558139535, - "high_school_government_and_politics": 0.8125, - "college_chemistry": 0.41414141414141414, - "logical_fallacies": 0.6419753086419753, - "high_school_geography": 0.8020304568527918, - "elementary_mathematics": 0.4244031830238727, - "human_aging": 0.6486486486486487, - "college_mathematics": 0.32323232323232326, - "high_school_psychology": 0.7867647058823529, - "formal_logic": 0.392, - "high_school_statistics": 0.5162790697674419, - "international_law": 0.7083333333333334, - "high_school_mathematics": 0.3048327137546468, - "high_school_computer_science": 0.6262626262626263, - "conceptual_physics": 0.5726495726495726, - "miscellaneous": 0.7928388746803069, - "high_school_chemistry": 0.5099009900990099, - "marketing": 0.8369098712446352, - "professional_law": 0.4598825831702544, - "management": 0.7352941176470589, - "college_physics": 0.32673267326732675, - "jurisprudence": 0.7289719626168224, - "world_religions": 0.8117647058823529, - "sociology": 0.725, - "us_foreign_policy": 0.7878787878787878, - "high_school_macroeconomics": 0.5501285347043702, - "computer_security": 0.6666666666666666, - "moral_scenarios": 0.24049217002237136, - "moral_disputes": 0.6405797101449275, - "electrical_engineering": 0.5347222222222222, - "astronomy": 0.7086092715231788, - "college_biology": 0.7342657342657343 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.4011887072808321 - }, - "prompt_2": { - "accuracy": 0.4264487369985141 - }, - "prompt_3": { - "accuracy": 0.4049034175334324 - }, - "prompt_4": { - "accuracy": 0.40564635958395245 - }, - "prompt_5": { - "accuracy": 0.3202080237741456 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.3991282689912827, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.5952380952380952, - "college_physics": 0.5, - "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.4583333333333333, - "probability_and_statistics": 0.08695652173913043, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.21428571428571427, - "metrology_engineer": 0.4827586206896552, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.25, - "high_school_chemistry": 0.5, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.4, - "veterinary_medicine": 0.5357142857142857, - "college_economics": 0.38333333333333336, - "business_administration": 0.2631578947368421, - "marxism": 0.5, - "mao_zedong_thought": 0.5517241379310345, - "education_science": 0.5882352941176471, - "teacher_qualification": 0.5306122448979592, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.375, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.4583333333333333, - "logic": 0.2222222222222222, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.48, - "middle_school_history": 0.48148148148148145, - "civil_servant": 0.3269230769230769, - "sports_science": 0.5416666666666666, - "plant_protection": 0.5925925925925926, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.4117647058823529, - "accountant": 0.3148148148148148, - "fire_engineer": 0.4722222222222222, - "environmental_impact_assessment_engineer": 0.3611111111111111, - "tax_accountant": 0.35185185185185186, - "physician": 0.48148148148148145 - } - }, - "prompt_2": { - "accuracy": 0.42714819427148193, - "category_acc": { - "computer_network": 0.5, - "operating_system": 0.5833333333333334, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.4523809523809524, - "college_physics": 0.5, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.4583333333333333, - "probability_and_statistics": 0.13043478260869565, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.375, - "high_school_chemistry": 0.5, - "high_school_biology": 0.4166666666666667, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.6153846153846154, - "middle_school_physics": 0.625, - "middle_school_chemistry": 0.52, - "veterinary_medicine": 0.5714285714285714, - "college_economics": 0.4666666666666667, - "business_administration": 0.34210526315789475, - "marxism": 0.5416666666666666, - "mao_zedong_thought": 0.5862068965517241, - "education_science": 0.5882352941176471, - "teacher_qualification": 0.5918367346938775, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.5, - "middle_school_politics": 0.5769230769230769, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.32142857142857145, - "ideological_and_moral_cultivation": 0.5, - "logic": 0.18518518518518517, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.6, - "middle_school_history": 0.5925925925925926, - "civil_servant": 0.28846153846153844, - "sports_science": 0.5416666666666666, - "plant_protection": 0.5925925925925926, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.5294117647058824, - "accountant": 0.3333333333333333, - "fire_engineer": 0.4444444444444444, - "environmental_impact_assessment_engineer": 0.3888888888888889, - "tax_accountant": 0.4444444444444444, - "physician": 0.5 - } - }, - "prompt_3": { - "accuracy": 0.4165628891656289, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.5238095238095238, - "college_physics": 0.4583333333333333, - "college_chemistry": 0.4482758620689655, - "advanced_mathematics": 0.4583333333333333, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.41379310344827586, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.25, - "high_school_chemistry": 0.375, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.5357142857142857, - "college_economics": 0.4166666666666667, - "business_administration": 0.3684210526315789, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.5517241379310345, - "education_science": 0.5588235294117647, - "teacher_qualification": 0.5510204081632653, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.5416666666666666, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.35714285714285715, - "ideological_and_moral_cultivation": 0.5, - "logic": 0.3333333333333333, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.5, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.4411764705882353, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.52, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.3076923076923077, - "sports_science": 0.5416666666666666, - "plant_protection": 0.5925925925925926, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.5686274509803921, - "accountant": 0.4074074074074074, - "fire_engineer": 0.4444444444444444, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.3888888888888889, - "physician": 0.46296296296296297 - } - }, - "prompt_4": { - "accuracy": 0.41344956413449563, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.46153846153846156, - "college_programming": 0.5714285714285714, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.4583333333333333, - "probability_and_statistics": 0.13043478260869565, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.4482758620689655, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.52, - "veterinary_medicine": 0.6071428571428571, - "college_economics": 0.4666666666666667, - "business_administration": 0.2894736842105263, - "marxism": 0.4583333333333333, - "mao_zedong_thought": 0.5862068965517241, - "education_science": 0.5588235294117647, - "teacher_qualification": 0.5714285714285714, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.5, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.4583333333333333, - "logic": 0.14814814814814814, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.42857142857142855, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.64, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.3076923076923077, - "sports_science": 0.5, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.49019607843137253, - "accountant": 0.37037037037037035, - "fire_engineer": 0.4444444444444444, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.35185185185185186, - "physician": 0.4444444444444444 - } - }, - "prompt_5": { - "accuracy": 0.31444582814445826, - "category_acc": { - "computer_network": 0.375, - "operating_system": 0.25, - "computer_architecture": 0.5769230769230769, - "college_programming": 0.38095238095238093, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.5, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.09523809523809523, - "electrical_engineer": 0.21428571428571427, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.25, - "high_school_chemistry": 0.375, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.36666666666666664, - "business_administration": 0.3157894736842105, - "marxism": 0.4583333333333333, - "mao_zedong_thought": 0.5172413793103449, - "education_science": 0.3235294117647059, - "teacher_qualification": 0.40816326530612246, - "high_school_politics": 0.16666666666666666, - "high_school_geography": 0.375, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.375, - "logic": 0.25925925925925924, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.5, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.10714285714285714, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.44, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.23076923076923078, - "sports_science": 0.3333333333333333, - "plant_protection": 0.48148148148148145, - "basic_medicine": 0.375, - "clinical_medicine": 0.18518518518518517, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.2037037037037037, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.2222222222222222, - "physician": 0.3888888888888889 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.46236559139784944 - }, - "prompt_2": { - "accuracy": 0.4910394265232975 - }, - "prompt_3": { - "accuracy": 0.4946236559139785 - }, - "prompt_4": { - "accuracy": 0.4731182795698925 - }, - "prompt_5": { - "accuracy": 0.3870967741935484 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.39500949749611464, - "category_acc": { - "agronomy": 0.39644970414201186, - "anatomy": 0.3108108108108108, - "ancient_chinese": 0.23170731707317074, - "arts": 0.46875, - "astronomy": 0.3151515151515151, - "business_ethics": 0.37320574162679426, - "chinese_civil_service_exam": 0.3875, - "chinese_driving_rule": 0.5877862595419847, - "chinese_food_culture": 0.3014705882352941, - "chinese_foreign_policy": 0.4205607476635514, - "chinese_history": 0.4179566563467492, - "chinese_literature": 0.3382352941176471, - "chinese_teacher_qualification": 0.5195530726256983, - "clinical_knowledge": 0.3670886075949367, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.4766355140186916, - "college_engineering_hydrology": 0.4811320754716981, - "college_law": 0.28703703703703703, - "college_mathematics": 0.2857142857142857, - "college_medical_statistics": 0.3867924528301887, - "college_medicine": 0.4175824175824176, - "computer_science": 0.4117647058823529, - "computer_security": 0.5029239766081871, - "conceptual_physics": 0.3333333333333333, - "construction_project_management": 0.39568345323741005, - "economics": 0.4088050314465409, - "education": 0.38650306748466257, - "electrical_engineering": 0.436046511627907, - "elementary_chinese": 0.3055555555555556, - "elementary_commonsense": 0.35353535353535354, - "elementary_information_and_technology": 0.5798319327731093, - "elementary_mathematics": 0.36086956521739133, - "ethnology": 0.34814814814814815, - "food_science": 0.4405594405594406, - "genetics": 0.42613636363636365, - "global_facts": 0.4161073825503356, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.29545454545454547, - "high_school_geography": 0.3559322033898305, - "high_school_mathematics": 0.29878048780487804, - "high_school_physics": 0.3181818181818182, - "high_school_politics": 0.3356643356643357, - "human_sexuality": 0.3968253968253968, - "international_law": 0.3675675675675676, - "journalism": 0.4069767441860465, - "jurisprudence": 0.36982968369829683, - "legal_and_moral_basis": 0.5747663551401869, - "logical": 0.5365853658536586, - "machine_learning": 0.45081967213114754, - "management": 0.3904761904761905, - "marketing": 0.40555555555555556, - "marxist_theory": 0.4126984126984127, - "modern_chinese": 0.3448275862068966, - "nutrition": 0.4413793103448276, - "philosophy": 0.3619047619047619, - "professional_accounting": 0.3942857142857143, - "professional_law": 0.32701421800947866, - "professional_medicine": 0.34308510638297873, - "professional_psychology": 0.41810344827586204, - "public_relations": 0.4942528735632184, - "security_study": 0.45925925925925926, - "sociology": 0.4469026548672566, - "sports_science": 0.36363636363636365, - "traditional_chinese_medicine": 0.34594594594594597, - "virology": 0.46745562130177515, - "world_history": 0.4161490683229814, - "world_religions": 0.4375 - } - }, - "prompt_2": { - "accuracy": 0.4340355724399931, - "category_acc": { - "agronomy": 0.46153846153846156, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.2073170731707317, - "arts": 0.5, - "astronomy": 0.4, - "business_ethics": 0.430622009569378, - "chinese_civil_service_exam": 0.36875, - "chinese_driving_rule": 0.5648854961832062, - "chinese_food_culture": 0.3382352941176471, - "chinese_foreign_policy": 0.5046728971962616, - "chinese_history": 0.4613003095975232, - "chinese_literature": 0.3431372549019608, - "chinese_teacher_qualification": 0.5586592178770949, - "clinical_knowledge": 0.42616033755274263, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.4766355140186916, - "college_engineering_hydrology": 0.41509433962264153, - "college_law": 0.3888888888888889, - "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.4056603773584906, - "college_medicine": 0.43223443223443225, - "computer_science": 0.44607843137254904, - "computer_security": 0.6432748538011696, - "conceptual_physics": 0.4013605442176871, - "construction_project_management": 0.3669064748201439, - "economics": 0.5408805031446541, - "education": 0.43558282208588955, - "electrical_engineering": 0.47093023255813954, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.3484848484848485, - "elementary_information_and_technology": 0.6554621848739496, - "elementary_mathematics": 0.34782608695652173, - "ethnology": 0.3851851851851852, - "food_science": 0.5104895104895105, - "genetics": 0.48295454545454547, - "global_facts": 0.4563758389261745, - "high_school_biology": 0.39644970414201186, - "high_school_chemistry": 0.30303030303030304, - "high_school_geography": 0.4152542372881356, - "high_school_mathematics": 0.3597560975609756, - "high_school_physics": 0.32727272727272727, - "high_school_politics": 0.44755244755244755, - "human_sexuality": 0.47619047619047616, - "international_law": 0.3567567567567568, - "journalism": 0.45930232558139533, - "jurisprudence": 0.39416058394160586, - "legal_and_moral_basis": 0.677570093457944, - "logical": 0.44715447154471544, - "machine_learning": 0.47540983606557374, - "management": 0.5142857142857142, - "marketing": 0.5, - "marxist_theory": 0.48148148148148145, - "modern_chinese": 0.3103448275862069, - "nutrition": 0.5241379310344828, - "philosophy": 0.4666666666666667, - "professional_accounting": 0.5142857142857142, - "professional_law": 0.33649289099526064, - "professional_medicine": 0.38563829787234044, - "professional_psychology": 0.43103448275862066, - "public_relations": 0.5402298850574713, - "security_study": 0.5037037037037037, - "sociology": 0.43805309734513276, - "sports_science": 0.3878787878787879, - "traditional_chinese_medicine": 0.34054054054054056, - "virology": 0.5384615384615384, - "world_history": 0.4658385093167702, - "world_religions": 0.46875 - } - }, - "prompt_3": { - "accuracy": 0.4153859437057503, - "category_acc": { - "agronomy": 0.46745562130177515, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.21341463414634146, - "arts": 0.5375, - "astronomy": 0.36363636363636365, - "business_ethics": 0.4354066985645933, - "chinese_civil_service_exam": 0.31875, - "chinese_driving_rule": 0.5419847328244275, - "chinese_food_culture": 0.38235294117647056, - "chinese_foreign_policy": 0.5233644859813084, - "chinese_history": 0.4148606811145511, - "chinese_literature": 0.3284313725490196, - "chinese_teacher_qualification": 0.48044692737430167, - "clinical_knowledge": 0.38396624472573837, - "college_actuarial_science": 0.22641509433962265, - "college_education": 0.5327102803738317, - "college_engineering_hydrology": 0.41509433962264153, - "college_law": 0.37962962962962965, - "college_mathematics": 0.2857142857142857, - "college_medical_statistics": 0.4056603773584906, - "college_medicine": 0.4175824175824176, - "computer_science": 0.4411764705882353, - "computer_security": 0.5321637426900585, - "conceptual_physics": 0.3469387755102041, - "construction_project_management": 0.34532374100719426, - "economics": 0.4779874213836478, - "education": 0.4662576687116564, - "electrical_engineering": 0.47093023255813954, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.3888888888888889, - "elementary_information_and_technology": 0.6008403361344538, - "elementary_mathematics": 0.36086956521739133, - "ethnology": 0.4, - "food_science": 0.48951048951048953, - "genetics": 0.4602272727272727, - "global_facts": 0.4899328859060403, - "high_school_biology": 0.31952662721893493, - "high_school_chemistry": 0.25, - "high_school_geography": 0.3728813559322034, - "high_school_mathematics": 0.31097560975609756, - "high_school_physics": 0.3181818181818182, - "high_school_politics": 0.35664335664335667, - "human_sexuality": 0.42063492063492064, - "international_law": 0.32432432432432434, - "journalism": 0.42441860465116277, - "jurisprudence": 0.36739659367396593, - "legal_and_moral_basis": 0.6682242990654206, - "logical": 0.4146341463414634, - "machine_learning": 0.4016393442622951, - "management": 0.48095238095238096, - "marketing": 0.5, - "marxist_theory": 0.4656084656084656, - "modern_chinese": 0.31896551724137934, - "nutrition": 0.4689655172413793, - "philosophy": 0.5142857142857142, - "professional_accounting": 0.4742857142857143, - "professional_law": 0.3175355450236967, - "professional_medicine": 0.3377659574468085, - "professional_psychology": 0.44396551724137934, - "public_relations": 0.46551724137931033, - "security_study": 0.5111111111111111, - "sociology": 0.4469026548672566, - "sports_science": 0.4, - "traditional_chinese_medicine": 0.372972972972973, - "virology": 0.5207100591715976, - "world_history": 0.43478260869565216, - "world_religions": 0.49375 - } - }, - "prompt_4": { - "accuracy": 0.4041616301156968, - "category_acc": { - "agronomy": 0.4319526627218935, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.23170731707317074, - "arts": 0.56875, - "astronomy": 0.3515151515151515, - "business_ethics": 0.39712918660287083, - "chinese_civil_service_exam": 0.33125, - "chinese_driving_rule": 0.5038167938931297, - "chinese_food_culture": 0.3235294117647059, - "chinese_foreign_policy": 0.4672897196261682, - "chinese_history": 0.37770897832817335, - "chinese_literature": 0.31862745098039214, - "chinese_teacher_qualification": 0.4972067039106145, - "clinical_knowledge": 0.35864978902953587, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.48598130841121495, - "college_engineering_hydrology": 0.4528301886792453, - "college_law": 0.3425925925925926, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.39622641509433965, - "college_medicine": 0.42124542124542125, - "computer_science": 0.43137254901960786, - "computer_security": 0.52046783625731, - "conceptual_physics": 0.3673469387755102, - "construction_project_management": 0.38848920863309355, - "economics": 0.48427672955974843, - "education": 0.3558282208588957, - "electrical_engineering": 0.47093023255813954, - "elementary_chinese": 0.29365079365079366, - "elementary_commonsense": 0.4090909090909091, - "elementary_information_and_technology": 0.6302521008403361, - "elementary_mathematics": 0.3347826086956522, - "ethnology": 0.37777777777777777, - "food_science": 0.4755244755244755, - "genetics": 0.4772727272727273, - "global_facts": 0.436241610738255, - "high_school_biology": 0.35502958579881655, - "high_school_chemistry": 0.25, - "high_school_geography": 0.3983050847457627, - "high_school_mathematics": 0.3048780487804878, - "high_school_physics": 0.2909090909090909, - "high_school_politics": 0.3916083916083916, - "human_sexuality": 0.4444444444444444, - "international_law": 0.32972972972972975, - "journalism": 0.38372093023255816, - "jurisprudence": 0.35523114355231145, - "legal_and_moral_basis": 0.6074766355140186, - "logical": 0.45528455284552843, - "machine_learning": 0.4426229508196721, - "management": 0.46190476190476193, - "marketing": 0.45, - "marxist_theory": 0.4126984126984127, - "modern_chinese": 0.3620689655172414, - "nutrition": 0.4896551724137931, - "philosophy": 0.4380952380952381, - "professional_accounting": 0.41714285714285715, - "professional_law": 0.33175355450236965, - "professional_medicine": 0.3351063829787234, - "professional_psychology": 0.43103448275862066, - "public_relations": 0.45977011494252873, - "security_study": 0.5259259259259259, - "sociology": 0.3938053097345133, - "sports_science": 0.42424242424242425, - "traditional_chinese_medicine": 0.32432432432432434, - "virology": 0.48520710059171596, - "world_history": 0.39751552795031053, - "world_religions": 0.425 - } - }, - "prompt_5": { - "accuracy": 0.3279226385771024, - "category_acc": { - "agronomy": 0.3254437869822485, - "anatomy": 0.28378378378378377, - "ancient_chinese": 0.2682926829268293, - "arts": 0.425, - "astronomy": 0.26666666666666666, - "business_ethics": 0.3253588516746411, - "chinese_civil_service_exam": 0.3125, - "chinese_driving_rule": 0.3816793893129771, - "chinese_food_culture": 0.3382352941176471, - "chinese_foreign_policy": 0.3364485981308411, - "chinese_history": 0.3126934984520124, - "chinese_literature": 0.3137254901960784, - "chinese_teacher_qualification": 0.4022346368715084, - "clinical_knowledge": 0.32489451476793246, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.3644859813084112, - "college_engineering_hydrology": 0.25471698113207547, - "college_law": 0.3333333333333333, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.36792452830188677, - "college_medicine": 0.37362637362637363, - "computer_science": 0.3872549019607843, - "computer_security": 0.34502923976608185, - "conceptual_physics": 0.29931972789115646, - "construction_project_management": 0.2589928057553957, - "economics": 0.3522012578616352, - "education": 0.34355828220858897, - "electrical_engineering": 0.37790697674418605, - "elementary_chinese": 0.25396825396825395, - "elementary_commonsense": 0.3434343434343434, - "elementary_information_and_technology": 0.3319327731092437, - "elementary_mathematics": 0.3130434782608696, - "ethnology": 0.362962962962963, - "food_science": 0.34965034965034963, - "genetics": 0.3465909090909091, - "global_facts": 0.3422818791946309, - "high_school_biology": 0.2958579881656805, - "high_school_chemistry": 0.25, - "high_school_geography": 0.2796610169491525, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.2867132867132867, - "human_sexuality": 0.35714285714285715, - "international_law": 0.2918918918918919, - "journalism": 0.38372093023255816, - "jurisprudence": 0.30900243309002434, - "legal_and_moral_basis": 0.4439252336448598, - "logical": 0.34959349593495936, - "machine_learning": 0.3360655737704918, - "management": 0.3476190476190476, - "marketing": 0.35, - "marxist_theory": 0.35978835978835977, - "modern_chinese": 0.28448275862068967, - "nutrition": 0.30344827586206896, - "philosophy": 0.3904761904761905, - "professional_accounting": 0.35428571428571426, - "professional_law": 0.2843601895734597, - "professional_medicine": 0.2632978723404255, - "professional_psychology": 0.3448275862068966, - "public_relations": 0.3563218390804598, - "security_study": 0.32592592592592595, - "sociology": 0.3584070796460177, - "sports_science": 0.2787878787878788, - "traditional_chinese_medicine": 0.2972972972972973, - "virology": 0.40828402366863903, - "world_history": 0.3105590062111801, - "world_religions": 0.39375 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.24242424242424243 - }, - "prompt_2": { - "accuracy": 0.24242424242424243 - }, - "prompt_3": { - "accuracy": 0.24242424242424243 - }, - "prompt_4": { - "accuracy": 0.21212121212121213 - }, - "prompt_5": { - "accuracy": 0.2727272727272727 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.5613636363636364 - }, - "prompt_2": { - "accuracy": 0.2409090909090909 - }, - "prompt_3": { - "accuracy": 0.20681818181818182 - }, - "prompt_4": { - "accuracy": 0.5931818181818181 - }, - "prompt_5": { - "accuracy": 0.5318181818181819 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3369491525423729 - }, - "prompt_2": { - "accuracy": 0.3403389830508475 - }, - "prompt_3": { - "accuracy": 0.3284745762711864 - }, - "prompt_4": { - "accuracy": 0.34440677966101696 - }, - "prompt_5": { - "accuracy": 0.32372881355932204 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.655946148092745 - }, - "prompt_2": { - "accuracy": 0.6413612565445026 - }, - "prompt_3": { - "accuracy": 0.6140613313388182 - }, - "prompt_4": { - "accuracy": 0.6488406881077038 - }, - "prompt_5": { - "accuracy": 0.5325355272999253 - } - }, - "dream": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "samsum": { - "prompt_1": { - "rouge1": 0.31697316930282315, - "rouge2": 0.13616141789953073, - "rougeL": 0.25148587514125503, - "avg_rouge": 0.23487348744786965 - }, - "prompt_2": { - "rouge1": 0.38139959009581176, - "rouge2": 0.17322914962132516, - "rougeL": 0.30731322151748236, - "avg_rouge": 0.28731398707820643 - }, - "prompt_3": { - "rouge1": 0.3260889902412858, - "rouge2": 0.13479031945744196, - "rougeL": 0.25861822211719976, - "avg_rouge": 0.2398325106053092 - }, - "prompt_4": { - "rouge1": 0.34927586879072603, - "rouge2": 0.155865262289952, - "rougeL": 0.2788107400543373, - "avg_rouge": 0.26131729037833845 - }, - "prompt_5": { - "rouge1": 0.3582894310943466, - "rouge2": 0.15370536672602358, - "rougeL": 0.2846835797341718, - "avg_rouge": 0.2655594591848473 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.21745599501133733, - "rouge2": 0.06063153720338593, - "rougeL": 0.16176158743038177, - "avg_rouge": 0.146616373215035 - }, - "prompt_2": { - "rouge1": 0.21965344892741037, - "rouge2": 0.06104769711469251, - "rougeL": 0.16238242725486898, - "avg_rouge": 0.14769452443232398 - }, - "prompt_3": { - "rouge1": 0.2168643396875201, - "rouge2": 0.062069682838943215, - "rougeL": 0.16115542880359557, - "avg_rouge": 0.1466964837766863 - }, - "prompt_4": { - "rouge1": 0.2236060194566211, - "rouge2": 0.06310493699305976, - "rougeL": 0.16557401047905387, - "avg_rouge": 0.15076165564291158 - }, - "prompt_5": { - "rouge1": 0.19951219765483352, - "rouge2": 0.06396457846831104, - "rougeL": 0.15026171874325178, - "avg_rouge": 0.1379128316221321 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.8658256880733946 - }, - "prompt_2": { - "accuracy": 0.7775229357798165 - }, - "prompt_3": { - "accuracy": 0.8864678899082569 - }, - "prompt_4": { - "accuracy": 0.8715596330275229 - }, - "prompt_5": { - "accuracy": 0.5676605504587156 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.32790028763183127 - }, - "prompt_2": { - "accuracy": 0.3221476510067114 - }, - "prompt_3": { - "accuracy": 0.3537871524448706 - }, - "prompt_4": { - "accuracy": 0.31831255992329816 - }, - "prompt_5": { - "accuracy": 0.5445829338446788 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.612 - }, - "prompt_2": { - "accuracy": 0.5355 - }, - "prompt_3": { - "accuracy": 0.5625 - }, - "prompt_4": -1, - "prompt_5": -1 - }, - "mnli": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "qnli": { - "prompt_1": { - "accuracy": 0.513 - }, - "prompt_2": { - "accuracy": 0.51 - }, - "prompt_3": { - "accuracy": 0.516 - }, - "prompt_4": { - "accuracy": 0.5195 - }, - "prompt_5": { - "accuracy": 0.524 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.6338028169014085 - }, - "prompt_2": { - "accuracy": 0.5915492957746479 - }, - "prompt_3": { - "accuracy": 0.5774647887323944 - }, - "prompt_4": { - "accuracy": 0.43661971830985913 - }, - "prompt_5": { - "accuracy": 0.5070422535211268 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.6173285198555957 - }, - "prompt_2": { - "accuracy": 0.5992779783393501 - }, - "prompt_3": { - "accuracy": 0.6462093862815884 - }, - "prompt_4": { - "accuracy": 0.555956678700361 - }, - "prompt_5": { - "accuracy": 0.5740072202166066 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.6862745098039216 - }, - "prompt_2": { - "accuracy": 0.6225490196078431 - }, - "prompt_3": { - "accuracy": 0.6715686274509803 - }, - "prompt_4": { - "accuracy": 0.6813725490196079 - }, - "prompt_5": { - "accuracy": 0.6715686274509803 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.863655462184874, - "language_acc": { - "English": 0.8647058823529412, - "Chinese": 0.8621848739495799, - "Spanish": 0.8647058823529412, - "Vietnamese": 0.8630252100840337 - }, - "consistency_score_2": 0.8095238095238094, - "consistency_score_3": 0.7218487394957983, - "consistency_score_4": 0.6579831932773109, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8134453781512605, - "English,Spanish": 0.8042016806722689, - "English,Vietnamese": 0.8117647058823529, - "Chinese,Spanish": 0.8067226890756303, - "Chinese,Vietnamese": 0.8151260504201681, - "Spanish,Vietnamese": 0.8058823529411765 - }, - "3_combine": { - "English,Chinese,Spanish": 0.7210084033613445, - "English,Chinese,Vietnamese": 0.7277310924369748, - "English,Spanish,Vietnamese": 0.7168067226890756, - "Chinese,Spanish,Vietnamese": 0.7218487394957983 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.6579831932773109 - } - }, - "AC3_2": 0.8357139867124633, - "AC3_3": 0.7864105387253375, - "AC3_4": 0.7469194829173709 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.41047619047619055, - "language_acc": { - "Spanish": 0.4533333333333333, - "English": 0.43333333333333335, - "Chinese": 0.43333333333333335, - "Vietnamese": 0.36666666666666664, - "Filipino": 0.38, - "Indonesian": 0.41333333333333333, - "Malay": 0.3933333333333333 - }, - "consistency_score_2": 0.47936507936507944, - "consistency_score_3": 0.2828571428571428, - "consistency_score_4": 0.18780952380952376, - "consistency_score_5": 0.13555555555555557, - "consistency_score_6": 0.10285714285714286, - "consistency_score_7": 0.08, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.4666666666666667, - "Spanish,Chinese": 0.5333333333333333, - "Spanish,Vietnamese": 0.48, - "Spanish,Filipino": 0.44666666666666666, - "Spanish,Indonesian": 0.47333333333333333, - "Spanish,Malay": 0.4533333333333333, - "English,Chinese": 0.5733333333333334, - "English,Vietnamese": 0.52, - "English,Filipino": 0.42, - "English,Indonesian": 0.44666666666666666, - "English,Malay": 0.52, - "Chinese,Vietnamese": 0.47333333333333333, - "Chinese,Filipino": 0.4, - "Chinese,Indonesian": 0.47333333333333333, - "Chinese,Malay": 0.5066666666666667, - "Vietnamese,Filipino": 0.43333333333333335, - "Vietnamese,Indonesian": 0.47333333333333333, - "Vietnamese,Malay": 0.46, - "Filipino,Indonesian": 0.49333333333333335, - "Filipino,Malay": 0.5133333333333333, - "Indonesian,Malay": 0.5066666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.36, - "Spanish,English,Vietnamese": 0.3, - "Spanish,English,Filipino": 0.22, - "Spanish,English,Indonesian": 0.2733333333333333, - "Spanish,English,Malay": 0.28, - "Spanish,Chinese,Vietnamese": 0.30666666666666664, - "Spanish,Chinese,Filipino": 0.23333333333333334, - "Spanish,Chinese,Indonesian": 0.3, - "Spanish,Chinese,Malay": 0.30666666666666664, - "Spanish,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Filipino,Malay": 0.26, - "Spanish,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese": 0.34, - "English,Chinese,Filipino": 0.26, - "English,Chinese,Indonesian": 0.32666666666666666, - "English,Chinese,Malay": 0.36, - "English,Vietnamese,Filipino": 0.26, - "English,Vietnamese,Indonesian": 0.3, - "English,Vietnamese,Malay": 0.32666666666666666, - "English,Filipino,Indonesian": 0.24, - "English,Filipino,Malay": 0.28, - "English,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Chinese,Vietnamese,Indonesian": 0.28, - "Chinese,Vietnamese,Malay": 0.29333333333333333, - "Chinese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Filipino,Malay": 0.26666666666666666, - "Chinese,Indonesian,Malay": 0.29333333333333333, - "Vietnamese,Filipino,Indonesian": 0.28, - "Vietnamese,Filipino,Malay": 0.24666666666666667, - "Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Filipino,Indonesian,Malay": 0.31333333333333335 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.23333333333333334, - "Spanish,English,Chinese,Filipino": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.2, - "Spanish,English,Vietnamese,Malay": 0.2, - "Spanish,English,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Filipino,Malay": 0.16, - "Spanish,English,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.18666666666666668, - "Spanish,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.16666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.18, - "English,Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "English,Chinese,Vietnamese,Malay": 0.24666666666666667, - "English,Chinese,Filipino,Indonesian": 0.18, - "English,Chinese,Filipino,Malay": 0.20666666666666667, - "English,Chinese,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "English,Vietnamese,Filipino,Malay": 0.18, - "English,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "English,Filipino,Indonesian,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Vietnamese,Filipino,Indonesian,Malay": 0.18 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - } - }, - "AC3_2": 0.4422540474844044, - "AC3_3": 0.3349215070160582, - "AC3_4": 0.2577074331371382, - "AC3_5": 0.20380620151306264, - "AC3_6": 0.1644950967079502, - "AC3_7": 0.13390291259405862 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.5032467532467533, - "language_acc": { - "English": 0.5340909090909091, - "Filipino": 0.4715909090909091, - "Spanish": 0.5170454545454546, - "Malay": 0.5170454545454546, - "Chinese": 0.5227272727272727, - "Indonesian": 0.4772727272727273, - "Vietnamese": 0.48295454545454547 - }, - "consistency_score_2": 0.5100108225108224, - "consistency_score_3": 0.3202922077922078, - "consistency_score_4": 0.22987012987012986, - "consistency_score_5": 0.18019480519480519, - "consistency_score_6": 0.15016233766233764, - "consistency_score_7": 0.13068181818181818, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4318181818181818, - "English,Spanish": 0.5397727272727273, - "English,Malay": 0.5397727272727273, - "English,Chinese": 0.4772727272727273, - "English,Indonesian": 0.5340909090909091, - "English,Vietnamese": 0.5340909090909091, - "Filipino,Spanish": 0.5284090909090909, - "Filipino,Malay": 0.5, - "Filipino,Chinese": 0.4715909090909091, - "Filipino,Indonesian": 0.5340909090909091, - "Filipino,Vietnamese": 0.4431818181818182, - "Spanish,Malay": 0.5397727272727273, - "Spanish,Chinese": 0.44886363636363635, - "Spanish,Indonesian": 0.5681818181818182, - "Spanish,Vietnamese": 0.5170454545454546, - "Malay,Chinese": 0.4772727272727273, - "Malay,Indonesian": 0.6136363636363636, - "Malay,Vietnamese": 0.4772727272727273, - "Chinese,Indonesian": 0.4715909090909091, - "Chinese,Vietnamese": 0.5284090909090909, - "Indonesian,Vietnamese": 0.5340909090909091 - }, - "3_combine": { - "English,Filipino,Spanish": 0.30113636363636365, - "English,Filipino,Malay": 0.2784090909090909, - "English,Filipino,Chinese": 0.26704545454545453, - "English,Filipino,Indonesian": 0.3068181818181818, - "English,Filipino,Vietnamese": 0.2784090909090909, - "English,Spanish,Malay": 0.36363636363636365, - "English,Spanish,Chinese": 0.2840909090909091, - "English,Spanish,Indonesian": 0.375, - "English,Spanish,Vietnamese": 0.3352272727272727, - "English,Malay,Chinese": 0.29545454545454547, - "English,Malay,Indonesian": 0.3806818181818182, - "English,Malay,Vietnamese": 0.32954545454545453, - "English,Chinese,Indonesian": 0.30113636363636365, - "English,Chinese,Vietnamese": 0.32954545454545453, - "English,Indonesian,Vietnamese": 0.3465909090909091, - "Filipino,Spanish,Malay": 0.32954545454545453, - "Filipino,Spanish,Chinese": 0.2897727272727273, - "Filipino,Spanish,Indonesian": 0.36363636363636365, - "Filipino,Spanish,Vietnamese": 0.3125, - "Filipino,Malay,Chinese": 0.2840909090909091, - "Filipino,Malay,Indonesian": 0.35795454545454547, - "Filipino,Malay,Vietnamese": 0.26704545454545453, - "Filipino,Chinese,Indonesian": 0.3068181818181818, - "Filipino,Chinese,Vietnamese": 0.29545454545454547, - "Filipino,Indonesian,Vietnamese": 0.32386363636363635, - "Spanish,Malay,Chinese": 0.29545454545454547, - "Spanish,Malay,Indonesian": 0.39204545454545453, - "Spanish,Malay,Vietnamese": 0.3181818181818182, - "Spanish,Chinese,Indonesian": 0.29545454545454547, - "Spanish,Chinese,Vietnamese": 0.3181818181818182, - "Spanish,Indonesian,Vietnamese": 0.35795454545454547, - "Malay,Chinese,Indonesian": 0.3409090909090909, - "Malay,Chinese,Vietnamese": 0.29545454545454547, - "Malay,Indonesian,Vietnamese": 0.35795454545454547, - "Chinese,Indonesian,Vietnamese": 0.3352272727272727 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.2215909090909091, - "English,Filipino,Spanish,Chinese": 0.1875, - "English,Filipino,Spanish,Indonesian": 0.26136363636363635, - "English,Filipino,Spanish,Vietnamese": 0.2159090909090909, - "English,Filipino,Malay,Chinese": 0.18181818181818182, - "English,Filipino,Malay,Indonesian": 0.22727272727272727, - "English,Filipino,Malay,Vietnamese": 0.2159090909090909, - "English,Filipino,Chinese,Indonesian": 0.20454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.20454545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay,Chinese": 0.2159090909090909, - "English,Spanish,Malay,Indonesian": 0.2897727272727273, - "English,Spanish,Malay,Vietnamese": 0.23295454545454544, - "English,Spanish,Chinese,Indonesian": 0.2215909090909091, - "English,Spanish,Chinese,Vietnamese": 0.2215909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.26136363636363635, - "English,Malay,Chinese,Indonesian": 0.22727272727272727, - "English,Malay,Chinese,Vietnamese": 0.2215909090909091, - "English,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "English,Chinese,Indonesian,Vietnamese": 0.25, - "Filipino,Spanish,Malay,Chinese": 0.19318181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.2159090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.1875, - "Filipino,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Indonesian": 0.20454545454545456, - "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Chinese,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.19886363636363635, - "English,Filipino,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.1875, - "English,Spanish,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - } - }, - "AC3_2": 0.5066062108285763, - "AC3_3": 0.3914472083718819, - "AC3_4": 0.31558786647484566, - "AC3_5": 0.26537002186325986, - "AC3_6": 0.23130596108422583, - "AC3_7": 0.20748457685005436 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.32038834951456313 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.4095238095238095 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.45794392523364486 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.31, - "category_acc": { - "brand": 0.2, - "demographics": 0.4, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.2, - "politics": 0.5, - "culture": 0.3, - "film": 0.4, - "law": 0.2, - "geography": 0.3 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.08737430789111206 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.09343303322887905 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.07335122941784139 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.09382423884531595 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.11764822497739133 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.425904317386231 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.4167322130854487, - "category_acc": { - "high_school_european_history": 0.5853658536585366, - "business_ethics": 0.48484848484848486, - "clinical_knowledge": 0.42803030303030304, - "medical_genetics": 0.494949494949495, - "high_school_us_history": 0.5665024630541872, - "high_school_physics": 0.2866666666666667, - "high_school_world_history": 0.6271186440677966, - "virology": 0.3878787878787879, - "high_school_microeconomics": 0.43037974683544306, - "econometrics": 0.3805309734513274, - "college_computer_science": 0.43434343434343436, - "high_school_biology": 0.459546925566343, - "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.40213523131672596, - "philosophy": 0.44516129032258067, - "professional_medicine": 0.5055350553505535, - "nutrition": 0.4098360655737705, - "global_facts": 0.25252525252525254, - "machine_learning": 0.3963963963963964, - "security_studies": 0.4918032786885246, - "public_relations": 0.3577981651376147, - "professional_psychology": 0.4386252045826514, - "prehistory": 0.43034055727554177, - "anatomy": 0.30597014925373134, - "human_sexuality": 0.5307692307692308, - "college_medicine": 0.436046511627907, - "high_school_government_and_politics": 0.53125, - "college_chemistry": 0.35353535353535354, - "logical_fallacies": 0.4074074074074074, - "high_school_geography": 0.4365482233502538, - "elementary_mathematics": 0.32625994694960214, - "human_aging": 0.46396396396396394, - "college_mathematics": 0.2222222222222222, - "high_school_psychology": 0.48161764705882354, - "formal_logic": 0.384, - "high_school_statistics": 0.4232558139534884, - "international_law": 0.35, - "high_school_mathematics": 0.32342007434944237, - "high_school_computer_science": 0.42424242424242425, - "conceptual_physics": 0.45726495726495725, - "miscellaneous": 0.4539641943734015, - "high_school_chemistry": 0.3613861386138614, - "marketing": 0.5536480686695279, - "professional_law": 0.3796477495107632, - "management": 0.37254901960784315, - "college_physics": 0.26732673267326734, - "jurisprudence": 0.4953271028037383, - "world_religions": 0.5058823529411764, - "sociology": 0.475, - "us_foreign_policy": 0.3939393939393939, - "high_school_macroeconomics": 0.42930591259640105, - "computer_security": 0.40404040404040403, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.46956521739130436, - "electrical_engineering": 0.4305555555555556, - "astronomy": 0.4105960264900662, - "college_biology": 0.46853146853146854 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.3967310549777117 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.3866749688667497, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.375, - "computer_architecture": 0.5, - "college_programming": 0.40476190476190477, - "college_physics": 0.08333333333333333, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.09523809523809523, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.6666666666666666, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.6, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.38333333333333336, - "business_administration": 0.3157894736842105, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.4827586206896552, - "education_science": 0.5, - "teacher_qualification": 0.5510204081632653, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.5, - "middle_school_geography": 0.5294117647058824, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.5416666666666666, - "logic": 0.48148148148148145, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.4411764705882353, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.44, - "middle_school_history": 0.48148148148148145, - "civil_servant": 0.40384615384615385, - "sports_science": 0.5, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.48148148148148145, - "urban_and_rural_planner": 0.39215686274509803, - "accountant": 0.3333333333333333, - "fire_engineer": 0.3611111111111111, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.2962962962962963, - "physician": 0.3888888888888889 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.5376344086021505 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.46943533068554655, - "category_acc": { - "agronomy": 0.46153846153846156, - "anatomy": 0.32432432432432434, - "ancient_chinese": 0.2682926829268293, - "arts": 0.58125, - "astronomy": 0.38181818181818183, - "business_ethics": 0.45454545454545453, - "chinese_civil_service_exam": 0.39375, - "chinese_driving_rule": 0.549618320610687, - "chinese_food_culture": 0.40441176470588236, - "chinese_foreign_policy": 0.48598130841121495, - "chinese_history": 0.4953560371517028, - "chinese_literature": 0.35784313725490197, - "chinese_teacher_qualification": 0.5921787709497207, - "clinical_knowledge": 0.39662447257383965, - "college_actuarial_science": 0.3018867924528302, - "college_education": 0.5981308411214953, - "college_engineering_hydrology": 0.5094339622641509, - "college_law": 0.3333333333333333, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.5094339622641509, - "college_medicine": 0.4542124542124542, - "computer_science": 0.5049019607843137, - "computer_security": 0.6432748538011696, - "conceptual_physics": 0.35374149659863946, - "construction_project_management": 0.4316546762589928, - "economics": 0.5031446540880503, - "education": 0.5276073619631901, - "electrical_engineering": 0.4941860465116279, - "elementary_chinese": 0.31746031746031744, - "elementary_commonsense": 0.5050505050505051, - "elementary_information_and_technology": 0.6932773109243697, - "elementary_mathematics": 0.3391304347826087, - "ethnology": 0.42962962962962964, - "food_science": 0.5734265734265734, - "genetics": 0.4943181818181818, - "global_facts": 0.5234899328859061, - "high_school_biology": 0.38461538461538464, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.4576271186440678, - "high_school_mathematics": 0.2926829268292683, - "high_school_physics": 0.38181818181818183, - "high_school_politics": 0.48951048951048953, - "human_sexuality": 0.5158730158730159, - "international_law": 0.3675675675675676, - "journalism": 0.5697674418604651, - "jurisprudence": 0.4282238442822384, - "legal_and_moral_basis": 0.780373831775701, - "logical": 0.4959349593495935, - "machine_learning": 0.5245901639344263, - "management": 0.6238095238095238, - "marketing": 0.5722222222222222, - "marxist_theory": 0.5502645502645502, - "modern_chinese": 0.4051724137931034, - "nutrition": 0.5172413793103449, - "philosophy": 0.5523809523809524, - "professional_accounting": 0.49142857142857144, - "professional_law": 0.33649289099526064, - "professional_medicine": 0.3696808510638298, - "professional_psychology": 0.521551724137931, - "public_relations": 0.5804597701149425, - "security_study": 0.5925925925925926, - "sociology": 0.5265486725663717, - "sports_science": 0.46060606060606063, - "traditional_chinese_medicine": 0.35135135135135137, - "virology": 0.514792899408284, - "world_history": 0.5652173913043478, - "world_religions": 0.49375 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.3333333333333333 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.509090909090909 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3806779661016949 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.6170531039640987 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.8652621264086232 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.20010225175871652, - "rouge2": 0.08121969964716567, - "rougeL": 0.15574863418845444, - "avg_rouge": 0.14569019519811222 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.18176419657208084, - "rouge2": 0.06525231482072144, - "rougeL": 0.14259200040536582, - "avg_rouge": 0.1298695039327227 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.49770642201834864 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.3221476510067114 - } - }, - "qqp": { - "prompt_1": -1 - }, - "mnli": { - "prompt_1": -1 - }, - "qnli": { - "prompt_1": { - "accuracy": 0.4835 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.6197183098591549 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.6137184115523465 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.5343137254901961 - } - } - } - }, - "gemma_7b_it": { - "model_size": "7B", - "model_link": "https://huggingface.co/google/gemma-7b-it", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.3855042016806723, - "language_acc": { - "English": 0.39915966386554624, - "Chinese": 0.3941176470588235, - "Spanish": 0.3722689075630252, - "Vietnamese": 0.3764705882352941 - }, - "consistency_score_2": 0.380672268907563, - "consistency_score_3": 0.21533613445378152, - "consistency_score_4": 0.16470588235294117, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.37563025210084033, - "English,Spanish": 0.4134453781512605, - "English,Vietnamese": 0.37058823529411766, - "Chinese,Spanish": 0.3831932773109244, - "Chinese,Vietnamese": 0.36050420168067226, - "Spanish,Vietnamese": 0.380672268907563 - }, - "3_combine": { - "English,Chinese,Spanish": 0.22941176470588234, - "English,Chinese,Vietnamese": 0.1991596638655462, - "English,Spanish,Vietnamese": 0.22521008403361345, - "Chinese,Spanish,Vietnamese": 0.20756302521008405 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.16470588235294117 - } - }, - "AC3_2": 0.3830729988232557, - "AC3_3": 0.27632294171812877, - "AC3_4": 0.2308020573216618 - }, - "prompt_2": { - "overall_acc": 0.4004201680672269, - "language_acc": { - "English": 0.4134453781512605, - "Chinese": 0.3831932773109244, - "Spanish": 0.41596638655462187, - "Vietnamese": 0.38907563025210085 - }, - "consistency_score_2": 0.3899159663865546, - "consistency_score_3": 0.22815126050420168, - "consistency_score_4": 0.17563025210084032, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.38823529411764707, - "English,Spanish": 0.40252100840336136, - "English,Vietnamese": 0.3815126050420168, - "Chinese,Spanish": 0.4, - "Chinese,Vietnamese": 0.3873949579831933, - "Spanish,Vietnamese": 0.3798319327731092 - }, - "3_combine": { - "English,Chinese,Spanish": 0.2411764705882353, - "English,Chinese,Vietnamese": 0.2184873949579832, - "English,Spanish,Vietnamese": 0.226890756302521, - "Chinese,Spanish,Vietnamese": 0.22605042016806723 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.17563025210084032 - } - }, - "AC3_2": 0.39509826254053143, - "AC3_3": 0.2906793465587077, - "AC3_4": 0.24416576254273542 - }, - "prompt_3": { - "overall_acc": 0.39642857142857146, - "language_acc": { - "English": 0.41680672268907565, - "Chinese": 0.3722689075630252, - "Spanish": 0.4016806722689076, - "Vietnamese": 0.3949579831932773 - }, - "consistency_score_2": 0.3861344537815126, - "consistency_score_3": 0.22647058823529412, - "consistency_score_4": 0.17899159663865546, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3865546218487395, - "English,Spanish": 0.4176470588235294, - "English,Vietnamese": 0.3764705882352941, - "Chinese,Spanish": 0.37142857142857144, - "Chinese,Vietnamese": 0.36722689075630255, - "Spanish,Vietnamese": 0.39747899159663863 - }, - "3_combine": { - "English,Chinese,Spanish": 0.22857142857142856, - "English,Chinese,Vietnamese": 0.22268907563025211, - "English,Spanish,Vietnamese": 0.23697478991596638, - "Chinese,Spanish,Vietnamese": 0.21764705882352942 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.17899159663865546 - } - }, - "AC3_2": 0.39121380627790897, - "AC3_3": 0.28826306909368865, - "AC3_4": 0.24662807068159576 - }, - "prompt_4": { - "overall_acc": 0.39054621848739496, - "language_acc": { - "English": 0.4117647058823529, - "Chinese": 0.3815126050420168, - "Spanish": 0.3899159663865546, - "Vietnamese": 0.37899159663865545 - }, - "consistency_score_2": 0.39593837535014004, - "consistency_score_3": 0.2357142857142857, - "consistency_score_4": 0.18571428571428572, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3865546218487395, - "English,Spanish": 0.4084033613445378, - "English,Vietnamese": 0.41092436974789914, - "Chinese,Spanish": 0.39915966386554624, - "Chinese,Vietnamese": 0.3949579831932773, - "Spanish,Vietnamese": 0.37563025210084033 - }, - "3_combine": { - "English,Chinese,Spanish": 0.23781512605042016, - "English,Chinese,Vietnamese": 0.2361344537815126, - "English,Spanish,Vietnamese": 0.24033613445378152, - "Chinese,Spanish,Vietnamese": 0.22857142857142856 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.18571428571428572 - } - }, - "AC3_2": 0.3932238124916693, - "AC3_3": 0.293990511286741, - "AC3_4": 0.25172647253587044 - }, - "prompt_5": { - "overall_acc": 0.38949579831932774, - "language_acc": { - "English": 0.3983193277310924, - "Chinese": 0.38907563025210085, - "Spanish": 0.3907563025210084, - "Vietnamese": 0.3798319327731092 - }, - "consistency_score_2": 0.3927170868347339, - "consistency_score_3": 0.2310924369747899, - "consistency_score_4": 0.1773109243697479, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.37563025210084033, - "English,Spanish": 0.4151260504201681, - "English,Vietnamese": 0.39915966386554624, - "Chinese,Spanish": 0.3722689075630252, - "Chinese,Vietnamese": 0.39243697478991596, - "Spanish,Vietnamese": 0.4016806722689076 - }, - "3_combine": { - "English,Chinese,Spanish": 0.22521008403361345, - "English,Chinese,Vietnamese": 0.22436974789915967, - "English,Spanish,Vietnamese": 0.24873949579831933, - "Chinese,Spanish,Vietnamese": 0.22605042016806723 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.1773109243697479 - } - }, - "AC3_2": 0.3910998096142425, - "AC3_3": 0.2900781165079363, - "AC3_4": 0.2436875120263269 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.44, - "language_acc": { - "Spanish": 0.5066666666666667, - "English": 0.5666666666666667, - "Chinese": 0.35333333333333333, - "Vietnamese": 0.3933333333333333, - "Filipino": 0.42, - "Indonesian": 0.4066666666666667, - "Malay": 0.43333333333333335 - }, - "consistency_score_2": 0.4717460317460318, - "consistency_score_3": 0.29866666666666664, - "consistency_score_4": 0.22704761904761908, - "consistency_score_5": 0.19142857142857142, - "consistency_score_6": 0.16952380952380955, - "consistency_score_7": 0.15333333333333332, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6066666666666667, - "Spanish,Chinese": 0.4, - "Spanish,Vietnamese": 0.46, - "Spanish,Filipino": 0.5133333333333333, - "Spanish,Indonesian": 0.5066666666666667, - "Spanish,Malay": 0.5333333333333333, - "English,Chinese": 0.4066666666666667, - "English,Vietnamese": 0.46, - "English,Filipino": 0.48, - "English,Indonesian": 0.44666666666666666, - "English,Malay": 0.4666666666666667, - "Chinese,Vietnamese": 0.43333333333333335, - "Chinese,Filipino": 0.4, - "Chinese,Indonesian": 0.4533333333333333, - "Chinese,Malay": 0.37333333333333335, - "Vietnamese,Filipino": 0.5266666666666666, - "Vietnamese,Indonesian": 0.5066666666666667, - "Vietnamese,Malay": 0.4866666666666667, - "Filipino,Indonesian": 0.4866666666666667, - "Filipino,Malay": 0.43333333333333335, - "Indonesian,Malay": 0.5266666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.2866666666666667, - "Spanish,English,Vietnamese": 0.34, - "Spanish,English,Filipino": 0.38, - "Spanish,English,Indonesian": 0.36, - "Spanish,English,Malay": 0.36, - "Spanish,Chinese,Vietnamese": 0.26, - "Spanish,Chinese,Filipino": 0.2733333333333333, - "Spanish,Chinese,Indonesian": 0.2733333333333333, - "Spanish,Chinese,Malay": 0.24, - "Spanish,Vietnamese,Filipino": 0.34, - "Spanish,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,Vietnamese,Malay": 0.32666666666666666, - "Spanish,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Filipino,Malay": 0.3333333333333333, - "Spanish,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese": 0.25333333333333335, - "English,Chinese,Filipino": 0.24666666666666667, - "English,Chinese,Indonesian": 0.26, - "English,Chinese,Malay": 0.25333333333333335, - "English,Vietnamese,Filipino": 0.32666666666666666, - "English,Vietnamese,Indonesian": 0.2866666666666667, - "English,Vietnamese,Malay": 0.28, - "English,Filipino,Indonesian": 0.3, - "English,Filipino,Malay": 0.30666666666666664, - "English,Indonesian,Malay": 0.3, - "Chinese,Vietnamese,Filipino": 0.2866666666666667, - "Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Malay": 0.24666666666666667, - "Chinese,Filipino,Indonesian": 0.26, - "Chinese,Filipino,Malay": 0.22, - "Chinese,Indonesian,Malay": 0.26, - "Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "Vietnamese,Filipino,Malay": 0.32666666666666666, - "Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Filipino,Indonesian,Malay": 0.3 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.22, - "Spanish,English,Chinese,Filipino": 0.22666666666666666, - "Spanish,English,Chinese,Indonesian": 0.22, - "Spanish,English,Chinese,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.2733333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,English,Vietnamese,Malay": 0.24, - "Spanish,English,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino": 0.22, - "Spanish,Chinese,Vietnamese,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.19333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.2, - "Spanish,Chinese,Indonesian,Malay": 0.2, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.2, - "English,Chinese,Vietnamese,Malay": 0.18666666666666668, - "English,Chinese,Filipino,Indonesian": 0.2, - "English,Chinese,Filipino,Malay": 0.18666666666666668, - "English,Chinese,Indonesian,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.24, - "English,Vietnamese,Filipino,Malay": 0.24, - "English,Vietnamese,Indonesian,Malay": 0.24, - "English,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian": 0.22, - "Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.2, - "Chinese,Filipino,Indonesian,Malay": 0.18, - "Vietnamese,Filipino,Indonesian,Malay": 0.26 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Filipino,Malay": 0.18, - "Spanish,English,Chinese,Indonesian,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,English,Filipino,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "English,Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - } - }, - "AC3_2": 0.4553203342118991, - "AC3_3": 0.3558122743200615, - "AC3_4": 0.299531696128711, - "AC3_5": 0.2667873302744907, - "AC3_6": 0.24474999995984573, - "AC3_7": 0.22741573029875017 - }, - "prompt_2": { - "overall_acc": 0.4371428571428572, - "language_acc": { - "Spanish": 0.4533333333333333, - "English": 0.5666666666666667, - "Chinese": 0.4066666666666667, - "Vietnamese": 0.36, - "Filipino": 0.37333333333333335, - "Indonesian": 0.4666666666666667, - "Malay": 0.43333333333333335 - }, - "consistency_score_2": 0.47142857142857136, - "consistency_score_3": 0.29619047619047617, - "consistency_score_4": 0.2211428571428571, - "consistency_score_5": 0.18253968253968253, - "consistency_score_6": 0.1580952380952381, - "consistency_score_7": 0.14, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5266666666666666, - "Spanish,Chinese": 0.44, - "Spanish,Vietnamese": 0.5133333333333333, - "Spanish,Filipino": 0.5133333333333333, - "Spanish,Indonesian": 0.5, - "Spanish,Malay": 0.5133333333333333, - "English,Chinese": 0.48, - "English,Vietnamese": 0.4666666666666667, - "English,Filipino": 0.5, - "English,Indonesian": 0.4866666666666667, - "English,Malay": 0.47333333333333333, - "Chinese,Vietnamese": 0.38, - "Chinese,Filipino": 0.4, - "Chinese,Indonesian": 0.4266666666666667, - "Chinese,Malay": 0.37333333333333335, - "Vietnamese,Filipino": 0.49333333333333335, - "Vietnamese,Indonesian": 0.44, - "Vietnamese,Malay": 0.44666666666666666, - "Filipino,Indonesian": 0.5333333333333333, - "Filipino,Malay": 0.47333333333333333, - "Indonesian,Malay": 0.52 - }, - "3_combine": { - "Spanish,English,Chinese": 0.3, - "Spanish,English,Vietnamese": 0.32666666666666666, - "Spanish,English,Filipino": 0.35333333333333333, - "Spanish,English,Indonesian": 0.3466666666666667, - "Spanish,English,Malay": 0.3333333333333333, - "Spanish,Chinese,Vietnamese": 0.26666666666666666, - "Spanish,Chinese,Filipino": 0.2866666666666667, - "Spanish,Chinese,Indonesian": 0.26666666666666666, - "Spanish,Chinese,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Filipino": 0.3466666666666667, - "Spanish,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,Vietnamese,Malay": 0.31333333333333335, - "Spanish,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Filipino,Malay": 0.3333333333333333, - "Spanish,Indonesian,Malay": 0.3333333333333333, - "English,Chinese,Vietnamese": 0.26, - "English,Chinese,Filipino": 0.2866666666666667, - "English,Chinese,Indonesian": 0.29333333333333333, - "English,Chinese,Malay": 0.26, - "English,Vietnamese,Filipino": 0.3, - "English,Vietnamese,Indonesian": 0.29333333333333333, - "English,Vietnamese,Malay": 0.2866666666666667, - "English,Filipino,Indonesian": 0.34, - "English,Filipino,Malay": 0.30666666666666664, - "English,Indonesian,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Filipino": 0.24, - "Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "Chinese,Vietnamese,Malay": 0.22666666666666666, - "Chinese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Filipino,Malay": 0.21333333333333335, - "Chinese,Indonesian,Malay": 0.25333333333333335, - "Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Vietnamese,Filipino,Malay": 0.31333333333333335, - "Vietnamese,Indonesian,Malay": 0.3, - "Filipino,Indonesian,Malay": 0.30666666666666664 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,English,Chinese,Filipino": 0.23333333333333334, - "Spanish,English,Chinese,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.24, - "Spanish,English,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,English,Vietnamese,Malay": 0.24666666666666667, - "Spanish,English,Filipino,Indonesian": 0.2733333333333333, - "Spanish,English,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.2, - "Spanish,Chinese,Vietnamese,Malay": 0.2, - "Spanish,Chinese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Indonesian,Malay": 0.18, - "Spanish,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.26, - "Spanish,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.2, - "English,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "English,Chinese,Vietnamese,Malay": 0.19333333333333333, - "English,Chinese,Filipino,Indonesian": 0.22, - "English,Chinese,Filipino,Malay": 0.18, - "English,Chinese,Indonesian,Malay": 0.20666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Vietnamese,Filipino,Malay": 0.23333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Malay": 0.18, - "Chinese,Vietnamese,Indonesian,Malay": 0.18, - "Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.18, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Vietnamese,Malay": 0.18, - "Spanish,English,Chinese,Filipino,Indonesian": 0.2, - "Spanish,English,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,English,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.16, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14 - } - }, - "AC3_2": 0.4536388139662437, - "AC3_3": 0.35312059364386944, - "AC3_4": 0.2937053570982404, - "AC3_5": 0.25753805616452935, - "AC3_6": 0.23221028567527438, - "AC3_7": 0.21207920788404572 - }, - "prompt_3": { - "overall_acc": 0.44380952380952376, - "language_acc": { - "Spanish": 0.49333333333333335, - "English": 0.5733333333333334, - "Chinese": 0.4066666666666667, - "Vietnamese": 0.4066666666666667, - "Filipino": 0.4266666666666667, - "Indonesian": 0.4, - "Malay": 0.4 - }, - "consistency_score_2": 0.46825396825396826, - "consistency_score_3": 0.29180952380952374, - "consistency_score_4": 0.21352380952380956, - "consistency_score_5": 0.17206349206349206, - "consistency_score_6": 0.1457142857142857, - "consistency_score_7": 0.12666666666666668, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5733333333333334, - "Spanish,Chinese": 0.4533333333333333, - "Spanish,Vietnamese": 0.4866666666666667, - "Spanish,Filipino": 0.4666666666666667, - "Spanish,Indonesian": 0.5133333333333333, - "Spanish,Malay": 0.5333333333333333, - "English,Chinese": 0.41333333333333333, - "English,Vietnamese": 0.4666666666666667, - "English,Filipino": 0.4866666666666667, - "English,Indonesian": 0.5, - "English,Malay": 0.4666666666666667, - "Chinese,Vietnamese": 0.4, - "Chinese,Filipino": 0.4, - "Chinese,Indonesian": 0.4266666666666667, - "Chinese,Malay": 0.37333333333333335, - "Vietnamese,Filipino": 0.47333333333333333, - "Vietnamese,Indonesian": 0.46, - "Vietnamese,Malay": 0.44, - "Filipino,Indonesian": 0.44, - "Filipino,Malay": 0.4866666666666667, - "Indonesian,Malay": 0.5733333333333334 - }, - "3_combine": { - "Spanish,English,Chinese": 0.30666666666666664, - "Spanish,English,Vietnamese": 0.3333333333333333, - "Spanish,English,Filipino": 0.35333333333333333, - "Spanish,English,Indonesian": 0.3466666666666667, - "Spanish,English,Malay": 0.36, - "Spanish,Chinese,Vietnamese": 0.26666666666666666, - "Spanish,Chinese,Filipino": 0.25333333333333335, - "Spanish,Chinese,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,Vietnamese,Indonesian": 0.3466666666666667, - "Spanish,Vietnamese,Malay": 0.31333333333333335, - "Spanish,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Filipino,Malay": 0.32666666666666666, - "Spanish,Indonesian,Malay": 0.38, - "English,Chinese,Vietnamese": 0.25333333333333335, - "English,Chinese,Filipino": 0.26666666666666666, - "English,Chinese,Indonesian": 0.2733333333333333, - "English,Chinese,Malay": 0.24, - "English,Vietnamese,Filipino": 0.2866666666666667, - "English,Vietnamese,Indonesian": 0.30666666666666664, - "English,Vietnamese,Malay": 0.2733333333333333, - "English,Filipino,Indonesian": 0.29333333333333333, - "English,Filipino,Malay": 0.32, - "English,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino": 0.26, - "Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "Chinese,Vietnamese,Malay": 0.22666666666666666, - "Chinese,Filipino,Indonesian": 0.20666666666666667, - "Chinese,Filipino,Malay": 0.21333333333333335, - "Chinese,Indonesian,Malay": 0.26, - "Vietnamese,Filipino,Indonesian": 0.28, - "Vietnamese,Filipino,Malay": 0.2866666666666667, - "Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Filipino,Indonesian,Malay": 0.3333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.22, - "Spanish,English,Chinese,Filipino": 0.22, - "Spanish,English,Chinese,Indonesian": 0.21333333333333335, - "Spanish,English,Chinese,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,English,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,English,Vietnamese,Malay": 0.24666666666666667, - "Spanish,English,Filipino,Indonesian": 0.24, - "Spanish,English,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.2, - "Spanish,Chinese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.18, - "Spanish,Chinese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.28, - "Spanish,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "English,Chinese,Vietnamese,Indonesian": 0.2, - "English,Chinese,Vietnamese,Malay": 0.18, - "English,Chinese,Filipino,Indonesian": 0.17333333333333334, - "English,Chinese,Filipino,Malay": 0.18666666666666668, - "English,Chinese,Indonesian,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino,Indonesian": 0.22, - "English,Vietnamese,Filipino,Malay": 0.22, - "English,Vietnamese,Indonesian,Malay": 0.24, - "English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.22 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16, - "Spanish,English,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - } - }, - "AC3_2": 0.45570417511122463, - "AC3_3": 0.35210574333700806, - "AC3_4": 0.28832829679211247, - "AC3_5": 0.2479842905832004, - "AC3_6": 0.2193953380657153, - "AC3_7": 0.19708402890257015 - }, - "prompt_4": { - "overall_acc": 0.419047619047619, - "language_acc": { - "Spanish": 0.48, - "English": 0.4866666666666667, - "Chinese": 0.38666666666666666, - "Vietnamese": 0.36666666666666664, - "Filipino": 0.37333333333333335, - "Indonesian": 0.43333333333333335, - "Malay": 0.4066666666666667 - }, - "consistency_score_2": 0.45968253968253964, - "consistency_score_3": 0.28914285714285703, - "consistency_score_4": 0.21771428571428575, - "consistency_score_5": 0.17777777777777778, - "consistency_score_6": 0.14952380952380953, - "consistency_score_7": 0.12666666666666668, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5466666666666666, - "Spanish,Chinese": 0.44, - "Spanish,Vietnamese": 0.5066666666666667, - "Spanish,Filipino": 0.48, - "Spanish,Indonesian": 0.48, - "Spanish,Malay": 0.5133333333333333, - "English,Chinese": 0.4533333333333333, - "English,Vietnamese": 0.43333333333333335, - "English,Filipino": 0.46, - "English,Indonesian": 0.4866666666666667, - "English,Malay": 0.4533333333333333, - "Chinese,Vietnamese": 0.42, - "Chinese,Filipino": 0.41333333333333333, - "Chinese,Indonesian": 0.3933333333333333, - "Chinese,Malay": 0.4066666666666667, - "Vietnamese,Filipino": 0.49333333333333335, - "Vietnamese,Indonesian": 0.44666666666666666, - "Vietnamese,Malay": 0.4266666666666667, - "Filipino,Indonesian": 0.44666666666666666, - "Filipino,Malay": 0.46, - "Indonesian,Malay": 0.49333333333333335 - }, - "3_combine": { - "Spanish,English,Chinese": 0.3, - "Spanish,English,Vietnamese": 0.3333333333333333, - "Spanish,English,Filipino": 0.3333333333333333, - "Spanish,English,Indonesian": 0.34, - "Spanish,English,Malay": 0.3333333333333333, - "Spanish,Chinese,Vietnamese": 0.28, - "Spanish,Chinese,Filipino": 0.2733333333333333, - "Spanish,Chinese,Indonesian": 0.26, - "Spanish,Chinese,Malay": 0.26, - "Spanish,Vietnamese,Filipino": 0.32666666666666666, - "Spanish,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,Vietnamese,Malay": 0.30666666666666664, - "Spanish,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Filipino,Malay": 0.3, - "Spanish,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese": 0.28, - "English,Chinese,Filipino": 0.28, - "English,Chinese,Indonesian": 0.28, - "English,Chinese,Malay": 0.26, - "English,Vietnamese,Filipino": 0.30666666666666664, - "English,Vietnamese,Indonesian": 0.3, - "English,Vietnamese,Malay": 0.26666666666666666, - "English,Filipino,Indonesian": 0.29333333333333333, - "English,Filipino,Malay": 0.31333333333333335, - "English,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Chinese,Vietnamese,Indonesian": 0.26, - "Chinese,Vietnamese,Malay": 0.23333333333333334, - "Chinese,Filipino,Indonesian": 0.23333333333333334, - "Chinese,Filipino,Malay": 0.21333333333333335, - "Chinese,Indonesian,Malay": 0.24666666666666667, - "Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Vietnamese,Filipino,Malay": 0.28, - "Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Filipino,Indonesian,Malay": 0.30666666666666664 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.23333333333333334, - "Spanish,English,Chinese,Filipino": 0.22666666666666666, - "Spanish,English,Chinese,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,English,Vietnamese,Malay": 0.24, - "Spanish,English,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.22, - "Spanish,Chinese,Vietnamese,Indonesian": 0.22, - "Spanish,Chinese,Vietnamese,Malay": 0.2, - "Spanish,Chinese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,Chinese,Indonesian,Malay": 0.2, - "Spanish,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino": 0.22, - "English,Chinese,Vietnamese,Indonesian": 0.20666666666666667, - "English,Chinese,Vietnamese,Malay": 0.18666666666666668, - "English,Chinese,Filipino,Indonesian": 0.2, - "English,Chinese,Filipino,Malay": 0.18666666666666668, - "English,Chinese,Indonesian,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Vietnamese,Filipino,Malay": 0.22, - "English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Malay": 0.16, - "Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.18, - "Spanish,English,Chinese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - } - }, - "AC3_2": 0.43842554357796265, - "AC3_3": 0.34218089597873214, - "AC3_4": 0.2865518567132568, - "AC3_5": 0.24964539002909308, - "AC3_6": 0.22040360528946398, - "AC3_7": 0.1945317044437013 - }, - "prompt_5": { - "overall_acc": 0.4257142857142857, - "language_acc": { - "Spanish": 0.44666666666666666, - "English": 0.5733333333333334, - "Chinese": 0.4266666666666667, - "Vietnamese": 0.34, - "Filipino": 0.38666666666666666, - "Indonesian": 0.4, - "Malay": 0.4066666666666667 - }, - "consistency_score_2": 0.4653968253968254, - "consistency_score_3": 0.2921904761904762, - "consistency_score_4": 0.21809523809523806, - "consistency_score_5": 0.17682539682539686, - "consistency_score_6": 0.14857142857142858, - "consistency_score_7": 0.12666666666666668, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5133333333333333, - "Spanish,Chinese": 0.4666666666666667, - "Spanish,Vietnamese": 0.46, - "Spanish,Filipino": 0.46, - "Spanish,Indonesian": 0.5, - "Spanish,Malay": 0.44, - "English,Chinese": 0.4866666666666667, - "English,Vietnamese": 0.4266666666666667, - "English,Filipino": 0.44666666666666666, - "English,Indonesian": 0.4666666666666667, - "English,Malay": 0.46, - "Chinese,Vietnamese": 0.48, - "Chinese,Filipino": 0.4066666666666667, - "Chinese,Indonesian": 0.49333333333333335, - "Chinese,Malay": 0.41333333333333333, - "Vietnamese,Filipino": 0.49333333333333335, - "Vietnamese,Indonesian": 0.48, - "Vietnamese,Malay": 0.44666666666666666, - "Filipino,Indonesian": 0.4533333333333333, - "Filipino,Malay": 0.4533333333333333, - "Indonesian,Malay": 0.5266666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.30666666666666664, - "Spanish,English,Vietnamese": 0.3, - "Spanish,English,Filipino": 0.30666666666666664, - "Spanish,English,Indonesian": 0.3333333333333333, - "Spanish,English,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese": 0.2866666666666667, - "Spanish,Chinese,Filipino": 0.26, - "Spanish,Chinese,Indonesian": 0.30666666666666664, - "Spanish,Chinese,Malay": 0.26, - "Spanish,Vietnamese,Filipino": 0.29333333333333333, - "Spanish,Vietnamese,Indonesian": 0.32666666666666666, - "Spanish,Vietnamese,Malay": 0.26666666666666666, - "Spanish,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Filipino,Malay": 0.29333333333333333, - "Spanish,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese": 0.2866666666666667, - "English,Chinese,Filipino": 0.28, - "English,Chinese,Indonesian": 0.31333333333333335, - "English,Chinese,Malay": 0.28, - "English,Vietnamese,Filipino": 0.28, - "English,Vietnamese,Indonesian": 0.3, - "English,Vietnamese,Malay": 0.25333333333333335, - "English,Filipino,Indonesian": 0.2866666666666667, - "English,Filipino,Malay": 0.2866666666666667, - "English,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino": 0.28, - "Chinese,Vietnamese,Indonesian": 0.3, - "Chinese,Vietnamese,Malay": 0.26666666666666666, - "Chinese,Filipino,Indonesian": 0.25333333333333335, - "Chinese,Filipino,Malay": 0.24, - "Chinese,Indonesian,Malay": 0.28, - "Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Vietnamese,Filipino,Malay": 0.3, - "Vietnamese,Indonesian,Malay": 0.32, - "Filipino,Indonesian,Malay": 0.32 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.22, - "Spanish,English,Chinese,Filipino": 0.22, - "Spanish,English,Chinese,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Malay": 0.22, - "Spanish,English,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,English,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Filipino,Indonesian": 0.23333333333333334, - "Spanish,English,Filipino,Malay": 0.24, - "Spanish,English,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.2, - "Spanish,Chinese,Filipino,Indonesian": 0.2, - "Spanish,Chinese,Filipino,Malay": 0.19333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.22666666666666666, - "English,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "English,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "English,Chinese,Vietnamese,Malay": 0.20666666666666667, - "English,Chinese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Filipino,Malay": 0.20666666666666667, - "English,Chinese,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.22, - "English,Vietnamese,Filipino,Malay": 0.21333333333333335, - "English,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "English,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2, - "Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.18, - "Vietnamese,Filipino,Indonesian,Malay": 0.24 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.18, - "Spanish,English,Chinese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Filipino,Malay": 0.18, - "Spanish,English,Chinese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,English,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "English,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - } - }, - "AC3_2": 0.4446719934357727, - "AC3_3": 0.3465352688748999, - "AC3_4": 0.2884277260752336, - "AC3_5": 0.24986602434510355, - "AC3_6": 0.22027007814217048, - "AC3_7": 0.1952413792749994 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.3717532467532468, - "language_acc": { - "English": 0.45454545454545453, - "Filipino": 0.2897727272727273, - "Spanish": 0.4147727272727273, - "Malay": 0.38636363636363635, - "Chinese": 0.3352272727272727, - "Indonesian": 0.32386363636363635, - "Vietnamese": 0.3977272727272727 - }, - "consistency_score_2": 0.46347402597402576, - "consistency_score_3": 0.2748376623376622, - "consistency_score_4": 0.1891233766233766, - "consistency_score_5": 0.14204545454545456, - "consistency_score_6": 0.11201298701298702, - "consistency_score_7": 0.09090909090909091, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.45454545454545453, - "English,Spanish": 0.6306818181818182, - "English,Malay": 0.48295454545454547, - "English,Chinese": 0.4034090909090909, - "English,Indonesian": 0.4431818181818182, - "English,Vietnamese": 0.5397727272727273, - "Filipino,Spanish": 0.42045454545454547, - "Filipino,Malay": 0.4659090909090909, - "Filipino,Chinese": 0.3522727272727273, - "Filipino,Indonesian": 0.4318181818181818, - "Filipino,Vietnamese": 0.45454545454545453, - "Spanish,Malay": 0.5284090909090909, - "Spanish,Chinese": 0.375, - "Spanish,Indonesian": 0.5, - "Spanish,Vietnamese": 0.5227272727272727, - "Malay,Chinese": 0.375, - "Malay,Indonesian": 0.5511363636363636, - "Malay,Vietnamese": 0.5454545454545454, - "Chinese,Indonesian": 0.3465909090909091, - "Chinese,Vietnamese": 0.42045454545454547, - "Indonesian,Vietnamese": 0.48863636363636365 - }, - "3_combine": { - "English,Filipino,Spanish": 0.3125, - "English,Filipino,Malay": 0.2840909090909091, - "English,Filipino,Chinese": 0.19318181818181818, - "English,Filipino,Indonesian": 0.2556818181818182, - "English,Filipino,Vietnamese": 0.30113636363636365, - "English,Spanish,Malay": 0.3806818181818182, - "English,Spanish,Chinese": 0.2784090909090909, - "English,Spanish,Indonesian": 0.36363636363636365, - "English,Spanish,Vietnamese": 0.4034090909090909, - "English,Malay,Chinese": 0.22727272727272727, - "English,Malay,Indonesian": 0.3125, - "English,Malay,Vietnamese": 0.3522727272727273, - "English,Chinese,Indonesian": 0.1875, - "English,Chinese,Vietnamese": 0.26704545454545453, - "English,Indonesian,Vietnamese": 0.30113636363636365, - "Filipino,Spanish,Malay": 0.2897727272727273, - "Filipino,Spanish,Chinese": 0.17613636363636365, - "Filipino,Spanish,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Vietnamese": 0.2784090909090909, - "Filipino,Malay,Chinese": 0.20454545454545456, - "Filipino,Malay,Indonesian": 0.2897727272727273, - "Filipino,Malay,Vietnamese": 0.29545454545454547, - "Filipino,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Indonesian,Vietnamese": 0.2727272727272727, - "Spanish,Malay,Chinese": 0.2215909090909091, - "Spanish,Malay,Indonesian": 0.36363636363636365, - "Spanish,Malay,Vietnamese": 0.3693181818181818, - "Spanish,Chinese,Indonesian": 0.20454545454545456, - "Spanish,Chinese,Vietnamese": 0.23295454545454544, - "Spanish,Indonesian,Vietnamese": 0.3352272727272727, - "Malay,Chinese,Indonesian": 0.2215909090909091, - "Malay,Chinese,Vietnamese": 0.23863636363636365, - "Malay,Indonesian,Vietnamese": 0.36363636363636365, - "Chinese,Indonesian,Vietnamese": 0.1875 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.24431818181818182, - "English,Filipino,Spanish,Chinese": 0.14204545454545456, - "English,Filipino,Spanish,Indonesian": 0.23295454545454544, - "English,Filipino,Spanish,Vietnamese": 0.24431818181818182, - "English,Filipino,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Malay,Indonesian": 0.2159090909090909, - "English,Filipino,Malay,Vietnamese": 0.22727272727272727, - "English,Filipino,Chinese,Indonesian": 0.125, - "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay,Chinese": 0.17613636363636365, - "English,Spanish,Malay,Indonesian": 0.2897727272727273, - "English,Spanish,Malay,Vietnamese": 0.30113636363636365, - "English,Spanish,Chinese,Indonesian": 0.17045454545454544, - "English,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.26704545454545453, - "English,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "Filipino,Spanish,Malay,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.125, - "Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, - "English,Filipino,Spanish,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - } - }, - "AC3_2": 0.41257746136188356, - "AC3_3": 0.31603226043480226, - "AC3_4": 0.2507047944372303, - "AC3_5": 0.2055507683069708, - "AC3_6": 0.17215418805819918, - "AC3_7": 0.14609250395566434 - }, - "prompt_2": { - "overall_acc": 0.3628246753246754, - "language_acc": { - "English": 0.4034090909090909, - "Filipino": 0.2897727272727273, - "Spanish": 0.38636363636363635, - "Malay": 0.38636363636363635, - "Chinese": 0.32954545454545453, - "Indonesian": 0.32954545454545453, - "Vietnamese": 0.4147727272727273 - }, - "consistency_score_2": 0.46239177489177485, - "consistency_score_3": 0.2670454545454545, - "consistency_score_4": 0.17548701298701297, - "consistency_score_5": 0.12716450216450212, - "consistency_score_6": 0.10064935064935066, - "consistency_score_7": 0.08522727272727272, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4318181818181818, - "English,Spanish": 0.5568181818181818, - "English,Malay": 0.44886363636363635, - "English,Chinese": 0.375, - "English,Indonesian": 0.48295454545454547, - "English,Vietnamese": 0.5170454545454546, - "Filipino,Spanish": 0.4659090909090909, - "Filipino,Malay": 0.3806818181818182, - "Filipino,Chinese": 0.39204545454545453, - "Filipino,Indonesian": 0.4034090909090909, - "Filipino,Vietnamese": 0.4715909090909091, - "Spanish,Malay": 0.5, - "Spanish,Chinese": 0.4090909090909091, - "Spanish,Indonesian": 0.4943181818181818, - "Spanish,Vietnamese": 0.5397727272727273, - "Malay,Chinese": 0.4090909090909091, - "Malay,Indonesian": 0.5738636363636364, - "Malay,Vietnamese": 0.5625, - "Chinese,Indonesian": 0.35795454545454547, - "Chinese,Vietnamese": 0.4318181818181818, - "Indonesian,Vietnamese": 0.5056818181818182 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2784090909090909, - "English,Filipino,Malay": 0.21022727272727273, - "English,Filipino,Chinese": 0.17045454545454544, - "English,Filipino,Indonesian": 0.23863636363636365, - "English,Filipino,Vietnamese": 0.2727272727272727, - "English,Spanish,Malay": 0.3352272727272727, - "English,Spanish,Chinese": 0.23863636363636365, - "English,Spanish,Indonesian": 0.3522727272727273, - "English,Spanish,Vietnamese": 0.3693181818181818, - "English,Malay,Chinese": 0.2159090909090909, - "English,Malay,Indonesian": 0.3068181818181818, - "English,Malay,Vietnamese": 0.32954545454545453, - "English,Chinese,Indonesian": 0.2159090909090909, - "English,Chinese,Vietnamese": 0.24431818181818182, - "English,Indonesian,Vietnamese": 0.3181818181818182, - "Filipino,Spanish,Malay": 0.24431818181818182, - "Filipino,Spanish,Chinese": 0.2215909090909091, - "Filipino,Spanish,Indonesian": 0.2556818181818182, - "Filipino,Spanish,Vietnamese": 0.30113636363636365, - "Filipino,Malay,Chinese": 0.18181818181818182, - "Filipino,Malay,Indonesian": 0.25, - "Filipino,Malay,Vietnamese": 0.2784090909090909, - "Filipino,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Chinese,Vietnamese": 0.22727272727272727, - "Filipino,Indonesian,Vietnamese": 0.26704545454545453, - "Spanish,Malay,Chinese": 0.23295454545454544, - "Spanish,Malay,Indonesian": 0.36363636363636365, - "Spanish,Malay,Vietnamese": 0.3522727272727273, - "Spanish,Chinese,Indonesian": 0.21022727272727273, - "Spanish,Chinese,Vietnamese": 0.23863636363636365, - "Spanish,Indonesian,Vietnamese": 0.3409090909090909, - "Malay,Chinese,Indonesian": 0.23295454545454544, - "Malay,Chinese,Vietnamese": 0.2556818181818182, - "Malay,Indonesian,Vietnamese": 0.3806818181818182, - "Chinese,Indonesian,Vietnamese": 0.23295454545454544 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17045454545454544, - "English,Filipino,Spanish,Chinese": 0.125, - "English,Filipino,Spanish,Indonesian": 0.19318181818181818, - "English,Filipino,Spanish,Vietnamese": 0.2215909090909091, - "English,Filipino,Malay,Chinese": 0.10227272727272728, - "English,Filipino,Malay,Indonesian": 0.17045454545454544, - "English,Filipino,Malay,Vietnamese": 0.19318181818181818, - "English,Filipino,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Chinese": 0.1590909090909091, - "English,Spanish,Malay,Indonesian": 0.26136363636363635, - "English,Spanish,Malay,Vietnamese": 0.26136363636363635, - "English,Spanish,Chinese,Indonesian": 0.16477272727272727, - "English,Spanish,Chinese,Vietnamese": 0.18181818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.25, - "English,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "English,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.11931818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.16477272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.18181818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 - } - }, - "AC3_2": 0.40660155413424925, - "AC3_3": 0.3076528818630586, - "AC3_4": 0.23655818694397715, - "AC3_5": 0.18832423784405858, - "AC3_6": 0.1575840971297813, - "AC3_7": 0.1380311264514065 - }, - "prompt_3": { - "overall_acc": 0.3482142857142857, - "language_acc": { - "English": 0.3806818181818182, - "Filipino": 0.2727272727272727, - "Spanish": 0.36363636363636365, - "Malay": 0.3522727272727273, - "Chinese": 0.3409090909090909, - "Indonesian": 0.3068181818181818, - "Vietnamese": 0.42045454545454547 - }, - "consistency_score_2": 0.4558982683982683, - "consistency_score_3": 0.2629870129870128, - "consistency_score_4": 0.17012987012987013, - "consistency_score_5": 0.11661255411255408, - "consistency_score_6": 0.0836038961038961, - "consistency_score_7": 0.0625, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4147727272727273, - "English,Spanish": 0.6022727272727273, - "English,Malay": 0.42045454545454547, - "English,Chinese": 0.38636363636363635, - "English,Indonesian": 0.5170454545454546, - "English,Vietnamese": 0.5113636363636364, - "Filipino,Spanish": 0.4602272727272727, - "Filipino,Malay": 0.4034090909090909, - "Filipino,Chinese": 0.3181818181818182, - "Filipino,Indonesian": 0.3977272727272727, - "Filipino,Vietnamese": 0.45454545454545453, - "Spanish,Malay": 0.5284090909090909, - "Spanish,Chinese": 0.3693181818181818, - "Spanish,Indonesian": 0.5056818181818182, - "Spanish,Vietnamese": 0.5738636363636364, - "Malay,Chinese": 0.3977272727272727, - "Malay,Indonesian": 0.5170454545454546, - "Malay,Vietnamese": 0.5170454545454546, - "Chinese,Indonesian": 0.3522727272727273, - "Chinese,Vietnamese": 0.4147727272727273, - "Indonesian,Vietnamese": 0.5113636363636364 - }, - "3_combine": { - "English,Filipino,Spanish": 0.30113636363636365, - "English,Filipino,Malay": 0.20454545454545456, - "English,Filipino,Chinese": 0.17045454545454544, - "English,Filipino,Indonesian": 0.25, - "English,Filipino,Vietnamese": 0.26136363636363635, - "English,Spanish,Malay": 0.32954545454545453, - "English,Spanish,Chinese": 0.24431818181818182, - "English,Spanish,Indonesian": 0.3806818181818182, - "English,Spanish,Vietnamese": 0.4090909090909091, - "English,Malay,Chinese": 0.20454545454545456, - "English,Malay,Indonesian": 0.2897727272727273, - "English,Malay,Vietnamese": 0.30113636363636365, - "English,Chinese,Indonesian": 0.23295454545454544, - "English,Chinese,Vietnamese": 0.24431818181818182, - "English,Indonesian,Vietnamese": 0.3352272727272727, - "Filipino,Spanish,Malay": 0.26704545454545453, - "Filipino,Spanish,Chinese": 0.17613636363636365, - "Filipino,Spanish,Indonesian": 0.26136363636363635, - "Filipino,Spanish,Vietnamese": 0.30113636363636365, - "Filipino,Malay,Chinese": 0.17045454545454544, - "Filipino,Malay,Indonesian": 0.23863636363636365, - "Filipino,Malay,Vietnamese": 0.26704545454545453, - "Filipino,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Chinese,Vietnamese": 0.18181818181818182, - "Filipino,Indonesian,Vietnamese": 0.25, - "Spanish,Malay,Chinese": 0.2215909090909091, - "Spanish,Malay,Indonesian": 0.36363636363636365, - "Spanish,Malay,Vietnamese": 0.375, - "Spanish,Chinese,Indonesian": 0.20454545454545456, - "Spanish,Chinese,Vietnamese": 0.23863636363636365, - "Spanish,Indonesian,Vietnamese": 0.35795454545454547, - "Malay,Chinese,Indonesian": 0.2215909090909091, - "Malay,Chinese,Vietnamese": 0.23295454545454544, - "Malay,Indonesian,Vietnamese": 0.3465909090909091, - "Chinese,Indonesian,Vietnamese": 0.22727272727272727 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.125, - "English,Filipino,Spanish,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Vietnamese": 0.23295454545454544, - "English,Filipino,Malay,Chinese": 0.10795454545454546, - "English,Filipino,Malay,Indonesian": 0.16477272727272727, - "English,Filipino,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Chinese": 0.1534090909090909, - "English,Spanish,Malay,Indonesian": 0.26136363636363635, - "English,Spanish,Malay,Vietnamese": 0.26704545454545453, - "English,Spanish,Chinese,Indonesian": 0.16477272727272727, - "English,Spanish,Chinese,Vietnamese": 0.1875, - "English,Spanish,Indonesian,Vietnamese": 0.2784090909090909, - "English,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay,Chinese": 0.11931818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.125, - "Filipino,Spanish,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Malay,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.2897727272727273, - "Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - } - }, - "AC3_2": 0.39484594303876436, - "AC3_3": 0.29965850877282757, - "AC3_4": 0.22858037667282724, - "AC3_5": 0.174715200361548, - "AC3_6": 0.13483485496340453, - "AC3_7": 0.10597826084376182 - }, - "prompt_4": { - "overall_acc": 0.37581168831168826, - "language_acc": { - "English": 0.42613636363636365, - "Filipino": 0.32386363636363635, - "Spanish": 0.38636363636363635, - "Malay": 0.375, - "Chinese": 0.375, - "Indonesian": 0.35795454545454547, - "Vietnamese": 0.38636363636363635 - }, - "consistency_score_2": 0.46699134199134196, - "consistency_score_3": 0.2806818181818182, - "consistency_score_4": 0.1991883116883117, - "consistency_score_5": 0.1574675324675325, - "consistency_score_6": 0.13392857142857142, - "consistency_score_7": 0.11931818181818182, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4090909090909091, - "English,Spanish": 0.5511363636363636, - "English,Malay": 0.48295454545454547, - "English,Chinese": 0.4375, - "English,Indonesian": 0.4943181818181818, - "English,Vietnamese": 0.5511363636363636, - "Filipino,Spanish": 0.45454545454545453, - "Filipino,Malay": 0.4147727272727273, - "Filipino,Chinese": 0.375, - "Filipino,Indonesian": 0.39204545454545453, - "Filipino,Vietnamese": 0.4602272727272727, - "Spanish,Malay": 0.48863636363636365, - "Spanish,Chinese": 0.3977272727272727, - "Spanish,Indonesian": 0.5284090909090909, - "Spanish,Vietnamese": 0.5625, - "Malay,Chinese": 0.44886363636363635, - "Malay,Indonesian": 0.5284090909090909, - "Malay,Vietnamese": 0.5284090909090909, - "Chinese,Indonesian": 0.38636363636363635, - "Chinese,Vietnamese": 0.4147727272727273, - "Indonesian,Vietnamese": 0.5 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2727272727272727, - "English,Filipino,Malay": 0.23295454545454544, - "English,Filipino,Chinese": 0.1875, - "English,Filipino,Indonesian": 0.24431818181818182, - "English,Filipino,Vietnamese": 0.29545454545454547, - "English,Spanish,Malay": 0.32954545454545453, - "English,Spanish,Chinese": 0.2840909090909091, - "English,Spanish,Indonesian": 0.36363636363636365, - "English,Spanish,Vietnamese": 0.3977272727272727, - "English,Malay,Chinese": 0.2840909090909091, - "English,Malay,Indonesian": 0.32386363636363635, - "English,Malay,Vietnamese": 0.3409090909090909, - "English,Chinese,Indonesian": 0.26136363636363635, - "English,Chinese,Vietnamese": 0.2840909090909091, - "English,Indonesian,Vietnamese": 0.32954545454545453, - "Filipino,Spanish,Malay": 0.26136363636363635, - "Filipino,Spanish,Chinese": 0.19886363636363635, - "Filipino,Spanish,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Vietnamese": 0.29545454545454547, - "Filipino,Malay,Chinese": 0.2215909090909091, - "Filipino,Malay,Indonesian": 0.25, - "Filipino,Malay,Vietnamese": 0.2727272727272727, - "Filipino,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Indonesian,Vietnamese": 0.2556818181818182, - "Spanish,Malay,Chinese": 0.2556818181818182, - "Spanish,Malay,Indonesian": 0.3522727272727273, - "Spanish,Malay,Vietnamese": 0.36363636363636365, - "Spanish,Chinese,Indonesian": 0.2556818181818182, - "Spanish,Chinese,Vietnamese": 0.26136363636363635, - "Spanish,Indonesian,Vietnamese": 0.375, - "Malay,Chinese,Indonesian": 0.25, - "Malay,Chinese,Vietnamese": 0.26704545454545453, - "Malay,Indonesian,Vietnamese": 0.3522727272727273, - "Chinese,Indonesian,Vietnamese": 0.23863636363636365 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.19886363636363635, - "English,Filipino,Spanish,Chinese": 0.14204545454545456, - "English,Filipino,Spanish,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Vietnamese": 0.23295454545454544, - "English,Filipino,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Malay,Indonesian": 0.1875, - "English,Filipino,Malay,Vietnamese": 0.2159090909090909, - "English,Filipino,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.20454545454545456, - "English,Spanish,Malay,Chinese": 0.21022727272727273, - "English,Spanish,Malay,Indonesian": 0.2727272727272727, - "English,Spanish,Malay,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese,Indonesian": 0.21022727272727273, - "English,Spanish,Chinese,Vietnamese": 0.2159090909090909, - "English,Spanish,Indonesian,Vietnamese": 0.2840909090909091, - "English,Malay,Chinese,Indonesian": 0.20454545454545456, - "English,Malay,Chinese,Vietnamese": 0.19318181818181818, - "English,Malay,Indonesian,Vietnamese": 0.25, - "English,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese": 0.16477272727272727, - "Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "Spanish,Malay,Chinese,Vietnamese": 0.19886363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.2897727272727273, - "Spanish,Chinese,Indonesian,Vietnamese": 0.1875, - "Malay,Chinese,Indonesian,Vietnamese": 0.1875 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, - "English,Filipino,Spanish,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Chinese,Indonesian": 0.17613636363636365, - "English,Spanish,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - } - }, - "AC3_2": 0.4164693251679039, - "AC3_3": 0.3213543071176716, - "AC3_4": 0.26037320241572365, - "AC3_5": 0.2219405403920731, - "AC3_6": 0.19748066420147692, - "AC3_7": 0.18112891203495293 - }, - "prompt_5": { - "overall_acc": 0.3547077922077922, - "language_acc": { - "English": 0.42045454545454547, - "Filipino": 0.24431818181818182, - "Spanish": 0.3806818181818182, - "Malay": 0.3806818181818182, - "Chinese": 0.35795454545454547, - "Indonesian": 0.3352272727272727, - "Vietnamese": 0.36363636363636365 - }, - "consistency_score_2": 0.4862012987012986, - "consistency_score_3": 0.30227272727272725, - "consistency_score_4": 0.21801948051948056, - "consistency_score_5": 0.17261904761904762, - "consistency_score_6": 0.1444805194805195, - "consistency_score_7": 0.125, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4318181818181818, - "English,Spanish": 0.6022727272727273, - "English,Malay": 0.4602272727272727, - "English,Chinese": 0.5, - "English,Indonesian": 0.45454545454545453, - "English,Vietnamese": 0.5056818181818182, - "Filipino,Spanish": 0.4772727272727273, - "Filipino,Malay": 0.4090909090909091, - "Filipino,Chinese": 0.38636363636363635, - "Filipino,Indonesian": 0.39204545454545453, - "Filipino,Vietnamese": 0.4659090909090909, - "Spanish,Malay": 0.5340909090909091, - "Spanish,Chinese": 0.4943181818181818, - "Spanish,Indonesian": 0.5454545454545454, - "Spanish,Vietnamese": 0.4943181818181818, - "Malay,Chinese": 0.48295454545454547, - "Malay,Indonesian": 0.5852272727272727, - "Malay,Vietnamese": 0.5340909090909091, - "Chinese,Indonesian": 0.4318181818181818, - "Chinese,Vietnamese": 0.4772727272727273, - "Indonesian,Vietnamese": 0.5454545454545454 - }, - "3_combine": { - "English,Filipino,Spanish": 0.3068181818181818, - "English,Filipino,Malay": 0.22727272727272727, - "English,Filipino,Chinese": 0.23863636363636365, - "English,Filipino,Indonesian": 0.24431818181818182, - "English,Filipino,Vietnamese": 0.2840909090909091, - "English,Spanish,Malay": 0.36363636363636365, - "English,Spanish,Chinese": 0.36363636363636365, - "English,Spanish,Indonesian": 0.375, - "English,Spanish,Vietnamese": 0.3806818181818182, - "English,Malay,Chinese": 0.29545454545454547, - "English,Malay,Indonesian": 0.3181818181818182, - "English,Malay,Vietnamese": 0.32386363636363635, - "English,Chinese,Indonesian": 0.2784090909090909, - "English,Chinese,Vietnamese": 0.3181818181818182, - "English,Indonesian,Vietnamese": 0.32954545454545453, - "Filipino,Spanish,Malay": 0.2784090909090909, - "Filipino,Spanish,Chinese": 0.26704545454545453, - "Filipino,Spanish,Indonesian": 0.2897727272727273, - "Filipino,Spanish,Vietnamese": 0.2897727272727273, - "Filipino,Malay,Chinese": 0.2215909090909091, - "Filipino,Malay,Indonesian": 0.2556818181818182, - "Filipino,Malay,Vietnamese": 0.26136363636363635, - "Filipino,Chinese,Indonesian": 0.20454545454545456, - "Filipino,Chinese,Vietnamese": 0.24431818181818182, - "Filipino,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Chinese": 0.3181818181818182, - "Spanish,Malay,Indonesian": 0.39204545454545453, - "Spanish,Malay,Vietnamese": 0.3522727272727273, - "Spanish,Chinese,Indonesian": 0.29545454545454547, - "Spanish,Chinese,Vietnamese": 0.3181818181818182, - "Spanish,Indonesian,Vietnamese": 0.375, - "Malay,Chinese,Indonesian": 0.3125, - "Malay,Chinese,Vietnamese": 0.30113636363636365, - "Malay,Indonesian,Vietnamese": 0.3806818181818182, - "Chinese,Indonesian,Vietnamese": 0.2897727272727273 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.19886363636363635, - "English,Filipino,Spanish,Chinese": 0.19886363636363635, - "English,Filipino,Spanish,Indonesian": 0.22727272727272727, - "English,Filipino,Spanish,Vietnamese": 0.23863636363636365, - "English,Filipino,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Malay,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian": 0.17613636363636365, - "English,Filipino,Chinese,Vietnamese": 0.1875, - "English,Filipino,Indonesian,Vietnamese": 0.20454545454545456, - "English,Spanish,Malay,Chinese": 0.24431818181818182, - "English,Spanish,Malay,Indonesian": 0.2897727272727273, - "English,Spanish,Malay,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese,Indonesian": 0.23863636363636365, - "English,Spanish,Chinese,Vietnamese": 0.2727272727272727, - "English,Spanish,Indonesian,Vietnamese": 0.2897727272727273, - "English,Malay,Chinese,Indonesian": 0.2159090909090909, - "English,Malay,Chinese,Vietnamese": 0.2215909090909091, - "English,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Malay,Chinese": 0.19318181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Chinese,Indonesian": 0.23295454545454544, - "Spanish,Malay,Chinese,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125 - } - }, - "AC3_2": 0.41017368244633357, - "AC3_3": 0.3263977805734512, - "AC3_4": 0.2700524744058327, - "AC3_5": 0.23222531696834908, - "AC3_6": 0.20532678699297197, - "AC3_7": 0.18485617593439208 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.5048543689320388 - }, - "prompt_2": { - "accuracy": 0.5242718446601942 - }, - "prompt_3": { - "accuracy": 0.5242718446601942 - }, - "prompt_4": { - "accuracy": 0.5533980582524272 - }, - "prompt_5": { - "accuracy": 0.5436893203883495 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.34285714285714286 - }, - "prompt_2": { - "accuracy": 0.3238095238095238 - }, - "prompt_3": { - "accuracy": 0.3619047619047619 - }, - "prompt_4": { - "accuracy": 0.3333333333333333 - }, - "prompt_5": { - "accuracy": 0.3904761904761905 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.4953271028037383 - }, - "prompt_2": { - "accuracy": 0.4205607476635514 - }, - "prompt_3": { - "accuracy": 0.4953271028037383 - }, - "prompt_4": { - "accuracy": 0.42990654205607476 - }, - "prompt_5": { - "accuracy": 0.4392523364485981 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.36, - "category_acc": { - "brand": 0.2, - "demographics": 0.0, - "biology": 0.5, - "history": 0.4, - "literature": 0.4, - "politics": 0.3, - "culture": 0.5, - "film": 0.4, - "law": 0.2, - "geography": 0.5 - } - }, - "prompt_2": { - "accuracy": 0.39, - "category_acc": { - "brand": 0.2, - "demographics": 0.4, - "biology": 0.6, - "history": 0.3333333333333333, - "literature": 0.4, - "politics": 0.2, - "culture": 0.7, - "film": 0.3, - "law": 0.2, - "geography": 0.6 - } - }, - "prompt_3": { - "accuracy": 0.43, - "category_acc": { - "brand": 0.2, - "demographics": 0.4, - "biology": 0.6, - "history": 0.5333333333333333, - "literature": 0.4, - "politics": 0.2, - "culture": 0.7, - "film": 0.4, - "law": 0.3, - "geography": 0.5 - } - }, - "prompt_4": { - "accuracy": 0.41, - "category_acc": { - "brand": 0.1, - "demographics": 0.4, - "biology": 0.5, - "history": 0.4666666666666667, - "literature": 0.4, - "politics": 0.4, - "culture": 0.6, - "film": 0.3, - "law": 0.4, - "geography": 0.5 - } - }, - "prompt_5": { - "accuracy": 0.37, - "category_acc": { - "brand": 0.2, - "demographics": 0.0, - "biology": 0.6, - "history": 0.4, - "literature": 0.4, - "politics": 0.5, - "culture": 0.5, - "film": 0.3, - "law": 0.2, - "geography": 0.4 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.15003646870123308 - }, - "prompt_2": { - "bleu_score": 0.14875390724922957 - }, - "prompt_3": { - "bleu_score": 0.14782317043740328 - }, - "prompt_4": { - "bleu_score": 0.14866819782233143 - }, - "prompt_5": { - "bleu_score": 0.12304874225269755 - } - }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.10823112534585214 - }, - "prompt_2": { - "bleu_score": 0.14139634434370052 - }, - "prompt_3": { - "bleu_score": 0.14801525832435974 - }, - "prompt_4": { - "bleu_score": 0.13397563432241336 - }, - "prompt_5": { - "bleu_score": 0.13669775401061685 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.0924438388966177 - }, - "prompt_2": { - "bleu_score": 0.11764840854886445 - }, - "prompt_3": { - "bleu_score": 0.11850518439188602 - }, - "prompt_4": { - "bleu_score": 0.11605886891298912 - }, - "prompt_5": { - "bleu_score": 0.11534262005196816 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.09734728829049691 - }, - "prompt_2": { - "bleu_score": 0.10529238804843469 - }, - "prompt_3": { - "bleu_score": 0.10464192774147724 - }, - "prompt_4": { - "bleu_score": 0.09828305220239039 - }, - "prompt_5": { - "bleu_score": 0.10722517148638896 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.09617723180676328 - }, - "prompt_2": { - "bleu_score": 0.13330860192761337 - }, - "prompt_3": { - "bleu_score": 0.1284813563772998 - }, - "prompt_4": { - "bleu_score": 0.12211843746412185 - }, - "prompt_5": { - "bleu_score": 0.11799488707198048 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.4492415402567094 - }, - "prompt_2": { - "accuracy": 0.396732788798133 - }, - "prompt_3": { - "accuracy": 0.4644107351225204 - }, - "prompt_4": { - "accuracy": 0.4644107351225204 - }, - "prompt_5": { - "accuracy": 0.45857642940490084 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.431748301751877, - "category_acc": { - "high_school_european_history": 0.20121951219512196, - "business_ethics": 0.494949494949495, - "clinical_knowledge": 0.4659090909090909, - "medical_genetics": 0.494949494949495, - "high_school_us_history": 0.26108374384236455, - "high_school_physics": 0.3, - "high_school_world_history": 0.2711864406779661, - "virology": 0.3515151515151515, - "high_school_microeconomics": 0.48945147679324896, - "econometrics": 0.3185840707964602, - "college_computer_science": 0.42424242424242425, - "high_school_biology": 0.56957928802589, - "abstract_algebra": 0.25252525252525254, - "professional_accounting": 0.35231316725978645, - "philosophy": 0.5193548387096775, - "professional_medicine": 0.25092250922509224, - "nutrition": 0.43934426229508194, - "global_facts": 0.29292929292929293, - "machine_learning": 0.40540540540540543, - "security_studies": 0.430327868852459, - "public_relations": 0.6055045871559633, - "professional_psychology": 0.45171849427168576, - "prehistory": 0.5077399380804953, - "anatomy": 0.4552238805970149, - "human_sexuality": 0.46153846153846156, - "college_medicine": 0.4011627906976744, - "high_school_government_and_politics": 0.59375, - "college_chemistry": 0.3838383838383838, - "logical_fallacies": 0.5308641975308642, - "high_school_geography": 0.6192893401015228, - "elementary_mathematics": 0.2572944297082228, - "human_aging": 0.5855855855855856, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.6452205882352942, - "formal_logic": 0.336, - "high_school_statistics": 0.27906976744186046, - "international_law": 0.5583333333333333, - "high_school_mathematics": 0.22676579925650558, - "high_school_computer_science": 0.43434343434343436, - "conceptual_physics": 0.405982905982906, - "miscellaneous": 0.6112531969309463, - "high_school_chemistry": 0.3910891089108911, - "marketing": 0.7682403433476395, - "professional_law": 0.33659491193737767, - "management": 0.6176470588235294, - "college_physics": 0.3069306930693069, - "jurisprudence": 0.5887850467289719, - "world_religions": 0.6176470588235294, - "sociology": 0.515, - "us_foreign_policy": 0.5555555555555556, - "high_school_macroeconomics": 0.43444730077120824, - "computer_security": 0.5858585858585859, - "moral_scenarios": 0.26174496644295303, - "moral_disputes": 0.4666666666666667, - "electrical_engineering": 0.4305555555555556, - "astronomy": 0.4503311258278146, - "college_biology": 0.5454545454545454 - } - }, - "prompt_2": { - "accuracy": 0.38941723274937434, - "category_acc": { - "high_school_european_history": 0.22560975609756098, - "business_ethics": 0.46464646464646464, - "clinical_knowledge": 0.4318181818181818, - "medical_genetics": 0.35353535353535354, - "high_school_us_history": 0.23645320197044334, - "high_school_physics": 0.36666666666666664, - "high_school_world_history": 0.2754237288135593, - "virology": 0.3696969696969697, - "high_school_microeconomics": 0.4472573839662447, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.36363636363636365, - "high_school_biology": 0.49838187702265374, - "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.3665480427046263, - "philosophy": 0.4096774193548387, - "professional_medicine": 0.2878228782287823, - "nutrition": 0.3639344262295082, - "global_facts": 0.25252525252525254, - "machine_learning": 0.34234234234234234, - "security_studies": 0.42213114754098363, - "public_relations": 0.5871559633027523, - "professional_psychology": 0.41734860883797054, - "prehistory": 0.4055727554179567, - "anatomy": 0.39552238805970147, - "human_sexuality": 0.4, - "college_medicine": 0.37790697674418605, - "high_school_government_and_politics": 0.5572916666666666, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.4876543209876543, - "high_school_geography": 0.5279187817258884, - "elementary_mathematics": 0.2546419098143236, - "human_aging": 0.4099099099099099, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.6029411764705882, - "formal_logic": 0.336, - "high_school_statistics": 0.27906976744186046, - "international_law": 0.6333333333333333, - "high_school_mathematics": 0.29739776951672864, - "high_school_computer_science": 0.41414141414141414, - "conceptual_physics": 0.3034188034188034, - "miscellaneous": 0.4194373401534527, - "high_school_chemistry": 0.24257425742574257, - "marketing": 0.7424892703862661, - "professional_law": 0.3431180691454664, - "management": 0.5588235294117647, - "college_physics": 0.297029702970297, - "jurisprudence": 0.5327102803738317, - "world_religions": 0.4117647058823529, - "sociology": 0.545, - "us_foreign_policy": 0.36363636363636365, - "high_school_macroeconomics": 0.39331619537275064, - "computer_security": 0.5151515151515151, - "moral_scenarios": 0.2595078299776286, - "moral_disputes": 0.4115942028985507, - "electrical_engineering": 0.2638888888888889, - "astronomy": 0.4900662251655629, - "college_biology": 0.44755244755244755 - } - }, - "prompt_3": { - "accuracy": 0.4371111905613157, - "category_acc": { - "high_school_european_history": 0.18902439024390244, - "business_ethics": 0.47474747474747475, - "clinical_knowledge": 0.5, - "medical_genetics": 0.46464646464646464, - "high_school_us_history": 0.22167487684729065, - "high_school_physics": 0.3466666666666667, - "high_school_world_history": 0.288135593220339, - "virology": 0.37575757575757573, - "high_school_microeconomics": 0.510548523206751, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.43434343434343436, - "high_school_biology": 0.6019417475728155, - "abstract_algebra": 0.35353535353535354, - "professional_accounting": 0.3309608540925267, - "philosophy": 0.5419354838709678, - "professional_medicine": 0.23985239852398524, - "nutrition": 0.42295081967213116, - "global_facts": 0.21212121212121213, - "machine_learning": 0.32432432432432434, - "security_studies": 0.45901639344262296, - "public_relations": 0.6238532110091743, - "professional_psychology": 0.43044189852700493, - "prehistory": 0.5139318885448917, - "anatomy": 0.48507462686567165, - "human_sexuality": 0.4846153846153846, - "college_medicine": 0.4127906976744186, - "high_school_government_and_politics": 0.59375, - "college_chemistry": 0.36363636363636365, - "logical_fallacies": 0.5370370370370371, - "high_school_geography": 0.6649746192893401, - "elementary_mathematics": 0.2519893899204244, - "human_aging": 0.6261261261261262, - "college_mathematics": 0.18181818181818182, - "high_school_psychology": 0.6544117647058824, - "formal_logic": 0.336, - "high_school_statistics": 0.2837209302325581, - "international_law": 0.6083333333333333, - "high_school_mathematics": 0.22676579925650558, - "high_school_computer_science": 0.45454545454545453, - "conceptual_physics": 0.4358974358974359, - "miscellaneous": 0.6445012787723785, - "high_school_chemistry": 0.33663366336633666, - "marketing": 0.7553648068669528, - "professional_law": 0.33985649054142203, - "management": 0.6274509803921569, - "college_physics": 0.26732673267326734, - "jurisprudence": 0.6448598130841121, - "world_religions": 0.6647058823529411, - "sociology": 0.52, - "us_foreign_policy": 0.6262626262626263, - "high_school_macroeconomics": 0.4498714652956298, - "computer_security": 0.5353535353535354, - "moral_scenarios": 0.22818791946308725, - "moral_disputes": 0.5246376811594203, - "electrical_engineering": 0.4375, - "astronomy": 0.46357615894039733, - "college_biology": 0.5664335664335665 - } - }, - "prompt_4": { - "accuracy": 0.44082946013585983, - "category_acc": { - "high_school_european_history": 0.29878048780487804, - "business_ethics": 0.5252525252525253, - "clinical_knowledge": 0.48484848484848486, - "medical_genetics": 0.46464646464646464, - "high_school_us_history": 0.28078817733990147, - "high_school_physics": 0.28, - "high_school_world_history": 0.2796610169491525, - "virology": 0.3878787878787879, - "high_school_microeconomics": 0.4936708860759494, - "econometrics": 0.34513274336283184, - "college_computer_science": 0.3939393939393939, - "high_school_biology": 0.5631067961165048, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.3487544483985765, - "philosophy": 0.5387096774193548, - "professional_medicine": 0.3025830258302583, - "nutrition": 0.4360655737704918, - "global_facts": 0.23232323232323232, - "machine_learning": 0.38738738738738737, - "security_studies": 0.45081967213114754, - "public_relations": 0.6330275229357798, - "professional_psychology": 0.44189852700491, - "prehistory": 0.5077399380804953, - "anatomy": 0.4925373134328358, - "human_sexuality": 0.5, - "college_medicine": 0.4127906976744186, - "high_school_government_and_politics": 0.6041666666666666, - "college_chemistry": 0.32323232323232326, - "logical_fallacies": 0.5185185185185185, - "high_school_geography": 0.6598984771573604, - "elementary_mathematics": 0.26525198938992045, - "human_aging": 0.6261261261261262, - "college_mathematics": 0.1717171717171717, - "high_school_psychology": 0.6544117647058824, - "formal_logic": 0.32, - "high_school_statistics": 0.28837209302325584, - "international_law": 0.6083333333333333, - "high_school_mathematics": 0.241635687732342, - "high_school_computer_science": 0.3838383838383838, - "conceptual_physics": 0.44017094017094016, - "miscellaneous": 0.639386189258312, - "high_school_chemistry": 0.3069306930693069, - "marketing": 0.7725321888412017, - "professional_law": 0.350293542074364, - "management": 0.6764705882352942, - "college_physics": 0.2871287128712871, - "jurisprudence": 0.6074766355140186, - "world_religions": 0.6705882352941176, - "sociology": 0.51, - "us_foreign_policy": 0.6161616161616161, - "high_school_macroeconomics": 0.4473007712082262, - "computer_security": 0.494949494949495, - "moral_scenarios": 0.24272930648769575, - "moral_disputes": 0.5217391304347826, - "electrical_engineering": 0.4583333333333333, - "astronomy": 0.4503311258278146, - "college_biology": 0.5524475524475524 - } - }, - "prompt_5": { - "accuracy": 0.4437611726850197, - "category_acc": { - "high_school_european_history": 0.21951219512195122, - "business_ethics": 0.494949494949495, - "clinical_knowledge": 0.49242424242424243, - "medical_genetics": 0.5252525252525253, - "high_school_us_history": 0.270935960591133, - "high_school_physics": 0.30666666666666664, - "high_school_world_history": 0.2923728813559322, - "virology": 0.3696969696969697, - "high_school_microeconomics": 0.47257383966244726, - "econometrics": 0.3008849557522124, - "college_computer_science": 0.40404040404040403, - "high_school_biology": 0.5792880258899676, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.40213523131672596, - "philosophy": 0.532258064516129, - "professional_medicine": 0.23985239852398524, - "nutrition": 0.4360655737704918, - "global_facts": 0.29292929292929293, - "machine_learning": 0.40540540540540543, - "security_studies": 0.4672131147540984, - "public_relations": 0.6788990825688074, - "professional_psychology": 0.4369885433715221, - "prehistory": 0.5139318885448917, - "anatomy": 0.5, - "human_sexuality": 0.5307692307692308, - "college_medicine": 0.38953488372093026, - "high_school_government_and_politics": 0.5885416666666666, - "college_chemistry": 0.3939393939393939, - "logical_fallacies": 0.5308641975308642, - "high_school_geography": 0.6294416243654822, - "elementary_mathematics": 0.2917771883289125, - "human_aging": 0.6126126126126126, - "college_mathematics": 0.20202020202020202, - "high_school_psychology": 0.65625, - "formal_logic": 0.328, - "high_school_statistics": 0.3302325581395349, - "international_law": 0.6166666666666667, - "high_school_mathematics": 0.2527881040892193, - "high_school_computer_science": 0.40404040404040403, - "conceptual_physics": 0.4358974358974359, - "miscellaneous": 0.6585677749360613, - "high_school_chemistry": 0.3217821782178218, - "marketing": 0.7553648068669528, - "professional_law": 0.33724722765818654, - "management": 0.6666666666666666, - "college_physics": 0.27722772277227725, - "jurisprudence": 0.6261682242990654, - "world_religions": 0.6647058823529411, - "sociology": 0.535, - "us_foreign_policy": 0.6565656565656566, - "high_school_macroeconomics": 0.42930591259640105, - "computer_security": 0.5656565656565656, - "moral_scenarios": 0.25279642058165547, - "moral_disputes": 0.518840579710145, - "electrical_engineering": 0.3958333333333333, - "astronomy": 0.4503311258278146, - "college_biology": 0.5874125874125874 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.3491827637444279 - }, - "prompt_2": { - "accuracy": 0.337295690936107 - }, - "prompt_3": { - "accuracy": 0.33060921248142644 - }, - "prompt_4": { - "accuracy": 0.32689450222882616 - }, - "prompt_5": { - "accuracy": 0.3350668647845468 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.34744707347447074, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.5833333333333334, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.40476190476190477, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.21428571428571427, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.5416666666666666, - "high_school_chemistry": 0.375, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.46153846153846156, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.24, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.3, - "business_administration": 0.21052631578947367, - "marxism": 0.375, - "mao_zedong_thought": 0.4827586206896552, - "education_science": 0.5882352941176471, - "teacher_qualification": 0.5510204081632653, - "high_school_politics": 0.25, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.34615384615384615, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.5, - "logic": 0.4444444444444444, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.25, - "high_school_history": 0.4, - "middle_school_history": 0.2222222222222222, - "civil_servant": 0.28846153846153844, - "sports_science": 0.3333333333333333, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.20833333333333334, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.43137254901960786, - "accountant": 0.3148148148148148, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.37037037037037035, - "physician": 0.3333333333333333 - } - }, - "prompt_2": { - "accuracy": 0.33748443337484435, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.4523809523809524, - "college_physics": 0.125, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.125, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.3333333333333333, - "business_administration": 0.18421052631578946, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.4482758620689655, - "education_science": 0.6470588235294118, - "teacher_qualification": 0.46938775510204084, - "high_school_politics": 0.25, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.4583333333333333, - "logic": 0.37037037037037035, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.25, - "high_school_chinese": 0.25, - "high_school_history": 0.24, - "middle_school_history": 0.2222222222222222, - "civil_servant": 0.40384615384615385, - "sports_science": 0.4166666666666667, - "plant_protection": 0.48148148148148145, - "basic_medicine": 0.25, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.39215686274509803, - "accountant": 0.3333333333333333, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.3148148148148148, - "physician": 0.3888888888888889 - } - }, - "prompt_3": { - "accuracy": 0.35118306351183065, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.4523809523809524, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.16666666666666666, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.4827586206896552, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.375, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.4166666666666667, - "middle_school_chemistry": 0.4, - "veterinary_medicine": 0.4642857142857143, - "college_economics": 0.35, - "business_administration": 0.21052631578947367, - "marxism": 0.375, - "mao_zedong_thought": 0.4827586206896552, - "education_science": 0.7352941176470589, - "teacher_qualification": 0.4897959183673469, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.4166666666666667, - "logic": 0.4074074074074074, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.25, - "high_school_chinese": 0.25, - "high_school_history": 0.2, - "middle_school_history": 0.18518518518518517, - "civil_servant": 0.28846153846153844, - "sports_science": 0.4166666666666667, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.2962962962962963, - "fire_engineer": 0.4166666666666667, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.35185185185185186, - "physician": 0.37037037037037035 - } - }, - "prompt_4": { - "accuracy": 0.3343711083437111, - "category_acc": { - "computer_network": 0.16666666666666666, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.47619047619047616, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.09523809523809523, - "electrical_engineer": 0.19047619047619047, - "metrology_engineer": 0.5172413793103449, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.35, - "business_administration": 0.18421052631578946, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.4827586206896552, - "education_science": 0.6176470588235294, - "teacher_qualification": 0.5714285714285714, - "high_school_politics": 0.16666666666666666, - "high_school_geography": 0.4583333333333333, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.5, - "logic": 0.25925925925925924, - "law": 0.1724137931034483, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.32, - "middle_school_history": 0.2222222222222222, - "civil_servant": 0.34615384615384615, - "sports_science": 0.20833333333333334, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.16666666666666666, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.25925925925925924, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.3888888888888889, - "physician": 0.4074074074074074 - } - }, - "prompt_5": { - "accuracy": 0.3318804483188045, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.5, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.38095238095238093, - "college_physics": 0.5, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.41379310344827586, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.125, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.4, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.3333333333333333, - "business_administration": 0.18421052631578946, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.4482758620689655, - "education_science": 0.6176470588235294, - "teacher_qualification": 0.42857142857142855, - "high_school_politics": 0.08333333333333333, - "high_school_geography": 0.5, - "middle_school_politics": 0.5, - "middle_school_geography": 0.11764705882352941, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.14814814814814814, - "law": 0.13793103448275862, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.21052631578947367, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.4, - "middle_school_history": 0.25925925925925924, - "civil_servant": 0.34615384615384615, - "sports_science": 0.3333333333333333, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.20833333333333334, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.37254901960784315, - "accountant": 0.2962962962962963, - "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.37037037037037035, - "physician": 0.37037037037037035 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.4050179211469534 - }, - "prompt_2": { - "accuracy": 0.4050179211469534 - }, - "prompt_3": { - "accuracy": 0.3978494623655914 - }, - "prompt_4": { - "accuracy": 0.3763440860215054 - }, - "prompt_5": { - "accuracy": 0.3727598566308244 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.3577965809013987, - "category_acc": { - "agronomy": 0.3727810650887574, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.2804878048780488, - "arts": 0.38125, - "astronomy": 0.2606060606060606, - "business_ethics": 0.4449760765550239, - "chinese_civil_service_exam": 0.325, - "chinese_driving_rule": 0.40458015267175573, - "chinese_food_culture": 0.29411764705882354, - "chinese_foreign_policy": 0.29906542056074764, - "chinese_history": 0.3157894736842105, - "chinese_literature": 0.27941176470588236, - "chinese_teacher_qualification": 0.41899441340782123, - "clinical_knowledge": 0.2616033755274262, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.48598130841121495, - "college_engineering_hydrology": 0.4528301886792453, - "college_law": 0.2962962962962963, - "college_mathematics": 0.2, - "college_medical_statistics": 0.3490566037735849, - "college_medicine": 0.26373626373626374, - "computer_science": 0.3627450980392157, - "computer_security": 0.4269005847953216, - "conceptual_physics": 0.3469387755102041, - "construction_project_management": 0.3597122302158273, - "economics": 0.4025157232704403, - "education": 0.4171779141104294, - "electrical_engineering": 0.4011627906976744, - "elementary_chinese": 0.27380952380952384, - "elementary_commonsense": 0.3333333333333333, - "elementary_information_and_technology": 0.5966386554621849, - "elementary_mathematics": 0.3, - "ethnology": 0.35555555555555557, - "food_science": 0.3916083916083916, - "genetics": 0.30113636363636365, - "global_facts": 0.348993288590604, - "high_school_biology": 0.31952662721893493, - "high_school_chemistry": 0.20454545454545456, - "high_school_geography": 0.3474576271186441, - "high_school_mathematics": 0.1951219512195122, - "high_school_physics": 0.21818181818181817, - "high_school_politics": 0.32867132867132864, - "human_sexuality": 0.3888888888888889, - "international_law": 0.3675675675675676, - "journalism": 0.38953488372093026, - "jurisprudence": 0.36982968369829683, - "legal_and_moral_basis": 0.6214953271028038, - "logical": 0.36585365853658536, - "machine_learning": 0.319672131147541, - "management": 0.43333333333333335, - "marketing": 0.5, - "marxist_theory": 0.4021164021164021, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.3310344827586207, - "philosophy": 0.5142857142857142, - "professional_accounting": 0.4057142857142857, - "professional_law": 0.2985781990521327, - "professional_medicine": 0.2632978723404255, - "professional_psychology": 0.4353448275862069, - "public_relations": 0.3735632183908046, - "security_study": 0.35555555555555557, - "sociology": 0.40707964601769914, - "sports_science": 0.44242424242424244, - "traditional_chinese_medicine": 0.2972972972972973, - "virology": 0.42011834319526625, - "world_history": 0.37888198757763975, - "world_religions": 0.375 - } - }, - "prompt_2": { - "accuracy": 0.35745121740632013, - "category_acc": { - "agronomy": 0.38461538461538464, - "anatomy": 0.24324324324324326, - "ancient_chinese": 0.31097560975609756, - "arts": 0.35625, - "astronomy": 0.2909090909090909, - "business_ethics": 0.4354066985645933, - "chinese_civil_service_exam": 0.25, - "chinese_driving_rule": 0.4351145038167939, - "chinese_food_culture": 0.3014705882352941, - "chinese_foreign_policy": 0.308411214953271, - "chinese_history": 0.3622291021671827, - "chinese_literature": 0.28921568627450983, - "chinese_teacher_qualification": 0.40782122905027934, - "clinical_knowledge": 0.3037974683544304, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.4766355140186916, - "college_engineering_hydrology": 0.36792452830188677, - "college_law": 0.25925925925925924, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.44339622641509435, - "college_medicine": 0.2857142857142857, - "computer_science": 0.37254901960784315, - "computer_security": 0.4093567251461988, - "conceptual_physics": 0.3741496598639456, - "construction_project_management": 0.3381294964028777, - "economics": 0.3836477987421384, - "education": 0.4233128834355828, - "electrical_engineering": 0.43023255813953487, - "elementary_chinese": 0.29365079365079366, - "elementary_commonsense": 0.3333333333333333, - "elementary_information_and_technology": 0.6134453781512605, - "elementary_mathematics": 0.2782608695652174, - "ethnology": 0.31851851851851853, - "food_science": 0.36363636363636365, - "genetics": 0.3125, - "global_facts": 0.3422818791946309, - "high_school_biology": 0.33727810650887574, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.288135593220339, - "high_school_mathematics": 0.21951219512195122, - "high_school_physics": 0.3090909090909091, - "high_school_politics": 0.3006993006993007, - "human_sexuality": 0.36507936507936506, - "international_law": 0.3081081081081081, - "journalism": 0.3953488372093023, - "jurisprudence": 0.36009732360097324, - "legal_and_moral_basis": 0.602803738317757, - "logical": 0.34959349593495936, - "machine_learning": 0.28688524590163933, - "management": 0.4142857142857143, - "marketing": 0.45, - "marxist_theory": 0.42328042328042326, - "modern_chinese": 0.29310344827586204, - "nutrition": 0.33793103448275863, - "philosophy": 0.4666666666666667, - "professional_accounting": 0.4114285714285714, - "professional_law": 0.3222748815165877, - "professional_medicine": 0.26063829787234044, - "professional_psychology": 0.43103448275862066, - "public_relations": 0.43103448275862066, - "security_study": 0.37777777777777777, - "sociology": 0.39823008849557523, - "sports_science": 0.40606060606060607, - "traditional_chinese_medicine": 0.32972972972972975, - "virology": 0.41420118343195267, - "world_history": 0.2857142857142857, - "world_religions": 0.36875 - } - }, - "prompt_3": { - "accuracy": 0.3552926955620791, - "category_acc": { - "agronomy": 0.3431952662721893, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.2804878048780488, - "arts": 0.35625, - "astronomy": 0.2606060606060606, - "business_ethics": 0.45454545454545453, - "chinese_civil_service_exam": 0.28125, - "chinese_driving_rule": 0.48091603053435117, - "chinese_food_culture": 0.2867647058823529, - "chinese_foreign_policy": 0.308411214953271, - "chinese_history": 0.3591331269349845, - "chinese_literature": 0.27941176470588236, - "chinese_teacher_qualification": 0.41899441340782123, - "clinical_knowledge": 0.27848101265822783, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.4672897196261682, - "college_engineering_hydrology": 0.36792452830188677, - "college_law": 0.24074074074074073, - "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.37735849056603776, - "college_medicine": 0.30036630036630035, - "computer_science": 0.35294117647058826, - "computer_security": 0.4327485380116959, - "conceptual_physics": 0.35374149659863946, - "construction_project_management": 0.31654676258992803, - "economics": 0.4025157232704403, - "education": 0.3987730061349693, - "electrical_engineering": 0.38372093023255816, - "elementary_chinese": 0.27380952380952384, - "elementary_commonsense": 0.3383838383838384, - "elementary_information_and_technology": 0.5756302521008403, - "elementary_mathematics": 0.23478260869565218, - "ethnology": 0.34814814814814815, - "food_science": 0.38461538461538464, - "genetics": 0.2897727272727273, - "global_facts": 0.33557046979865773, - "high_school_biology": 0.28402366863905326, - "high_school_chemistry": 0.3333333333333333, - "high_school_geography": 0.3305084745762712, - "high_school_mathematics": 0.1951219512195122, - "high_school_physics": 0.36363636363636365, - "high_school_politics": 0.32167832167832167, - "human_sexuality": 0.3333333333333333, - "international_law": 0.32972972972972975, - "journalism": 0.38953488372093026, - "jurisprudence": 0.3309002433090024, - "legal_and_moral_basis": 0.616822429906542, - "logical": 0.4065040650406504, - "machine_learning": 0.319672131147541, - "management": 0.42857142857142855, - "marketing": 0.4777777777777778, - "marxist_theory": 0.455026455026455, - "modern_chinese": 0.28448275862068967, - "nutrition": 0.3103448275862069, - "philosophy": 0.41904761904761906, - "professional_accounting": 0.41714285714285715, - "professional_law": 0.27488151658767773, - "professional_medicine": 0.27925531914893614, - "professional_psychology": 0.4353448275862069, - "public_relations": 0.39080459770114945, - "security_study": 0.3925925925925926, - "sociology": 0.42035398230088494, - "sports_science": 0.4, - "traditional_chinese_medicine": 0.2972972972972973, - "virology": 0.41420118343195267, - "world_history": 0.3167701863354037, - "world_religions": 0.4125 - } - }, - "prompt_4": { - "accuracy": 0.3514073562424452, - "category_acc": { - "agronomy": 0.39644970414201186, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.2682926829268293, - "arts": 0.4, - "astronomy": 0.2727272727272727, - "business_ethics": 0.40669856459330145, - "chinese_civil_service_exam": 0.34375, - "chinese_driving_rule": 0.3816793893129771, - "chinese_food_culture": 0.3088235294117647, - "chinese_foreign_policy": 0.2616822429906542, - "chinese_history": 0.3622291021671827, - "chinese_literature": 0.29901960784313725, - "chinese_teacher_qualification": 0.4245810055865922, - "clinical_knowledge": 0.2489451476793249, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.4205607476635514, - "college_engineering_hydrology": 0.37735849056603776, - "college_law": 0.28703703703703703, - "college_mathematics": 0.21904761904761905, - "college_medical_statistics": 0.33962264150943394, - "college_medicine": 0.29304029304029305, - "computer_science": 0.35294117647058826, - "computer_security": 0.4152046783625731, - "conceptual_physics": 0.40816326530612246, - "construction_project_management": 0.381294964028777, - "economics": 0.4339622641509434, - "education": 0.4233128834355828, - "electrical_engineering": 0.36627906976744184, - "elementary_chinese": 0.27380952380952384, - "elementary_commonsense": 0.3282828282828283, - "elementary_information_and_technology": 0.5756302521008403, - "elementary_mathematics": 0.2608695652173913, - "ethnology": 0.3037037037037037, - "food_science": 0.3916083916083916, - "genetics": 0.3068181818181818, - "global_facts": 0.35570469798657717, - "high_school_biology": 0.2485207100591716, - "high_school_chemistry": 0.2878787878787879, - "high_school_geography": 0.288135593220339, - "high_school_mathematics": 0.2073170731707317, - "high_school_physics": 0.3090909090909091, - "high_school_politics": 0.23776223776223776, - "human_sexuality": 0.35714285714285715, - "international_law": 0.34054054054054056, - "journalism": 0.36046511627906974, - "jurisprudence": 0.3309002433090024, - "legal_and_moral_basis": 0.6121495327102804, - "logical": 0.3983739837398374, - "machine_learning": 0.3524590163934426, - "management": 0.4142857142857143, - "marketing": 0.4666666666666667, - "marxist_theory": 0.42328042328042326, - "modern_chinese": 0.2672413793103448, - "nutrition": 0.35172413793103446, - "philosophy": 0.49523809523809526, - "professional_accounting": 0.42857142857142855, - "professional_law": 0.2938388625592417, - "professional_medicine": 0.27393617021276595, - "professional_psychology": 0.4396551724137931, - "public_relations": 0.3850574712643678, - "security_study": 0.37037037037037035, - "sociology": 0.3893805309734513, - "sports_science": 0.3878787878787879, - "traditional_chinese_medicine": 0.3081081081081081, - "virology": 0.3668639053254438, - "world_history": 0.2919254658385093, - "world_religions": 0.3625 - } - }, - "prompt_5": { - "accuracy": 0.3443274045933345, - "category_acc": { - "agronomy": 0.33136094674556216, - "anatomy": 0.25, - "ancient_chinese": 0.2804878048780488, - "arts": 0.35, - "astronomy": 0.24242424242424243, - "business_ethics": 0.430622009569378, - "chinese_civil_service_exam": 0.3125, - "chinese_driving_rule": 0.44274809160305345, - "chinese_food_culture": 0.29411764705882354, - "chinese_foreign_policy": 0.3364485981308411, - "chinese_history": 0.3065015479876161, - "chinese_literature": 0.30392156862745096, - "chinese_teacher_qualification": 0.39664804469273746, - "clinical_knowledge": 0.270042194092827, - "college_actuarial_science": 0.20754716981132076, - "college_education": 0.45794392523364486, - "college_engineering_hydrology": 0.2641509433962264, - "college_law": 0.2962962962962963, - "college_mathematics": 0.21904761904761905, - "college_medical_statistics": 0.3113207547169811, - "college_medicine": 0.304029304029304, - "computer_science": 0.35294117647058826, - "computer_security": 0.4093567251461988, - "conceptual_physics": 0.3333333333333333, - "construction_project_management": 0.34532374100719426, - "economics": 0.3836477987421384, - "education": 0.4294478527607362, - "electrical_engineering": 0.36627906976744184, - "elementary_chinese": 0.26587301587301587, - "elementary_commonsense": 0.31313131313131315, - "elementary_information_and_technology": 0.5840336134453782, - "elementary_mathematics": 0.26521739130434785, - "ethnology": 0.3037037037037037, - "food_science": 0.3986013986013986, - "genetics": 0.32386363636363635, - "global_facts": 0.3624161073825503, - "high_school_biology": 0.3254437869822485, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.25, - "high_school_physics": 0.34545454545454546, - "high_school_politics": 0.2867132867132867, - "human_sexuality": 0.35714285714285715, - "international_law": 0.31891891891891894, - "journalism": 0.37209302325581395, - "jurisprudence": 0.3309002433090024, - "legal_and_moral_basis": 0.5607476635514018, - "logical": 0.35772357723577236, - "machine_learning": 0.3524590163934426, - "management": 0.4142857142857143, - "marketing": 0.4777777777777778, - "marxist_theory": 0.42857142857142855, - "modern_chinese": 0.2672413793103448, - "nutrition": 0.3103448275862069, - "philosophy": 0.4666666666666667, - "professional_accounting": 0.4, - "professional_law": 0.2843601895734597, - "professional_medicine": 0.26861702127659576, - "professional_psychology": 0.4051724137931034, - "public_relations": 0.39080459770114945, - "security_study": 0.34814814814814815, - "sociology": 0.3893805309734513, - "sports_science": 0.38181818181818183, - "traditional_chinese_medicine": 0.2810810810810811, - "virology": 0.3905325443786982, - "world_history": 0.2919254658385093, - "world_religions": 0.35625 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.21212121212121213 - }, - "prompt_2": { - "accuracy": 0.18181818181818182 - }, - "prompt_3": { - "accuracy": 0.2727272727272727 - }, - "prompt_4": { - "accuracy": 0.24242424242424243 - }, - "prompt_5": { - "accuracy": 0.2727272727272727 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.4590909090909091 - }, - "prompt_2": { - "accuracy": 0.43636363636363634 - }, - "prompt_3": { - "accuracy": 0.4068181818181818 - }, - "prompt_4": { - "accuracy": 0.42727272727272725 - }, - "prompt_5": { - "accuracy": 0.42954545454545456 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3783050847457627 - }, - "prompt_2": { - "accuracy": 0.36542372881355933 - }, - "prompt_3": { - "accuracy": 0.3657627118644068 - }, - "prompt_4": { - "accuracy": 0.33728813559322035 - }, - "prompt_5": { - "accuracy": 0.37457627118644066 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.587135377711294 - }, - "prompt_2": { - "accuracy": 0.6103216155572176 - }, - "prompt_3": { - "accuracy": 0.581151832460733 - }, - "prompt_4": { - "accuracy": 0.5815258040388931 - }, - "prompt_5": { - "accuracy": 0.6020942408376964 - } - }, - "dream": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "samsum": { - "prompt_1": { - "rouge1": 0.15389877113313594, - "rouge2": 0.05727857183008742, - "rougeL": 0.11648435497304192, - "avg_rouge": 0.10922056597875508 - }, - "prompt_2": { - "rouge1": 0.15602369438466948, - "rouge2": 0.05498223992166002, - "rougeL": 0.11715064247949278, - "avg_rouge": 0.10938552559527409 - }, - "prompt_3": { - "rouge1": 0.15677195605313232, - "rouge2": 0.05502129126626263, - "rougeL": 0.11668319751480999, - "avg_rouge": 0.1094921482780683 - }, - "prompt_4": { - "rouge1": 0.14933593363778747, - "rouge2": 0.05102144111638376, - "rougeL": 0.11000934660045925, - "avg_rouge": 0.10345557378487684 - }, - "prompt_5": { - "rouge1": 0.1548081416063613, - "rouge2": 0.05314660241435149, - "rougeL": 0.11530996015597572, - "avg_rouge": 0.1077549013922295 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.14197960895649211, - "rouge2": 0.04249189635338857, - "rougeL": 0.10672450032909402, - "avg_rouge": 0.09706533521299156 - }, - "prompt_2": { - "rouge1": 0.15562557935372212, - "rouge2": 0.049191683362015466, - "rougeL": 0.11984095072605849, - "avg_rouge": 0.10821940448059869 - }, - "prompt_3": { - "rouge1": 0.15203907428391147, - "rouge2": 0.04696273491114106, - "rougeL": 0.11764385924486252, - "avg_rouge": 0.10554855614663834 - }, - "prompt_4": { - "rouge1": 0.1362087169552435, - "rouge2": 0.03934969945702414, - "rougeL": 0.10212088781063179, - "avg_rouge": 0.09255976807429982 - }, - "prompt_5": { - "rouge1": 0.15722853362625938, - "rouge2": 0.03903219436388582, - "rougeL": 0.11675411118981595, - "avg_rouge": 0.10433827972665372 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.8681192660550459 - }, - "prompt_2": { - "accuracy": 0.8669724770642202 - }, - "prompt_3": { - "accuracy": 0.8738532110091743 - }, - "prompt_4": { - "accuracy": 0.8543577981651376 - }, - "prompt_5": { - "accuracy": 0.8612385321100917 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.5819750719079578 - }, - "prompt_2": { - "accuracy": 0.585810162991371 - }, - "prompt_3": { - "accuracy": 0.5992329817833174 - }, - "prompt_4": { - "accuracy": 0.5896452540747843 - }, - "prompt_5": { - "accuracy": 0.5656759348034516 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.4905 - }, - "prompt_2": { - "accuracy": 0.496 - }, - "prompt_3": { - "accuracy": 0.5025 - }, - "prompt_4": { - "accuracy": 0.492 - }, - "prompt_5": { - "accuracy": 0.5025 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.486 - }, - "prompt_2": { - "accuracy": 0.4815 - }, - "prompt_3": { - "accuracy": 0.494 - }, - "prompt_4": { - "accuracy": 0.4595 - }, - "prompt_5": { - "accuracy": 0.4805 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.5375 - }, - "prompt_2": { - "accuracy": 0.555 - }, - "prompt_3": { - "accuracy": 0.556 - }, - "prompt_4": { - "accuracy": 0.5455 - }, - "prompt_5": { - "accuracy": 0.5565 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.49295774647887325 - }, - "prompt_2": { - "accuracy": 0.5915492957746479 - }, - "prompt_3": { - "accuracy": 0.43661971830985913 - }, - "prompt_4": { - "accuracy": 0.49295774647887325 - }, - "prompt_5": { - "accuracy": 0.43661971830985913 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.5018050541516246 - }, - "prompt_2": { - "accuracy": 0.4729241877256318 - }, - "prompt_3": { - "accuracy": 0.5703971119133574 - }, - "prompt_4": { - "accuracy": 0.5054151624548736 - }, - "prompt_5": { - "accuracy": 0.4620938628158845 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.5294117647058824 - }, - "prompt_2": { - "accuracy": 0.4877450980392157 - }, - "prompt_3": { - "accuracy": 0.5465686274509803 - }, - "prompt_4": { - "accuracy": 0.46568627450980393 - }, - "prompt_5": { - "accuracy": 0.4681372549019608 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.2485294117647059, - "language_acc": { - "English": 0.253781512605042, - "Chinese": 0.23781512605042016, - "Spanish": 0.25630252100840334, - "Vietnamese": 0.246218487394958 - }, - "consistency_score_2": 0.2544817927170868, - "consistency_score_3": 0.06596638655462185, - "consistency_score_4": 0.01680672268907563, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.253781512605042, - "English,Spanish": 0.2672268907563025, - "English,Vietnamese": 0.25126050420168067, - "Chinese,Spanish": 0.26134453781512607, - "Chinese,Vietnamese": 0.24033613445378152, - "Spanish,Vietnamese": 0.2529411764705882 - }, - "3_combine": { - "English,Chinese,Spanish": 0.073109243697479, - "English,Chinese,Vietnamese": 0.06134453781512605, - "English,Spanish,Vietnamese": 0.07142857142857142, - "Chinese,Spanish,Vietnamese": 0.05798319327731093 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.01680672268907563 - } - }, - "AC3_2": 0.25147038345354067, - "AC3_3": 0.10425949935778743, - "AC3_4": 0.03148432768595914 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.26571428571428574, - "language_acc": { - "Spanish": 0.2733333333333333, - "English": 0.26666666666666666, - "Chinese": 0.26, - "Vietnamese": 0.24666666666666667, - "Filipino": 0.26, - "Indonesian": 0.3, - "Malay": 0.25333333333333335 - }, - "consistency_score_2": 0.25206349206349205, - "consistency_score_3": 0.06342857142857145, - "consistency_score_4": 0.01580952380952381, - "consistency_score_5": 0.0031746031746031746, - "consistency_score_6": 0.0, - "consistency_score_7": 0.0, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.28, - "Spanish,Chinese": 0.24666666666666667, - "Spanish,Vietnamese": 0.25333333333333335, - "Spanish,Filipino": 0.2, - "Spanish,Indonesian": 0.25333333333333335, - "Spanish,Malay": 0.24, - "English,Chinese": 0.29333333333333333, - "English,Vietnamese": 0.32, - "English,Filipino": 0.22, - "English,Indonesian": 0.22, - "English,Malay": 0.19333333333333333, - "Chinese,Vietnamese": 0.30666666666666664, - "Chinese,Filipino": 0.20666666666666667, - "Chinese,Indonesian": 0.25333333333333335, - "Chinese,Malay": 0.30666666666666664, - "Vietnamese,Filipino": 0.21333333333333335, - "Vietnamese,Indonesian": 0.30666666666666664, - "Vietnamese,Malay": 0.21333333333333335, - "Filipino,Indonesian": 0.22, - "Filipino,Malay": 0.26666666666666666, - "Indonesian,Malay": 0.28 - }, - "3_combine": { - "Spanish,English,Chinese": 0.09333333333333334, - "Spanish,English,Vietnamese": 0.07333333333333333, - "Spanish,English,Filipino": 0.04, - "Spanish,English,Indonesian": 0.04, - "Spanish,English,Malay": 0.04666666666666667, - "Spanish,Chinese,Vietnamese": 0.08, - "Spanish,Chinese,Filipino": 0.02, - "Spanish,Chinese,Indonesian": 0.05333333333333334, - "Spanish,Chinese,Malay": 0.07333333333333333, - "Spanish,Vietnamese,Filipino": 0.03333333333333333, - "Spanish,Vietnamese,Indonesian": 0.08, - "Spanish,Vietnamese,Malay": 0.06666666666666667, - "Spanish,Filipino,Indonesian": 0.02666666666666667, - "Spanish,Filipino,Malay": 0.05333333333333334, - "Spanish,Indonesian,Malay": 0.08, - "English,Chinese,Vietnamese": 0.11333333333333333, - "English,Chinese,Filipino": 0.04666666666666667, - "English,Chinese,Indonesian": 0.07333333333333333, - "English,Chinese,Malay": 0.08, - "English,Vietnamese,Filipino": 0.06, - "English,Vietnamese,Indonesian": 0.07333333333333333, - "English,Vietnamese,Malay": 0.05333333333333334, - "English,Filipino,Indonesian": 0.03333333333333333, - "English,Filipino,Malay": 0.06666666666666667, - "English,Indonesian,Malay": 0.04666666666666667, - "Chinese,Vietnamese,Filipino": 0.06, - "Chinese,Vietnamese,Indonesian": 0.10666666666666667, - "Chinese,Vietnamese,Malay": 0.1, - "Chinese,Filipino,Indonesian": 0.04, - "Chinese,Filipino,Malay": 0.06666666666666667, - "Chinese,Indonesian,Malay": 0.1, - "Vietnamese,Filipino,Indonesian": 0.05333333333333334, - "Vietnamese,Filipino,Malay": 0.04, - "Vietnamese,Indonesian,Malay": 0.08, - "Filipino,Indonesian,Malay": 0.06666666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.02666666666666667, - "Spanish,English,Chinese,Filipino": 0.006666666666666667, - "Spanish,English,Chinese,Indonesian": 0.02, - "Spanish,English,Chinese,Malay": 0.02, - "Spanish,English,Vietnamese,Filipino": 0.006666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,English,Vietnamese,Malay": 0.013333333333333334, - "Spanish,English,Filipino,Indonesian": 0.0, - "Spanish,English,Filipino,Malay": 0.006666666666666667, - "Spanish,English,Indonesian,Malay": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Filipino": 0.0, - "Spanish,Chinese,Vietnamese,Indonesian": 0.03333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.02666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.0, - "Spanish,Chinese,Filipino,Malay": 0.0, - "Spanish,Chinese,Indonesian,Malay": 0.02666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.006666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.02666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.013333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.02, - "English,Chinese,Vietnamese,Indonesian": 0.03333333333333333, - "English,Chinese,Vietnamese,Malay": 0.03333333333333333, - "English,Chinese,Filipino,Indonesian": 0.006666666666666667, - "English,Chinese,Filipino,Malay": 0.02, - "English,Chinese,Indonesian,Malay": 0.02666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.0, - "English,Vietnamese,Filipino,Malay": 0.02, - "English,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "English,Filipino,Indonesian,Malay": 0.013333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.02, - "Chinese,Vietnamese,Indonesian,Malay": 0.03333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.02666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.013333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.013333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "English,Chinese,Vietnamese,Filipino,Malay": 0.013333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "English,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.0, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.0, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.0, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 - } - }, - "AC3_2": 0.2587089427546244, - "AC3_3": 0.10241071425459994, - "AC3_4": 0.029843417735388116, - "AC3_5": 0.0062742452329508, - "AC3_6": 0.0, - "AC3_7": 0.0 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.24269480519480519, - "language_acc": { - "English": 0.20454545454545456, - "Filipino": 0.26704545454545453, - "Spanish": 0.24431818181818182, - "Malay": 0.2215909090909091, - "Chinese": 0.20454545454545456, - "Indonesian": 0.26136363636363635, - "Vietnamese": 0.29545454545454547 - }, - "consistency_score_2": 0.26488095238095233, - "consistency_score_3": 0.07451298701298702, - "consistency_score_4": 0.023538961038961036, - "consistency_score_5": 0.008928571428571428, - "consistency_score_6": 0.003246753246753247, - "consistency_score_7": 0.0, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.32954545454545453, - "English,Spanish": 0.26704545454545453, - "English,Malay": 0.19318181818181818, - "English,Chinese": 0.29545454545454547, - "English,Indonesian": 0.23863636363636365, - "English,Vietnamese": 0.2727272727272727, - "Filipino,Spanish": 0.3409090909090909, - "Filipino,Malay": 0.23863636363636365, - "Filipino,Chinese": 0.25, - "Filipino,Indonesian": 0.2215909090909091, - "Filipino,Vietnamese": 0.23295454545454544, - "Spanish,Malay": 0.25, - "Spanish,Chinese": 0.26136363636363635, - "Spanish,Indonesian": 0.26704545454545453, - "Spanish,Vietnamese": 0.30113636363636365, - "Malay,Chinese": 0.2556818181818182, - "Malay,Indonesian": 0.3068181818181818, - "Malay,Vietnamese": 0.26704545454545453, - "Chinese,Indonesian": 0.2840909090909091, - "Chinese,Vietnamese": 0.23863636363636365, - "Indonesian,Vietnamese": 0.25 - }, - "3_combine": { - "English,Filipino,Spanish": 0.11363636363636363, - "English,Filipino,Malay": 0.06818181818181818, - "English,Filipino,Chinese": 0.11363636363636363, - "English,Filipino,Indonesian": 0.0625, - "English,Filipino,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay": 0.05113636363636364, - "English,Spanish,Chinese": 0.08522727272727272, - "English,Spanish,Indonesian": 0.06818181818181818, - "English,Spanish,Vietnamese": 0.09659090909090909, - "English,Malay,Chinese": 0.05113636363636364, - "English,Malay,Indonesian": 0.06818181818181818, - "English,Malay,Vietnamese": 0.056818181818181816, - "English,Chinese,Indonesian": 0.06818181818181818, - "English,Chinese,Vietnamese": 0.0625, - "English,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay": 0.09659090909090909, - "Filipino,Spanish,Chinese": 0.09090909090909091, - "Filipino,Spanish,Indonesian": 0.0625, - "Filipino,Spanish,Vietnamese": 0.09090909090909091, - "Filipino,Malay,Chinese": 0.045454545454545456, - "Filipino,Malay,Indonesian": 0.0625, - "Filipino,Malay,Vietnamese": 0.06818181818181818, - "Filipino,Chinese,Indonesian": 0.0625, - "Filipino,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Indonesian,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Chinese": 0.06818181818181818, - "Spanish,Malay,Indonesian": 0.07386363636363637, - "Spanish,Malay,Vietnamese": 0.08522727272727272, - "Spanish,Chinese,Indonesian": 0.07386363636363637, - "Spanish,Chinese,Vietnamese": 0.06818181818181818, - "Spanish,Indonesian,Vietnamese": 0.06818181818181818, - "Malay,Chinese,Indonesian": 0.09659090909090909, - "Malay,Chinese,Vietnamese": 0.07954545454545454, - "Malay,Indonesian,Vietnamese": 0.07386363636363637, - "Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.028409090909090908, - "English,Filipino,Spanish,Chinese": 0.03977272727272727, - "English,Filipino,Spanish,Indonesian": 0.022727272727272728, - "English,Filipino,Spanish,Vietnamese": 0.045454545454545456, - "English,Filipino,Malay,Chinese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian": 0.011363636363636364, - "English,Filipino,Malay,Vietnamese": 0.022727272727272728, - "English,Filipino,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Indonesian,Vietnamese": 0.022727272727272728, - "English,Spanish,Malay,Chinese": 0.017045454545454544, - "English,Spanish,Malay,Indonesian": 0.017045454545454544, - "English,Spanish,Malay,Vietnamese": 0.028409090909090908, - "English,Spanish,Chinese,Indonesian": 0.011363636363636364, - "English,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Spanish,Indonesian,Vietnamese": 0.022727272727272728, - "English,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Malay,Indonesian,Vietnamese": 0.017045454545454544, - "English,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese": 0.03409090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.022727272727272728, - "Filipino,Spanish,Malay,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Chinese,Indonesian": 0.017045454545454544, - "Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Malay,Chinese,Indonesian": 0.017045454545454544, - "Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.017045454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 - } - }, - "AC3_2": 0.25330300025469493, - "AC3_3": 0.11401936084882049, - "AC3_4": 0.04291554480491312, - "AC3_5": 0.017223502297302507, - "AC3_6": 0.006407783632900999, - "AC3_7": 0.0 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.2815533980582524 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.2571428571428571 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.24299065420560748 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.24, - "category_acc": { - "brand": 0.4, - "demographics": 0.6, - "biology": 0.1, - "history": 0.06666666666666667, - "literature": 0.2, - "politics": 0.4, - "culture": 0.1, - "film": 0.3, - "law": 0.1, - "geography": 0.4 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.0014052738317716826 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.0013820347660288234 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.0012878439719912157 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.0015370557254796619 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.0012883686712926471 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.23453908984830804 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.2528423310690025, - "category_acc": { - "high_school_european_history": 0.24390243902439024, - "business_ethics": 0.29292929292929293, - "clinical_knowledge": 0.25757575757575757, - "medical_genetics": 0.29292929292929293, - "high_school_us_history": 0.28078817733990147, - "high_school_physics": 0.25333333333333335, - "high_school_world_history": 0.2627118644067797, - "virology": 0.30303030303030304, - "high_school_microeconomics": 0.21940928270042195, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.2222222222222222, - "high_school_biology": 0.28802588996763756, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.2526690391459075, - "philosophy": 0.22580645161290322, - "professional_medicine": 0.2767527675276753, - "nutrition": 0.29508196721311475, - "global_facts": 0.30303030303030304, - "machine_learning": 0.3063063063063063, - "security_studies": 0.25, - "public_relations": 0.30275229357798167, - "professional_psychology": 0.24713584288052373, - "prehistory": 0.23529411764705882, - "anatomy": 0.291044776119403, - "human_sexuality": 0.34615384615384615, - "college_medicine": 0.2441860465116279, - "high_school_government_and_politics": 0.21354166666666666, - "college_chemistry": 0.21212121212121213, - "logical_fallacies": 0.24691358024691357, - "high_school_geography": 0.233502538071066, - "elementary_mathematics": 0.259946949602122, - "human_aging": 0.21171171171171171, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.26286764705882354, - "formal_logic": 0.24, - "high_school_statistics": 0.22790697674418606, - "international_law": 0.24166666666666667, - "high_school_mathematics": 0.22304832713754646, - "high_school_computer_science": 0.26262626262626265, - "conceptual_physics": 0.21367521367521367, - "miscellaneous": 0.26342710997442453, - "high_school_chemistry": 0.29207920792079206, - "marketing": 0.19742489270386265, - "professional_law": 0.24396607958251793, - "management": 0.18627450980392157, - "college_physics": 0.26732673267326734, - "jurisprudence": 0.2897196261682243, - "world_religions": 0.21176470588235294, - "sociology": 0.255, - "us_foreign_policy": 0.30303030303030304, - "high_school_macroeconomics": 0.1953727506426735, - "computer_security": 0.2222222222222222, - "moral_scenarios": 0.2785234899328859, - "moral_disputes": 0.24347826086956523, - "electrical_engineering": 0.2013888888888889, - "astronomy": 0.304635761589404, - "college_biology": 0.22377622377622378 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.2526002971768202 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.261519302615193, - "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.15384615384615385, - "college_programming": 0.2857142857142857, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.25, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.125, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.19230769230769232, - "middle_school_physics": 0.16666666666666666, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.2833333333333333, - "business_administration": 0.21052631578947367, - "marxism": 0.08333333333333333, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.29411764705882354, - "teacher_qualification": 0.32653061224489793, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.15384615384615385, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.20833333333333334, - "logic": 0.2962962962962963, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.15789473684210525, - "professional_tour_guide": 0.14705882352941177, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.25, - "high_school_history": 0.2, - "middle_school_history": 0.2222222222222222, - "civil_servant": 0.25, - "sports_science": 0.25, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.25, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.23529411764705882, - "accountant": 0.2222222222222222, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.3333333333333333, - "physician": 0.24074074074074073 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.2974910394265233 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.2582455534450009, - "category_acc": { - "agronomy": 0.2958579881656805, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.25, - "arts": 0.275, - "astronomy": 0.2909090909090909, - "business_ethics": 0.3014354066985646, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.2595419847328244, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.29906542056074764, - "chinese_history": 0.2755417956656347, - "chinese_literature": 0.25980392156862747, - "chinese_teacher_qualification": 0.2737430167597765, - "clinical_knowledge": 0.23628691983122363, - "college_actuarial_science": 0.2169811320754717, - "college_education": 0.27102803738317754, - "college_engineering_hydrology": 0.3490566037735849, - "college_law": 0.3055555555555556, - "college_mathematics": 0.29523809523809524, - "college_medical_statistics": 0.24528301886792453, - "college_medicine": 0.2857142857142857, - "computer_science": 0.2647058823529412, - "computer_security": 0.25146198830409355, - "conceptual_physics": 0.2857142857142857, - "construction_project_management": 0.23741007194244604, - "economics": 0.2641509433962264, - "education": 0.22699386503067484, - "electrical_engineering": 0.21511627906976744, - "elementary_chinese": 0.25, - "elementary_commonsense": 0.3282828282828283, - "elementary_information_and_technology": 0.24789915966386555, - "elementary_mathematics": 0.24347826086956523, - "ethnology": 0.26666666666666666, - "food_science": 0.3076923076923077, - "genetics": 0.29545454545454547, - "global_facts": 0.21476510067114093, - "high_school_biology": 0.3136094674556213, - "high_school_chemistry": 0.2196969696969697, - "high_school_geography": 0.23728813559322035, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.2, - "high_school_politics": 0.23076923076923078, - "human_sexuality": 0.23015873015873015, - "international_law": 0.31891891891891894, - "journalism": 0.2558139534883721, - "jurisprudence": 0.25790754257907544, - "legal_and_moral_basis": 0.2570093457943925, - "logical": 0.2926829268292683, - "machine_learning": 0.22131147540983606, - "management": 0.2571428571428571, - "marketing": 0.18888888888888888, - "marxist_theory": 0.2328042328042328, - "modern_chinese": 0.1810344827586207, - "nutrition": 0.2413793103448276, - "philosophy": 0.3142857142857143, - "professional_accounting": 0.26857142857142857, - "professional_law": 0.2037914691943128, - "professional_medicine": 0.2553191489361702, - "professional_psychology": 0.23275862068965517, - "public_relations": 0.2471264367816092, - "security_study": 0.2074074074074074, - "sociology": 0.23893805309734514, - "sports_science": 0.30303030303030304, - "traditional_chinese_medicine": 0.24324324324324326, - "virology": 0.23076923076923078, - "world_history": 0.2732919254658385, - "world_religions": 0.275 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.30303030303030304 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.18181818181818182 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3467796610169491 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.2629020194465221 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.3238608525232729 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.0021507616397528984, - "rouge2": 6.805852039198668e-05, - "rougeL": 0.00182156377897003, - "avg_rouge": 0.0013467946463716384 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.004787471188829816, - "rouge2": 0.00010416587105252379, - "rougeL": 0.004262908833934965, - "avg_rouge": 0.0030515152979391016 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.536697247706422 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.5551294343240653 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.4935 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.3355 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.497 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.5352112676056338 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.49458483754512633 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.47794117647058826 - } - } - } - }, - "bloomz_7b1": { - "model_size": "7.1B", - "model_link": "https://huggingface.co/bigscience/bloomz-7b1", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.9523109243697478, - "language_acc": { - "English": 0.9689075630252101, - "Chinese": 0.9378151260504202, - "Spanish": 0.9537815126050421, - "Vietnamese": 0.9487394957983193 - }, - "consistency_score_2": 0.9380952380952382, - "consistency_score_3": 0.9092436974789916, - "consistency_score_4": 0.8873949579831932, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.9310924369747899, - "English,Spanish": 0.9571428571428572, - "English,Vietnamese": 0.9563025210084034, - "Chinese,Spanish": 0.9193277310924369, - "Chinese,Vietnamese": 0.9201680672268907, - "Spanish,Vietnamese": 0.9445378151260504 - }, - "3_combine": { - "English,Chinese,Spanish": 0.9058823529411765, - "English,Chinese,Vietnamese": 0.9067226890756303, - "English,Spanish,Vietnamese": 0.9294117647058824, - "Chinese,Spanish,Vietnamese": 0.8949579831932774 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8873949579831932 - } - }, - "AC3_2": 0.945149630834029, - "AC3_3": 0.9302791288682681, - "AC3_4": 0.9187076268857285 - }, - "prompt_2": { - "overall_acc": 0.953781512605042, - "language_acc": { - "English": 0.9647058823529412, - "Chinese": 0.9403361344537815, - "Spanish": 0.9588235294117647, - "Vietnamese": 0.9512605042016806 - }, - "consistency_score_2": 0.9411764705882354, - "consistency_score_3": 0.9140756302521008, - "consistency_score_4": 0.8932773109243698, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.938655462184874, - "English,Spanish": 0.9546218487394958, - "English,Vietnamese": 0.9588235294117647, - "Chinese,Spanish": 0.9243697478991597, - "Chinese,Vietnamese": 0.9218487394957983, - "Spanish,Vietnamese": 0.9487394957983193 - }, - "3_combine": { - "English,Chinese,Spanish": 0.9117647058823529, - "English,Chinese,Vietnamese": 0.9117647058823529, - "English,Spanish,Vietnamese": 0.9319327731092437, - "Chinese,Spanish,Vietnamese": 0.9008403361344538 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8932773109243698 - } - }, - "AC3_2": 0.9474370679035695, - "AC3_3": 0.9335065484472754, - "AC3_4": 0.9225384420784973 - }, - "prompt_3": { - "overall_acc": 0.9516806722689075, - "language_acc": { - "English": 0.9680672268907563, - "Chinese": 0.9394957983193277, - "Spanish": 0.9546218487394958, - "Vietnamese": 0.9445378151260504 - }, - "consistency_score_2": 0.9385154061624651, - "consistency_score_3": 0.9100840336134454, - "consistency_score_4": 0.8890756302521008, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.9378151260504202, - "English,Spanish": 0.9621848739495799, - "English,Vietnamese": 0.9495798319327731, - "Chinese,Spanish": 0.9226890756302522, - "Chinese,Vietnamese": 0.9100840336134454, - "Spanish,Vietnamese": 0.9487394957983193 - }, - "3_combine": { - "English,Chinese,Spanish": 0.9142857142857143, - "English,Chinese,Vietnamese": 0.9008403361344538, - "English,Spanish,Vietnamese": 0.9319327731092437, - "Chinese,Spanish,Vietnamese": 0.8932773109243698 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8890756302521008 - } - }, - "AC3_2": 0.9450521909508841, - "AC3_3": 0.9304176646462352, - "AC3_4": 0.9193135368233782 - }, - "prompt_4": { - "overall_acc": 0.9493697478991596, - "language_acc": { - "English": 0.9638655462184874, - "Chinese": 0.9403361344537815, - "Spanish": 0.9512605042016806, - "Vietnamese": 0.9420168067226891 - }, - "consistency_score_2": 0.9368347338935575, - "consistency_score_3": 0.9077731092436975, - "consistency_score_4": 0.8865546218487395, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.9327731092436975, - "English,Spanish": 0.9571428571428572, - "English,Vietnamese": 0.9487394957983193, - "Chinese,Spanish": 0.9226890756302522, - "Chinese,Vietnamese": 0.9235294117647059, - "Spanish,Vietnamese": 0.9361344537815126 - }, - "3_combine": { - "English,Chinese,Spanish": 0.9100840336134454, - "English,Chinese,Vietnamese": 0.9042016806722689, - "English,Spanish,Vietnamese": 0.9226890756302522, - "Chinese,Spanish,Vietnamese": 0.8941176470588236 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.8865546218487395 - } - }, - "AC3_2": 0.9430605893240478, - "AC3_3": 0.9281055838125293, - "AC3_4": 0.9168875926115653 - }, - "prompt_5": { - "overall_acc": 0.9478991596638655, - "language_acc": { - "English": 0.9621848739495799, - "Chinese": 0.9369747899159664, - "Spanish": 0.9512605042016806, - "Vietnamese": 0.9411764705882353 - }, - "consistency_score_2": 0.9375350140056021, - "consistency_score_3": 0.9081932773109244, - "consistency_score_4": 0.888235294117647, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.9319327731092437, - "English,Spanish": 0.9588235294117647, - "English,Vietnamese": 0.9470588235294117, - "Chinese,Spanish": 0.9218487394957983, - "Chinese,Vietnamese": 0.9218487394957983, - "Spanish,Vietnamese": 0.9436974789915966 - }, - "3_combine": { - "English,Chinese,Spanish": 0.9084033613445378, - "English,Chinese,Vietnamese": 0.9025210084033614, - "English,Spanish,Vietnamese": 0.9260504201680673, - "Chinese,Spanish,Vietnamese": 0.8957983193277311 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.888235294117647 - } - }, - "AC3_2": 0.9426886011667471, - "AC3_3": 0.9276215205446784, - "AC3_4": 0.917097859688914 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.39428571428571424, - "language_acc": { - "Spanish": 0.46, - "English": 0.4666666666666667, - "Chinese": 0.44666666666666666, - "Vietnamese": 0.36666666666666664, - "Filipino": 0.26, - "Indonesian": 0.38, - "Malay": 0.38 - }, - "consistency_score_2": 0.6187301587301588, - "consistency_score_3": 0.4659047619047619, - "consistency_score_4": 0.3761904761904762, - "consistency_score_5": 0.31428571428571417, - "consistency_score_6": 0.26857142857142857, - "consistency_score_7": 0.23333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.7666666666666667, - "Spanish,Chinese": 0.6333333333333333, - "Spanish,Vietnamese": 0.6333333333333333, - "Spanish,Filipino": 0.48, - "Spanish,Indonesian": 0.7266666666666667, - "Spanish,Malay": 0.6466666666666666, - "English,Chinese": 0.7266666666666667, - "English,Vietnamese": 0.6533333333333333, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.6866666666666666, - "English,Malay": 0.6666666666666666, - "Chinese,Vietnamese": 0.66, - "Chinese,Filipino": 0.44666666666666666, - "Chinese,Indonesian": 0.6933333333333334, - "Chinese,Malay": 0.7066666666666667, - "Vietnamese,Filipino": 0.41333333333333333, - "Vietnamese,Indonesian": 0.6666666666666666, - "Vietnamese,Malay": 0.7133333333333334, - "Filipino,Indonesian": 0.44, - "Filipino,Malay": 0.44666666666666666, - "Indonesian,Malay": 0.72 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5733333333333334, - "Spanish,English,Vietnamese": 0.54, - "Spanish,English,Filipino": 0.4, - "Spanish,English,Indonesian": 0.6, - "Spanish,English,Malay": 0.5533333333333333, - "Spanish,Chinese,Vietnamese": 0.5, - "Spanish,Chinese,Filipino": 0.3466666666666667, - "Spanish,Chinese,Indonesian": 0.5466666666666666, - "Spanish,Chinese,Malay": 0.5133333333333333, - "Spanish,Vietnamese,Filipino": 0.32666666666666666, - "Spanish,Vietnamese,Indonesian": 0.5266666666666666, - "Spanish,Vietnamese,Malay": 0.52, - "Spanish,Filipino,Indonesian": 0.38, - "Spanish,Filipino,Malay": 0.35333333333333333, - "Spanish,Indonesian,Malay": 0.56, - "English,Chinese,Vietnamese": 0.5466666666666666, - "English,Chinese,Filipino": 0.37333333333333335, - "English,Chinese,Indonesian": 0.5866666666666667, - "English,Chinese,Malay": 0.5666666666666667, - "English,Vietnamese,Filipino": 0.32666666666666666, - "English,Vietnamese,Indonesian": 0.5266666666666666, - "English,Vietnamese,Malay": 0.5466666666666666, - "English,Filipino,Indonesian": 0.36666666666666664, - "English,Filipino,Malay": 0.36, - "English,Indonesian,Malay": 0.5666666666666667, - "Chinese,Vietnamese,Filipino": 0.32, - "Chinese,Vietnamese,Indonesian": 0.5466666666666666, - "Chinese,Vietnamese,Malay": 0.5733333333333334, - "Chinese,Filipino,Indonesian": 0.35333333333333333, - "Chinese,Filipino,Malay": 0.36, - "Chinese,Indonesian,Malay": 0.5866666666666667, - "Vietnamese,Filipino,Indonesian": 0.32, - "Vietnamese,Filipino,Malay": 0.32666666666666666, - "Vietnamese,Indonesian,Malay": 0.56, - "Filipino,Indonesian,Malay": 0.35333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.46, - "Spanish,English,Chinese,Filipino": 0.32, - "Spanish,English,Chinese,Indonesian": 0.5066666666666667, - "Spanish,English,Chinese,Malay": 0.47333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.29333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.4666666666666667, - "Spanish,English,Vietnamese,Malay": 0.46, - "Spanish,English,Filipino,Indonesian": 0.34, - "Spanish,English,Filipino,Malay": 0.32666666666666666, - "Spanish,English,Indonesian,Malay": 0.4866666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.28, - "Spanish,Chinese,Vietnamese,Indonesian": 0.44666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.4533333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.32, - "Spanish,Chinese,Filipino,Malay": 0.31333333333333335, - "Spanish,Chinese,Indonesian,Malay": 0.4666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.28, - "Spanish,Vietnamese,Indonesian,Malay": 0.4533333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese,Filipino": 0.2866666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.4666666666666667, - "English,Chinese,Vietnamese,Malay": 0.48, - "English,Chinese,Filipino,Indonesian": 0.32666666666666666, - "English,Chinese,Filipino,Malay": 0.32666666666666666, - "English,Chinese,Indonesian,Malay": 0.5066666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.28, - "English,Vietnamese,Filipino,Malay": 0.28, - "English,Vietnamese,Indonesian,Malay": 0.4666666666666667, - "English,Filipino,Indonesian,Malay": 0.32, - "Chinese,Vietnamese,Filipino,Indonesian": 0.28, - "Chinese,Vietnamese,Filipino,Malay": 0.3, - "Chinese,Vietnamese,Indonesian,Malay": 0.49333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian,Malay": 0.28 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.26, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.41333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.41333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.3, - "Spanish,English,Chinese,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.43333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.26, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.3, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "English,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.3, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.28, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - } - }, - "AC3_2": 0.4816439091613902, - "AC3_3": 0.42711375422087666, - "AC3_4": 0.38502560475313546, - "AC3_5": 0.34976958520409346, - "AC3_6": 0.31950738911436, - "AC3_7": 0.2931714718804507 - }, - "prompt_2": { - "overall_acc": 0.41619047619047617, - "language_acc": { - "Spanish": 0.48, - "English": 0.49333333333333335, - "Chinese": 0.4533333333333333, - "Vietnamese": 0.38666666666666666, - "Filipino": 0.32, - "Indonesian": 0.4, - "Malay": 0.38 - }, - "consistency_score_2": 0.6231746031746032, - "consistency_score_3": 0.47085714285714275, - "consistency_score_4": 0.3782857142857143, - "consistency_score_5": 0.31269841269841264, - "consistency_score_6": 0.2638095238095238, - "consistency_score_7": 0.22666666666666666, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.7933333333333333, - "Spanish,Chinese": 0.6266666666666667, - "Spanish,Vietnamese": 0.68, - "Spanish,Filipino": 0.46, - "Spanish,Indonesian": 0.7466666666666667, - "Spanish,Malay": 0.66, - "English,Chinese": 0.7266666666666667, - "English,Vietnamese": 0.66, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.7133333333333334, - "English,Malay": 0.6466666666666666, - "Chinese,Vietnamese": 0.6666666666666666, - "Chinese,Filipino": 0.47333333333333333, - "Chinese,Indonesian": 0.68, - "Chinese,Malay": 0.64, - "Vietnamese,Filipino": 0.42, - "Vietnamese,Indonesian": 0.7, - "Vietnamese,Malay": 0.7133333333333334, - "Filipino,Indonesian": 0.46, - "Filipino,Malay": 0.4533333333333333, - "Indonesian,Malay": 0.7 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5866666666666667, - "Spanish,English,Vietnamese": 0.5866666666666667, - "Spanish,English,Filipino": 0.3933333333333333, - "Spanish,English,Indonesian": 0.6466666666666666, - "Spanish,English,Malay": 0.5733333333333334, - "Spanish,Chinese,Vietnamese": 0.52, - "Spanish,Chinese,Filipino": 0.3466666666666667, - "Spanish,Chinese,Indonesian": 0.5533333333333333, - "Spanish,Chinese,Malay": 0.4866666666666667, - "Spanish,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian": 0.5666666666666667, - "Spanish,Vietnamese,Malay": 0.5466666666666666, - "Spanish,Filipino,Indonesian": 0.3933333333333333, - "Spanish,Filipino,Malay": 0.3466666666666667, - "Spanish,Indonesian,Malay": 0.5666666666666667, - "English,Chinese,Vietnamese": 0.5466666666666666, - "English,Chinese,Filipino": 0.37333333333333335, - "English,Chinese,Indonesian": 0.58, - "English,Chinese,Malay": 0.52, - "English,Vietnamese,Filipino": 0.3333333333333333, - "English,Vietnamese,Indonesian": 0.5533333333333333, - "English,Vietnamese,Malay": 0.5466666666666666, - "English,Filipino,Indonesian": 0.37333333333333335, - "English,Filipino,Malay": 0.34, - "English,Indonesian,Malay": 0.56, - "Chinese,Vietnamese,Filipino": 0.3466666666666667, - "Chinese,Vietnamese,Indonesian": 0.5533333333333333, - "Chinese,Vietnamese,Malay": 0.5533333333333333, - "Chinese,Filipino,Indonesian": 0.37333333333333335, - "Chinese,Filipino,Malay": 0.3466666666666667, - "Chinese,Indonesian,Malay": 0.54, - "Vietnamese,Filipino,Indonesian": 0.34, - "Vietnamese,Filipino,Malay": 0.3333333333333333, - "Vietnamese,Indonesian,Malay": 0.5666666666666667, - "Filipino,Indonesian,Malay": 0.35333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4866666666666667, - "Spanish,English,Chinese,Filipino": 0.32, - "Spanish,English,Chinese,Indonesian": 0.52, - "Spanish,English,Chinese,Malay": 0.46, - "Spanish,English,Vietnamese,Filipino": 0.3, - "Spanish,English,Vietnamese,Indonesian": 0.5133333333333333, - "Spanish,English,Vietnamese,Malay": 0.49333333333333335, - "Spanish,English,Filipino,Indonesian": 0.35333333333333333, - "Spanish,English,Filipino,Malay": 0.30666666666666664, - "Spanish,English,Indonesian,Malay": 0.5066666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.28, - "Spanish,Chinese,Vietnamese,Indonesian": 0.4666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.44666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.2866666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.44666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,Vietnamese,Filipino,Malay": 0.28, - "Spanish,Vietnamese,Indonesian,Malay": 0.48, - "Spanish,Filipino,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.2866666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.48, - "English,Chinese,Vietnamese,Malay": 0.4666666666666667, - "English,Chinese,Filipino,Indonesian": 0.32666666666666666, - "English,Chinese,Filipino,Malay": 0.29333333333333333, - "English,Chinese,Indonesian,Malay": 0.48, - "English,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "English,Vietnamese,Filipino,Malay": 0.2733333333333333, - "English,Vietnamese,Indonesian,Malay": 0.48, - "English,Filipino,Indonesian,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.3, - "Chinese,Vietnamese,Indonesian,Malay": 0.48, - "Chinese,Filipino,Indonesian,Malay": 0.30666666666666664, - "Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.26, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.44, - "Spanish,English,Chinese,Vietnamese,Malay": 0.42, - "Spanish,English,Chinese,Filipino,Indonesian": 0.30666666666666664, - "Spanish,English,Chinese,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.4266666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.44, - "Spanish,English,Filipino,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.26, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "English,Chinese,Vietnamese,Filipino,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.28, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.26, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 - } - }, - "AC3_2": 0.4990726357259015, - "AC3_3": 0.4418393201747366, - "AC3_4": 0.3963338698682561, - "AC3_5": 0.3570972290865646, - "AC3_6": 0.3229265038874181, - "AC3_7": 0.29349135797903714 - }, - "prompt_3": { - "overall_acc": 0.42095238095238097, - "language_acc": { - "Spanish": 0.4866666666666667, - "English": 0.5133333333333333, - "Chinese": 0.4666666666666667, - "Vietnamese": 0.42, - "Filipino": 0.29333333333333333, - "Indonesian": 0.4066666666666667, - "Malay": 0.36 - }, - "consistency_score_2": 0.6247619047619049, - "consistency_score_3": 0.47257142857142853, - "consistency_score_4": 0.3805714285714287, - "consistency_score_5": 0.3165079365079366, - "consistency_score_6": 0.2695238095238095, - "consistency_score_7": 0.23333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.7666666666666667, - "Spanish,Chinese": 0.6733333333333333, - "Spanish,Vietnamese": 0.6666666666666666, - "Spanish,Filipino": 0.4666666666666667, - "Spanish,Indonesian": 0.68, - "Spanish,Malay": 0.6133333333333333, - "English,Chinese": 0.7866666666666666, - "English,Vietnamese": 0.68, - "English,Filipino": 0.47333333333333333, - "English,Indonesian": 0.6866666666666666, - "English,Malay": 0.6333333333333333, - "Chinese,Vietnamese": 0.7066666666666667, - "Chinese,Filipino": 0.47333333333333333, - "Chinese,Indonesian": 0.7, - "Chinese,Malay": 0.68, - "Vietnamese,Filipino": 0.44, - "Vietnamese,Indonesian": 0.6866666666666666, - "Vietnamese,Malay": 0.6933333333333334, - "Filipino,Indonesian": 0.46, - "Filipino,Malay": 0.44666666666666666, - "Indonesian,Malay": 0.7066666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.6333333333333333, - "Spanish,English,Vietnamese": 0.58, - "Spanish,English,Filipino": 0.38666666666666666, - "Spanish,English,Indonesian": 0.5866666666666667, - "Spanish,English,Malay": 0.5333333333333333, - "Spanish,Chinese,Vietnamese": 0.5533333333333333, - "Spanish,Chinese,Filipino": 0.35333333333333333, - "Spanish,Chinese,Indonesian": 0.5533333333333333, - "Spanish,Chinese,Malay": 0.5133333333333333, - "Spanish,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian": 0.5266666666666666, - "Spanish,Vietnamese,Malay": 0.5133333333333333, - "Spanish,Filipino,Indonesian": 0.36, - "Spanish,Filipino,Malay": 0.3333333333333333, - "Spanish,Indonesian,Malay": 0.5266666666666666, - "English,Chinese,Vietnamese": 0.5933333333333334, - "English,Chinese,Filipino": 0.38666666666666666, - "English,Chinese,Indonesian": 0.6066666666666667, - "English,Chinese,Malay": 0.5666666666666667, - "English,Vietnamese,Filipino": 0.3466666666666667, - "English,Vietnamese,Indonesian": 0.5466666666666666, - "English,Vietnamese,Malay": 0.5266666666666666, - "English,Filipino,Indonesian": 0.36666666666666664, - "English,Filipino,Malay": 0.3333333333333333, - "English,Indonesian,Malay": 0.5466666666666666, - "Chinese,Vietnamese,Filipino": 0.36, - "Chinese,Vietnamese,Indonesian": 0.5733333333333334, - "Chinese,Vietnamese,Malay": 0.5666666666666667, - "Chinese,Filipino,Indonesian": 0.37333333333333335, - "Chinese,Filipino,Malay": 0.35333333333333333, - "Chinese,Indonesian,Malay": 0.5733333333333334, - "Vietnamese,Filipino,Indonesian": 0.36666666666666664, - "Vietnamese,Filipino,Malay": 0.3466666666666667, - "Vietnamese,Indonesian,Malay": 0.56, - "Filipino,Indonesian,Malay": 0.36 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.52, - "Spanish,English,Chinese,Filipino": 0.32666666666666666, - "Spanish,English,Chinese,Indonesian": 0.52, - "Spanish,English,Chinese,Malay": 0.48, - "Spanish,English,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,English,Vietnamese,Indonesian": 0.47333333333333333, - "Spanish,English,Vietnamese,Malay": 0.46, - "Spanish,English,Filipino,Indonesian": 0.3333333333333333, - "Spanish,English,Filipino,Malay": 0.30666666666666664, - "Spanish,English,Indonesian,Malay": 0.4666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.47333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.46, - "Spanish,Chinese,Filipino,Indonesian": 0.32, - "Spanish,Chinese,Filipino,Malay": 0.29333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.4666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,Vietnamese,Filipino,Malay": 0.28, - "Spanish,Vietnamese,Indonesian,Malay": 0.4533333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.3, - "English,Chinese,Vietnamese,Filipino": 0.29333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.5066666666666667, - "English,Chinese,Vietnamese,Malay": 0.48, - "English,Chinese,Filipino,Indonesian": 0.32666666666666666, - "English,Chinese,Filipino,Malay": 0.3, - "English,Chinese,Indonesian,Malay": 0.5133333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "English,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.46, - "English,Filipino,Indonesian,Malay": 0.3, - "Chinese,Vietnamese,Filipino,Indonesian": 0.32, - "Chinese,Vietnamese,Filipino,Malay": 0.3, - "Chinese,Vietnamese,Indonesian,Malay": 0.49333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.44666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.4266666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.3, - "Spanish,English,Chinese,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.44, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.28, - "Spanish,English,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.42, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.28, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.26, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.28, - "English,Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.44, - "English,Chinese,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.28 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.3933333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - } - }, - "AC3_2": 0.5029959232756914, - "AC3_3": 0.4452708834048065, - "AC3_4": 0.39974470391536326, - "AC3_5": 0.3613340712463313, - "AC3_6": 0.3286331690821257, - "AC3_7": 0.30024260063339325 - }, - "prompt_4": { - "overall_acc": 0.40571428571428575, - "language_acc": { - "Spanish": 0.4533333333333333, - "English": 0.4666666666666667, - "Chinese": 0.4533333333333333, - "Vietnamese": 0.3933333333333333, - "Filipino": 0.28, - "Indonesian": 0.4066666666666667, - "Malay": 0.38666666666666666 - }, - "consistency_score_2": 0.6073015873015873, - "consistency_score_3": 0.4485714285714286, - "consistency_score_4": 0.35390476190476194, - "consistency_score_5": 0.28920634920634924, - "consistency_score_6": 0.2419047619047619, - "consistency_score_7": 0.20666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.7133333333333334, - "Spanish,Chinese": 0.6466666666666666, - "Spanish,Vietnamese": 0.6266666666666667, - "Spanish,Filipino": 0.4666666666666667, - "Spanish,Indonesian": 0.7333333333333333, - "Spanish,Malay": 0.64, - "English,Chinese": 0.7333333333333333, - "English,Vietnamese": 0.6133333333333333, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.7066666666666667, - "English,Malay": 0.6333333333333333, - "Chinese,Vietnamese": 0.6666666666666666, - "Chinese,Filipino": 0.44666666666666666, - "Chinese,Indonesian": 0.7, - "Chinese,Malay": 0.6666666666666666, - "Vietnamese,Filipino": 0.42, - "Vietnamese,Indonesian": 0.6266666666666667, - "Vietnamese,Malay": 0.6733333333333333, - "Filipino,Indonesian": 0.43333333333333335, - "Filipino,Malay": 0.44666666666666666, - "Indonesian,Malay": 0.6933333333333334 - }, - "3_combine": { - "Spanish,English,Chinese": 0.56, - "Spanish,English,Vietnamese": 0.5, - "Spanish,English,Filipino": 0.37333333333333335, - "Spanish,English,Indonesian": 0.5933333333333334, - "Spanish,English,Malay": 0.5133333333333333, - "Spanish,Chinese,Vietnamese": 0.5066666666666667, - "Spanish,Chinese,Filipino": 0.34, - "Spanish,Chinese,Indonesian": 0.56, - "Spanish,Chinese,Malay": 0.5, - "Spanish,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,Vietnamese,Indonesian": 0.5066666666666667, - "Spanish,Vietnamese,Malay": 0.5, - "Spanish,Filipino,Indonesian": 0.36, - "Spanish,Filipino,Malay": 0.3333333333333333, - "Spanish,Indonesian,Malay": 0.5333333333333333, - "English,Chinese,Vietnamese": 0.5333333333333333, - "English,Chinese,Filipino": 0.38, - "English,Chinese,Indonesian": 0.5933333333333334, - "English,Chinese,Malay": 0.54, - "English,Vietnamese,Filipino": 0.31333333333333335, - "English,Vietnamese,Indonesian": 0.5, - "English,Vietnamese,Malay": 0.5, - "English,Filipino,Indonesian": 0.36666666666666664, - "English,Filipino,Malay": 0.3333333333333333, - "English,Indonesian,Malay": 0.5466666666666666, - "Chinese,Vietnamese,Filipino": 0.32, - "Chinese,Vietnamese,Indonesian": 0.52, - "Chinese,Vietnamese,Malay": 0.5333333333333333, - "Chinese,Filipino,Indonesian": 0.3466666666666667, - "Chinese,Filipino,Malay": 0.32666666666666666, - "Chinese,Indonesian,Malay": 0.5666666666666667, - "Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Vietnamese,Filipino,Malay": 0.32666666666666666, - "Vietnamese,Indonesian,Malay": 0.52, - "Filipino,Indonesian,Malay": 0.34 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.44, - "Spanish,English,Chinese,Filipino": 0.30666666666666664, - "Spanish,English,Chinese,Indonesian": 0.49333333333333335, - "Spanish,English,Chinese,Malay": 0.44, - "Spanish,English,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.4266666666666667, - "Spanish,English,Vietnamese,Malay": 0.4266666666666667, - "Spanish,English,Filipino,Indonesian": 0.32666666666666666, - "Spanish,English,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Indonesian,Malay": 0.44, - "Spanish,Chinese,Vietnamese,Filipino": 0.26, - "Spanish,Chinese,Vietnamese,Indonesian": 0.44, - "Spanish,Chinese,Vietnamese,Malay": 0.4266666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Chinese,Filipino,Malay": 0.2866666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.4533333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.26, - "Spanish,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.28, - "English,Chinese,Vietnamese,Indonesian": 0.4533333333333333, - "English,Chinese,Vietnamese,Malay": 0.44666666666666666, - "English,Chinese,Filipino,Indonesian": 0.32, - "English,Chinese,Filipino,Malay": 0.2866666666666667, - "English,Chinese,Indonesian,Malay": 0.49333333333333335, - "English,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "English,Vietnamese,Filipino,Malay": 0.26, - "English,Vietnamese,Indonesian,Malay": 0.44, - "English,Filipino,Indonesian,Malay": 0.3, - "Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "Chinese,Vietnamese,Filipino,Malay": 0.28, - "Chinese,Vietnamese,Indonesian,Malay": 0.4533333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.24, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.38666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.38, - "Spanish,English,Chinese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.26, - "Spanish,English,Chinese,Indonesian,Malay": 0.4, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,English,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Spanish,English,Filipino,Indonesian,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.38, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.22, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.34, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 - } - }, - "AC3_2": 0.4864502842336651, - "AC3_3": 0.42606784514840584, - "AC3_4": 0.37804269947738556, - "AC3_5": 0.3376936630722043, - "AC3_6": 0.3030924369279889, - "AC3_7": 0.27384136854004143 - }, - "prompt_5": { - "overall_acc": 0.4142857142857143, - "language_acc": { - "Spanish": 0.48, - "English": 0.5066666666666667, - "Chinese": 0.46, - "Vietnamese": 0.38666666666666666, - "Filipino": 0.2866666666666667, - "Indonesian": 0.3933333333333333, - "Malay": 0.38666666666666666 - }, - "consistency_score_2": 0.62, - "consistency_score_3": 0.4676190476190477, - "consistency_score_4": 0.37809523809523826, - "consistency_score_5": 0.31682539682539673, - "consistency_score_6": 0.2723809523809524, - "consistency_score_7": 0.24, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.7666666666666667, - "Spanish,Chinese": 0.7066666666666667, - "Spanish,Vietnamese": 0.6333333333333333, - "Spanish,Filipino": 0.44666666666666666, - "Spanish,Indonesian": 0.7333333333333333, - "Spanish,Malay": 0.6533333333333333, - "English,Chinese": 0.7666666666666667, - "English,Vietnamese": 0.64, - "English,Filipino": 0.46, - "English,Indonesian": 0.68, - "English,Malay": 0.6266666666666667, - "Chinese,Vietnamese": 0.68, - "Chinese,Filipino": 0.44, - "Chinese,Indonesian": 0.7133333333333334, - "Chinese,Malay": 0.6733333333333333, - "Vietnamese,Filipino": 0.44, - "Vietnamese,Indonesian": 0.6333333333333333, - "Vietnamese,Malay": 0.72, - "Filipino,Indonesian": 0.44666666666666666, - "Filipino,Malay": 0.46, - "Indonesian,Malay": 0.7 - }, - "3_combine": { - "Spanish,English,Chinese": 0.6266666666666667, - "Spanish,English,Vietnamese": 0.54, - "Spanish,English,Filipino": 0.37333333333333335, - "Spanish,English,Indonesian": 0.6066666666666667, - "Spanish,English,Malay": 0.5533333333333333, - "Spanish,Chinese,Vietnamese": 0.54, - "Spanish,Chinese,Filipino": 0.35333333333333333, - "Spanish,Chinese,Indonesian": 0.6, - "Spanish,Chinese,Malay": 0.5466666666666666, - "Spanish,Vietnamese,Filipino": 0.31333333333333335, - "Spanish,Vietnamese,Indonesian": 0.5133333333333333, - "Spanish,Vietnamese,Malay": 0.5333333333333333, - "Spanish,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Filipino,Malay": 0.34, - "Spanish,Indonesian,Malay": 0.56, - "English,Chinese,Vietnamese": 0.5533333333333333, - "English,Chinese,Filipino": 0.38, - "English,Chinese,Indonesian": 0.6, - "English,Chinese,Malay": 0.56, - "English,Vietnamese,Filipino": 0.3333333333333333, - "English,Vietnamese,Indonesian": 0.49333333333333335, - "English,Vietnamese,Malay": 0.5133333333333333, - "English,Filipino,Indonesian": 0.36, - "English,Filipino,Malay": 0.32666666666666666, - "English,Indonesian,Malay": 0.54, - "Chinese,Vietnamese,Filipino": 0.32666666666666666, - "Chinese,Vietnamese,Indonesian": 0.54, - "Chinese,Vietnamese,Malay": 0.56, - "Chinese,Filipino,Indonesian": 0.37333333333333335, - "Chinese,Filipino,Malay": 0.3466666666666667, - "Chinese,Indonesian,Malay": 0.58, - "Vietnamese,Filipino,Indonesian": 0.34, - "Vietnamese,Filipino,Malay": 0.36, - "Vietnamese,Indonesian,Malay": 0.5533333333333333, - "Filipino,Indonesian,Malay": 0.37333333333333335 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.4866666666666667, - "Spanish,English,Chinese,Filipino": 0.32666666666666666, - "Spanish,English,Chinese,Indonesian": 0.5333333333333333, - "Spanish,English,Chinese,Malay": 0.49333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.44666666666666666, - "Spanish,English,Vietnamese,Malay": 0.46, - "Spanish,English,Filipino,Indonesian": 0.32666666666666666, - "Spanish,English,Filipino,Malay": 0.30666666666666664, - "Spanish,English,Indonesian,Malay": 0.48, - "Spanish,Chinese,Vietnamese,Filipino": 0.28, - "Spanish,Chinese,Vietnamese,Indonesian": 0.4666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.4666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.30666666666666664, - "Spanish,Chinese,Indonesian,Malay": 0.5, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.2866666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.4666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino": 0.2866666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.46, - "English,Chinese,Vietnamese,Malay": 0.4666666666666667, - "English,Chinese,Filipino,Indonesian": 0.3333333333333333, - "English,Chinese,Filipino,Malay": 0.3, - "English,Chinese,Indonesian,Malay": 0.5133333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "English,Vietnamese,Filipino,Malay": 0.2733333333333333, - "English,Vietnamese,Indonesian,Malay": 0.44666666666666666, - "English,Filipino,Indonesian,Malay": 0.3, - "Chinese,Vietnamese,Filipino,Indonesian": 0.3, - "Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.4866666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.3333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.31333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.26, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.42, - "Spanish,English,Chinese,Vietnamese,Malay": 0.42, - "Spanish,English,Chinese,Filipino,Indonesian": 0.30666666666666664, - "Spanish,English,Chinese,Filipino,Malay": 0.28, - "Spanish,English,Chinese,Indonesian,Malay": 0.4533333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.26, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.26, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.24, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24 - } - }, - "AC3_2": 0.4966850828249062, - "AC3_3": 0.43933970991624277, - "AC3_4": 0.3953640109391154, - "AC3_5": 0.35905961163750844, - "AC3_6": 0.328670497277279, - "AC3_7": 0.3039301309579146 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.40340909090909083, - "language_acc": { - "English": 0.4431818181818182, - "Filipino": 0.3068181818181818, - "Spanish": 0.4147727272727273, - "Malay": 0.42045454545454547, - "Chinese": 0.4375, - "Indonesian": 0.4034090909090909, - "Vietnamese": 0.3977272727272727 - }, - "consistency_score_2": 0.5221861471861472, - "consistency_score_3": 0.3366883116883117, - "consistency_score_4": 0.2327922077922079, - "consistency_score_5": 0.16612554112554112, - "consistency_score_6": 0.11931818181818184, - "consistency_score_7": 0.08522727272727272, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.7329545454545454, - "English,Malay": 0.5284090909090909, - "English,Chinese": 0.5852272727272727, - "English,Indonesian": 0.5795454545454546, - "English,Vietnamese": 0.5681818181818182, - "Filipino,Spanish": 0.3806818181818182, - "Filipino,Malay": 0.35795454545454547, - "Filipino,Chinese": 0.3522727272727273, - "Filipino,Indonesian": 0.3806818181818182, - "Filipino,Vietnamese": 0.3181818181818182, - "Spanish,Malay": 0.5795454545454546, - "Spanish,Chinese": 0.5852272727272727, - "Spanish,Indonesian": 0.625, - "Spanish,Vietnamese": 0.5852272727272727, - "Malay,Chinese": 0.5738636363636364, - "Malay,Indonesian": 0.6363636363636364, - "Malay,Vietnamese": 0.6079545454545454, - "Chinese,Indonesian": 0.5568181818181818, - "Chinese,Vietnamese": 0.5056818181818182, - "Indonesian,Vietnamese": 0.5909090909090909 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2784090909090909, - "English,Filipino,Malay": 0.19886363636363635, - "English,Filipino,Chinese": 0.2159090909090909, - "English,Filipino,Indonesian": 0.2215909090909091, - "English,Filipino,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay": 0.4431818181818182, - "English,Spanish,Chinese": 0.48863636363636365, - "English,Spanish,Indonesian": 0.4943181818181818, - "English,Spanish,Vietnamese": 0.4715909090909091, - "English,Malay,Chinese": 0.38636363636363635, - "English,Malay,Indonesian": 0.4034090909090909, - "English,Malay,Vietnamese": 0.3977272727272727, - "English,Chinese,Indonesian": 0.4034090909090909, - "English,Chinese,Vietnamese": 0.375, - "English,Indonesian,Vietnamese": 0.4090909090909091, - "Filipino,Spanish,Malay": 0.22727272727272727, - "Filipino,Spanish,Chinese": 0.22727272727272727, - "Filipino,Spanish,Indonesian": 0.25, - "Filipino,Spanish,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese": 0.22727272727272727, - "Filipino,Malay,Indonesian": 0.24431818181818182, - "Filipino,Malay,Vietnamese": 0.2159090909090909, - "Filipino,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Indonesian,Vietnamese": 0.21022727272727273, - "Spanish,Malay,Chinese": 0.42045454545454547, - "Spanish,Malay,Indonesian": 0.44886363636363635, - "Spanish,Malay,Vietnamese": 0.42613636363636365, - "Spanish,Chinese,Indonesian": 0.3977272727272727, - "Spanish,Chinese,Vietnamese": 0.38636363636363635, - "Spanish,Indonesian,Vietnamese": 0.42613636363636365, - "Malay,Chinese,Indonesian": 0.42613636363636365, - "Malay,Chinese,Vietnamese": 0.3977272727272727, - "Malay,Indonesian,Vietnamese": 0.45454545454545453, - "Chinese,Indonesian,Vietnamese": 0.3693181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.19886363636363635, - "English,Filipino,Spanish,Indonesian": 0.19318181818181818, - "English,Filipino,Spanish,Vietnamese": 0.18181818181818182, - "English,Filipino,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese": 0.3465909090909091, - "English,Spanish,Malay,Indonesian": 0.3693181818181818, - "English,Spanish,Malay,Vietnamese": 0.3465909090909091, - "English,Spanish,Chinese,Indonesian": 0.3522727272727273, - "English,Spanish,Chinese,Vietnamese": 0.32954545454545453, - "English,Spanish,Indonesian,Vietnamese": 0.375, - "English,Malay,Chinese,Indonesian": 0.30113636363636365, - "English,Malay,Chinese,Vietnamese": 0.29545454545454547, - "English,Malay,Indonesian,Vietnamese": 0.3181818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Malay,Chinese": 0.1590909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.32386363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.32386363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.3409090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.29545454545454547, - "Malay,Chinese,Indonesian,Vietnamese": 0.3125 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian": 0.2784090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.2727272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 - } - }, - "AC3_2": 0.4551765830870674, - "AC3_3": 0.36704121712323134, - "AC3_4": 0.2952225752858933, - "AC3_5": 0.23533793992844784, - "AC3_6": 0.18416501972761426, - "AC3_7": 0.14072410145111605 - }, - "prompt_2": { - "overall_acc": 0.4074675324675324, - "language_acc": { - "English": 0.4715909090909091, - "Filipino": 0.29545454545454547, - "Spanish": 0.39204545454545453, - "Malay": 0.4147727272727273, - "Chinese": 0.4431818181818182, - "Indonesian": 0.4147727272727273, - "Vietnamese": 0.42045454545454547 - }, - "consistency_score_2": 0.5181277056277057, - "consistency_score_3": 0.3344155844155844, - "consistency_score_4": 0.23392857142857137, - "consistency_score_5": 0.16856060606060605, - "consistency_score_6": 0.12012987012987011, - "consistency_score_7": 0.07954545454545454, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.32386363636363635, - "English,Spanish": 0.7045454545454546, - "English,Malay": 0.5625, - "English,Chinese": 0.5397727272727273, - "English,Indonesian": 0.5965909090909091, - "English,Vietnamese": 0.5568181818181818, - "Filipino,Spanish": 0.3465909090909091, - "Filipino,Malay": 0.3522727272727273, - "Filipino,Chinese": 0.35795454545454547, - "Filipino,Indonesian": 0.3693181818181818, - "Filipino,Vietnamese": 0.32954545454545453, - "Spanish,Malay": 0.5681818181818182, - "Spanish,Chinese": 0.5738636363636364, - "Spanish,Indonesian": 0.6306818181818182, - "Spanish,Vietnamese": 0.5681818181818182, - "Malay,Chinese": 0.5625, - "Malay,Indonesian": 0.6193181818181818, - "Malay,Vietnamese": 0.5795454545454546, - "Chinese,Indonesian": 0.5852272727272727, - "Chinese,Vietnamese": 0.5511363636363636, - "Indonesian,Vietnamese": 0.6022727272727273 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2556818181818182, - "English,Filipino,Malay": 0.19318181818181818, - "English,Filipino,Chinese": 0.19886363636363635, - "English,Filipino,Indonesian": 0.2159090909090909, - "English,Filipino,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay": 0.44886363636363635, - "English,Spanish,Chinese": 0.4375, - "English,Spanish,Indonesian": 0.48295454545454547, - "English,Spanish,Vietnamese": 0.4375, - "English,Malay,Chinese": 0.38636363636363635, - "English,Malay,Indonesian": 0.4375, - "English,Malay,Vietnamese": 0.3977272727272727, - "English,Chinese,Indonesian": 0.4090909090909091, - "English,Chinese,Vietnamese": 0.375, - "English,Indonesian,Vietnamese": 0.42045454545454547, - "Filipino,Spanish,Malay": 0.20454545454545456, - "Filipino,Spanish,Chinese": 0.2215909090909091, - "Filipino,Spanish,Indonesian": 0.2556818181818182, - "Filipino,Spanish,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Chinese": 0.2215909090909091, - "Filipino,Malay,Indonesian": 0.25, - "Filipino,Malay,Vietnamese": 0.20454545454545456, - "Filipino,Chinese,Indonesian": 0.25, - "Filipino,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Chinese": 0.4090909090909091, - "Spanish,Malay,Indonesian": 0.4375, - "Spanish,Malay,Vietnamese": 0.4147727272727273, - "Spanish,Chinese,Indonesian": 0.42045454545454547, - "Spanish,Chinese,Vietnamese": 0.39204545454545453, - "Spanish,Indonesian,Vietnamese": 0.4147727272727273, - "Malay,Chinese,Indonesian": 0.42613636363636365, - "Malay,Chinese,Vietnamese": 0.3977272727272727, - "Malay,Indonesian,Vietnamese": 0.4318181818181818, - "Chinese,Indonesian,Vietnamese": 0.4034090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17045454545454544, - "English,Filipino,Spanish,Chinese": 0.18181818181818182, - "English,Filipino,Spanish,Indonesian": 0.19886363636363635, - "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, - "English,Filipino,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Chinese,Indonesian": 0.16477272727272727, - "English,Filipino,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese": 0.3352272727272727, - "English,Spanish,Malay,Indonesian": 0.375, - "English,Spanish,Malay,Vietnamese": 0.3352272727272727, - "English,Spanish,Chinese,Indonesian": 0.35795454545454547, - "English,Spanish,Chinese,Vietnamese": 0.3181818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.3522727272727273, - "English,Malay,Chinese,Indonesian": 0.32954545454545453, - "English,Malay,Chinese,Vietnamese": 0.2897727272727273, - "English,Malay,Indonesian,Vietnamese": 0.3352272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.30113636363636365, - "Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "Filipino,Spanish,Malay,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Chinese,Indonesian": 0.3352272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.3181818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.3352272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.29545454545454547, - "Malay,Chinese,Indonesian,Vietnamese": 0.3181818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese,Indonesian": 0.125, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.29545454545454547, - "English,Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.29545454545454547, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - } - }, - "AC3_2": 0.45618259257002175, - "AC3_3": 0.36734490887088606, - "AC3_4": 0.2972213182455935, - "AC3_5": 0.23847089965542817, - "AC3_6": 0.18555444551927583, - "AC3_7": 0.13310606057872948 - }, - "prompt_3": { - "overall_acc": 0.38961038961038963, - "language_acc": { - "English": 0.4431818181818182, - "Filipino": 0.3125, - "Spanish": 0.375, - "Malay": 0.38636363636363635, - "Chinese": 0.4147727272727273, - "Indonesian": 0.3806818181818182, - "Vietnamese": 0.4147727272727273 - }, - "consistency_score_2": 0.5194805194805195, - "consistency_score_3": 0.3373376623376623, - "consistency_score_4": 0.2321428571428572, - "consistency_score_5": 0.15990259740259744, - "consistency_score_6": 0.10551948051948053, - "consistency_score_7": 0.0625, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.35795454545454547, - "English,Spanish": 0.7386363636363636, - "English,Malay": 0.5795454545454546, - "English,Chinese": 0.5681818181818182, - "English,Indonesian": 0.625, - "English,Vietnamese": 0.6022727272727273, - "Filipino,Spanish": 0.35795454545454547, - "Filipino,Malay": 0.3806818181818182, - "Filipino,Chinese": 0.2727272727272727, - "Filipino,Indonesian": 0.32954545454545453, - "Filipino,Vietnamese": 0.3352272727272727, - "Spanish,Malay": 0.625, - "Spanish,Chinese": 0.5568181818181818, - "Spanish,Indonesian": 0.6647727272727273, - "Spanish,Vietnamese": 0.6022727272727273, - "Malay,Chinese": 0.5, - "Malay,Indonesian": 0.6363636363636364, - "Malay,Vietnamese": 0.5738636363636364, - "Chinese,Indonesian": 0.5056818181818182, - "Chinese,Vietnamese": 0.5113636363636364, - "Indonesian,Vietnamese": 0.5852272727272727 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2840909090909091, - "English,Filipino,Malay": 0.25, - "English,Filipino,Chinese": 0.20454545454545456, - "English,Filipino,Indonesian": 0.24431818181818182, - "English,Filipino,Vietnamese": 0.23295454545454544, - "English,Spanish,Malay": 0.5056818181818182, - "English,Spanish,Chinese": 0.4659090909090909, - "English,Spanish,Indonesian": 0.5397727272727273, - "English,Spanish,Vietnamese": 0.48863636363636365, - "English,Malay,Chinese": 0.38636363636363635, - "English,Malay,Indonesian": 0.45454545454545453, - "English,Malay,Vietnamese": 0.42613636363636365, - "English,Chinese,Indonesian": 0.39204545454545453, - "English,Chinese,Vietnamese": 0.3806818181818182, - "English,Indonesian,Vietnamese": 0.44886363636363635, - "Filipino,Spanish,Malay": 0.24431818181818182, - "Filipino,Spanish,Chinese": 0.19318181818181818, - "Filipino,Spanish,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese": 0.18181818181818182, - "Filipino,Malay,Indonesian": 0.24431818181818182, - "Filipino,Malay,Vietnamese": 0.21022727272727273, - "Filipino,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese": 0.39204545454545453, - "Spanish,Malay,Indonesian": 0.48863636363636365, - "Spanish,Malay,Vietnamese": 0.4375, - "Spanish,Chinese,Indonesian": 0.39204545454545453, - "Spanish,Chinese,Vietnamese": 0.3693181818181818, - "Spanish,Indonesian,Vietnamese": 0.4431818181818182, - "Malay,Chinese,Indonesian": 0.3693181818181818, - "Malay,Chinese,Vietnamese": 0.36363636363636365, - "Malay,Indonesian,Vietnamese": 0.42613636363636365, - "Chinese,Indonesian,Vietnamese": 0.3352272727272727 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.20454545454545456, - "English,Filipino,Spanish,Chinese": 0.17613636363636365, - "English,Filipino,Spanish,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Vietnamese": 0.19886363636363635, - "English,Filipino,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Malay,Indonesian": 0.1875, - "English,Filipino,Malay,Vietnamese": 0.17045454545454544, - "English,Filipino,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Chinese": 0.3409090909090909, - "English,Spanish,Malay,Indonesian": 0.4147727272727273, - "English,Spanish,Malay,Vietnamese": 0.375, - "English,Spanish,Chinese,Indonesian": 0.3522727272727273, - "English,Spanish,Chinese,Vietnamese": 0.3181818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.3977272727272727, - "English,Malay,Chinese,Indonesian": 0.3068181818181818, - "English,Malay,Chinese,Vietnamese": 0.29545454545454547, - "English,Malay,Indonesian,Vietnamese": 0.3465909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "Filipino,Spanish,Malay,Chinese": 0.13068181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Chinese,Vietnamese": 0.125, - "Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Spanish,Malay,Chinese,Indonesian": 0.3068181818181818, - "Spanish,Malay,Chinese,Vietnamese": 0.30113636363636365, - "Spanish,Malay,Indonesian,Vietnamese": 0.3522727272727273, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.2784090909090909 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Indonesian": 0.2784090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.3181818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - } - }, - "AC3_2": 0.4452690166486086, - "AC3_3": 0.3615946357744611, - "AC3_4": 0.2909362177823754, - "AC3_5": 0.226745189776305, - "AC3_6": 0.16606344471843, - "AC3_7": 0.10771992816288851 - }, - "prompt_4": { - "overall_acc": 0.3944805194805195, - "language_acc": { - "English": 0.4375, - "Filipino": 0.2897727272727273, - "Spanish": 0.4034090909090909, - "Malay": 0.38636363636363635, - "Chinese": 0.4375, - "Indonesian": 0.38636363636363635, - "Vietnamese": 0.42045454545454547 - }, - "consistency_score_2": 0.5081168831168831, - "consistency_score_3": 0.326948051948052, - "consistency_score_4": 0.226461038961039, - "consistency_score_5": 0.1599025974025974, - "consistency_score_6": 0.1112012987012987, - "consistency_score_7": 0.07386363636363637, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3522727272727273, - "English,Spanish": 0.6875, - "English,Malay": 0.5511363636363636, - "English,Chinese": 0.5454545454545454, - "English,Indonesian": 0.5681818181818182, - "English,Vietnamese": 0.5795454545454546, - "Filipino,Spanish": 0.35795454545454547, - "Filipino,Malay": 0.3693181818181818, - "Filipino,Chinese": 0.3352272727272727, - "Filipino,Indonesian": 0.3522727272727273, - "Filipino,Vietnamese": 0.3181818181818182, - "Spanish,Malay": 0.5568181818181818, - "Spanish,Chinese": 0.5511363636363636, - "Spanish,Indonesian": 0.5852272727272727, - "Spanish,Vietnamese": 0.5795454545454546, - "Malay,Chinese": 0.5397727272727273, - "Malay,Indonesian": 0.6534090909090909, - "Malay,Vietnamese": 0.6022727272727273, - "Chinese,Indonesian": 0.5284090909090909, - "Chinese,Vietnamese": 0.48295454545454547, - "Indonesian,Vietnamese": 0.5738636363636364 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2784090909090909, - "English,Filipino,Malay": 0.23295454545454544, - "English,Filipino,Chinese": 0.22727272727272727, - "English,Filipino,Indonesian": 0.22727272727272727, - "English,Filipino,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay": 0.42613636363636365, - "English,Spanish,Chinese": 0.4375, - "English,Spanish,Indonesian": 0.45454545454545453, - "English,Spanish,Vietnamese": 0.44886363636363635, - "English,Malay,Chinese": 0.375, - "English,Malay,Indonesian": 0.42045454545454547, - "English,Malay,Vietnamese": 0.42045454545454547, - "English,Chinese,Indonesian": 0.375, - "English,Chinese,Vietnamese": 0.35795454545454547, - "English,Indonesian,Vietnamese": 0.4147727272727273, - "Filipino,Spanish,Malay": 0.22727272727272727, - "Filipino,Spanish,Chinese": 0.2215909090909091, - "Filipino,Spanish,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese": 0.2159090909090909, - "Filipino,Malay,Indonesian": 0.2556818181818182, - "Filipino,Malay,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian": 0.21022727272727273, - "Filipino,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese": 0.38636363636363635, - "Spanish,Malay,Indonesian": 0.4375, - "Spanish,Malay,Vietnamese": 0.4147727272727273, - "Spanish,Chinese,Indonesian": 0.375, - "Spanish,Chinese,Vietnamese": 0.36363636363636365, - "Spanish,Indonesian,Vietnamese": 0.4034090909090909, - "Malay,Chinese,Indonesian": 0.3977272727272727, - "Malay,Chinese,Vietnamese": 0.36363636363636365, - "Malay,Indonesian,Vietnamese": 0.4431818181818182, - "Chinese,Indonesian,Vietnamese": 0.3352272727272727 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.19886363636363635, - "English,Filipino,Spanish,Chinese": 0.19318181818181818, - "English,Filipino,Spanish,Indonesian": 0.19318181818181818, - "English,Filipino,Spanish,Vietnamese": 0.20454545454545456, - "English,Filipino,Malay,Chinese": 0.16477272727272727, - "English,Filipino,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese": 0.32386363636363635, - "English,Spanish,Malay,Indonesian": 0.3465909090909091, - "English,Spanish,Malay,Vietnamese": 0.3409090909090909, - "English,Spanish,Chinese,Indonesian": 0.32386363636363635, - "English,Spanish,Chinese,Vietnamese": 0.3068181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.3409090909090909, - "English,Malay,Chinese,Indonesian": 0.29545454545454547, - "English,Malay,Chinese,Vietnamese": 0.2784090909090909, - "English,Malay,Indonesian,Vietnamese": 0.32386363636363635, - "English,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Malay,Chinese": 0.1534090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian": 0.3125, - "Spanish,Malay,Chinese,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.3352272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.2840909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Indonesian": 0.26704545454545453, - "English,Spanish,Malay,Chinese,Vietnamese": 0.25, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 - } - }, - "AC3_2": 0.4441453330349745, - "AC3_3": 0.35755345010064793, - "AC3_4": 0.28773873181003984, - "AC3_5": 0.2275627008013015, - "AC3_6": 0.17349544513047785, - "AC3_7": 0.1244288640036827 - }, - "prompt_5": { - "overall_acc": 0.38961038961038963, - "language_acc": { - "English": 0.45454545454545453, - "Filipino": 0.29545454545454547, - "Spanish": 0.3806818181818182, - "Malay": 0.375, - "Chinese": 0.4318181818181818, - "Indonesian": 0.375, - "Vietnamese": 0.4147727272727273 - }, - "consistency_score_2": 0.545995670995671, - "consistency_score_3": 0.3753246753246754, - "consistency_score_4": 0.27694805194805205, - "consistency_score_5": 0.20968614718614717, - "consistency_score_6": 0.15909090909090912, - "consistency_score_7": 0.11931818181818182, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.39204545454545453, - "English,Spanish": 0.75, - "English,Malay": 0.625, - "English,Chinese": 0.5965909090909091, - "English,Indonesian": 0.6534090909090909, - "English,Vietnamese": 0.6704545454545454, - "Filipino,Spanish": 0.4034090909090909, - "Filipino,Malay": 0.4090909090909091, - "Filipino,Chinese": 0.3068181818181818, - "Filipino,Indonesian": 0.3409090909090909, - "Filipino,Vietnamese": 0.36363636363636365, - "Spanish,Malay": 0.6079545454545454, - "Spanish,Chinese": 0.5681818181818182, - "Spanish,Indonesian": 0.6988636363636364, - "Spanish,Vietnamese": 0.6306818181818182, - "Malay,Chinese": 0.5227272727272727, - "Malay,Indonesian": 0.6363636363636364, - "Malay,Vietnamese": 0.6022727272727273, - "Chinese,Indonesian": 0.5170454545454546, - "Chinese,Vietnamese": 0.5454545454545454, - "Indonesian,Vietnamese": 0.625 - }, - "3_combine": { - "English,Filipino,Spanish": 0.32954545454545453, - "English,Filipino,Malay": 0.2897727272727273, - "English,Filipino,Chinese": 0.25, - "English,Filipino,Indonesian": 0.26704545454545453, - "English,Filipino,Vietnamese": 0.2727272727272727, - "English,Spanish,Malay": 0.5056818181818182, - "English,Spanish,Chinese": 0.48863636363636365, - "English,Spanish,Indonesian": 0.5681818181818182, - "English,Spanish,Vietnamese": 0.5454545454545454, - "English,Malay,Chinese": 0.4147727272727273, - "English,Malay,Indonesian": 0.48863636363636365, - "English,Malay,Vietnamese": 0.48295454545454547, - "English,Chinese,Indonesian": 0.4318181818181818, - "English,Chinese,Vietnamese": 0.44886363636363635, - "English,Indonesian,Vietnamese": 0.5113636363636364, - "Filipino,Spanish,Malay": 0.2784090909090909, - "Filipino,Spanish,Chinese": 0.2556818181818182, - "Filipino,Spanish,Indonesian": 0.2784090909090909, - "Filipino,Spanish,Vietnamese": 0.29545454545454547, - "Filipino,Malay,Chinese": 0.21022727272727273, - "Filipino,Malay,Indonesian": 0.26704545454545453, - "Filipino,Malay,Vietnamese": 0.26136363636363635, - "Filipino,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Indonesian,Vietnamese": 0.23863636363636365, - "Spanish,Malay,Chinese": 0.4034090909090909, - "Spanish,Malay,Indonesian": 0.5, - "Spanish,Malay,Vietnamese": 0.4659090909090909, - "Spanish,Chinese,Indonesian": 0.42613636363636365, - "Spanish,Chinese,Vietnamese": 0.4147727272727273, - "Spanish,Indonesian,Vietnamese": 0.5, - "Malay,Chinese,Indonesian": 0.3806818181818182, - "Malay,Chinese,Vietnamese": 0.3977272727272727, - "Malay,Indonesian,Vietnamese": 0.48295454545454547, - "Chinese,Indonesian,Vietnamese": 0.38636363636363635 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.25, - "English,Filipino,Spanish,Chinese": 0.22727272727272727, - "English,Filipino,Spanish,Indonesian": 0.25, - "English,Filipino,Spanish,Vietnamese": 0.2556818181818182, - "English,Filipino,Malay,Chinese": 0.1875, - "English,Filipino,Malay,Indonesian": 0.22727272727272727, - "English,Filipino,Malay,Vietnamese": 0.21022727272727273, - "English,Filipino,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Chinese,Vietnamese": 0.18181818181818182, - "English,Filipino,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Chinese": 0.35795454545454547, - "English,Spanish,Malay,Indonesian": 0.4431818181818182, - "English,Spanish,Malay,Vietnamese": 0.42045454545454547, - "English,Spanish,Chinese,Indonesian": 0.3977272727272727, - "English,Spanish,Chinese,Vietnamese": 0.3806818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.4602272727272727, - "English,Malay,Chinese,Indonesian": 0.3465909090909091, - "English,Malay,Chinese,Vietnamese": 0.3352272727272727, - "English,Malay,Indonesian,Vietnamese": 0.4090909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.3465909090909091, - "Filipino,Spanish,Malay,Chinese": 0.18181818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.23295454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.3352272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.32954545454545453, - "Spanish,Malay,Indonesian,Vietnamese": 0.3977272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.32954545454545453, - "Malay,Chinese,Indonesian,Vietnamese": 0.3181818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.17045454545454544, - "English,Filipino,Spanish,Malay,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, - "English,Filipino,Spanish,Chinese,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "English,Filipino,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Chinese,Indonesian": 0.32386363636363635, - "English,Spanish,Malay,Chinese,Vietnamese": 0.30113636363636365, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.375, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.32386363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2784090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - } - }, - "AC3_2": 0.45473323663952087, - "AC3_3": 0.382334134406393, - "AC3_4": 0.32375807334418283, - "AC3_5": 0.2726393245125672, - "AC3_6": 0.22592791819443558, - "AC3_7": 0.18268812523596073 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.49514563106796117 - }, - "prompt_2": { - "accuracy": 0.4854368932038835 - }, - "prompt_3": { - "accuracy": 0.5242718446601942 - }, - "prompt_4": { - "accuracy": 0.49514563106796117 - }, - "prompt_5": { - "accuracy": 0.4563106796116505 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.5238095238095238 - }, - "prompt_2": { - "accuracy": 0.5523809523809524 - }, - "prompt_3": { - "accuracy": 0.4857142857142857 - }, - "prompt_4": { - "accuracy": 0.4666666666666667 - }, - "prompt_5": { - "accuracy": 0.45714285714285713 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.4205607476635514 - }, - "prompt_2": { - "accuracy": 0.3925233644859813 - }, - "prompt_3": { - "accuracy": 0.411214953271028 - }, - "prompt_4": { - "accuracy": 0.4392523364485981 - }, - "prompt_5": { - "accuracy": 0.4485981308411215 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.38, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.6, - "history": 0.4666666666666667, - "literature": 0.3, - "politics": 0.5, - "culture": 0.1, - "film": 0.2, - "law": 0.4, - "geography": 0.7 - } - }, - "prompt_2": { - "accuracy": 0.35, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.6, - "history": 0.4666666666666667, - "literature": 0.3, - "politics": 0.4, - "culture": 0.1, - "film": 0.2, - "law": 0.4, - "geography": 0.5 - } - }, - "prompt_3": { - "accuracy": 0.33, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.3, - "history": 0.4, - "literature": 0.4, - "politics": 0.4, - "culture": 0.1, - "film": 0.1, - "law": 0.3, - "geography": 0.6 - } - }, - "prompt_4": { - "accuracy": 0.33, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.4, - "history": 0.4666666666666667, - "literature": 0.3, - "politics": 0.4, - "culture": 0.1, - "film": 0.2, - "law": 0.4, - "geography": 0.5 - } - }, - "prompt_5": { - "accuracy": 0.34, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.4, - "history": 0.4666666666666667, - "literature": 0.4, - "politics": 0.4, - "culture": 0.1, - "film": 0.1, - "law": 0.3, - "geography": 0.5 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.14618315842245094 - }, - "prompt_2": { - "bleu_score": 0.14519000035944152 - }, - "prompt_3": { - "bleu_score": 0.17180008149380094 - }, - "prompt_4": { - "bleu_score": 0.17078112298363443 - }, - "prompt_5": { - "bleu_score": 0.16094636120600622 - } - }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.5748763476671741 - }, - "prompt_2": { - "bleu_score": 0.622584367383388 - }, - "prompt_3": { - "bleu_score": 0.5925270961508073 - }, - "prompt_4": { - "bleu_score": 0.5737109374343466 - }, - "prompt_5": { - "bleu_score": 0.571873048730087 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.5391203927442048 - }, - "prompt_2": { - "bleu_score": 0.5812876995281306 - }, - "prompt_3": { - "bleu_score": 0.5408430647844015 - }, - "prompt_4": { - "bleu_score": 0.5297692639851893 - }, - "prompt_5": { - "bleu_score": 0.5497739616341655 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.44458667481447495 - }, - "prompt_2": { - "bleu_score": 0.49736816900002 - }, - "prompt_3": { - "bleu_score": 0.45711269556272305 - }, - "prompt_4": { - "bleu_score": 0.44840905369974166 - }, - "prompt_5": { - "bleu_score": 0.4465748141900384 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.5228465588271887 - }, - "prompt_2": { - "bleu_score": 0.5743818110209153 - }, - "prompt_3": { - "bleu_score": 0.5341692367695634 - }, - "prompt_4": { - "bleu_score": 0.5032260776608347 - }, - "prompt_5": { - "bleu_score": 0.5243479232082423 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.4457409568261377 - }, - "prompt_2": { - "accuracy": 0.45274212368728123 - }, - "prompt_3": { - "accuracy": 0.4539089848308051 - }, - "prompt_4": { - "accuracy": 0.43757292882147025 - }, - "prompt_5": { - "accuracy": 0.45040840140023336 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.4391848409009653, - "category_acc": { - "high_school_european_history": 0.5853658536585366, - "business_ethics": 0.5151515151515151, - "clinical_knowledge": 0.5037878787878788, - "medical_genetics": 0.5252525252525253, - "high_school_us_history": 0.5221674876847291, - "high_school_physics": 0.35333333333333333, - "high_school_world_history": 0.6779661016949152, - "virology": 0.3333333333333333, - "high_school_microeconomics": 0.459915611814346, - "econometrics": 0.336283185840708, - "college_computer_science": 0.41414141414141414, - "high_school_biology": 0.511326860841424, - "abstract_algebra": 0.3434343434343434, - "professional_accounting": 0.3274021352313167, - "philosophy": 0.44516129032258067, - "professional_medicine": 0.45387453874538747, - "nutrition": 0.46229508196721314, - "global_facts": 0.29292929292929293, - "machine_learning": 0.2882882882882883, - "security_studies": 0.45081967213114754, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.45171849427168576, - "prehistory": 0.43653250773993807, - "anatomy": 0.4626865671641791, - "human_sexuality": 0.3769230769230769, - "college_medicine": 0.436046511627907, - "high_school_government_and_politics": 0.5520833333333334, - "college_chemistry": 0.3838383838383838, - "logical_fallacies": 0.4567901234567901, - "high_school_geography": 0.5786802030456852, - "elementary_mathematics": 0.2864721485411141, - "human_aging": 0.536036036036036, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.6452205882352942, - "formal_logic": 0.328, - "high_school_statistics": 0.3581395348837209, - "international_law": 0.5666666666666667, - "high_school_mathematics": 0.26394052044609667, - "high_school_computer_science": 0.5252525252525253, - "conceptual_physics": 0.41452991452991456, - "miscellaneous": 0.5639386189258312, - "high_school_chemistry": 0.3613861386138614, - "marketing": 0.6824034334763949, - "professional_law": 0.32093933463796476, - "management": 0.6078431372549019, - "college_physics": 0.32673267326732675, - "jurisprudence": 0.4766355140186916, - "world_religions": 0.5470588235294118, - "sociology": 0.56, - "us_foreign_policy": 0.5151515151515151, - "high_school_macroeconomics": 0.4318766066838046, - "computer_security": 0.5151515151515151, - "moral_scenarios": 0.2785234899328859, - "moral_disputes": 0.4144927536231884, - "electrical_engineering": 0.4097222222222222, - "astronomy": 0.46357615894039733, - "college_biology": 0.48951048951048953 - } - }, - "prompt_2": { - "accuracy": 0.44097247050411154, - "category_acc": { - "high_school_european_history": 0.5975609756097561, - "business_ethics": 0.47474747474747475, - "clinical_knowledge": 0.5, - "medical_genetics": 0.5252525252525253, - "high_school_us_history": 0.5123152709359606, - "high_school_physics": 0.3466666666666667, - "high_school_world_history": 0.6779661016949152, - "virology": 0.34545454545454546, - "high_school_microeconomics": 0.45569620253164556, - "econometrics": 0.336283185840708, - "college_computer_science": 0.42424242424242425, - "high_school_biology": 0.5080906148867314, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.3167259786476868, - "philosophy": 0.45806451612903226, - "professional_medicine": 0.44649446494464945, - "nutrition": 0.46557377049180326, - "global_facts": 0.2828282828282828, - "machine_learning": 0.32432432432432434, - "security_studies": 0.47950819672131145, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.4386252045826514, - "prehistory": 0.44272445820433437, - "anatomy": 0.4701492537313433, - "human_sexuality": 0.36153846153846153, - "college_medicine": 0.4186046511627907, - "high_school_government_and_politics": 0.5625, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.4691358024691358, - "high_school_geography": 0.5634517766497462, - "elementary_mathematics": 0.3076923076923077, - "human_aging": 0.527027027027027, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.6470588235294118, - "formal_logic": 0.32, - "high_school_statistics": 0.39069767441860465, - "international_law": 0.5416666666666666, - "high_school_mathematics": 0.27137546468401486, - "high_school_computer_science": 0.5151515151515151, - "conceptual_physics": 0.3974358974358974, - "miscellaneous": 0.5664961636828645, - "high_school_chemistry": 0.3613861386138614, - "marketing": 0.6781115879828327, - "professional_law": 0.32811480756686234, - "management": 0.6274509803921569, - "college_physics": 0.33663366336633666, - "jurisprudence": 0.48598130841121495, - "world_religions": 0.5058823529411764, - "sociology": 0.585, - "us_foreign_policy": 0.5151515151515151, - "high_school_macroeconomics": 0.43444730077120824, - "computer_security": 0.5353535353535354, - "moral_scenarios": 0.2897091722595078, - "moral_disputes": 0.4289855072463768, - "electrical_engineering": 0.4027777777777778, - "astronomy": 0.46357615894039733, - "college_biology": 0.48951048951048953 - } - }, - "prompt_3": { - "accuracy": 0.43689667500893814, - "category_acc": { - "high_school_european_history": 0.5853658536585366, - "business_ethics": 0.46464646464646464, - "clinical_knowledge": 0.4962121212121212, - "medical_genetics": 0.5353535353535354, - "high_school_us_history": 0.5073891625615764, - "high_school_physics": 0.36, - "high_school_world_history": 0.6694915254237288, - "virology": 0.3393939393939394, - "high_school_microeconomics": 0.4430379746835443, - "econometrics": 0.3008849557522124, - "college_computer_science": 0.40404040404040403, - "high_school_biology": 0.49514563106796117, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.30604982206405695, - "philosophy": 0.44516129032258067, - "professional_medicine": 0.46494464944649444, - "nutrition": 0.46557377049180326, - "global_facts": 0.2727272727272727, - "machine_learning": 0.35135135135135137, - "security_studies": 0.45081967213114754, - "public_relations": 0.5504587155963303, - "professional_psychology": 0.4369885433715221, - "prehistory": 0.43343653250773995, - "anatomy": 0.4626865671641791, - "human_sexuality": 0.36153846153846153, - "college_medicine": 0.42441860465116277, - "high_school_government_and_politics": 0.5677083333333334, - "college_chemistry": 0.3838383838383838, - "logical_fallacies": 0.47530864197530864, - "high_school_geography": 0.5736040609137056, - "elementary_mathematics": 0.2917771883289125, - "human_aging": 0.5180180180180181, - "college_mathematics": 0.35353535353535354, - "high_school_psychology": 0.6397058823529411, - "formal_logic": 0.32, - "high_school_statistics": 0.37209302325581395, - "international_law": 0.55, - "high_school_mathematics": 0.275092936802974, - "high_school_computer_science": 0.5151515151515151, - "conceptual_physics": 0.41025641025641024, - "miscellaneous": 0.5588235294117647, - "high_school_chemistry": 0.3613861386138614, - "marketing": 0.6738197424892703, - "professional_law": 0.33007175472928896, - "management": 0.6078431372549019, - "college_physics": 0.32673267326732675, - "jurisprudence": 0.4953271028037383, - "world_religions": 0.5, - "sociology": 0.575, - "us_foreign_policy": 0.494949494949495, - "high_school_macroeconomics": 0.43958868894601544, - "computer_security": 0.5050505050505051, - "moral_scenarios": 0.2829977628635347, - "moral_disputes": 0.4318840579710145, - "electrical_engineering": 0.3958333333333333, - "astronomy": 0.4370860927152318, - "college_biology": 0.4755244755244755 - } - }, - "prompt_4": { - "accuracy": 0.42631390775831246, - "category_acc": { - "high_school_european_history": 0.5914634146341463, - "business_ethics": 0.43434343434343436, - "clinical_knowledge": 0.5037878787878788, - "medical_genetics": 0.5353535353535354, - "high_school_us_history": 0.4827586206896552, - "high_school_physics": 0.36, - "high_school_world_history": 0.6483050847457628, - "virology": 0.3393939393939394, - "high_school_microeconomics": 0.43037974683544306, - "econometrics": 0.3185840707964602, - "college_computer_science": 0.42424242424242425, - "high_school_biology": 0.4919093851132686, - "abstract_algebra": 0.3333333333333333, - "professional_accounting": 0.3096085409252669, - "philosophy": 0.43870967741935485, - "professional_medicine": 0.43911439114391143, - "nutrition": 0.4524590163934426, - "global_facts": 0.2727272727272727, - "machine_learning": 0.3153153153153153, - "security_studies": 0.4139344262295082, - "public_relations": 0.5321100917431193, - "professional_psychology": 0.4238952536824877, - "prehistory": 0.4458204334365325, - "anatomy": 0.4626865671641791, - "human_sexuality": 0.38461538461538464, - "college_medicine": 0.43023255813953487, - "high_school_government_and_politics": 0.5364583333333334, - "college_chemistry": 0.35353535353535354, - "logical_fallacies": 0.4691358024691358, - "high_school_geography": 0.5685279187817259, - "elementary_mathematics": 0.27320954907161804, - "human_aging": 0.509009009009009, - "college_mathematics": 0.35353535353535354, - "high_school_psychology": 0.6213235294117647, - "formal_logic": 0.328, - "high_school_statistics": 0.3488372093023256, - "international_law": 0.5333333333333333, - "high_school_mathematics": 0.24907063197026022, - "high_school_computer_science": 0.494949494949495, - "conceptual_physics": 0.41025641025641024, - "miscellaneous": 0.5498721227621484, - "high_school_chemistry": 0.3316831683168317, - "marketing": 0.6351931330472103, - "professional_law": 0.31833007175472927, - "management": 0.6078431372549019, - "college_physics": 0.3465346534653465, - "jurisprudence": 0.4392523364485981, - "world_religions": 0.5176470588235295, - "sociology": 0.555, - "us_foreign_policy": 0.48484848484848486, - "high_school_macroeconomics": 0.4267352185089974, - "computer_security": 0.5151515151515151, - "moral_scenarios": 0.2651006711409396, - "moral_disputes": 0.4144927536231884, - "electrical_engineering": 0.375, - "astronomy": 0.48344370860927155, - "college_biology": 0.45454545454545453 - } - }, - "prompt_5": { - "accuracy": 0.43267786914551304, - "category_acc": { - "high_school_european_history": 0.5853658536585366, - "business_ethics": 0.43434343434343436, - "clinical_knowledge": 0.5075757575757576, - "medical_genetics": 0.5555555555555556, - "high_school_us_history": 0.49261083743842365, - "high_school_physics": 0.34, - "high_school_world_history": 0.6567796610169492, - "virology": 0.3333333333333333, - "high_school_microeconomics": 0.43037974683544306, - "econometrics": 0.35398230088495575, - "college_computer_science": 0.3838383838383838, - "high_school_biology": 0.49514563106796117, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.3167259786476868, - "philosophy": 0.45806451612903226, - "professional_medicine": 0.45018450184501846, - "nutrition": 0.4557377049180328, - "global_facts": 0.2828282828282828, - "machine_learning": 0.3153153153153153, - "security_studies": 0.45901639344262296, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.4238952536824877, - "prehistory": 0.43962848297213625, - "anatomy": 0.4626865671641791, - "human_sexuality": 0.36923076923076925, - "college_medicine": 0.4186046511627907, - "high_school_government_and_politics": 0.5625, - "college_chemistry": 0.35353535353535354, - "logical_fallacies": 0.47530864197530864, - "high_school_geography": 0.5431472081218274, - "elementary_mathematics": 0.27055702917771884, - "human_aging": 0.509009009009009, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.6378676470588235, - "formal_logic": 0.304, - "high_school_statistics": 0.33488372093023255, - "international_law": 0.55, - "high_school_mathematics": 0.24907063197026022, - "high_school_computer_science": 0.5252525252525253, - "conceptual_physics": 0.41452991452991456, - "miscellaneous": 0.5664961636828645, - "high_school_chemistry": 0.33663366336633666, - "marketing": 0.6652360515021459, - "professional_law": 0.3268101761252446, - "management": 0.5882352941176471, - "college_physics": 0.297029702970297, - "jurisprudence": 0.45794392523364486, - "world_religions": 0.4647058823529412, - "sociology": 0.6, - "us_foreign_policy": 0.47474747474747475, - "high_school_macroeconomics": 0.43958868894601544, - "computer_security": 0.5353535353535354, - "moral_scenarios": 0.28635346756152125, - "moral_disputes": 0.42028985507246375, - "electrical_engineering": 0.4027777777777778, - "astronomy": 0.46357615894039733, - "college_biology": 0.4825174825174825 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.43016344725111444 - }, - "prompt_2": { - "accuracy": 0.4138187221396731 - }, - "prompt_3": { - "accuracy": 0.4101040118870728 - }, - "prompt_4": { - "accuracy": 0.41679049034175336 - }, - "prompt_5": { - "accuracy": 0.42199108469539376 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.40597758405977585, - "category_acc": { - "computer_network": 0.5, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.40476190476190477, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.5238095238095238, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.5172413793103449, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.375, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.36666666666666664, - "business_administration": 0.3684210526315789, - "marxism": 0.5833333333333334, - "mao_zedong_thought": 0.6551724137931034, - "education_science": 0.5882352941176471, - "teacher_qualification": 0.5510204081632653, - "high_school_politics": 0.5, - "high_school_geography": 0.375, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.7058823529411765, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.2962962962962963, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.4642857142857143, - "art_studies": 0.5526315789473685, - "professional_tour_guide": 0.4117647058823529, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.52, - "middle_school_history": 0.4074074074074074, - "civil_servant": 0.3269230769230769, - "sports_science": 0.4583333333333333, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.35294117647058826, - "accountant": 0.42592592592592593, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.4074074074074074, - "physician": 0.4444444444444444 - } - }, - "prompt_2": { - "accuracy": 0.3991282689912827, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.15384615384615385, - "college_programming": 0.40476190476190477, - "college_physics": 0.25, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.4827586206896552, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.375, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.625, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.36666666666666664, - "business_administration": 0.3684210526315789, - "marxism": 0.5833333333333334, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.5588235294117647, - "teacher_qualification": 0.6122448979591837, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.5882352941176471, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.2962962962962963, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.4642857142857143, - "art_studies": 0.5789473684210527, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.48, - "middle_school_history": 0.4074074074074074, - "civil_servant": 0.36538461538461536, - "sports_science": 0.4583333333333333, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.5, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.35294117647058826, - "accountant": 0.3888888888888889, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.4074074074074074, - "physician": 0.4444444444444444 - } - }, - "prompt_3": { - "accuracy": 0.40473225404732255, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.15384615384615385, - "college_programming": 0.38095238095238093, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.47619047619047616, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.5172413793103449, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.4166666666666667, - "high_school_chemistry": 0.25, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.5769230769230769, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.38333333333333336, - "business_administration": 0.42105263157894735, - "marxism": 0.5833333333333334, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.5588235294117647, - "teacher_qualification": 0.6122448979591837, - "high_school_politics": 0.375, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.37037037037037035, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.5789473684210527, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.52, - "middle_school_history": 0.48148148148148145, - "civil_servant": 0.40384615384615385, - "sports_science": 0.4583333333333333, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.35294117647058826, - "accountant": 0.37037037037037035, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.3888888888888889, - "tax_accountant": 0.4074074074074074, - "physician": 0.4444444444444444 - } - }, - "prompt_4": { - "accuracy": 0.3972602739726027, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.35714285714285715, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.47619047619047616, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.4482758620689655, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.25, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.38333333333333336, - "business_administration": 0.39473684210526316, - "marxism": 0.5, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.5588235294117647, - "teacher_qualification": 0.5102040816326531, - "high_school_politics": 0.4583333333333333, - "high_school_geography": 0.4583333333333333, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.6470588235294118, - "modern_chinese_history": 0.5, - "ideological_and_moral_cultivation": 0.5416666666666666, - "logic": 0.37037037037037035, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.42857142857142855, - "art_studies": 0.6052631578947368, - "professional_tour_guide": 0.4411764705882353, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.52, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.36538461538461536, - "sports_science": 0.5416666666666666, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.37254901960784315, - "accountant": 0.3333333333333333, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.3888888888888889, - "tax_accountant": 0.35185185185185186, - "physician": 0.4444444444444444 - } - }, - "prompt_5": { - "accuracy": 0.3972602739726027, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.375, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.35714285714285715, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.47619047619047616, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.41379310344827586, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.375, - "high_school_chemistry": 0.25, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.5769230769230769, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.42857142857142855, - "college_economics": 0.36666666666666664, - "business_administration": 0.34210526315789475, - "marxism": 0.5416666666666666, - "mao_zedong_thought": 0.5517241379310345, - "education_science": 0.6470588235294118, - "teacher_qualification": 0.5714285714285714, - "high_school_politics": 0.4583333333333333, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.5882352941176471, - "modern_chinese_history": 0.5, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.18518518518518517, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.42857142857142855, - "art_studies": 0.47368421052631576, - "professional_tour_guide": 0.4411764705882353, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.52, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.36538461538461536, - "sports_science": 0.5, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.35294117647058826, - "accountant": 0.35185185185185186, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.3888888888888889, - "physician": 0.42592592592592593 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.4050179211469534 - }, - "prompt_2": { - "accuracy": 0.43727598566308246 - }, - "prompt_3": { - "accuracy": 0.41935483870967744 - }, - "prompt_4": { - "accuracy": 0.4014336917562724 - }, - "prompt_5": { - "accuracy": 0.41935483870967744 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.42117078224831633, - "category_acc": { - "agronomy": 0.4437869822485207, - "anatomy": 0.27702702702702703, - "ancient_chinese": 0.29878048780487804, - "arts": 0.5625, - "astronomy": 0.3575757575757576, - "business_ethics": 0.4784688995215311, - "chinese_civil_service_exam": 0.35, - "chinese_driving_rule": 0.5877862595419847, - "chinese_food_culture": 0.3897058823529412, - "chinese_foreign_policy": 0.5046728971962616, - "chinese_history": 0.5108359133126935, - "chinese_literature": 0.3088235294117647, - "chinese_teacher_qualification": 0.553072625698324, - "clinical_knowledge": 0.33755274261603374, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.5514018691588785, - "college_engineering_hydrology": 0.33962264150943394, - "college_law": 0.32407407407407407, - "college_mathematics": 0.2, - "college_medical_statistics": 0.36792452830188677, - "college_medicine": 0.38095238095238093, - "computer_science": 0.44607843137254904, - "computer_security": 0.5087719298245614, - "conceptual_physics": 0.36054421768707484, - "construction_project_management": 0.35251798561151076, - "economics": 0.4276729559748428, - "education": 0.5337423312883436, - "electrical_engineering": 0.38953488372093026, - "elementary_chinese": 0.3611111111111111, - "elementary_commonsense": 0.41414141414141414, - "elementary_information_and_technology": 0.6050420168067226, - "elementary_mathematics": 0.28695652173913044, - "ethnology": 0.4222222222222222, - "food_science": 0.4125874125874126, - "genetics": 0.35795454545454547, - "global_facts": 0.48322147651006714, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.21212121212121213, - "high_school_geography": 0.423728813559322, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.36363636363636365, - "high_school_politics": 0.3986013986013986, - "human_sexuality": 0.46825396825396826, - "international_law": 0.372972972972973, - "journalism": 0.5290697674418605, - "jurisprudence": 0.4233576642335766, - "legal_and_moral_basis": 0.794392523364486, - "logical": 0.3902439024390244, - "machine_learning": 0.28688524590163933, - "management": 0.4523809523809524, - "marketing": 0.4888888888888889, - "marxist_theory": 0.5132275132275133, - "modern_chinese": 0.33620689655172414, - "nutrition": 0.4206896551724138, - "philosophy": 0.4380952380952381, - "professional_accounting": 0.48, - "professional_law": 0.3459715639810427, - "professional_medicine": 0.3058510638297872, - "professional_psychology": 0.4870689655172414, - "public_relations": 0.5172413793103449, - "security_study": 0.5037037037037037, - "sociology": 0.4778761061946903, - "sports_science": 0.44242424242424244, - "traditional_chinese_medicine": 0.372972972972973, - "virology": 0.4437869822485207, - "world_history": 0.45962732919254656, - "world_religions": 0.49375 - } - }, - "prompt_2": { - "accuracy": 0.422465895354861, - "category_acc": { - "agronomy": 0.4319526627218935, - "anatomy": 0.2905405405405405, - "ancient_chinese": 0.32926829268292684, - "arts": 0.575, - "astronomy": 0.3575757575757576, - "business_ethics": 0.46411483253588515, - "chinese_civil_service_exam": 0.3625, - "chinese_driving_rule": 0.5877862595419847, - "chinese_food_culture": 0.4117647058823529, - "chinese_foreign_policy": 0.4953271028037383, - "chinese_history": 0.5201238390092879, - "chinese_literature": 0.3284313725490196, - "chinese_teacher_qualification": 0.5418994413407822, - "clinical_knowledge": 0.3333333333333333, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.5233644859813084, - "college_engineering_hydrology": 0.39622641509433965, - "college_law": 0.3148148148148148, - "college_mathematics": 0.18095238095238095, - "college_medical_statistics": 0.3584905660377358, - "college_medicine": 0.39194139194139194, - "computer_science": 0.4264705882352941, - "computer_security": 0.5263157894736842, - "conceptual_physics": 0.35374149659863946, - "construction_project_management": 0.4172661870503597, - "economics": 0.44654088050314467, - "education": 0.50920245398773, - "electrical_engineering": 0.3953488372093023, - "elementary_chinese": 0.373015873015873, - "elementary_commonsense": 0.3939393939393939, - "elementary_information_and_technology": 0.592436974789916, - "elementary_mathematics": 0.28695652173913044, - "ethnology": 0.45925925925925926, - "food_science": 0.4405594405594406, - "genetics": 0.3409090909090909, - "global_facts": 0.42953020134228187, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.4322033898305085, - "high_school_mathematics": 0.25, - "high_school_physics": 0.37272727272727274, - "high_school_politics": 0.42657342657342656, - "human_sexuality": 0.47619047619047616, - "international_law": 0.3621621621621622, - "journalism": 0.5, - "jurisprudence": 0.4233576642335766, - "legal_and_moral_basis": 0.8037383177570093, - "logical": 0.4065040650406504, - "machine_learning": 0.28688524590163933, - "management": 0.4666666666666667, - "marketing": 0.4722222222222222, - "marxist_theory": 0.5026455026455027, - "modern_chinese": 0.3103448275862069, - "nutrition": 0.42758620689655175, - "philosophy": 0.4857142857142857, - "professional_accounting": 0.46285714285714286, - "professional_law": 0.35071090047393366, - "professional_medicine": 0.31382978723404253, - "professional_psychology": 0.4827586206896552, - "public_relations": 0.4942528735632184, - "security_study": 0.5185185185185185, - "sociology": 0.4823008849557522, - "sports_science": 0.3939393939393939, - "traditional_chinese_medicine": 0.372972972972973, - "virology": 0.4437869822485207, - "world_history": 0.4720496894409938, - "world_religions": 0.4875 - } - }, - "prompt_3": { - "accuracy": 0.42082541875323776, - "category_acc": { - "agronomy": 0.4260355029585799, - "anatomy": 0.2905405405405405, - "ancient_chinese": 0.2926829268292683, - "arts": 0.5875, - "astronomy": 0.34545454545454546, - "business_ethics": 0.45933014354066987, - "chinese_civil_service_exam": 0.375, - "chinese_driving_rule": 0.5954198473282443, - "chinese_food_culture": 0.4117647058823529, - "chinese_foreign_policy": 0.5046728971962616, - "chinese_history": 0.4984520123839009, - "chinese_literature": 0.3137254901960784, - "chinese_teacher_qualification": 0.5698324022346368, - "clinical_knowledge": 0.3333333333333333, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.5607476635514018, - "college_engineering_hydrology": 0.3867924528301887, - "college_law": 0.35185185185185186, - "college_mathematics": 0.19047619047619047, - "college_medical_statistics": 0.33962264150943394, - "college_medicine": 0.39194139194139194, - "computer_science": 0.45588235294117646, - "computer_security": 0.49707602339181284, - "conceptual_physics": 0.3333333333333333, - "construction_project_management": 0.39568345323741005, - "economics": 0.44654088050314467, - "education": 0.5030674846625767, - "electrical_engineering": 0.3953488372093023, - "elementary_chinese": 0.373015873015873, - "elementary_commonsense": 0.4090909090909091, - "elementary_information_and_technology": 0.5882352941176471, - "elementary_mathematics": 0.2956521739130435, - "ethnology": 0.4074074074074074, - "food_science": 0.4195804195804196, - "genetics": 0.3522727272727273, - "global_facts": 0.46308724832214765, - "high_school_biology": 0.22485207100591717, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.4322033898305085, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.36363636363636365, - "high_school_politics": 0.4125874125874126, - "human_sexuality": 0.4444444444444444, - "international_law": 0.3621621621621622, - "journalism": 0.5116279069767442, - "jurisprudence": 0.40875912408759124, - "legal_and_moral_basis": 0.7757009345794392, - "logical": 0.42276422764227645, - "machine_learning": 0.29508196721311475, - "management": 0.47619047619047616, - "marketing": 0.48333333333333334, - "marxist_theory": 0.48148148148148145, - "modern_chinese": 0.3103448275862069, - "nutrition": 0.45517241379310347, - "philosophy": 0.44761904761904764, - "professional_accounting": 0.4742857142857143, - "professional_law": 0.33175355450236965, - "professional_medicine": 0.30851063829787234, - "professional_psychology": 0.4827586206896552, - "public_relations": 0.4885057471264368, - "security_study": 0.5259259259259259, - "sociology": 0.49557522123893805, - "sports_science": 0.42424242424242425, - "traditional_chinese_medicine": 0.3945945945945946, - "virology": 0.4556213017751479, - "world_history": 0.4968944099378882, - "world_religions": 0.4875 - } - }, - "prompt_4": { - "accuracy": 0.4156449663270592, - "category_acc": { - "agronomy": 0.4437869822485207, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.2926829268292683, - "arts": 0.56875, - "astronomy": 0.34545454545454546, - "business_ethics": 0.4784688995215311, - "chinese_civil_service_exam": 0.35625, - "chinese_driving_rule": 0.5954198473282443, - "chinese_food_culture": 0.3897058823529412, - "chinese_foreign_policy": 0.48598130841121495, - "chinese_history": 0.4984520123839009, - "chinese_literature": 0.3382352941176471, - "chinese_teacher_qualification": 0.547486033519553, - "clinical_knowledge": 0.32489451476793246, - "college_actuarial_science": 0.3018867924528302, - "college_education": 0.5700934579439252, - "college_engineering_hydrology": 0.36792452830188677, - "college_law": 0.35185185185185186, - "college_mathematics": 0.19047619047619047, - "college_medical_statistics": 0.3490566037735849, - "college_medicine": 0.3882783882783883, - "computer_science": 0.4362745098039216, - "computer_security": 0.49122807017543857, - "conceptual_physics": 0.32653061224489793, - "construction_project_management": 0.3381294964028777, - "economics": 0.4528301886792453, - "education": 0.48466257668711654, - "electrical_engineering": 0.37209302325581395, - "elementary_chinese": 0.376984126984127, - "elementary_commonsense": 0.41414141414141414, - "elementary_information_and_technology": 0.6218487394957983, - "elementary_mathematics": 0.26521739130434785, - "ethnology": 0.4148148148148148, - "food_science": 0.44755244755244755, - "genetics": 0.3693181818181818, - "global_facts": 0.4563758389261745, - "high_school_biology": 0.20710059171597633, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.4491525423728814, - "high_school_mathematics": 0.25609756097560976, - "high_school_physics": 0.37272727272727274, - "high_school_politics": 0.34965034965034963, - "human_sexuality": 0.42857142857142855, - "international_law": 0.35135135135135137, - "journalism": 0.4883720930232558, - "jurisprudence": 0.41119221411192214, - "legal_and_moral_basis": 0.7710280373831776, - "logical": 0.3821138211382114, - "machine_learning": 0.26229508196721313, - "management": 0.46190476190476193, - "marketing": 0.49444444444444446, - "marxist_theory": 0.5185185185185185, - "modern_chinese": 0.3017241379310345, - "nutrition": 0.43448275862068964, - "philosophy": 0.4380952380952381, - "professional_accounting": 0.4857142857142857, - "professional_law": 0.32701421800947866, - "professional_medicine": 0.300531914893617, - "professional_psychology": 0.46120689655172414, - "public_relations": 0.5057471264367817, - "security_study": 0.5111111111111111, - "sociology": 0.4646017699115044, - "sports_science": 0.43636363636363634, - "traditional_chinese_medicine": 0.3675675675675676, - "virology": 0.4378698224852071, - "world_history": 0.484472049689441, - "world_religions": 0.4875 - } - }, - "prompt_5": { - "accuracy": 0.42376100846140563, - "category_acc": { - "agronomy": 0.4437869822485207, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.3048780487804878, - "arts": 0.58125, - "astronomy": 0.3939393939393939, - "business_ethics": 0.48325358851674644, - "chinese_civil_service_exam": 0.375, - "chinese_driving_rule": 0.6030534351145038, - "chinese_food_culture": 0.4338235294117647, - "chinese_foreign_policy": 0.4672897196261682, - "chinese_history": 0.5170278637770898, - "chinese_literature": 0.3284313725490196, - "chinese_teacher_qualification": 0.547486033519553, - "clinical_knowledge": 0.3291139240506329, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.5794392523364486, - "college_engineering_hydrology": 0.330188679245283, - "college_law": 0.3148148148148148, - "college_mathematics": 0.19047619047619047, - "college_medical_statistics": 0.3584905660377358, - "college_medicine": 0.36996336996337, - "computer_science": 0.4362745098039216, - "computer_security": 0.49707602339181284, - "conceptual_physics": 0.3401360544217687, - "construction_project_management": 0.381294964028777, - "economics": 0.4088050314465409, - "education": 0.5276073619631901, - "electrical_engineering": 0.42441860465116277, - "elementary_chinese": 0.39285714285714285, - "elementary_commonsense": 0.4090909090909091, - "elementary_information_and_technology": 0.6092436974789915, - "elementary_mathematics": 0.27391304347826084, - "ethnology": 0.4222222222222222, - "food_science": 0.4195804195804196, - "genetics": 0.3693181818181818, - "global_facts": 0.4563758389261745, - "high_school_biology": 0.26627218934911245, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.423728813559322, - "high_school_mathematics": 0.25609756097560976, - "high_school_physics": 0.34545454545454546, - "high_school_politics": 0.3986013986013986, - "human_sexuality": 0.46825396825396826, - "international_law": 0.3621621621621622, - "journalism": 0.48255813953488375, - "jurisprudence": 0.4257907542579075, - "legal_and_moral_basis": 0.7990654205607477, - "logical": 0.3902439024390244, - "machine_learning": 0.27049180327868855, - "management": 0.4666666666666667, - "marketing": 0.4777777777777778, - "marxist_theory": 0.5026455026455027, - "modern_chinese": 0.3620689655172414, - "nutrition": 0.42758620689655175, - "philosophy": 0.4857142857142857, - "professional_accounting": 0.4742857142857143, - "professional_law": 0.3886255924170616, - "professional_medicine": 0.30319148936170215, - "professional_psychology": 0.47413793103448276, - "public_relations": 0.5172413793103449, - "security_study": 0.5185185185185185, - "sociology": 0.47345132743362833, - "sports_science": 0.43636363636363634, - "traditional_chinese_medicine": 0.372972972972973, - "virology": 0.46153846153846156, - "world_history": 0.4906832298136646, - "world_religions": 0.5 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.2727272727272727 - }, - "prompt_2": { - "accuracy": 0.30303030303030304 - }, - "prompt_3": { - "accuracy": 0.36363636363636365 - }, - "prompt_4": { - "accuracy": 0.30303030303030304 - }, - "prompt_5": { - "accuracy": 0.30303030303030304 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.49772727272727274 - }, - "prompt_2": { - "accuracy": 0.425 - }, - "prompt_3": { - "accuracy": 0.4340909090909091 - }, - "prompt_4": { - "accuracy": 0.5363636363636364 - }, - "prompt_5": { - "accuracy": 0.5295454545454545 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3522033898305085 - }, - "prompt_2": { - "accuracy": 0.3410169491525424 - }, - "prompt_3": { - "accuracy": 0.3328813559322034 - }, - "prompt_4": { - "accuracy": 0.3616949152542373 - }, - "prompt_5": { - "accuracy": 0.3911864406779661 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.8459237097980553 - }, - "prompt_2": { - "accuracy": 0.8485415108451758 - }, - "prompt_3": { - "accuracy": 0.8448017950635752 - }, - "prompt_4": { - "accuracy": 0.8492894540014959 - }, - "prompt_5": { - "accuracy": 0.8477935676888556 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.8309652131308183 - }, - "prompt_2": { - "accuracy": 0.8221460068593827 - }, - "prompt_3": { - "accuracy": 0.8270455658990691 - }, - "prompt_4": { - "accuracy": 0.8187163155316022 - }, - "prompt_5": { - "accuracy": 0.8265556099951005 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.4824982636957889, - "rouge2": 0.24240392732475038, - "rougeL": 0.40689064942931014, - "avg_rouge": 0.3772642801499498 - }, - "prompt_2": { - "rouge1": 0.4841225122915221, - "rouge2": 0.24015981473394443, - "rougeL": 0.40618966072364265, - "avg_rouge": 0.3768239959163697 - }, - "prompt_3": { - "rouge1": 0.4679758322603248, - "rouge2": 0.2349836827348769, - "rougeL": 0.3956417618848214, - "avg_rouge": 0.3662004256266744 - }, - "prompt_4": { - "rouge1": 0.47500378958481515, - "rouge2": 0.2426076999238804, - "rougeL": 0.40211771289703824, - "avg_rouge": 0.373243067468578 - }, - "prompt_5": { - "rouge1": 0.46231426533241154, - "rouge2": 0.23522614702392788, - "rougeL": 0.39342188953545104, - "avg_rouge": 0.36365410063059683 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.2156247444832894, - "rouge2": 0.055880588253819474, - "rougeL": 0.16729005021198137, - "avg_rouge": 0.14626512764969676 - }, - "prompt_2": { - "rouge1": 0.2244342036191811, - "rouge2": 0.06184877775107199, - "rougeL": 0.1691756710710183, - "avg_rouge": 0.15181955081375711 - }, - "prompt_3": { - "rouge1": 0.2337468986078192, - "rouge2": 0.0642409607133074, - "rougeL": 0.18041099456245718, - "avg_rouge": 0.15946628462786125 - }, - "prompt_4": { - "rouge1": 0.23134012937219503, - "rouge2": 0.061619274357077744, - "rougeL": 0.17862699994554881, - "avg_rouge": 0.15719546789160718 - }, - "prompt_5": { - "rouge1": 0.22610837857502727, - "rouge2": 0.054894066260553806, - "rougeL": 0.18799105041444647, - "avg_rouge": 0.15633116508334252 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.9403669724770642 - }, - "prompt_2": { - "accuracy": 0.9392201834862385 - }, - "prompt_3": { - "accuracy": 0.9357798165137615 - }, - "prompt_4": { - "accuracy": 0.9311926605504587 - }, - "prompt_5": { - "accuracy": 0.9311926605504587 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.6059443911792906 - }, - "prompt_2": { - "accuracy": 0.6097794822627037 - }, - "prompt_3": { - "accuracy": 0.6164908916586769 - }, - "prompt_4": { - "accuracy": 0.6558005752636625 - }, - "prompt_5": { - "accuracy": 0.6289549376797698 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.692 - }, - "prompt_2": { - "accuracy": 0.7705 - }, - "prompt_3": { - "accuracy": 0.7375 - }, - "prompt_4": { - "accuracy": 0.723 - }, - "prompt_5": { - "accuracy": 0.7305 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.52 - }, - "prompt_2": { - "accuracy": 0.4655 - }, - "prompt_3": { - "accuracy": 0.488 - }, - "prompt_4": { - "accuracy": 0.496 - }, - "prompt_5": { - "accuracy": 0.482 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.6 - }, - "prompt_2": { - "accuracy": 0.644 - }, - "prompt_3": { - "accuracy": 0.63 - }, - "prompt_4": { - "accuracy": 0.6045 - }, - "prompt_5": { - "accuracy": 0.6075 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.49295774647887325 - }, - "prompt_2": { - "accuracy": 0.5352112676056338 - }, - "prompt_3": { - "accuracy": 0.5352112676056338 - }, - "prompt_4": { - "accuracy": 0.5633802816901409 - }, - "prompt_5": { - "accuracy": 0.5774647887323944 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.5848375451263538 - }, - "prompt_2": { - "accuracy": 0.6064981949458483 - }, - "prompt_3": { - "accuracy": 0.6389891696750902 - }, - "prompt_4": { - "accuracy": 0.6823104693140795 - }, - "prompt_5": { - "accuracy": 0.6245487364620939 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.7254901960784313 - }, - "prompt_2": { - "accuracy": 0.6004901960784313 - }, - "prompt_3": { - "accuracy": 0.7132352941176471 - }, - "prompt_4": { - "accuracy": 0.4411764705882353 - }, - "prompt_5": { - "accuracy": 0.6568627450980392 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.8029411764705883, - "language_acc": { - "English": 0.8445378151260504, - "Chinese": 0.7873949579831933, - "Spanish": 0.8092436974789916, - "Vietnamese": 0.7705882352941177 - }, - "consistency_score_2": 0.7686274509803921, - "consistency_score_3": 0.6710084033613445, - "consistency_score_4": 0.607563025210084, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.7773109243697479, - "English,Spanish": 0.8159663865546218, - "English,Vietnamese": 0.7764705882352941, - "Chinese,Spanish": 0.7554621848739496, - "Chinese,Vietnamese": 0.7294117647058823, - "Spanish,Vietnamese": 0.7571428571428571 - }, - "3_combine": { - "English,Chinese,Spanish": 0.6882352941176471, - "English,Chinese,Vietnamese": 0.6605042016806723, - "English,Spanish,Vietnamese": 0.6915966386554622, - "Chinese,Spanish,Vietnamese": 0.6436974789915967 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.607563025210084 - } - }, - "AC3_2": 0.7854097096854464, - "AC3_3": 0.7310701589147844, - "AC3_4": 0.6917205487602954 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.35047619047619044, - "language_acc": { - "Spanish": 0.35333333333333333, - "English": 0.41333333333333333, - "Chinese": 0.37333333333333335, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.30666666666666664, - "Indonesian": 0.34, - "Malay": 0.38 - }, - "consistency_score_2": 0.5441269841269841, - "consistency_score_3": 0.36419047619047634, - "consistency_score_4": 0.2643809523809524, - "consistency_score_5": 0.19904761904761908, - "consistency_score_6": 0.15333333333333335, - "consistency_score_7": 0.12, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6733333333333333, - "Spanish,Chinese": 0.6, - "Spanish,Vietnamese": 0.54, - "Spanish,Filipino": 0.4066666666666667, - "Spanish,Indonesian": 0.56, - "Spanish,Malay": 0.62, - "English,Chinese": 0.5866666666666667, - "English,Vietnamese": 0.5533333333333333, - "English,Filipino": 0.43333333333333335, - "English,Indonesian": 0.58, - "English,Malay": 0.5533333333333333, - "Chinese,Vietnamese": 0.56, - "Chinese,Filipino": 0.5066666666666667, - "Chinese,Indonesian": 0.62, - "Chinese,Malay": 0.5866666666666667, - "Vietnamese,Filipino": 0.4, - "Vietnamese,Indonesian": 0.6466666666666666, - "Vietnamese,Malay": 0.5866666666666667, - "Filipino,Indonesian": 0.38, - "Filipino,Malay": 0.43333333333333335, - "Indonesian,Malay": 0.6 - }, - "3_combine": { - "Spanish,English,Chinese": 0.4666666666666667, - "Spanish,English,Vietnamese": 0.43333333333333335, - "Spanish,English,Filipino": 0.31333333333333335, - "Spanish,English,Indonesian": 0.43333333333333335, - "Spanish,English,Malay": 0.4533333333333333, - "Spanish,Chinese,Vietnamese": 0.4, - "Spanish,Chinese,Filipino": 0.30666666666666664, - "Spanish,Chinese,Indonesian": 0.4266666666666667, - "Spanish,Chinese,Malay": 0.43333333333333335, - "Spanish,Vietnamese,Filipino": 0.24, - "Spanish,Vietnamese,Indonesian": 0.41333333333333333, - "Spanish,Vietnamese,Malay": 0.4266666666666667, - "Spanish,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Filipino,Malay": 0.28, - "Spanish,Indonesian,Malay": 0.44, - "English,Chinese,Vietnamese": 0.4, - "English,Chinese,Filipino": 0.32, - "English,Chinese,Indonesian": 0.4266666666666667, - "English,Chinese,Malay": 0.4, - "English,Vietnamese,Filipino": 0.24666666666666667, - "English,Vietnamese,Indonesian": 0.42, - "English,Vietnamese,Malay": 0.4066666666666667, - "English,Filipino,Indonesian": 0.25333333333333335, - "English,Filipino,Malay": 0.2733333333333333, - "English,Indonesian,Malay": 0.4066666666666667, - "Chinese,Vietnamese,Filipino": 0.3, - "Chinese,Vietnamese,Indonesian": 0.44666666666666666, - "Chinese,Vietnamese,Malay": 0.4066666666666667, - "Chinese,Filipino,Indonesian": 0.3, - "Chinese,Filipino,Malay": 0.31333333333333335, - "Chinese,Indonesian,Malay": 0.43333333333333335, - "Vietnamese,Filipino,Indonesian": 0.28, - "Vietnamese,Filipino,Malay": 0.2733333333333333, - "Vietnamese,Indonesian,Malay": 0.48, - "Filipino,Indonesian,Malay": 0.26666666666666666 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.34, - "Spanish,English,Chinese,Filipino": 0.25333333333333335, - "Spanish,English,Chinese,Indonesian": 0.3333333333333333, - "Spanish,English,Chinese,Malay": 0.3466666666666667, - "Spanish,English,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.34, - "Spanish,English,Vietnamese,Malay": 0.35333333333333333, - "Spanish,English,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Indonesian,Malay": 0.34, - "Spanish,Chinese,Vietnamese,Filipino": 0.22, - "Spanish,Chinese,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.31333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.23333333333333334, - "Spanish,Chinese,Indonesian,Malay": 0.34, - "Spanish,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Indonesian,Malay": 0.36, - "Spanish,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "English,Chinese,Vietnamese,Indonesian": 0.32666666666666666, - "English,Chinese,Vietnamese,Malay": 0.32, - "English,Chinese,Filipino,Indonesian": 0.22, - "English,Chinese,Filipino,Malay": 0.22666666666666666, - "English,Chinese,Indonesian,Malay": 0.31333333333333335, - "English,Vietnamese,Filipino,Indonesian": 0.18, - "English,Vietnamese,Filipino,Malay": 0.2, - "English,Vietnamese,Indonesian,Malay": 0.35333333333333333, - "English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.22, - "Chinese,Vietnamese,Indonesian,Malay": 0.3466666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.28, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.26666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.3, - "Spanish,English,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "English,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - } - }, - "AC3_2": 0.42634222172394715, - "AC3_3": 0.3572017463202083, - "AC3_4": 0.30140083785739447, - "AC3_5": 0.25389782944954653, - "AC3_6": 0.21333333329098927, - "AC3_7": 0.1787854250632136 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.29788961038961037, - "language_acc": { - "English": 0.3409090909090909, - "Filipino": 0.25, - "Spanish": 0.29545454545454547, - "Malay": 0.2727272727272727, - "Chinese": 0.3522727272727273, - "Indonesian": 0.3068181818181818, - "Vietnamese": 0.26704545454545453 - }, - "consistency_score_2": 0.6534090909090909, - "consistency_score_3": 0.5134740259740259, - "consistency_score_4": 0.4256493506493506, - "consistency_score_5": 0.36066017316017324, - "consistency_score_6": 0.3092532467532467, - "consistency_score_7": 0.26704545454545453, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.5397727272727273, - "English,Spanish": 0.6193181818181818, - "English,Malay": 0.6306818181818182, - "English,Chinese": 0.6761363636363636, - "English,Indonesian": 0.6590909090909091, - "English,Vietnamese": 0.6647727272727273, - "Filipino,Spanish": 0.5284090909090909, - "Filipino,Malay": 0.5454545454545454, - "Filipino,Chinese": 0.5965909090909091, - "Filipino,Indonesian": 0.5625, - "Filipino,Vietnamese": 0.6022727272727273, - "Spanish,Malay": 0.6761363636363636, - "Spanish,Chinese": 0.7159090909090909, - "Spanish,Indonesian": 0.6988636363636364, - "Spanish,Vietnamese": 0.7159090909090909, - "Malay,Chinese": 0.6590909090909091, - "Malay,Indonesian": 0.7159090909090909, - "Malay,Vietnamese": 0.6818181818181818, - "Chinese,Indonesian": 0.7329545454545454, - "Chinese,Vietnamese": 0.75, - "Indonesian,Vietnamese": 0.75 - }, - "3_combine": { - "English,Filipino,Spanish": 0.39204545454545453, - "English,Filipino,Malay": 0.42613636363636365, - "English,Filipino,Chinese": 0.45454545454545453, - "English,Filipino,Indonesian": 0.42045454545454547, - "English,Filipino,Vietnamese": 0.4431818181818182, - "English,Spanish,Malay": 0.5, - "English,Spanish,Chinese": 0.5284090909090909, - "English,Spanish,Indonesian": 0.5170454545454546, - "English,Spanish,Vietnamese": 0.5340909090909091, - "English,Malay,Chinese": 0.5227272727272727, - "English,Malay,Indonesian": 0.5340909090909091, - "English,Malay,Vietnamese": 0.5170454545454546, - "English,Chinese,Indonesian": 0.5568181818181818, - "English,Chinese,Vietnamese": 0.5625, - "English,Indonesian,Vietnamese": 0.5625, - "Filipino,Spanish,Malay": 0.42045454545454547, - "Filipino,Spanish,Chinese": 0.4659090909090909, - "Filipino,Spanish,Indonesian": 0.4375, - "Filipino,Spanish,Vietnamese": 0.4659090909090909, - "Filipino,Malay,Chinese": 0.44886363636363635, - "Filipino,Malay,Indonesian": 0.45454545454545453, - "Filipino,Malay,Vietnamese": 0.45454545454545453, - "Filipino,Chinese,Indonesian": 0.4772727272727273, - "Filipino,Chinese,Vietnamese": 0.5113636363636364, - "Filipino,Indonesian,Vietnamese": 0.48295454545454547, - "Spanish,Malay,Chinese": 0.5511363636363636, - "Spanish,Malay,Indonesian": 0.5681818181818182, - "Spanish,Malay,Vietnamese": 0.5738636363636364, - "Spanish,Chinese,Indonesian": 0.6079545454545454, - "Spanish,Chinese,Vietnamese": 0.6136363636363636, - "Spanish,Indonesian,Vietnamese": 0.6136363636363636, - "Malay,Chinese,Indonesian": 0.5681818181818182, - "Malay,Chinese,Vietnamese": 0.5625, - "Malay,Indonesian,Vietnamese": 0.5909090909090909, - "Chinese,Indonesian,Vietnamese": 0.6306818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.3409090909090909, - "English,Filipino,Spanish,Chinese": 0.36363636363636365, - "English,Filipino,Spanish,Indonesian": 0.3409090909090909, - "English,Filipino,Spanish,Vietnamese": 0.36363636363636365, - "English,Filipino,Malay,Chinese": 0.3806818181818182, - "English,Filipino,Malay,Indonesian": 0.3806818181818182, - "English,Filipino,Malay,Vietnamese": 0.375, - "English,Filipino,Chinese,Indonesian": 0.375, - "English,Filipino,Chinese,Vietnamese": 0.38636363636363635, - "English,Filipino,Indonesian,Vietnamese": 0.3693181818181818, - "English,Spanish,Malay,Chinese": 0.45454545454545453, - "English,Spanish,Malay,Indonesian": 0.45454545454545453, - "English,Spanish,Malay,Vietnamese": 0.44886363636363635, - "English,Spanish,Chinese,Indonesian": 0.4659090909090909, - "English,Spanish,Chinese,Vietnamese": 0.48295454545454547, - "English,Spanish,Indonesian,Vietnamese": 0.4602272727272727, - "English,Malay,Chinese,Indonesian": 0.4659090909090909, - "English,Malay,Chinese,Vietnamese": 0.45454545454545453, - "English,Malay,Indonesian,Vietnamese": 0.4715909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.48863636363636365, - "Filipino,Spanish,Malay,Chinese": 0.375, - "Filipino,Spanish,Malay,Indonesian": 0.3693181818181818, - "Filipino,Spanish,Malay,Vietnamese": 0.375, - "Filipino,Spanish,Chinese,Indonesian": 0.4147727272727273, - "Filipino,Spanish,Chinese,Vietnamese": 0.4318181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.4090909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.3977272727272727, - "Filipino,Malay,Chinese,Vietnamese": 0.3977272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.4034090909090909, - "Filipino,Chinese,Indonesian,Vietnamese": 0.4318181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.5, - "Spanish,Malay,Chinese,Vietnamese": 0.5, - "Spanish,Malay,Indonesian,Vietnamese": 0.5170454545454546, - "Spanish,Chinese,Indonesian,Vietnamese": 0.5511363636363636, - "Malay,Chinese,Indonesian,Vietnamese": 0.5 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.3181818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.3125, - "English,Filipino,Spanish,Malay,Vietnamese": 0.3125, - "English,Filipino,Spanish,Chinese,Indonesian": 0.32386363636363635, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.3352272727272727, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.3125, - "English,Filipino,Malay,Chinese,Indonesian": 0.3409090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.32954545454545453, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.3409090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.32954545454545453, - "English,Spanish,Malay,Chinese,Indonesian": 0.4147727272727273, - "English,Spanish,Malay,Chinese,Vietnamese": 0.4147727272727273, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.4090909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.42613636363636365, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.4147727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.3465909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.3465909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.3409090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.38636363636363635, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.35795454545454547, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.4602272727272727 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.29545454545454547, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.2897727272727273, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.29545454545454547, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.30113636363636365, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.3806818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.3181818181818182 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.26704545454545453 - } - }, - "AC3_2": 0.4092169562089225, - "AC3_3": 0.3770407513190112, - "AC3_4": 0.3504898175156417, - "AC3_5": 0.32628336120711054, - "AC3_6": 0.30346508433087727, - "AC3_7": 0.28162552241618877 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.3300970873786408 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.4 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.34579439252336447 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.28, - "category_acc": { - "brand": 0.5, - "demographics": 0.2, - "biology": 0.4, - "history": 0.3333333333333333, - "literature": 0.2, - "politics": 0.2, - "culture": 0.3, - "film": 0.2, - "law": 0.3, - "geography": 0.1 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.22986451250263185 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.6107414242233284 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.5714705921925624 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.4989983902866432 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.5829326995118379 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.35355892648774795 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.35302109402931714, - "category_acc": { - "high_school_european_history": 0.5304878048780488, - "business_ethics": 0.25252525252525254, - "clinical_knowledge": 0.36742424242424243, - "medical_genetics": 0.3838383838383838, - "high_school_us_history": 0.4236453201970443, - "high_school_physics": 0.38666666666666666, - "high_school_world_history": 0.5169491525423728, - "virology": 0.2909090909090909, - "high_school_microeconomics": 0.3628691983122363, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.36363636363636365, - "high_school_biology": 0.3883495145631068, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.28825622775800713, - "philosophy": 0.3225806451612903, - "professional_medicine": 0.4132841328413284, - "nutrition": 0.3442622950819672, - "global_facts": 0.20202020202020202, - "machine_learning": 0.26126126126126126, - "security_studies": 0.3483606557377049, - "public_relations": 0.43119266055045874, - "professional_psychology": 0.33878887070376434, - "prehistory": 0.30959752321981426, - "anatomy": 0.39552238805970147, - "human_sexuality": 0.35384615384615387, - "college_medicine": 0.4011627906976744, - "high_school_government_and_politics": 0.4322916666666667, - "college_chemistry": 0.3434343434343434, - "logical_fallacies": 0.2962962962962963, - "high_school_geography": 0.45685279187817257, - "elementary_mathematics": 0.27320954907161804, - "human_aging": 0.32432432432432434, - "college_mathematics": 0.36363636363636365, - "high_school_psychology": 0.4852941176470588, - "formal_logic": 0.368, - "high_school_statistics": 0.42790697674418604, - "international_law": 0.325, - "high_school_mathematics": 0.2936802973977695, - "high_school_computer_science": 0.3434343434343434, - "conceptual_physics": 0.3034188034188034, - "miscellaneous": 0.4092071611253197, - "high_school_chemistry": 0.3316831683168317, - "marketing": 0.47639484978540775, - "professional_law": 0.2928897586431833, - "management": 0.4803921568627451, - "college_physics": 0.3465346534653465, - "jurisprudence": 0.34579439252336447, - "world_religions": 0.38823529411764707, - "sociology": 0.42, - "us_foreign_policy": 0.3838383838383838, - "high_school_macroeconomics": 0.36246786632390743, - "computer_security": 0.36363636363636365, - "moral_scenarios": 0.2483221476510067, - "moral_disputes": 0.33043478260869563, - "electrical_engineering": 0.3055555555555556, - "astronomy": 0.40397350993377484, - "college_biology": 0.35664335664335667 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.32317979197622587 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.3306351183063512, - "category_acc": { - "computer_network": 0.375, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.23809523809523808, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.125, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.08695652173913043, - "high_school_physics": 0.375, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.38461538461538464, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.36666666666666664, - "business_administration": 0.23684210526315788, - "marxism": 0.5, - "mao_zedong_thought": 0.5172413793103449, - "education_science": 0.35294117647058826, - "teacher_qualification": 0.5102040816326531, - "high_school_politics": 0.08333333333333333, - "high_school_geography": 0.25, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.37037037037037035, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.25, - "high_school_history": 0.4, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.3076923076923077, - "sports_science": 0.4583333333333333, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.2777777777777778, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.3611111111111111, - "tax_accountant": 0.25925925925925924, - "physician": 0.4074074074074074 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.36917562724014336 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.3419098601277845, - "category_acc": { - "agronomy": 0.33136094674556216, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.27439024390243905, - "arts": 0.41875, - "astronomy": 0.2, - "business_ethics": 0.45454545454545453, - "chinese_civil_service_exam": 0.31875, - "chinese_driving_rule": 0.48091603053435117, - "chinese_food_culture": 0.38235294117647056, - "chinese_foreign_policy": 0.35514018691588783, - "chinese_history": 0.38699690402476783, - "chinese_literature": 0.27941176470588236, - "chinese_teacher_qualification": 0.4022346368715084, - "clinical_knowledge": 0.34177215189873417, - "college_actuarial_science": 0.3018867924528302, - "college_education": 0.37383177570093457, - "college_engineering_hydrology": 0.32075471698113206, - "college_law": 0.2777777777777778, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.25471698113207547, - "college_medicine": 0.2967032967032967, - "computer_science": 0.3088235294117647, - "computer_security": 0.34502923976608185, - "conceptual_physics": 0.3129251700680272, - "construction_project_management": 0.33093525179856115, - "economics": 0.3584905660377358, - "education": 0.4294478527607362, - "electrical_engineering": 0.3313953488372093, - "elementary_chinese": 0.32936507936507936, - "elementary_commonsense": 0.35858585858585856, - "elementary_information_and_technology": 0.453781512605042, - "elementary_mathematics": 0.2391304347826087, - "ethnology": 0.37777777777777777, - "food_science": 0.34965034965034963, - "genetics": 0.3352272727272727, - "global_facts": 0.37583892617449666, - "high_school_biology": 0.28402366863905326, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.3559322033898305, - "high_school_mathematics": 0.21341463414634146, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.3146853146853147, - "human_sexuality": 0.36507936507936506, - "international_law": 0.31351351351351353, - "journalism": 0.37790697674418605, - "jurisprudence": 0.2895377128953771, - "legal_and_moral_basis": 0.5700934579439252, - "logical": 0.3333333333333333, - "machine_learning": 0.2459016393442623, - "management": 0.4238095238095238, - "marketing": 0.39444444444444443, - "marxist_theory": 0.3492063492063492, - "modern_chinese": 0.3275862068965517, - "nutrition": 0.33793103448275863, - "philosophy": 0.37142857142857144, - "professional_accounting": 0.33714285714285713, - "professional_law": 0.33175355450236965, - "professional_medicine": 0.28191489361702127, - "professional_psychology": 0.40086206896551724, - "public_relations": 0.3850574712643678, - "security_study": 0.37037037037037035, - "sociology": 0.3805309734513274, - "sports_science": 0.32727272727272727, - "traditional_chinese_medicine": 0.33513513513513515, - "virology": 0.33727810650887574, - "world_history": 0.391304347826087, - "world_religions": 0.31875 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.3333333333333333 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.22272727272727272 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3461016949152542 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.6866118175018698 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.679078882900539 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.441999759767901, - "rouge2": 0.20193231440486478, - "rougeL": 0.36870382725373957, - "avg_rouge": 0.3375453004755018 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.3651354157158801, - "rouge2": 0.12703088219230937, - "rougeL": 0.30085480636900147, - "avg_rouge": 0.26434036809239697 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.6181192660550459 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.5215723873441994 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.481 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.359 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.5075 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.4647887323943662 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.516245487364621 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.5465686274509803 - } - } - } - }, - "phi_2": { - "model_size": "2.7B", - "model_link": "https://huggingface.co/microsoft/phi-2", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.759873949579832, - "language_acc": { - "English": 0.8588235294117647, - "Chinese": 0.7411764705882353, - "Spanish": 0.8134453781512605, - "Vietnamese": 0.6260504201680672 - }, - "consistency_score_2": 0.6571428571428571, - "consistency_score_3": 0.5191176470588235, - "consistency_score_4": 0.43025210084033616, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.7, - "English,Spanish": 0.761344537815126, - "English,Vietnamese": 0.5957983193277311, - "Chinese,Spanish": 0.6941176470588235, - "Chinese,Vietnamese": 0.5983193277310924, - "Spanish,Vietnamese": 0.5932773109243697 - }, - "3_combine": { - "English,Chinese,Spanish": 0.5983193277310924, - "English,Chinese,Vietnamese": 0.4899159663865546, - "English,Spanish,Vietnamese": 0.5058823529411764, - "Chinese,Spanish,Vietnamese": 0.4823529411764706 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.43025210084033616 - } - }, - "AC3_2": 0.7047844964026615, - "AC3_3": 0.6168359163156891, - "AC3_4": 0.5494163631451843 - }, - "prompt_2": { - "overall_acc": 0.7880252100840336, - "language_acc": { - "English": 0.9218487394957983, - "Chinese": 0.7411764705882353, - "Spanish": 0.8563025210084033, - "Vietnamese": 0.6327731092436975 - }, - "consistency_score_2": 0.7007002801120449, - "consistency_score_3": 0.5802521008403362, - "consistency_score_4": 0.5050420168067227, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.7327731092436974, - "English,Spanish": 0.8529411764705882, - "English,Vietnamese": 0.6260504201680672, - "Chinese,Spanish": 0.7361344537815127, - "Chinese,Vietnamese": 0.6235294117647059, - "Spanish,Vietnamese": 0.6327731092436975 - }, - "3_combine": { - "English,Chinese,Spanish": 0.6764705882352942, - "English,Chinese,Vietnamese": 0.5310924369747899, - "English,Spanish,Vietnamese": 0.5781512605042017, - "Chinese,Spanish,Vietnamese": 0.5352941176470588 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.5050420168067227 - } - }, - "AC3_2": 0.7418016135821247, - "AC3_3": 0.6683634669408968, - "AC3_4": 0.6155686774640168 - }, - "prompt_3": { - "overall_acc": 0.7649159663865547, - "language_acc": { - "English": 0.8865546218487395, - "Chinese": 0.7151260504201681, - "Spanish": 0.8260504201680672, - "Vietnamese": 0.6319327731092437 - }, - "consistency_score_2": 0.6672268907563025, - "consistency_score_3": 0.538235294117647, - "consistency_score_4": 0.46218487394957986, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.6966386554621848, - "English,Spanish": 0.8050420168067227, - "English,Vietnamese": 0.5991596638655462, - "Chinese,Spanish": 0.6907563025210084, - "Chinese,Vietnamese": 0.6, - "Spanish,Vietnamese": 0.611764705882353 - }, - "3_combine": { - "English,Chinese,Spanish": 0.619327731092437, - "English,Chinese,Vietnamese": 0.49747899159663866, - "English,Spanish,Vietnamese": 0.5352941176470588, - "Chinese,Spanish,Vietnamese": 0.5008403361344538 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.46218487394957986 - } - }, - "AC3_2": 0.7127396535350052, - "AC3_3": 0.6318602952545447, - "AC3_4": 0.5762078842367001 - }, - "prompt_4": { - "overall_acc": 0.7924369747899159, - "language_acc": { - "English": 0.9176470588235294, - "Chinese": 0.7672268907563026, - "Spanish": 0.8487394957983193, - "Vietnamese": 0.6361344537815126 - }, - "consistency_score_2": 0.7022408963585435, - "consistency_score_3": 0.5834033613445379, - "consistency_score_4": 0.507563025210084, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.7470588235294118, - "English,Spanish": 0.8336134453781513, - "English,Vietnamese": 0.6277310924369748, - "Chinese,Spanish": 0.7487394957983193, - "Chinese,Vietnamese": 0.6294117647058823, - "Spanish,Vietnamese": 0.626890756302521 - }, - "3_combine": { - "English,Chinese,Spanish": 0.680672268907563, - "English,Chinese,Vietnamese": 0.542016806722689, - "English,Spanish,Vietnamese": 0.5714285714285714, - "Chinese,Spanish,Vietnamese": 0.5394957983193277 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.507563025210084 - } - }, - "AC3_2": 0.7446175021234652, - "AC3_3": 0.6720407631185376, - "AC3_4": 0.61878724335655 - }, - "prompt_5": { - "overall_acc": 0.7899159663865546, - "language_acc": { - "English": 0.9218487394957983, - "Chinese": 0.7512605042016807, - "Spanish": 0.8546218487394958, - "Vietnamese": 0.6319327731092437 - }, - "consistency_score_2": 0.6977591036414567, - "consistency_score_3": 0.5741596638655463, - "consistency_score_4": 0.49495798319327733, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.7394957983193278, - "English,Spanish": 0.8420168067226891, - "English,Vietnamese": 0.6151260504201681, - "Chinese,Spanish": 0.7453781512605042, - "Chinese,Vietnamese": 0.6260504201680672, - "Spanish,Vietnamese": 0.6184873949579832 - }, - "3_combine": { - "English,Chinese,Spanish": 0.6773109243697479, - "English,Chinese,Vietnamese": 0.526890756302521, - "English,Spanish,Vietnamese": 0.5605042016806723, - "Chinese,Spanish,Vietnamese": 0.5319327731092437 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.49495798319327733 - } - }, - "AC3_2": 0.7409831188614732, - "AC3_3": 0.664974691506604, - "AC3_4": 0.6085814312170963 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.3657142857142857, - "language_acc": { - "Spanish": 0.41333333333333333, - "English": 0.6266666666666667, - "Chinese": 0.3333333333333333, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.2866666666666667, - "Indonesian": 0.3, - "Malay": 0.31333333333333335 - }, - "consistency_score_2": 0.4682539682539681, - "consistency_score_3": 0.2811428571428571, - "consistency_score_4": 0.19295238095238096, - "consistency_score_5": 0.14158730158730157, - "consistency_score_6": 0.10666666666666667, - "consistency_score_7": 0.08, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5666666666666667, - "Spanish,Chinese": 0.4866666666666667, - "Spanish,Vietnamese": 0.44, - "Spanish,Filipino": 0.4266666666666667, - "Spanish,Indonesian": 0.49333333333333335, - "Spanish,Malay": 0.48, - "English,Chinese": 0.44666666666666666, - "English,Vietnamese": 0.37333333333333335, - "English,Filipino": 0.36666666666666664, - "English,Indonesian": 0.43333333333333335, - "English,Malay": 0.41333333333333333, - "Chinese,Vietnamese": 0.4066666666666667, - "Chinese,Filipino": 0.36, - "Chinese,Indonesian": 0.4066666666666667, - "Chinese,Malay": 0.4066666666666667, - "Vietnamese,Filipino": 0.5266666666666666, - "Vietnamese,Indonesian": 0.5066666666666667, - "Vietnamese,Malay": 0.5333333333333333, - "Filipino,Indonesian": 0.5533333333333333, - "Filipino,Malay": 0.54, - "Indonesian,Malay": 0.6666666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.30666666666666664, - "Spanish,English,Vietnamese": 0.2733333333333333, - "Spanish,English,Filipino": 0.25333333333333335, - "Spanish,English,Indonesian": 0.32666666666666666, - "Spanish,English,Malay": 0.32, - "Spanish,Chinese,Vietnamese": 0.24, - "Spanish,Chinese,Filipino": 0.24, - "Spanish,Chinese,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Malay": 0.28, - "Spanish,Vietnamese,Filipino": 0.26, - "Spanish,Vietnamese,Indonesian": 0.29333333333333333, - "Spanish,Vietnamese,Malay": 0.3, - "Spanish,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Filipino,Malay": 0.2866666666666667, - "Spanish,Indonesian,Malay": 0.37333333333333335, - "English,Chinese,Vietnamese": 0.21333333333333335, - "English,Chinese,Filipino": 0.2, - "English,Chinese,Indonesian": 0.24666666666666667, - "English,Chinese,Malay": 0.24, - "English,Vietnamese,Filipino": 0.22, - "English,Vietnamese,Indonesian": 0.26, - "English,Vietnamese,Malay": 0.26666666666666666, - "English,Filipino,Indonesian": 0.25333333333333335, - "English,Filipino,Malay": 0.22666666666666666, - "English,Indonesian,Malay": 0.32, - "Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "Chinese,Vietnamese,Malay": 0.26666666666666666, - "Chinese,Filipino,Indonesian": 0.23333333333333334, - "Chinese,Filipino,Malay": 0.23333333333333334, - "Chinese,Indonesian,Malay": 0.29333333333333333, - "Vietnamese,Filipino,Indonesian": 0.36, - "Vietnamese,Filipino,Malay": 0.38, - "Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Filipino,Indonesian,Malay": 0.4066666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.16666666666666666, - "Spanish,English,Chinese,Filipino": 0.15333333333333332, - "Spanish,English,Chinese,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian": 0.18, - "Spanish,English,Filipino,Malay": 0.18, - "Spanish,English,Indonesian,Malay": 0.2733333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.16, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Malay": 0.2, - "Spanish,Chinese,Filipino,Indonesian": 0.18, - "Spanish,Chinese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Indonesian,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,Vietnamese,Filipino,Malay": 0.22, - "Spanish,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "English,Chinese,Vietnamese,Malay": 0.16666666666666666, - "English,Chinese,Filipino,Indonesian": 0.14666666666666667, - "English,Chinese,Filipino,Malay": 0.14, - "English,Chinese,Indonesian,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino,Indonesian": 0.18, - "English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.22, - "English,Filipino,Indonesian,Malay": 0.2, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.18, - "Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Vietnamese,Filipino,Indonesian,Malay": 0.32 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.14, - "Spanish,English,Chinese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Indonesian,Malay": 0.16, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,English,Filipino,Indonesian,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.12, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.12, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - } - }, - "AC3_2": 0.41068029795499744, - "AC3_3": 0.3179000504304104, - "AC3_4": 0.2526209146691159, - "AC3_5": 0.20414089035847213, - "AC3_6": 0.16516129028761709, - "AC3_7": 0.13128205125259695 - }, - "prompt_2": { - "overall_acc": 0.3714285714285714, - "language_acc": { - "Spanish": 0.3933333333333333, - "English": 0.6333333333333333, - "Chinese": 0.36666666666666664, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.2866666666666667, - "Indonesian": 0.31333333333333335, - "Malay": 0.32 - }, - "consistency_score_2": 0.4974603174603175, - "consistency_score_3": 0.31657142857142867, - "consistency_score_4": 0.22533333333333336, - "consistency_score_5": 0.1707936507936508, - "consistency_score_6": 0.13619047619047622, - "consistency_score_7": 0.11333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.54, - "Spanish,Chinese": 0.4666666666666667, - "Spanish,Vietnamese": 0.4533333333333333, - "Spanish,Filipino": 0.52, - "Spanish,Indonesian": 0.54, - "Spanish,Malay": 0.52, - "English,Chinese": 0.44666666666666666, - "English,Vietnamese": 0.32, - "English,Filipino": 0.38, - "English,Indonesian": 0.4, - "English,Malay": 0.4066666666666667, - "Chinese,Vietnamese": 0.43333333333333335, - "Chinese,Filipino": 0.4533333333333333, - "Chinese,Indonesian": 0.4666666666666667, - "Chinese,Malay": 0.4666666666666667, - "Vietnamese,Filipino": 0.6, - "Vietnamese,Indonesian": 0.56, - "Vietnamese,Malay": 0.5466666666666666, - "Filipino,Indonesian": 0.6133333333333333, - "Filipino,Malay": 0.5866666666666667, - "Indonesian,Malay": 0.7266666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.31333333333333335, - "Spanish,English,Vietnamese": 0.24666666666666667, - "Spanish,English,Filipino": 0.28, - "Spanish,English,Indonesian": 0.32666666666666666, - "Spanish,English,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese": 0.26, - "Spanish,Chinese,Filipino": 0.2866666666666667, - "Spanish,Chinese,Indonesian": 0.30666666666666664, - "Spanish,Chinese,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino": 0.35333333333333333, - "Spanish,Vietnamese,Indonesian": 0.32666666666666666, - "Spanish,Vietnamese,Malay": 0.3333333333333333, - "Spanish,Filipino,Indonesian": 0.3933333333333333, - "Spanish,Filipino,Malay": 0.36, - "Spanish,Indonesian,Malay": 0.43333333333333335, - "English,Chinese,Vietnamese": 0.2, - "English,Chinese,Filipino": 0.24666666666666667, - "English,Chinese,Indonesian": 0.25333333333333335, - "English,Chinese,Malay": 0.26, - "English,Vietnamese,Filipino": 0.23333333333333334, - "English,Vietnamese,Indonesian": 0.22666666666666666, - "English,Vietnamese,Malay": 0.23333333333333334, - "English,Filipino,Indonesian": 0.26666666666666666, - "English,Filipino,Malay": 0.24666666666666667, - "English,Indonesian,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Filipino": 0.32, - "Chinese,Vietnamese,Indonesian": 0.31333333333333335, - "Chinese,Vietnamese,Malay": 0.30666666666666664, - "Chinese,Filipino,Indonesian": 0.32, - "Chinese,Filipino,Malay": 0.32, - "Chinese,Indonesian,Malay": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian": 0.44666666666666666, - "Vietnamese,Filipino,Malay": 0.43333333333333335, - "Vietnamese,Indonesian,Malay": 0.44, - "Filipino,Indonesian,Malay": 0.4866666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.16666666666666666, - "Spanish,English,Chinese,Filipino": 0.2, - "Spanish,English,Chinese,Indonesian": 0.22, - "Spanish,English,Chinese,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Indonesian,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian": 0.2, - "Spanish,Chinese,Vietnamese,Malay": 0.21333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.22, - "Spanish,Chinese,Filipino,Malay": 0.22, - "Spanish,Chinese,Indonesian,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.28, - "Spanish,Vietnamese,Filipino,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese,Filipino": 0.16666666666666666, - "English,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "English,Chinese,Vietnamese,Malay": 0.16, - "English,Chinese,Filipino,Indonesian": 0.18, - "English,Chinese,Filipino,Malay": 0.17333333333333334, - "English,Chinese,Indonesian,Malay": 0.22, - "English,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.2, - "English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "Chinese,Vietnamese,Filipino,Malay": 0.26, - "Chinese,Vietnamese,Indonesian,Malay": 0.26, - "Chinese,Filipino,Indonesian,Malay": 0.28, - "Vietnamese,Filipino,Indonesian,Malay": 0.37333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.14, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Indonesian,Malay": 0.2, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,English,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.2, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - } - }, - "AC3_2": 0.42530403460839317, - "AC3_3": 0.34181300422179395, - "AC3_4": 0.28049792526420003, - "AC3_5": 0.2339913013954541, - "AC3_6": 0.19930313584923942, - "AC3_7": 0.17367387029816156 - }, - "prompt_3": { - "overall_acc": 0.3676190476190476, - "language_acc": { - "Spanish": 0.4066666666666667, - "English": 0.5866666666666667, - "Chinese": 0.35333333333333333, - "Vietnamese": 0.26666666666666666, - "Filipino": 0.32666666666666666, - "Indonesian": 0.31333333333333335, - "Malay": 0.32 - }, - "consistency_score_2": 0.49968253968253973, - "consistency_score_3": 0.3163809523809524, - "consistency_score_4": 0.22266666666666668, - "consistency_score_5": 0.1657142857142857, - "consistency_score_6": 0.1276190476190476, - "consistency_score_7": 0.1, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5466666666666666, - "Spanish,Chinese": 0.47333333333333333, - "Spanish,Vietnamese": 0.47333333333333333, - "Spanish,Filipino": 0.49333333333333335, - "Spanish,Indonesian": 0.5733333333333334, - "Spanish,Malay": 0.5466666666666666, - "English,Chinese": 0.4266666666666667, - "English,Vietnamese": 0.36, - "English,Filipino": 0.4066666666666667, - "English,Indonesian": 0.38666666666666666, - "English,Malay": 0.38, - "Chinese,Vietnamese": 0.52, - "Chinese,Filipino": 0.4266666666666667, - "Chinese,Indonesian": 0.4533333333333333, - "Chinese,Malay": 0.46, - "Vietnamese,Filipino": 0.5533333333333333, - "Vietnamese,Indonesian": 0.6133333333333333, - "Vietnamese,Malay": 0.56, - "Filipino,Indonesian": 0.58, - "Filipino,Malay": 0.5333333333333333, - "Indonesian,Malay": 0.7266666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.2866666666666667, - "Spanish,English,Vietnamese": 0.26, - "Spanish,English,Filipino": 0.28, - "Spanish,English,Indonesian": 0.31333333333333335, - "Spanish,English,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese": 0.29333333333333333, - "Spanish,Chinese,Filipino": 0.26, - "Spanish,Chinese,Indonesian": 0.31333333333333335, - "Spanish,Chinese,Malay": 0.32, - "Spanish,Vietnamese,Filipino": 0.32, - "Spanish,Vietnamese,Indonesian": 0.38, - "Spanish,Vietnamese,Malay": 0.36, - "Spanish,Filipino,Indonesian": 0.37333333333333335, - "Spanish,Filipino,Malay": 0.35333333333333333, - "Spanish,Indonesian,Malay": 0.4666666666666667, - "English,Chinese,Vietnamese": 0.23333333333333334, - "English,Chinese,Filipino": 0.23333333333333334, - "English,Chinese,Indonesian": 0.22666666666666666, - "English,Chinese,Malay": 0.22, - "English,Vietnamese,Filipino": 0.24, - "English,Vietnamese,Indonesian": 0.26666666666666666, - "English,Vietnamese,Malay": 0.25333333333333335, - "English,Filipino,Indonesian": 0.26, - "English,Filipino,Malay": 0.24666666666666667, - "English,Indonesian,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Filipino": 0.30666666666666664, - "Chinese,Vietnamese,Indonesian": 0.35333333333333333, - "Chinese,Vietnamese,Malay": 0.3466666666666667, - "Chinese,Filipino,Indonesian": 0.2866666666666667, - "Chinese,Filipino,Malay": 0.28, - "Chinese,Indonesian,Malay": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian": 0.4266666666666667, - "Vietnamese,Filipino,Malay": 0.3933333333333333, - "Vietnamese,Indonesian,Malay": 0.48, - "Filipino,Indonesian,Malay": 0.4533333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.17333333333333334, - "Spanish,English,Chinese,Filipino": 0.18, - "Spanish,English,Chinese,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.18, - "Spanish,English,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Malay": 0.22, - "Spanish,English,Filipino,Indonesian": 0.22, - "Spanish,English,Filipino,Malay": 0.21333333333333335, - "Spanish,English,Indonesian,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Filipino,Malay": 0.20666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.26, - "Spanish,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.16, - "English,Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "English,Chinese,Vietnamese,Malay": 0.16, - "English,Chinese,Filipino,Indonesian": 0.15333333333333332, - "English,Chinese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Indonesian,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino,Indonesian": 0.2, - "English,Vietnamese,Filipino,Malay": 0.18, - "English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "English,Filipino,Indonesian,Malay": 0.21333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.24, - "Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Chinese,Filipino,Indonesian,Malay": 0.24, - "Vietnamese,Filipino,Indonesian,Malay": 0.36 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Malay": 0.14, - "Spanish,English,Chinese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "English,Chinese,Vietnamese,Filipino,Malay": 0.12, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.12, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 - } - }, - "AC3_2": 0.42359617927208243, - "AC3_3": 0.340080890006108, - "AC3_4": 0.2773453801831968, - "AC3_5": 0.22844897954900253, - "AC3_6": 0.18946520142694404, - "AC3_7": 0.15723014253256792 - }, - "prompt_4": { - "overall_acc": 0.3666666666666667, - "language_acc": { - "Spanish": 0.4066666666666667, - "English": 0.6066666666666667, - "Chinese": 0.36666666666666664, - "Vietnamese": 0.32, - "Filipino": 0.3, - "Indonesian": 0.28, - "Malay": 0.2866666666666667 - }, - "consistency_score_2": 0.4736507936507936, - "consistency_score_3": 0.2927619047619047, - "consistency_score_4": 0.20666666666666664, - "consistency_score_5": 0.15523809523809526, - "consistency_score_6": 0.12, - "consistency_score_7": 0.09333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5266666666666666, - "Spanish,Chinese": 0.4266666666666667, - "Spanish,Vietnamese": 0.44666666666666666, - "Spanish,Filipino": 0.4666666666666667, - "Spanish,Indonesian": 0.48, - "Spanish,Malay": 0.44666666666666666, - "English,Chinese": 0.5066666666666667, - "English,Vietnamese": 0.37333333333333335, - "English,Filipino": 0.36666666666666664, - "English,Indonesian": 0.4, - "English,Malay": 0.38666666666666666, - "Chinese,Vietnamese": 0.44666666666666666, - "Chinese,Filipino": 0.4866666666666667, - "Chinese,Indonesian": 0.44, - "Chinese,Malay": 0.46, - "Vietnamese,Filipino": 0.5333333333333333, - "Vietnamese,Indonesian": 0.5066666666666667, - "Vietnamese,Malay": 0.5266666666666666, - "Filipino,Indonesian": 0.52, - "Filipino,Malay": 0.5066666666666667, - "Indonesian,Malay": 0.6933333333333334 - }, - "3_combine": { - "Spanish,English,Chinese": 0.2733333333333333, - "Spanish,English,Vietnamese": 0.25333333333333335, - "Spanish,English,Filipino": 0.26666666666666666, - "Spanish,English,Indonesian": 0.2866666666666667, - "Spanish,English,Malay": 0.28, - "Spanish,Chinese,Vietnamese": 0.26, - "Spanish,Chinese,Filipino": 0.28, - "Spanish,Chinese,Indonesian": 0.28, - "Spanish,Chinese,Malay": 0.28, - "Spanish,Vietnamese,Filipino": 0.29333333333333333, - "Spanish,Vietnamese,Indonesian": 0.29333333333333333, - "Spanish,Vietnamese,Malay": 0.31333333333333335, - "Spanish,Filipino,Indonesian": 0.3, - "Spanish,Filipino,Malay": 0.30666666666666664, - "Spanish,Indonesian,Malay": 0.35333333333333333, - "English,Chinese,Vietnamese": 0.24666666666666667, - "English,Chinese,Filipino": 0.2733333333333333, - "English,Chinese,Indonesian": 0.2733333333333333, - "English,Chinese,Malay": 0.25333333333333335, - "English,Vietnamese,Filipino": 0.22666666666666666, - "English,Vietnamese,Indonesian": 0.24666666666666667, - "English,Vietnamese,Malay": 0.25333333333333335, - "English,Filipino,Indonesian": 0.22666666666666666, - "English,Filipino,Malay": 0.23333333333333334, - "English,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino": 0.30666666666666664, - "Chinese,Vietnamese,Indonesian": 0.29333333333333333, - "Chinese,Vietnamese,Malay": 0.30666666666666664, - "Chinese,Filipino,Indonesian": 0.2866666666666667, - "Chinese,Filipino,Malay": 0.31333333333333335, - "Chinese,Indonesian,Malay": 0.35333333333333333, - "Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "Vietnamese,Filipino,Malay": 0.37333333333333335, - "Vietnamese,Indonesian,Malay": 0.42, - "Filipino,Indonesian,Malay": 0.4 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.16666666666666666, - "Spanish,English,Chinese,Filipino": 0.2, - "Spanish,English,Chinese,Indonesian": 0.2, - "Spanish,English,Chinese,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,English,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian": 0.18, - "Spanish,English,Filipino,Malay": 0.18666666666666668, - "Spanish,English,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.21333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.22, - "Spanish,Chinese,Indonesian,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.24, - "Spanish,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "English,Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "English,Chinese,Vietnamese,Malay": 0.18, - "English,Chinese,Filipino,Indonesian": 0.18, - "English,Chinese,Filipino,Malay": 0.18, - "English,Chinese,Indonesian,Malay": 0.20666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "English,Vietnamese,Filipino,Malay": 0.18, - "English,Vietnamese,Indonesian,Malay": 0.2, - "English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.24, - "Chinese,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.26, - "Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.14, - "Spanish,English,Chinese,Filipino,Indonesian": 0.14, - "Spanish,English,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Filipino,Indonesian,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.18, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - } - }, - "AC3_2": 0.4133484447310648, - "AC3_3": 0.32557288652871386, - "AC3_4": 0.2643410852252118, - "AC3_5": 0.2181265206394709, - "AC3_6": 0.180821917771064, - "AC3_7": 0.14879227049905483 - }, - "prompt_5": { - "overall_acc": 0.3666666666666667, - "language_acc": { - "Spanish": 0.3933333333333333, - "English": 0.6, - "Chinese": 0.35333333333333333, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.29333333333333333, - "Indonesian": 0.31333333333333335, - "Malay": 0.32666666666666666 - }, - "consistency_score_2": 0.4584126984126983, - "consistency_score_3": 0.27123809523809517, - "consistency_score_4": 0.18476190476190477, - "consistency_score_5": 0.13523809523809524, - "consistency_score_6": 0.10285714285714286, - "consistency_score_7": 0.08, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5533333333333333, - "Spanish,Chinese": 0.43333333333333335, - "Spanish,Vietnamese": 0.4666666666666667, - "Spanish,Filipino": 0.4533333333333333, - "Spanish,Indonesian": 0.42, - "Spanish,Malay": 0.46, - "English,Chinese": 0.43333333333333335, - "English,Vietnamese": 0.3466666666666667, - "English,Filipino": 0.3333333333333333, - "English,Indonesian": 0.3933333333333333, - "English,Malay": 0.3933333333333333, - "Chinese,Vietnamese": 0.44, - "Chinese,Filipino": 0.4533333333333333, - "Chinese,Indonesian": 0.3933333333333333, - "Chinese,Malay": 0.41333333333333333, - "Vietnamese,Filipino": 0.48, - "Vietnamese,Indonesian": 0.4866666666666667, - "Vietnamese,Malay": 0.5133333333333333, - "Filipino,Indonesian": 0.5066666666666667, - "Filipino,Malay": 0.5533333333333333, - "Indonesian,Malay": 0.7 - }, - "3_combine": { - "Spanish,English,Chinese": 0.28, - "Spanish,English,Vietnamese": 0.28, - "Spanish,English,Filipino": 0.26, - "Spanish,English,Indonesian": 0.26, - "Spanish,English,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese": 0.25333333333333335, - "Spanish,Chinese,Filipino": 0.25333333333333335, - "Spanish,Chinese,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Filipino": 0.28, - "Spanish,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Malay": 0.29333333333333333, - "Spanish,Filipino,Indonesian": 0.25333333333333335, - "Spanish,Filipino,Malay": 0.30666666666666664, - "Spanish,Indonesian,Malay": 0.34, - "English,Chinese,Vietnamese": 0.21333333333333335, - "English,Chinese,Filipino": 0.22, - "English,Chinese,Indonesian": 0.22666666666666666, - "English,Chinese,Malay": 0.23333333333333334, - "English,Vietnamese,Filipino": 0.18666666666666668, - "English,Vietnamese,Indonesian": 0.22, - "English,Vietnamese,Malay": 0.22, - "English,Filipino,Indonesian": 0.19333333333333333, - "English,Filipino,Malay": 0.22666666666666666, - "English,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Chinese,Vietnamese,Malay": 0.28, - "Chinese,Filipino,Indonesian": 0.26666666666666666, - "Chinese,Filipino,Malay": 0.2866666666666667, - "Chinese,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Vietnamese,Filipino,Malay": 0.3466666666666667, - "Vietnamese,Indonesian,Malay": 0.3933333333333333, - "Filipino,Indonesian,Malay": 0.4066666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.16666666666666666, - "Spanish,English,Chinese,Filipino": 0.18, - "Spanish,English,Chinese,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino": 0.16, - "Spanish,English,Vietnamese,Indonesian": 0.18, - "Spanish,English,Vietnamese,Malay": 0.18666666666666668, - "Spanish,English,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Filipino,Malay": 0.18, - "Spanish,English,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Malay": 0.19333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.2, - "Spanish,Chinese,Indonesian,Malay": 0.2, - "Spanish,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.14, - "English,Chinese,Vietnamese,Indonesian": 0.16, - "English,Chinese,Vietnamese,Malay": 0.16, - "English,Chinese,Filipino,Indonesian": 0.14666666666666667, - "English,Chinese,Filipino,Malay": 0.16, - "English,Chinese,Indonesian,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "English,Vietnamese,Indonesian,Malay": 0.18, - "English,Filipino,Indonesian,Malay": 0.17333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.28 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.12, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - } - }, - "AC3_2": 0.40743875844747596, - "AC3_3": 0.311814471932805, - "AC3_4": 0.24571099592550433, - "AC3_5": 0.19759645789864458, - "AC3_6": 0.16064908718688, - "AC3_7": 0.13134328355268435 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.35470779220779225, - "language_acc": { - "English": 0.4943181818181818, - "Filipino": 0.2897727272727273, - "Spanish": 0.4147727272727273, - "Malay": 0.3522727272727273, - "Chinese": 0.3125, - "Indonesian": 0.2840909090909091, - "Vietnamese": 0.3352272727272727 - }, - "consistency_score_2": 0.4537337662337661, - "consistency_score_3": 0.26866883116883117, - "consistency_score_4": 0.18587662337662336, - "consistency_score_5": 0.14204545454545456, - "consistency_score_6": 0.11525974025974027, - "consistency_score_7": 0.09659090909090909, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.38636363636363635, - "English,Spanish": 0.5397727272727273, - "English,Malay": 0.3977272727272727, - "English,Chinese": 0.4090909090909091, - "English,Indonesian": 0.4090909090909091, - "English,Vietnamese": 0.38636363636363635, - "Filipino,Spanish": 0.4659090909090909, - "Filipino,Malay": 0.5170454545454546, - "Filipino,Chinese": 0.35795454545454547, - "Filipino,Indonesian": 0.5568181818181818, - "Filipino,Vietnamese": 0.4090909090909091, - "Spanish,Malay": 0.4375, - "Spanish,Chinese": 0.48863636363636365, - "Spanish,Indonesian": 0.4772727272727273, - "Spanish,Vietnamese": 0.4318181818181818, - "Malay,Chinese": 0.42613636363636365, - "Malay,Indonesian": 0.5852272727272727, - "Malay,Vietnamese": 0.5397727272727273, - "Chinese,Indonesian": 0.4147727272727273, - "Chinese,Vietnamese": 0.42045454545454547, - "Indonesian,Vietnamese": 0.4715909090909091 - }, - "3_combine": { - "English,Filipino,Spanish": 0.3068181818181818, - "English,Filipino,Malay": 0.2215909090909091, - "English,Filipino,Chinese": 0.19886363636363635, - "English,Filipino,Indonesian": 0.26136363636363635, - "English,Filipino,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay": 0.2727272727272727, - "English,Spanish,Chinese": 0.30113636363636365, - "English,Spanish,Indonesian": 0.2897727272727273, - "English,Spanish,Vietnamese": 0.26136363636363635, - "English,Malay,Chinese": 0.2159090909090909, - "English,Malay,Indonesian": 0.26136363636363635, - "English,Malay,Vietnamese": 0.2727272727272727, - "English,Chinese,Indonesian": 0.2215909090909091, - "English,Chinese,Vietnamese": 0.22727272727272727, - "English,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Malay": 0.29545454545454547, - "Filipino,Spanish,Chinese": 0.25, - "Filipino,Spanish,Indonesian": 0.32386363636363635, - "Filipino,Spanish,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese": 0.23295454545454544, - "Filipino,Malay,Indonesian": 0.3806818181818182, - "Filipino,Malay,Vietnamese": 0.3068181818181818, - "Filipino,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Chinese,Vietnamese": 0.21022727272727273, - "Filipino,Indonesian,Vietnamese": 0.3125, - "Spanish,Malay,Chinese": 0.26704545454545453, - "Spanish,Malay,Indonesian": 0.3068181818181818, - "Spanish,Malay,Vietnamese": 0.2840909090909091, - "Spanish,Chinese,Indonesian": 0.29545454545454547, - "Spanish,Chinese,Vietnamese": 0.2897727272727273, - "Spanish,Indonesian,Vietnamese": 0.2840909090909091, - "Malay,Chinese,Indonesian": 0.2897727272727273, - "Malay,Chinese,Vietnamese": 0.2784090909090909, - "Malay,Indonesian,Vietnamese": 0.3409090909090909, - "Chinese,Indonesian,Vietnamese": 0.2556818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.19318181818181818, - "English,Filipino,Spanish,Chinese": 0.18181818181818182, - "English,Filipino,Spanish,Indonesian": 0.2215909090909091, - "English,Filipino,Spanish,Vietnamese": 0.1590909090909091, - "English,Filipino,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Malay,Indonesian": 0.18181818181818182, - "English,Filipino,Malay,Vietnamese": 0.16477272727272727, - "English,Filipino,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese": 0.19318181818181818, - "English,Spanish,Malay,Indonesian": 0.2159090909090909, - "English,Spanish,Malay,Vietnamese": 0.21022727272727273, - "English,Spanish,Chinese,Indonesian": 0.19886363636363635, - "English,Spanish,Chinese,Vietnamese": 0.19886363636363635, - "English,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "English,Malay,Chinese,Indonesian": 0.16477272727272727, - "English,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Malay,Indonesian,Vietnamese": 0.1875, - "English,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Chinese": 0.17613636363636365, - "Filipino,Spanish,Malay,Indonesian": 0.23863636363636365, - "Filipino,Spanish,Malay,Vietnamese": 0.1875, - "Filipino,Spanish,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Indonesian,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Spanish,Malay,Chinese,Indonesian": 0.21022727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - } - }, - "AC3_2": 0.39815593538078203, - "AC3_3": 0.3057507270883984, - "AC3_4": 0.24392818138305838, - "AC3_5": 0.2028557634766919, - "AC3_6": 0.17398447835160294, - "AC3_7": 0.15183534986825256 - }, - "prompt_2": { - "overall_acc": 0.32305194805194803, - "language_acc": { - "English": 0.44886363636363635, - "Filipino": 0.24431818181818182, - "Spanish": 0.3409090909090909, - "Malay": 0.3409090909090909, - "Chinese": 0.30113636363636365, - "Indonesian": 0.2897727272727273, - "Vietnamese": 0.29545454545454547 - }, - "consistency_score_2": 0.5064935064935064, - "consistency_score_3": 0.32094155844155847, - "consistency_score_4": 0.22288961038961042, - "consistency_score_5": 0.16206709956709958, - "consistency_score_6": 0.12094155844155842, - "consistency_score_7": 0.09090909090909091, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.375, - "English,Spanish": 0.48295454545454547, - "English,Malay": 0.3977272727272727, - "English,Chinese": 0.4090909090909091, - "English,Indonesian": 0.4090909090909091, - "English,Vietnamese": 0.38636363636363635, - "Filipino,Spanish": 0.5, - "Filipino,Malay": 0.6306818181818182, - "Filipino,Chinese": 0.45454545454545453, - "Filipino,Indonesian": 0.6136363636363636, - "Filipino,Vietnamese": 0.5511363636363636, - "Spanish,Malay": 0.5284090909090909, - "Spanish,Chinese": 0.4602272727272727, - "Spanish,Indonesian": 0.5852272727272727, - "Spanish,Vietnamese": 0.4659090909090909, - "Malay,Chinese": 0.48295454545454547, - "Malay,Indonesian": 0.6818181818181818, - "Malay,Vietnamese": 0.6534090909090909, - "Chinese,Indonesian": 0.4715909090909091, - "Chinese,Vietnamese": 0.4715909090909091, - "Indonesian,Vietnamese": 0.625 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2556818181818182, - "English,Filipino,Malay": 0.26704545454545453, - "English,Filipino,Chinese": 0.20454545454545456, - "English,Filipino,Indonesian": 0.2840909090909091, - "English,Filipino,Vietnamese": 0.25, - "English,Spanish,Malay": 0.2784090909090909, - "English,Spanish,Chinese": 0.24431818181818182, - "English,Spanish,Indonesian": 0.2784090909090909, - "English,Spanish,Vietnamese": 0.23295454545454544, - "English,Malay,Chinese": 0.23295454545454544, - "English,Malay,Indonesian": 0.3068181818181818, - "English,Malay,Vietnamese": 0.2784090909090909, - "English,Chinese,Indonesian": 0.21022727272727273, - "English,Chinese,Vietnamese": 0.2159090909090909, - "English,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay": 0.3806818181818182, - "Filipino,Spanish,Chinese": 0.2840909090909091, - "Filipino,Spanish,Indonesian": 0.3977272727272727, - "Filipino,Spanish,Vietnamese": 0.32954545454545453, - "Filipino,Malay,Chinese": 0.3465909090909091, - "Filipino,Malay,Indonesian": 0.48295454545454547, - "Filipino,Malay,Vietnamese": 0.4431818181818182, - "Filipino,Chinese,Indonesian": 0.3352272727272727, - "Filipino,Chinese,Vietnamese": 0.29545454545454547, - "Filipino,Indonesian,Vietnamese": 0.4431818181818182, - "Spanish,Malay,Chinese": 0.3125, - "Spanish,Malay,Indonesian": 0.42613636363636365, - "Spanish,Malay,Vietnamese": 0.375, - "Spanish,Chinese,Indonesian": 0.32386363636363635, - "Spanish,Chinese,Vietnamese": 0.2727272727272727, - "Spanish,Indonesian,Vietnamese": 0.38636363636363635, - "Malay,Chinese,Indonesian": 0.3806818181818182, - "Malay,Chinese,Vietnamese": 0.35795454545454547, - "Malay,Indonesian,Vietnamese": 0.5056818181818182, - "Chinese,Indonesian,Vietnamese": 0.3465909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.19318181818181818, - "English,Filipino,Spanish,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Indonesian": 0.20454545454545456, - "English,Filipino,Spanish,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Chinese": 0.1590909090909091, - "English,Filipino,Malay,Indonesian": 0.23295454545454544, - "English,Filipino,Malay,Vietnamese": 0.20454545454545456, - "English,Filipino,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay,Chinese": 0.17045454545454544, - "English,Spanish,Malay,Indonesian": 0.22727272727272727, - "English,Spanish,Malay,Vietnamese": 0.19318181818181818, - "English,Spanish,Chinese,Indonesian": 0.1590909090909091, - "English,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "English,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Malay,Chinese,Indonesian": 0.18181818181818182, - "English,Malay,Chinese,Vietnamese": 0.17613636363636365, - "English,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Malay,Chinese": 0.24431818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.32386363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.2840909090909091, - "Filipino,Spanish,Chinese,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2897727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.30113636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.2556818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.3806818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.26704545454545453, - "Spanish,Malay,Chinese,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Indonesian,Vietnamese": 0.32386363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Malay,Chinese,Indonesian,Vietnamese": 0.30113636363636365 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Spanish,Malay,Chinese,Vietnamese": 0.125, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1875, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - } - }, - "AC3_2": 0.39449005001979454, - "AC3_3": 0.3219932952858721, - "AC3_4": 0.26378253023733556, - "AC3_5": 0.2158484292729817, - "AC3_6": 0.17599539396306743, - "AC3_7": 0.1418894830316776 - }, - "prompt_3": { - "overall_acc": 0.3392857142857143, - "language_acc": { - "English": 0.5, - "Filipino": 0.2727272727272727, - "Spanish": 0.375, - "Malay": 0.3409090909090909, - "Chinese": 0.2897727272727273, - "Indonesian": 0.29545454545454547, - "Vietnamese": 0.30113636363636365 - }, - "consistency_score_2": 0.49242424242424254, - "consistency_score_3": 0.3043831168831169, - "consistency_score_4": 0.20892857142857138, - "consistency_score_5": 0.1536796536796537, - "consistency_score_6": 0.11931818181818184, - "consistency_score_7": 0.09659090909090909, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3693181818181818, - "English,Spanish": 0.4431818181818182, - "English,Malay": 0.4602272727272727, - "English,Chinese": 0.4602272727272727, - "English,Indonesian": 0.4147727272727273, - "English,Vietnamese": 0.4090909090909091, - "Filipino,Spanish": 0.44886363636363635, - "Filipino,Malay": 0.6022727272727273, - "Filipino,Chinese": 0.4431818181818182, - "Filipino,Indonesian": 0.6193181818181818, - "Filipino,Vietnamese": 0.5113636363636364, - "Spanish,Malay": 0.5, - "Spanish,Chinese": 0.4715909090909091, - "Spanish,Indonesian": 0.5340909090909091, - "Spanish,Vietnamese": 0.4090909090909091, - "Malay,Chinese": 0.48863636363636365, - "Malay,Indonesian": 0.6875, - "Malay,Vietnamese": 0.5852272727272727, - "Chinese,Indonesian": 0.4659090909090909, - "Chinese,Vietnamese": 0.4659090909090909, - "Indonesian,Vietnamese": 0.5511363636363636 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2215909090909091, - "English,Filipino,Malay": 0.26704545454545453, - "English,Filipino,Chinese": 0.23863636363636365, - "English,Filipino,Indonesian": 0.2840909090909091, - "English,Filipino,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay": 0.26136363636363635, - "English,Spanish,Chinese": 0.2784090909090909, - "English,Spanish,Indonesian": 0.2556818181818182, - "English,Spanish,Vietnamese": 0.21022727272727273, - "English,Malay,Chinese": 0.2840909090909091, - "English,Malay,Indonesian": 0.3409090909090909, - "English,Malay,Vietnamese": 0.30113636363636365, - "English,Chinese,Indonesian": 0.26136363636363635, - "English,Chinese,Vietnamese": 0.25, - "English,Indonesian,Vietnamese": 0.2556818181818182, - "Filipino,Spanish,Malay": 0.3181818181818182, - "Filipino,Spanish,Chinese": 0.26136363636363635, - "Filipino,Spanish,Indonesian": 0.3409090909090909, - "Filipino,Spanish,Vietnamese": 0.26136363636363635, - "Filipino,Malay,Chinese": 0.32954545454545453, - "Filipino,Malay,Indonesian": 0.48295454545454547, - "Filipino,Malay,Vietnamese": 0.39204545454545453, - "Filipino,Chinese,Indonesian": 0.3352272727272727, - "Filipino,Chinese,Vietnamese": 0.2784090909090909, - "Filipino,Indonesian,Vietnamese": 0.39204545454545453, - "Spanish,Malay,Chinese": 0.2897727272727273, - "Spanish,Malay,Indonesian": 0.3977272727272727, - "Spanish,Malay,Vietnamese": 0.3068181818181818, - "Spanish,Chinese,Indonesian": 0.3068181818181818, - "Spanish,Chinese,Vietnamese": 0.2556818181818182, - "Spanish,Indonesian,Vietnamese": 0.3068181818181818, - "Malay,Chinese,Indonesian": 0.375, - "Malay,Chinese,Vietnamese": 0.3409090909090909, - "Malay,Indonesian,Vietnamese": 0.4375, - "Chinese,Indonesian,Vietnamese": 0.3068181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.16477272727272727, - "English,Filipino,Spanish,Chinese": 0.16477272727272727, - "English,Filipino,Spanish,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese": 0.19318181818181818, - "English,Filipino,Malay,Indonesian": 0.23863636363636365, - "English,Filipino,Malay,Vietnamese": 0.19318181818181818, - "English,Filipino,Chinese,Indonesian": 0.20454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay,Chinese": 0.18181818181818182, - "English,Spanish,Malay,Indonesian": 0.2159090909090909, - "English,Spanish,Malay,Vietnamese": 0.17045454545454544, - "English,Spanish,Chinese,Indonesian": 0.18181818181818182, - "English,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "English,Malay,Chinese,Indonesian": 0.22727272727272727, - "English,Malay,Chinese,Vietnamese": 0.20454545454545456, - "English,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay,Chinese": 0.2215909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.29545454545454547, - "Filipino,Malay,Chinese,Vietnamese": 0.24431818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.32954545454545453, - "Filipino,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.26704545454545453 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - } - }, - "AC3_2": 0.40175666878708877, - "AC3_3": 0.3208881282153245, - "AC3_4": 0.2586086551412868, - "AC3_5": 0.21154147714071664, - "AC3_6": 0.1765486725278747, - "AC3_7": 0.15037243944408588 - }, - "prompt_4": { - "overall_acc": 0.34983766233766234, - "language_acc": { - "English": 0.4943181818181818, - "Filipino": 0.30113636363636365, - "Spanish": 0.42045454545454547, - "Malay": 0.3522727272727273, - "Chinese": 0.3125, - "Indonesian": 0.26136363636363635, - "Vietnamese": 0.3068181818181818 - }, - "consistency_score_2": 0.4483225108225108, - "consistency_score_3": 0.2542207792207792, - "consistency_score_4": 0.16347402597402597, - "consistency_score_5": 0.11417748917748918, - "consistency_score_6": 0.0836038961038961, - "consistency_score_7": 0.0625, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3693181818181818, - "English,Spanish": 0.5397727272727273, - "English,Malay": 0.4090909090909091, - "English,Chinese": 0.375, - "English,Indonesian": 0.38636363636363635, - "English,Vietnamese": 0.3806818181818182, - "Filipino,Spanish": 0.4602272727272727, - "Filipino,Malay": 0.5284090909090909, - "Filipino,Chinese": 0.35795454545454547, - "Filipino,Indonesian": 0.5511363636363636, - "Filipino,Vietnamese": 0.4034090909090909, - "Spanish,Malay": 0.4375, - "Spanish,Chinese": 0.4602272727272727, - "Spanish,Indonesian": 0.48295454545454547, - "Spanish,Vietnamese": 0.375, - "Malay,Chinese": 0.42045454545454547, - "Malay,Indonesian": 0.6193181818181818, - "Malay,Vietnamese": 0.5170454545454546, - "Chinese,Indonesian": 0.42045454545454547, - "Chinese,Vietnamese": 0.38636363636363635, - "Indonesian,Vietnamese": 0.5340909090909091 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2784090909090909, - "English,Filipino,Malay": 0.2215909090909091, - "English,Filipino,Chinese": 0.17613636363636365, - "English,Filipino,Indonesian": 0.22727272727272727, - "English,Filipino,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay": 0.2840909090909091, - "English,Spanish,Chinese": 0.26136363636363635, - "English,Spanish,Indonesian": 0.2727272727272727, - "English,Spanish,Vietnamese": 0.2159090909090909, - "English,Malay,Chinese": 0.20454545454545456, - "English,Malay,Indonesian": 0.2727272727272727, - "English,Malay,Vietnamese": 0.25, - "English,Chinese,Indonesian": 0.18181818181818182, - "English,Chinese,Vietnamese": 0.19318181818181818, - "English,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Malay": 0.2784090909090909, - "Filipino,Spanish,Chinese": 0.25, - "Filipino,Spanish,Indonesian": 0.32386363636363635, - "Filipino,Spanish,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese": 0.23295454545454544, - "Filipino,Malay,Indonesian": 0.38636363636363635, - "Filipino,Malay,Vietnamese": 0.30113636363636365, - "Filipino,Chinese,Indonesian": 0.24431818181818182, - "Filipino,Chinese,Vietnamese": 0.1875, - "Filipino,Indonesian,Vietnamese": 0.3181818181818182, - "Spanish,Malay,Chinese": 0.24431818181818182, - "Spanish,Malay,Indonesian": 0.3125, - "Spanish,Malay,Vietnamese": 0.24431818181818182, - "Spanish,Chinese,Indonesian": 0.26704545454545453, - "Spanish,Chinese,Vietnamese": 0.2159090909090909, - "Spanish,Indonesian,Vietnamese": 0.2727272727272727, - "Malay,Chinese,Indonesian": 0.29545454545454547, - "Malay,Chinese,Vietnamese": 0.24431818181818182, - "Malay,Indonesian,Vietnamese": 0.35795454545454547, - "Chinese,Indonesian,Vietnamese": 0.2556818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17045454545454544, - "English,Filipino,Spanish,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Indonesian": 0.1875, - "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese": 0.11363636363636363, - "English,Filipino,Malay,Indonesian": 0.17613636363636365, - "English,Filipino,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Chinese,Indonesian": 0.125, - "English,Filipino,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Indonesian,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Chinese": 0.1534090909090909, - "English,Spanish,Malay,Indonesian": 0.21022727272727273, - "English,Spanish,Malay,Vietnamese": 0.17045454545454544, - "English,Spanish,Chinese,Indonesian": 0.1534090909090909, - "English,Spanish,Chinese,Vietnamese": 0.125, - "English,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "English,Malay,Chinese,Indonesian": 0.14772727272727273, - "English,Malay,Chinese,Vietnamese": 0.13068181818181818, - "English,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "English,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Chinese": 0.17045454545454544, - "Filipino,Spanish,Malay,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.1875, - "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.1875, - "Filipino,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Malay,Chinese,Indonesian": 0.1875, - "Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - } - }, - "AC3_2": 0.39300407214974875, - "AC3_3": 0.2944615851276995, - "AC3_4": 0.22282512707488666, - "AC3_5": 0.17216500696755385, - "AC3_6": 0.13495610191934804, - "AC3_7": 0.10605314958057924 - }, - "prompt_5": { - "overall_acc": 0.3457792207792208, - "language_acc": { - "English": 0.5113636363636364, - "Filipino": 0.3125, - "Spanish": 0.39204545454545453, - "Malay": 0.3409090909090909, - "Chinese": 0.29545454545454547, - "Indonesian": 0.25, - "Vietnamese": 0.3181818181818182 - }, - "consistency_score_2": 0.4437229437229438, - "consistency_score_3": 0.24853896103896106, - "consistency_score_4": 0.15941558441558437, - "consistency_score_5": 0.1130952380952381, - "consistency_score_6": 0.08603896103896103, - "consistency_score_7": 0.06818181818181818, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.35795454545454547, - "English,Spanish": 0.5511363636363636, - "English,Malay": 0.3977272727272727, - "English,Chinese": 0.3977272727272727, - "English,Indonesian": 0.36363636363636365, - "English,Vietnamese": 0.3465909090909091, - "Filipino,Spanish": 0.4431818181818182, - "Filipino,Malay": 0.5511363636363636, - "Filipino,Chinese": 0.3465909090909091, - "Filipino,Indonesian": 0.5397727272727273, - "Filipino,Vietnamese": 0.38636363636363635, - "Spanish,Malay": 0.44886363636363635, - "Spanish,Chinese": 0.48295454545454547, - "Spanish,Indonesian": 0.4431818181818182, - "Spanish,Vietnamese": 0.4147727272727273, - "Malay,Chinese": 0.4318181818181818, - "Malay,Indonesian": 0.5795454545454546, - "Malay,Vietnamese": 0.5170454545454546, - "Chinese,Indonesian": 0.4715909090909091, - "Chinese,Vietnamese": 0.3806818181818182, - "Indonesian,Vietnamese": 0.4659090909090909 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2727272727272727, - "English,Filipino,Malay": 0.22727272727272727, - "English,Filipino,Chinese": 0.17613636363636365, - "English,Filipino,Indonesian": 0.2159090909090909, - "English,Filipino,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay": 0.2727272727272727, - "English,Spanish,Chinese": 0.2840909090909091, - "English,Spanish,Indonesian": 0.23863636363636365, - "English,Spanish,Vietnamese": 0.23863636363636365, - "English,Malay,Chinese": 0.19886363636363635, - "English,Malay,Indonesian": 0.25, - "English,Malay,Vietnamese": 0.23295454545454544, - "English,Chinese,Indonesian": 0.2215909090909091, - "English,Chinese,Vietnamese": 0.17045454545454544, - "English,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay": 0.2784090909090909, - "Filipino,Spanish,Chinese": 0.23863636363636365, - "Filipino,Spanish,Indonesian": 0.2897727272727273, - "Filipino,Spanish,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese": 0.23863636363636365, - "Filipino,Malay,Indonesian": 0.38636363636363635, - "Filipino,Malay,Vietnamese": 0.29545454545454547, - "Filipino,Chinese,Indonesian": 0.23863636363636365, - "Filipino,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Chinese": 0.26136363636363635, - "Spanish,Malay,Indonesian": 0.2840909090909091, - "Spanish,Malay,Vietnamese": 0.26136363636363635, - "Spanish,Chinese,Indonesian": 0.29545454545454547, - "Spanish,Chinese,Vietnamese": 0.24431818181818182, - "Spanish,Indonesian,Vietnamese": 0.23863636363636365, - "Malay,Chinese,Indonesian": 0.3181818181818182, - "Malay,Chinese,Vietnamese": 0.23863636363636365, - "Malay,Indonesian,Vietnamese": 0.3125, - "Chinese,Indonesian,Vietnamese": 0.24431818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17045454545454544, - "English,Filipino,Spanish,Chinese": 0.1590909090909091, - "English,Filipino,Spanish,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese": 0.11931818181818182, - "English,Filipino,Malay,Indonesian": 0.17045454545454544, - "English,Filipino,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Chinese": 0.17045454545454544, - "English,Spanish,Malay,Indonesian": 0.18181818181818182, - "English,Spanish,Malay,Vietnamese": 0.17045454545454544, - "English,Spanish,Chinese,Indonesian": 0.17613636363636365, - "English,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "English,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "English,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Chinese": 0.17045454545454544, - "Filipino,Spanish,Malay,Indonesian": 0.21022727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.20454545454545456, - "Spanish,Malay,Chinese,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Malay,Chinese,Indonesian,Vietnamese": 0.18181818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - } - }, - "AC3_2": 0.38867575188903003, - "AC3_3": 0.289204035467857, - "AC3_4": 0.21822313286502698, - "AC3_5": 0.17044305926204767, - "AC3_6": 0.13779171952672717, - "AC3_7": 0.11390374328799245 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.4563106796116505 - }, - "prompt_2": { - "accuracy": 0.3883495145631068 - }, - "prompt_3": { - "accuracy": 0.44660194174757284 - }, - "prompt_4": { - "accuracy": 0.3592233009708738 - }, - "prompt_5": { - "accuracy": 0.4854368932038835 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.22857142857142856 - }, - "prompt_2": { - "accuracy": 0.24761904761904763 - }, - "prompt_3": { - "accuracy": 0.24761904761904763 - }, - "prompt_4": { - "accuracy": 0.23809523809523808 - }, - "prompt_5": { - "accuracy": 0.22857142857142856 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.5327102803738317 - }, - "prompt_2": { - "accuracy": 0.4392523364485981 - }, - "prompt_3": { - "accuracy": 0.5327102803738317 - }, - "prompt_4": { - "accuracy": 0.4485981308411215 - }, - "prompt_5": { - "accuracy": 0.5700934579439252 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.45, - "category_acc": { - "brand": 0.2, - "demographics": 0.4, - "biology": 0.6, - "history": 0.4, - "literature": 0.6, - "politics": 0.5, - "culture": 0.3, - "film": 0.5, - "law": 0.4, - "geography": 0.6 - } - }, - "prompt_2": { - "accuracy": 0.36, - "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.6, - "history": 0.3333333333333333, - "literature": 0.3, - "politics": 0.3, - "culture": 0.1, - "film": 0.4, - "law": 0.3, - "geography": 0.6 - } - }, - "prompt_3": { - "accuracy": 0.41, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.5, - "history": 0.2, - "literature": 0.5, - "politics": 0.4, - "culture": 0.2, - "film": 0.7, - "law": 0.3, - "geography": 0.7 - } - }, - "prompt_4": { - "accuracy": 0.39, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.4, - "history": 0.3333333333333333, - "literature": 0.3, - "politics": 0.5, - "culture": 0.1, - "film": 0.6, - "law": 0.3, - "geography": 0.7 - } - }, - "prompt_5": { - "accuracy": 0.36, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.6, - "culture": 0.2, - "film": 0.3, - "law": 0.3, - "geography": 0.5 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.20204613048715422 - }, - "prompt_2": { - "bleu_score": 0.21303573088120414 - }, - "prompt_3": { - "bleu_score": 0.2295259468241605 - }, - "prompt_4": { - "bleu_score": 0.2044146045784196 - }, - "prompt_5": { - "bleu_score": 0.0657817389105964 - } - }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.01631574332144387 - }, - "prompt_2": { - "bleu_score": 0.060075583839012 - }, - "prompt_3": { - "bleu_score": 0.03104112879587321 - }, - "prompt_4": { - "bleu_score": 0.015620288849299953 - }, - "prompt_5": { - "bleu_score": 0.04685998633132947 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.008965949828249793 - }, - "prompt_2": { - "bleu_score": 0.029453983983772546 - }, - "prompt_3": { - "bleu_score": 0.02264522602562453 - }, - "prompt_4": { - "bleu_score": 0.0102351698368996 - }, - "prompt_5": { - "bleu_score": 0.022773459101689857 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.025611075648526046 - }, - "prompt_2": { - "bleu_score": 0.0842365422580657 - }, - "prompt_3": { - "bleu_score": 0.07625021359654235 - }, - "prompt_4": { - "bleu_score": 0.04248903245108832 - }, - "prompt_5": { - "bleu_score": 0.06944944995384345 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.015900539398339016 - }, - "prompt_2": { - "bleu_score": 0.05431731782684005 - }, - "prompt_3": { - "bleu_score": 0.04655999429945362 - }, - "prompt_4": { - "bleu_score": 0.021687027658168666 - }, - "prompt_5": { - "bleu_score": 0.047256880776147536 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.514585764294049 - }, - "prompt_2": { - "accuracy": 0.544924154025671 - }, - "prompt_3": { - "accuracy": 0.5367561260210035 - }, - "prompt_4": { - "accuracy": 0.5425904317386231 - }, - "prompt_5": { - "accuracy": 0.5460910151691949 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.49910618519842687, - "category_acc": { - "high_school_european_history": 0.6890243902439024, - "business_ethics": 0.494949494949495, - "clinical_knowledge": 0.5, - "medical_genetics": 0.5858585858585859, - "high_school_us_history": 0.6650246305418719, - "high_school_physics": 0.36, - "high_school_world_history": 0.7415254237288136, - "virology": 0.4484848484848485, - "high_school_microeconomics": 0.5358649789029536, - "econometrics": 0.3185840707964602, - "college_computer_science": 0.46464646464646464, - "high_school_biology": 0.6148867313915858, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.4092526690391459, - "philosophy": 0.5290322580645161, - "professional_medicine": 0.47232472324723246, - "nutrition": 0.6163934426229508, - "global_facts": 0.2727272727272727, - "machine_learning": 0.44144144144144143, - "security_studies": 0.5204918032786885, - "public_relations": 0.5045871559633027, - "professional_psychology": 0.4828150572831424, - "prehistory": 0.4984520123839009, - "anatomy": 0.43283582089552236, - "human_sexuality": 0.5230769230769231, - "college_medicine": 0.5174418604651163, - "high_school_government_and_politics": 0.6875, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.6728395061728395, - "high_school_geography": 0.5634517766497462, - "elementary_mathematics": 0.38726790450928383, - "human_aging": 0.5225225225225225, - "college_mathematics": 0.42424242424242425, - "high_school_psychology": 0.6470588235294118, - "formal_logic": 0.392, - "high_school_statistics": 0.44651162790697674, - "international_law": 0.6583333333333333, - "high_school_mathematics": 0.3271375464684015, - "high_school_computer_science": 0.6161616161616161, - "conceptual_physics": 0.46153846153846156, - "miscellaneous": 0.6227621483375959, - "high_school_chemistry": 0.37623762376237624, - "marketing": 0.7381974248927039, - "professional_law": 0.40574037834311805, - "management": 0.6470588235294118, - "college_physics": 0.24752475247524752, - "jurisprudence": 0.6635514018691588, - "world_religions": 0.6176470588235294, - "sociology": 0.68, - "us_foreign_policy": 0.5454545454545454, - "high_school_macroeconomics": 0.506426735218509, - "computer_security": 0.5252525252525253, - "moral_scenarios": 0.24161073825503357, - "moral_disputes": 0.6202898550724638, - "electrical_engineering": 0.4513888888888889, - "astronomy": 0.5099337748344371, - "college_biology": 0.5314685314685315 - } - }, - "prompt_2": { - "accuracy": 0.5146228101537361, - "category_acc": { - "high_school_european_history": 0.5914634146341463, - "business_ethics": 0.5757575757575758, - "clinical_knowledge": 0.5757575757575758, - "medical_genetics": 0.5555555555555556, - "high_school_us_history": 0.6157635467980296, - "high_school_physics": 0.35333333333333333, - "high_school_world_history": 0.6694915254237288, - "virology": 0.4666666666666667, - "high_school_microeconomics": 0.5443037974683544, - "econometrics": 0.30973451327433627, - "college_computer_science": 0.3939393939393939, - "high_school_biology": 0.6343042071197411, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.3736654804270463, - "philosophy": 0.5645161290322581, - "professional_medicine": 0.4870848708487085, - "nutrition": 0.5934426229508196, - "global_facts": 0.3939393939393939, - "machine_learning": 0.43243243243243246, - "security_studies": 0.5, - "public_relations": 0.5504587155963303, - "professional_psychology": 0.5204582651391162, - "prehistory": 0.5789473684210527, - "anatomy": 0.5074626865671642, - "human_sexuality": 0.5615384615384615, - "college_medicine": 0.5290697674418605, - "high_school_government_and_politics": 0.71875, - "college_chemistry": 0.3333333333333333, - "logical_fallacies": 0.691358024691358, - "high_school_geography": 0.6446700507614214, - "elementary_mathematics": 0.40583554376657827, - "human_aging": 0.5585585585585585, - "college_mathematics": 0.42424242424242425, - "high_school_psychology": 0.7279411764705882, - "formal_logic": 0.328, - "high_school_statistics": 0.4372093023255814, - "international_law": 0.675, - "high_school_mathematics": 0.31226765799256506, - "high_school_computer_science": 0.6363636363636364, - "conceptual_physics": 0.452991452991453, - "miscellaneous": 0.6560102301790282, - "high_school_chemistry": 0.40594059405940597, - "marketing": 0.776824034334764, - "professional_law": 0.40769732550554466, - "management": 0.6568627450980392, - "college_physics": 0.2871287128712871, - "jurisprudence": 0.6448598130841121, - "world_religions": 0.6352941176470588, - "sociology": 0.745, - "us_foreign_policy": 0.7070707070707071, - "high_school_macroeconomics": 0.5269922879177378, - "computer_security": 0.6060606060606061, - "moral_scenarios": 0.22818791946308725, - "moral_disputes": 0.6260869565217392, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.543046357615894, - "college_biology": 0.4965034965034965 - } - }, - "prompt_3": { - "accuracy": 0.5131927064712192, - "category_acc": { - "high_school_european_history": 0.6280487804878049, - "business_ethics": 0.5353535353535354, - "clinical_knowledge": 0.5909090909090909, - "medical_genetics": 0.5858585858585859, - "high_school_us_history": 0.6305418719211823, - "high_school_physics": 0.3, - "high_school_world_history": 0.6864406779661016, - "virology": 0.4909090909090909, - "high_school_microeconomics": 0.5443037974683544, - "econometrics": 0.3185840707964602, - "college_computer_science": 0.42424242424242425, - "high_school_biology": 0.6310679611650486, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.398576512455516, - "philosophy": 0.5483870967741935, - "professional_medicine": 0.4870848708487085, - "nutrition": 0.5639344262295082, - "global_facts": 0.36363636363636365, - "machine_learning": 0.4864864864864865, - "security_studies": 0.47540983606557374, - "public_relations": 0.5871559633027523, - "professional_psychology": 0.5188216039279869, - "prehistory": 0.5479876160990712, - "anatomy": 0.48507462686567165, - "human_sexuality": 0.5692307692307692, - "college_medicine": 0.5348837209302325, - "high_school_government_and_politics": 0.7239583333333334, - "college_chemistry": 0.3333333333333333, - "logical_fallacies": 0.6666666666666666, - "high_school_geography": 0.649746192893401, - "elementary_mathematics": 0.38992042440318303, - "human_aging": 0.5675675675675675, - "college_mathematics": 0.3939393939393939, - "high_school_psychology": 0.7352941176470589, - "formal_logic": 0.392, - "high_school_statistics": 0.4232558139534884, - "international_law": 0.6583333333333333, - "high_school_mathematics": 0.2936802973977695, - "high_school_computer_science": 0.5959595959595959, - "conceptual_physics": 0.47435897435897434, - "miscellaneous": 0.6649616368286445, - "high_school_chemistry": 0.40594059405940597, - "marketing": 0.7639484978540773, - "professional_law": 0.40834964122635353, - "management": 0.6470588235294118, - "college_physics": 0.297029702970297, - "jurisprudence": 0.6355140186915887, - "world_religions": 0.611764705882353, - "sociology": 0.75, - "us_foreign_policy": 0.7171717171717171, - "high_school_macroeconomics": 0.5038560411311054, - "computer_security": 0.5959595959595959, - "moral_scenarios": 0.22706935123042504, - "moral_disputes": 0.5826086956521739, - "electrical_engineering": 0.5, - "astronomy": 0.5298013245033113, - "college_biology": 0.5874125874125874 - } - }, - "prompt_4": { - "accuracy": 0.5223453700393279, - "category_acc": { - "high_school_european_history": 0.6890243902439024, - "business_ethics": 0.5252525252525253, - "clinical_knowledge": 0.6022727272727273, - "medical_genetics": 0.6262626262626263, - "high_school_us_history": 0.6551724137931034, - "high_school_physics": 0.36, - "high_school_world_history": 0.75, - "virology": 0.47878787878787876, - "high_school_microeconomics": 0.5654008438818565, - "econometrics": 0.3185840707964602, - "college_computer_science": 0.42424242424242425, - "high_school_biology": 0.6504854368932039, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.39501779359430605, - "philosophy": 0.5419354838709678, - "professional_medicine": 0.4797047970479705, - "nutrition": 0.6, - "global_facts": 0.30303030303030304, - "machine_learning": 0.45045045045045046, - "security_studies": 0.6065573770491803, - "public_relations": 0.6055045871559633, - "professional_psychology": 0.4959083469721768, - "prehistory": 0.5634674922600619, - "anatomy": 0.5074626865671642, - "human_sexuality": 0.5769230769230769, - "college_medicine": 0.5581395348837209, - "high_school_government_and_politics": 0.7447916666666666, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.6975308641975309, - "high_school_geography": 0.6700507614213198, - "elementary_mathematics": 0.46153846153846156, - "human_aging": 0.5675675675675675, - "college_mathematics": 0.42424242424242425, - "high_school_psychology": 0.7279411764705882, - "formal_logic": 0.32, - "high_school_statistics": 0.4697674418604651, - "international_law": 0.6916666666666667, - "high_school_mathematics": 0.3048327137546468, - "high_school_computer_science": 0.6363636363636364, - "conceptual_physics": 0.4444444444444444, - "miscellaneous": 0.6457800511508951, - "high_school_chemistry": 0.40594059405940597, - "marketing": 0.7381974248927039, - "professional_law": 0.39399869536855836, - "management": 0.6568627450980392, - "college_physics": 0.3465346534653465, - "jurisprudence": 0.6542056074766355, - "world_religions": 0.6588235294117647, - "sociology": 0.725, - "us_foreign_policy": 0.696969696969697, - "high_school_macroeconomics": 0.5424164524421594, - "computer_security": 0.5757575757575758, - "moral_scenarios": 0.22706935123042504, - "moral_disputes": 0.6086956521739131, - "electrical_engineering": 0.5763888888888888, - "astronomy": 0.5629139072847682, - "college_biology": 0.5384615384615384 - } - }, - "prompt_5": { - "accuracy": 0.5269217018233822, - "category_acc": { - "high_school_european_history": 0.6890243902439024, - "business_ethics": 0.5454545454545454, - "clinical_knowledge": 0.5795454545454546, - "medical_genetics": 0.5858585858585859, - "high_school_us_history": 0.645320197044335, - "high_school_physics": 0.38, - "high_school_world_history": 0.75, - "virology": 0.47878787878787876, - "high_school_microeconomics": 0.5527426160337553, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.48484848484848486, - "high_school_biology": 0.6472491909385113, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.4412811387900356, - "philosophy": 0.5741935483870968, - "professional_medicine": 0.4612546125461255, - "nutrition": 0.6, - "global_facts": 0.29292929292929293, - "machine_learning": 0.44144144144144143, - "security_studies": 0.6147540983606558, - "public_relations": 0.5504587155963303, - "professional_psychology": 0.5057283142389526, - "prehistory": 0.5572755417956656, - "anatomy": 0.4925373134328358, - "human_sexuality": 0.5846153846153846, - "college_medicine": 0.5523255813953488, - "high_school_government_and_politics": 0.75, - "college_chemistry": 0.3939393939393939, - "logical_fallacies": 0.7283950617283951, - "high_school_geography": 0.6649746192893401, - "elementary_mathematics": 0.4482758620689655, - "human_aging": 0.5855855855855856, - "college_mathematics": 0.40404040404040403, - "high_school_psychology": 0.7426470588235294, - "formal_logic": 0.336, - "high_school_statistics": 0.4511627906976744, - "international_law": 0.6916666666666667, - "high_school_mathematics": 0.2936802973977695, - "high_school_computer_science": 0.6262626262626263, - "conceptual_physics": 0.452991452991453, - "miscellaneous": 0.6815856777493606, - "high_school_chemistry": 0.4504950495049505, - "marketing": 0.7467811158798283, - "professional_law": 0.3887801696020874, - "management": 0.6666666666666666, - "college_physics": 0.31683168316831684, - "jurisprudence": 0.6728971962616822, - "world_religions": 0.6705882352941176, - "sociology": 0.735, - "us_foreign_policy": 0.7171717171717171, - "high_school_macroeconomics": 0.5372750642673522, - "computer_security": 0.5353535353535354, - "moral_scenarios": 0.2360178970917226, - "moral_disputes": 0.6579710144927536, - "electrical_engineering": 0.5347222222222222, - "astronomy": 0.5364238410596026, - "college_biology": 0.5874125874125874 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.3202080237741456 - }, - "prompt_2": { - "accuracy": 0.31426448736998513 - }, - "prompt_3": { - "accuracy": 0.2986627043090639 - }, - "prompt_4": { - "accuracy": 0.2934621099554235 - }, - "prompt_5": { - "accuracy": 0.31946508172362553 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.3094645080946451, - "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.25, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.42857142857142855, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.27586206896551724, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.4583333333333333, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.38461538461538464, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.26666666666666666, - "business_administration": 0.2894736842105263, - "marxism": 0.375, - "mao_zedong_thought": 0.3448275862068966, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.32653061224489793, - "high_school_politics": 0.375, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.3333333333333333, - "law": 0.4482758620689655, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.25, - "high_school_chinese": 0.25, - "high_school_history": 0.36, - "middle_school_history": 0.25925925925925924, - "civil_servant": 0.3076923076923077, - "sports_science": 0.20833333333333334, - "plant_protection": 0.37037037037037035, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.2777777777777778, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.18518518518518517, - "physician": 0.3333333333333333 - } - }, - "prompt_2": { - "accuracy": 0.28580323785803236, - "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.4523809523809524, - "college_physics": 0.375, - "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.41379310344827586, - "high_school_mathematics": 0.13043478260869565, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.19230769230769232, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.25, - "business_administration": 0.23684210526315788, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.4482758620689655, - "education_science": 0.38235294117647056, - "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.07142857142857142, - "ideological_and_moral_cultivation": 0.25, - "logic": 0.2962962962962963, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.52, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.25, - "sports_science": 0.08333333333333333, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.2777777777777778, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.16666666666666666, - "tax_accountant": 0.18518518518518517, - "physician": 0.2962962962962963 - } - }, - "prompt_3": { - "accuracy": 0.2851805728518057, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.38095238095238093, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.25, - "high_school_chemistry": 0.375, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.23076923076923078, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.25, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.5172413793103449, - "education_science": 0.29411764705882354, - "teacher_qualification": 0.24489795918367346, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.14285714285714285, - "ideological_and_moral_cultivation": 0.25, - "logic": 0.2962962962962963, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.25, - "high_school_history": 0.44, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.25, - "sports_science": 0.16666666666666666, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.375, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.2962962962962963, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.08333333333333333, - "tax_accountant": 0.18518518518518517, - "physician": 0.3148148148148148 - } - }, - "prompt_4": { - "accuracy": 0.29389788293897884, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.16666666666666666, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.47619047619047616, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.23076923076923078, - "middle_school_physics": 0.4166666666666667, - "middle_school_chemistry": 0.24, - "veterinary_medicine": 0.42857142857142855, - "college_economics": 0.3, - "business_administration": 0.23684210526315788, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.4827586206896552, - "education_science": 0.38235294117647056, - "teacher_qualification": 0.30612244897959184, - "high_school_politics": 0.4166666666666667, - "high_school_geography": 0.4583333333333333, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.2962962962962963, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.10714285714285714, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.25, - "high_school_history": 0.32, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.3076923076923077, - "sports_science": 0.16666666666666666, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.3148148148148148, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.18518518518518517, - "physician": 0.25925925925925924 - } - }, - "prompt_5": { - "accuracy": 0.30510585305105853, - "category_acc": { - "computer_network": 0.375, - "operating_system": 0.25, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.4523809523809524, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.4583333333333333, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.41379310344827586, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.4166666666666667, - "high_school_biology": 0.16666666666666666, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.23076923076923078, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.42857142857142855, - "college_economics": 0.26666666666666666, - "business_administration": 0.34210526315789475, - "marxism": 0.5416666666666666, - "mao_zedong_thought": 0.3448275862068966, - "education_science": 0.38235294117647056, - "teacher_qualification": 0.3469387755102041, - "high_school_politics": 0.5, - "high_school_geography": 0.5416666666666666, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.375, - "logic": 0.3333333333333333, - "law": 0.5172413793103449, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.125, - "high_school_history": 0.36, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.3076923076923077, - "sports_science": 0.125, - "plant_protection": 0.37037037037037035, - "basic_medicine": 0.25, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.19607843137254902, - "accountant": 0.18518518518518517, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.16666666666666666, - "physician": 0.2777777777777778 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.2867383512544803 - }, - "prompt_2": { - "accuracy": 0.2831541218637993 - }, - "prompt_3": { - "accuracy": 0.30824372759856633 - }, - "prompt_4": { - "accuracy": 0.30824372759856633 - }, - "prompt_5": { - "accuracy": 0.2903225806451613 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.2861336556725954, - "category_acc": { - "agronomy": 0.26627218934911245, - "anatomy": 0.22972972972972974, - "ancient_chinese": 0.22560975609756098, - "arts": 0.29375, - "astronomy": 0.2, - "business_ethics": 0.3157894736842105, - "chinese_civil_service_exam": 0.28125, - "chinese_driving_rule": 0.31297709923664124, - "chinese_food_culture": 0.2867647058823529, - "chinese_foreign_policy": 0.2803738317757009, - "chinese_history": 0.2786377708978328, - "chinese_literature": 0.27941176470588236, - "chinese_teacher_qualification": 0.29608938547486036, - "clinical_knowledge": 0.25738396624472576, - "college_actuarial_science": 0.32075471698113206, - "college_education": 0.308411214953271, - "college_engineering_hydrology": 0.3113207547169811, - "college_law": 0.24074074074074073, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.37735849056603776, - "college_medicine": 0.32234432234432236, - "computer_science": 0.29901960784313725, - "computer_security": 0.26900584795321636, - "conceptual_physics": 0.2857142857142857, - "construction_project_management": 0.28776978417266186, - "economics": 0.3081761006289308, - "education": 0.31901840490797545, - "electrical_engineering": 0.3488372093023256, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.2878787878787879, - "elementary_information_and_technology": 0.31092436974789917, - "elementary_mathematics": 0.24347826086956523, - "ethnology": 0.3111111111111111, - "food_science": 0.34965034965034963, - "genetics": 0.3181818181818182, - "global_facts": 0.28187919463087246, - "high_school_biology": 0.22485207100591717, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.2627118644067797, - "high_school_mathematics": 0.24390243902439024, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.3076923076923077, - "human_sexuality": 0.31746031746031744, - "international_law": 0.2594594594594595, - "journalism": 0.3023255813953488, - "jurisprudence": 0.2895377128953771, - "legal_and_moral_basis": 0.34579439252336447, - "logical": 0.2764227642276423, - "machine_learning": 0.26229508196721313, - "management": 0.2571428571428571, - "marketing": 0.34444444444444444, - "marxist_theory": 0.31746031746031744, - "modern_chinese": 0.23275862068965517, - "nutrition": 0.296551724137931, - "philosophy": 0.3238095238095238, - "professional_accounting": 0.30857142857142855, - "professional_law": 0.27014218009478674, - "professional_medicine": 0.2473404255319149, - "professional_psychology": 0.2974137931034483, - "public_relations": 0.2988505747126437, - "security_study": 0.3037037037037037, - "sociology": 0.26991150442477874, - "sports_science": 0.30303030303030304, - "traditional_chinese_medicine": 0.2594594594594595, - "virology": 0.3609467455621302, - "world_history": 0.2422360248447205, - "world_religions": 0.2125 - } - }, - "prompt_2": { - "accuracy": 0.27853565878086683, - "category_acc": { - "agronomy": 0.27218934911242604, - "anatomy": 0.25, - "ancient_chinese": 0.21951219512195122, - "arts": 0.275, - "astronomy": 0.23636363636363636, - "business_ethics": 0.28708133971291866, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.31297709923664124, - "chinese_food_culture": 0.27205882352941174, - "chinese_foreign_policy": 0.3177570093457944, - "chinese_history": 0.2848297213622291, - "chinese_literature": 0.31862745098039214, - "chinese_teacher_qualification": 0.2849162011173184, - "clinical_knowledge": 0.29535864978902954, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.32710280373831774, - "college_engineering_hydrology": 0.2830188679245283, - "college_law": 0.2037037037037037, - "college_mathematics": 0.17142857142857143, - "college_medical_statistics": 0.27358490566037735, - "college_medicine": 0.23809523809523808, - "computer_science": 0.30392156862745096, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.2925170068027211, - "construction_project_management": 0.30935251798561153, - "economics": 0.31446540880503143, - "education": 0.27607361963190186, - "electrical_engineering": 0.36627906976744184, - "elementary_chinese": 0.29365079365079366, - "elementary_commonsense": 0.2777777777777778, - "elementary_information_and_technology": 0.3319327731092437, - "elementary_mathematics": 0.2782608695652174, - "ethnology": 0.37777777777777777, - "food_science": 0.34265734265734266, - "genetics": 0.24431818181818182, - "global_facts": 0.2751677852348993, - "high_school_biology": 0.25443786982248523, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.2711864406779661, - "high_school_mathematics": 0.25609756097560976, - "high_school_physics": 0.3, - "high_school_politics": 0.26573426573426573, - "human_sexuality": 0.2777777777777778, - "international_law": 0.2648648648648649, - "journalism": 0.313953488372093, - "jurisprudence": 0.27007299270072993, - "legal_and_moral_basis": 0.35046728971962615, - "logical": 0.2926829268292683, - "machine_learning": 0.29508196721311475, - "management": 0.2571428571428571, - "marketing": 0.3, - "marxist_theory": 0.2804232804232804, - "modern_chinese": 0.23275862068965517, - "nutrition": 0.296551724137931, - "philosophy": 0.29523809523809524, - "professional_accounting": 0.2857142857142857, - "professional_law": 0.26066350710900477, - "professional_medicine": 0.2154255319148936, - "professional_psychology": 0.2413793103448276, - "public_relations": 0.3045977011494253, - "security_study": 0.2814814814814815, - "sociology": 0.252212389380531, - "sports_science": 0.3090909090909091, - "traditional_chinese_medicine": 0.23783783783783785, - "virology": 0.3076923076923077, - "world_history": 0.2795031055900621, - "world_religions": 0.23125 - } - }, - "prompt_3": { - "accuracy": 0.2769815230530133, - "category_acc": { - "agronomy": 0.2603550295857988, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.25, - "arts": 0.26875, - "astronomy": 0.26666666666666666, - "business_ethics": 0.28708133971291866, - "chinese_civil_service_exam": 0.26875, - "chinese_driving_rule": 0.2824427480916031, - "chinese_food_culture": 0.22058823529411764, - "chinese_foreign_policy": 0.308411214953271, - "chinese_history": 0.2724458204334365, - "chinese_literature": 0.28431372549019607, - "chinese_teacher_qualification": 0.30726256983240224, - "clinical_knowledge": 0.2869198312236287, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.3364485981308411, - "college_engineering_hydrology": 0.25471698113207547, - "college_law": 0.25925925925925924, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.33962264150943394, - "college_medicine": 0.24175824175824176, - "computer_science": 0.28921568627450983, - "computer_security": 0.2573099415204678, - "conceptual_physics": 0.29931972789115646, - "construction_project_management": 0.2158273381294964, - "economics": 0.33962264150943394, - "education": 0.2883435582822086, - "electrical_engineering": 0.3430232558139535, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.21212121212121213, - "elementary_information_and_technology": 0.27310924369747897, - "elementary_mathematics": 0.2608695652173913, - "ethnology": 0.2740740740740741, - "food_science": 0.3776223776223776, - "genetics": 0.26136363636363635, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.2627118644067797, - "high_school_mathematics": 0.23170731707317074, - "high_school_physics": 0.24545454545454545, - "high_school_politics": 0.2517482517482518, - "human_sexuality": 0.31746031746031744, - "international_law": 0.2702702702702703, - "journalism": 0.3372093023255814, - "jurisprudence": 0.2749391727493917, - "legal_and_moral_basis": 0.32242990654205606, - "logical": 0.3333333333333333, - "machine_learning": 0.2540983606557377, - "management": 0.26666666666666666, - "marketing": 0.29444444444444445, - "marxist_theory": 0.3492063492063492, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.2896551724137931, - "philosophy": 0.3333333333333333, - "professional_accounting": 0.2857142857142857, - "professional_law": 0.22748815165876776, - "professional_medicine": 0.2393617021276596, - "professional_psychology": 0.2629310344827586, - "public_relations": 0.25862068965517243, - "security_study": 0.3037037037037037, - "sociology": 0.29646017699115046, - "sports_science": 0.2909090909090909, - "traditional_chinese_medicine": 0.23783783783783785, - "virology": 0.3076923076923077, - "world_history": 0.2857142857142857, - "world_religions": 0.21875 - } - }, - "prompt_4": { - "accuracy": 0.2776722500431704, - "category_acc": { - "agronomy": 0.30177514792899407, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.24390243902439024, - "arts": 0.275, - "astronomy": 0.22424242424242424, - "business_ethics": 0.3157894736842105, - "chinese_civil_service_exam": 0.26875, - "chinese_driving_rule": 0.29770992366412213, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.2803738317757009, - "chinese_history": 0.2848297213622291, - "chinese_literature": 0.2647058823529412, - "chinese_teacher_qualification": 0.27932960893854747, - "clinical_knowledge": 0.2869198312236287, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.2616822429906542, - "college_engineering_hydrology": 0.20754716981132076, - "college_law": 0.25, - "college_mathematics": 0.18095238095238095, - "college_medical_statistics": 0.36792452830188677, - "college_medicine": 0.3076923076923077, - "computer_science": 0.2647058823529412, - "computer_security": 0.27485380116959063, - "conceptual_physics": 0.3129251700680272, - "construction_project_management": 0.2589928057553957, - "economics": 0.3081761006289308, - "education": 0.26380368098159507, - "electrical_engineering": 0.313953488372093, - "elementary_chinese": 0.2857142857142857, - "elementary_commonsense": 0.24242424242424243, - "elementary_information_and_technology": 0.29831932773109243, - "elementary_mathematics": 0.29130434782608694, - "ethnology": 0.34814814814814815, - "food_science": 0.3076923076923077, - "genetics": 0.32386363636363635, - "global_facts": 0.19463087248322147, - "high_school_biology": 0.26627218934911245, - "high_school_chemistry": 0.25, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.2818181818181818, - "high_school_politics": 0.26573426573426573, - "human_sexuality": 0.2857142857142857, - "international_law": 0.21621621621621623, - "journalism": 0.27906976744186046, - "jurisprudence": 0.2944038929440389, - "legal_and_moral_basis": 0.3364485981308411, - "logical": 0.3089430894308943, - "machine_learning": 0.28688524590163933, - "management": 0.20476190476190476, - "marketing": 0.3055555555555556, - "marxist_theory": 0.3492063492063492, - "modern_chinese": 0.2672413793103448, - "nutrition": 0.27586206896551724, - "philosophy": 0.29523809523809524, - "professional_accounting": 0.2914285714285714, - "professional_law": 0.23222748815165878, - "professional_medicine": 0.2393617021276596, - "professional_psychology": 0.2672413793103448, - "public_relations": 0.2988505747126437, - "security_study": 0.31851851851851853, - "sociology": 0.3230088495575221, - "sports_science": 0.2545454545454545, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.33136094674556216, - "world_history": 0.21739130434782608, - "world_religions": 0.25 - } - }, - "prompt_5": { - "accuracy": 0.2912277672250043, - "category_acc": { - "agronomy": 0.33727810650887574, - "anatomy": 0.27702702702702703, - "ancient_chinese": 0.2682926829268293, - "arts": 0.3625, - "astronomy": 0.22424242424242424, - "business_ethics": 0.2679425837320574, - "chinese_civil_service_exam": 0.3, - "chinese_driving_rule": 0.35877862595419846, - "chinese_food_culture": 0.27205882352941174, - "chinese_foreign_policy": 0.2803738317757009, - "chinese_history": 0.26006191950464397, - "chinese_literature": 0.3431372549019608, - "chinese_teacher_qualification": 0.2849162011173184, - "clinical_knowledge": 0.2911392405063291, - "college_actuarial_science": 0.32075471698113206, - "college_education": 0.32710280373831774, - "college_engineering_hydrology": 0.2830188679245283, - "college_law": 0.24074074074074073, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.3584905660377358, - "college_medicine": 0.315018315018315, - "computer_science": 0.3235294117647059, - "computer_security": 0.27485380116959063, - "conceptual_physics": 0.2585034013605442, - "construction_project_management": 0.2805755395683453, - "economics": 0.32075471698113206, - "education": 0.31901840490797545, - "electrical_engineering": 0.31976744186046513, - "elementary_chinese": 0.26587301587301587, - "elementary_commonsense": 0.2777777777777778, - "elementary_information_and_technology": 0.31092436974789917, - "elementary_mathematics": 0.2608695652173913, - "ethnology": 0.28888888888888886, - "food_science": 0.32867132867132864, - "genetics": 0.2784090909090909, - "global_facts": 0.24161073825503357, - "high_school_biology": 0.26627218934911245, - "high_school_chemistry": 0.2803030303030303, - "high_school_geography": 0.211864406779661, - "high_school_mathematics": 0.25, - "high_school_physics": 0.2909090909090909, - "high_school_politics": 0.3006993006993007, - "human_sexuality": 0.31746031746031744, - "international_law": 0.2864864864864865, - "journalism": 0.3081395348837209, - "jurisprudence": 0.2749391727493917, - "legal_and_moral_basis": 0.3317757009345794, - "logical": 0.2845528455284553, - "machine_learning": 0.26229508196721313, - "management": 0.24285714285714285, - "marketing": 0.3611111111111111, - "marxist_theory": 0.36507936507936506, - "modern_chinese": 0.29310344827586204, - "nutrition": 0.2827586206896552, - "philosophy": 0.38095238095238093, - "professional_accounting": 0.3028571428571429, - "professional_law": 0.26066350710900477, - "professional_medicine": 0.3058510638297872, - "professional_psychology": 0.28448275862068967, - "public_relations": 0.3275862068965517, - "security_study": 0.22962962962962963, - "sociology": 0.27876106194690264, - "sports_science": 0.296969696969697, - "traditional_chinese_medicine": 0.2594594594594595, - "virology": 0.33727810650887574, - "world_history": 0.2857142857142857, - "world_religions": 0.2125 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.24242424242424243 - }, - "prompt_2": { - "accuracy": 0.30303030303030304 - }, - "prompt_3": { - "accuracy": 0.18181818181818182 - }, - "prompt_4": { - "accuracy": 0.15151515151515152 - }, - "prompt_5": { - "accuracy": 0.18181818181818182 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.1431818181818182 - }, - "prompt_2": { - "accuracy": 0.15227272727272728 - }, - "prompt_3": { - "accuracy": 0.15227272727272728 - }, - "prompt_4": { - "accuracy": 0.3795454545454545 - }, - "prompt_5": { - "accuracy": 0.3840909090909091 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.331864406779661 - }, - "prompt_2": { - "accuracy": 0.31966101694915255 - }, - "prompt_3": { - "accuracy": 0.31898305084745765 - }, - "prompt_4": { - "accuracy": 0.3213559322033898 - }, - "prompt_5": { - "accuracy": 0.3159322033898305 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.4850411368735976 - }, - "prompt_2": { - "accuracy": 0.42520568436798806 - }, - "prompt_3": { - "accuracy": 0.4199700822737472 - }, - "prompt_4": { - "accuracy": 0.4809274495138369 - }, - "prompt_5": { - "accuracy": 0.46409872849663425 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.8236158745712886 - }, - "prompt_2": { - "accuracy": 0.7951984321411073 - }, - "prompt_3": { - "accuracy": 0.761391474767271 - }, - "prompt_4": { - "accuracy": 0.7491425771680549 - }, - "prompt_5": { - "accuracy": 0.8157765801077903 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.2880792948478976, - "rouge2": 0.11662921909858297, - "rougeL": 0.2248611926418006, - "avg_rouge": 0.2098565688627604 - }, - "prompt_2": { - "rouge1": 0.3627042990240314, - "rouge2": 0.13519720754928125, - "rougeL": 0.2780418436585964, - "avg_rouge": 0.2586477834106364 - }, - "prompt_3": { - "rouge1": 0.37176823881116167, - "rouge2": 0.14283935859355634, - "rougeL": 0.2883784101676747, - "avg_rouge": 0.2676620025241309 - }, - "prompt_4": { - "rouge1": 0.36212097906203633, - "rouge2": 0.13738775402347686, - "rougeL": 0.27691939379891706, - "avg_rouge": 0.2588093756281434 - }, - "prompt_5": { - "rouge1": 0.31862421949185366, - "rouge2": 0.12830472239001003, - "rougeL": 0.24720194445417362, - "avg_rouge": 0.23137696211201245 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.21366983073434007, - "rouge2": 0.0602206777500302, - "rougeL": 0.15674706765647997, - "avg_rouge": 0.14354585871361672 - }, - "prompt_2": { - "rouge1": 0.22530731749207933, - "rouge2": 0.06318982668441006, - "rougeL": 0.16629542839880498, - "avg_rouge": 0.15159752419176478 - }, - "prompt_3": { - "rouge1": 0.22624826670785722, - "rouge2": 0.06403915836636667, - "rougeL": 0.16624848362022945, - "avg_rouge": 0.15217863623148445 - }, - "prompt_4": { - "rouge1": 0.21685190932527487, - "rouge2": 0.06013200941588291, - "rougeL": 0.15937278734868993, - "avg_rouge": 0.14545223536328258 - }, - "prompt_5": { - "rouge1": 0.20816790442787142, - "rouge2": 0.057331241565016275, - "rougeL": 0.15213509197968914, - "avg_rouge": 0.1392114126575256 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.7672018348623854 - }, - "prompt_2": { - "accuracy": 0.9036697247706422 - }, - "prompt_3": { - "accuracy": 0.893348623853211 - }, - "prompt_4": { - "accuracy": 0.7958715596330275 - }, - "prompt_5": { - "accuracy": 0.7408256880733946 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.535953978906999 - }, - "prompt_2": { - "accuracy": 0.5378715244487057 - }, - "prompt_3": { - "accuracy": 0.5656759348034516 - }, - "prompt_4": { - "accuracy": 0.5532118887823586 - }, - "prompt_5": { - "accuracy": 0.5282837967401726 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.5575 - }, - "prompt_2": { - "accuracy": 0.5475 - }, - "prompt_3": { - "accuracy": 0.5345 - }, - "prompt_4": { - "accuracy": 0.5435 - }, - "prompt_5": { - "accuracy": 0.5135 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.48 - }, - "prompt_2": { - "accuracy": 0.4045 - }, - "prompt_3": { - "accuracy": 0.4415 - }, - "prompt_4": { - "accuracy": 0.4825 - }, - "prompt_5": { - "accuracy": 0.364 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.564 - }, - "prompt_2": { - "accuracy": 0.5345 - }, - "prompt_3": { - "accuracy": 0.544 - }, - "prompt_4": { - "accuracy": 0.5155 - }, - "prompt_5": { - "accuracy": 0.5495 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.4084507042253521 - }, - "prompt_2": { - "accuracy": 0.5211267605633803 - }, - "prompt_3": { - "accuracy": 0.4647887323943662 - }, - "prompt_4": { - "accuracy": 0.4507042253521127 - }, - "prompt_5": { - "accuracy": 0.49295774647887325 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.5342960288808665 - }, - "prompt_2": { - "accuracy": 0.49097472924187724 - }, - "prompt_3": { - "accuracy": 0.5667870036101083 - }, - "prompt_4": { - "accuracy": 0.5667870036101083 - }, - "prompt_5": { - "accuracy": 0.5812274368231047 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.5122549019607843 - }, - "prompt_2": { - "accuracy": 0.5098039215686274 - }, - "prompt_3": { - "accuracy": 0.5318627450980392 - }, - "prompt_4": { - "accuracy": 0.6225490196078431 - }, - "prompt_5": { - "accuracy": 0.4877450980392157 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.3327731092436975, - "language_acc": { - "English": 0.4, - "Chinese": 0.28739495798319326, - "Spanish": 0.36470588235294116, - "Vietnamese": 0.27899159663865547 - }, - "consistency_score_2": 0.25322128851540615, - "consistency_score_3": 0.06722689075630252, - "consistency_score_4": 0.01764705882352941, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.24369747899159663, - "English,Spanish": 0.2672268907563025, - "English,Vietnamese": 0.2529411764705882, - "Chinese,Spanish": 0.2605042016806723, - "Chinese,Vietnamese": 0.253781512605042, - "Spanish,Vietnamese": 0.2411764705882353 - }, - "3_combine": { - "English,Chinese,Spanish": 0.07226890756302522, - "English,Chinese,Vietnamese": 0.06722689075630252, - "English,Spanish,Vietnamese": 0.06722689075630252, - "Chinese,Spanish,Vietnamese": 0.06218487394957983 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.01764705882352941 - } - }, - "AC3_2": 0.2875974098517843, - "AC3_3": 0.11185650728084175, - "AC3_4": 0.033516716029368836 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.2933333333333334, - "language_acc": { - "Spanish": 0.36, - "English": 0.35333333333333333, - "Chinese": 0.2733333333333333, - "Vietnamese": 0.24666666666666667, - "Filipino": 0.23333333333333334, - "Indonesian": 0.2733333333333333, - "Malay": 0.31333333333333335 - }, - "consistency_score_2": 0.4482539682539681, - "consistency_score_3": 0.25257142857142856, - "consistency_score_4": 0.1594285714285714, - "consistency_score_5": 0.10793650793650796, - "consistency_score_6": 0.0780952380952381, - "consistency_score_7": 0.06, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.42, - "Spanish,Chinese": 0.36, - "Spanish,Vietnamese": 0.44666666666666666, - "Spanish,Filipino": 0.38, - "Spanish,Indonesian": 0.49333333333333335, - "Spanish,Malay": 0.49333333333333335, - "English,Chinese": 0.4266666666666667, - "English,Vietnamese": 0.4266666666666667, - "English,Filipino": 0.44666666666666666, - "English,Indonesian": 0.5, - "English,Malay": 0.4266666666666667, - "Chinese,Vietnamese": 0.41333333333333333, - "Chinese,Filipino": 0.30666666666666664, - "Chinese,Indonesian": 0.38666666666666666, - "Chinese,Malay": 0.41333333333333333, - "Vietnamese,Filipino": 0.4533333333333333, - "Vietnamese,Indonesian": 0.5466666666666666, - "Vietnamese,Malay": 0.5266666666666666, - "Filipino,Indonesian": 0.5066666666666667, - "Filipino,Malay": 0.5066666666666667, - "Indonesian,Malay": 0.5333333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.2, - "Spanish,English,Vietnamese": 0.23333333333333334, - "Spanish,English,Filipino": 0.20666666666666667, - "Spanish,English,Indonesian": 0.28, - "Spanish,English,Malay": 0.24666666666666667, - "Spanish,Chinese,Vietnamese": 0.20666666666666667, - "Spanish,Chinese,Filipino": 0.12666666666666668, - "Spanish,Chinese,Indonesian": 0.20666666666666667, - "Spanish,Chinese,Malay": 0.22, - "Spanish,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,Vietnamese,Malay": 0.3, - "Spanish,Filipino,Indonesian": 0.25333333333333335, - "Spanish,Filipino,Malay": 0.25333333333333335, - "Spanish,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese": 0.24, - "English,Chinese,Filipino": 0.17333333333333334, - "English,Chinese,Indonesian": 0.24, - "English,Chinese,Malay": 0.21333333333333335, - "English,Vietnamese,Filipino": 0.25333333333333335, - "English,Vietnamese,Indonesian": 0.31333333333333335, - "English,Vietnamese,Malay": 0.2733333333333333, - "English,Filipino,Indonesian": 0.31333333333333335, - "English,Filipino,Malay": 0.2733333333333333, - "English,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino": 0.18, - "Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "Chinese,Vietnamese,Malay": 0.26, - "Chinese,Filipino,Indonesian": 0.20666666666666667, - "Chinese,Filipino,Malay": 0.2, - "Chinese,Indonesian,Malay": 0.23333333333333334, - "Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Vietnamese,Filipino,Malay": 0.31333333333333335, - "Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Filipino,Indonesian,Malay": 0.3466666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.14, - "Spanish,English,Chinese,Filipino": 0.08666666666666667, - "Spanish,English,Chinese,Indonesian": 0.14, - "Spanish,English,Chinese,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Malay": 0.16, - "Spanish,English,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Indonesian,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Filipino": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Malay": 0.16, - "Spanish,Chinese,Filipino,Indonesian": 0.1, - "Spanish,Chinese,Filipino,Malay": 0.10666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.14, - "Spanish,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.24, - "Spanish,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.12, - "English,Chinese,Vietnamese,Indonesian": 0.18, - "English,Chinese,Vietnamese,Malay": 0.17333333333333334, - "English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Filipino,Malay": 0.12666666666666668, - "English,Chinese,Indonesian,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "English,Filipino,Indonesian,Malay": 0.22, - "Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Indonesian,Malay": 0.18, - "Chinese,Filipino,Indonesian,Malay": 0.16, - "Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.12, - "Spanish,English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.06, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06 - } - }, - "AC3_2": 0.35461187209830075, - "AC3_3": 0.27143056519801084, - "AC3_4": 0.20657972229345628, - "AC3_5": 0.15780590713366918, - "AC3_6": 0.12335042731721764, - "AC3_7": 0.09962264148123887 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.2775974025974026, - "language_acc": { - "English": 0.3522727272727273, - "Filipino": 0.2840909090909091, - "Spanish": 0.3125, - "Malay": 0.2159090909090909, - "Chinese": 0.2215909090909091, - "Indonesian": 0.2840909090909091, - "Vietnamese": 0.2727272727272727 - }, - "consistency_score_2": 0.28977272727272724, - "consistency_score_3": 0.09561688311688314, - "consistency_score_4": 0.03165584415584416, - "consistency_score_5": 0.009740259740259737, - "consistency_score_6": 0.002435064935064935, - "consistency_score_7": 0.0, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3693181818181818, - "English,Spanish": 0.2840909090909091, - "English,Malay": 0.30113636363636365, - "English,Chinese": 0.25, - "English,Indonesian": 0.3522727272727273, - "English,Vietnamese": 0.32954545454545453, - "Filipino,Spanish": 0.30113636363636365, - "Filipino,Malay": 0.23295454545454544, - "Filipino,Chinese": 0.32954545454545453, - "Filipino,Indonesian": 0.2897727272727273, - "Filipino,Vietnamese": 0.32954545454545453, - "Spanish,Malay": 0.3352272727272727, - "Spanish,Chinese": 0.2159090909090909, - "Spanish,Indonesian": 0.32954545454545453, - "Spanish,Vietnamese": 0.26704545454545453, - "Malay,Chinese": 0.2784090909090909, - "Malay,Indonesian": 0.2840909090909091, - "Malay,Vietnamese": 0.2556818181818182, - "Chinese,Indonesian": 0.26136363636363635, - "Chinese,Vietnamese": 0.24431818181818182, - "Indonesian,Vietnamese": 0.24431818181818182 - }, - "3_combine": { - "English,Filipino,Spanish": 0.11363636363636363, - "English,Filipino,Malay": 0.10227272727272728, - "English,Filipino,Chinese": 0.11931818181818182, - "English,Filipino,Indonesian": 0.1534090909090909, - "English,Filipino,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay": 0.11363636363636363, - "English,Spanish,Chinese": 0.0625, - "English,Spanish,Indonesian": 0.11363636363636363, - "English,Spanish,Vietnamese": 0.11931818181818182, - "English,Malay,Chinese": 0.08522727272727272, - "English,Malay,Indonesian": 0.11363636363636363, - "English,Malay,Vietnamese": 0.11931818181818182, - "English,Chinese,Indonesian": 0.09090909090909091, - "English,Chinese,Vietnamese": 0.07386363636363637, - "English,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay": 0.08522727272727272, - "Filipino,Spanish,Chinese": 0.08522727272727272, - "Filipino,Spanish,Indonesian": 0.09659090909090909, - "Filipino,Spanish,Vietnamese": 0.10795454545454546, - "Filipino,Malay,Chinese": 0.09659090909090909, - "Filipino,Malay,Indonesian": 0.06818181818181818, - "Filipino,Malay,Vietnamese": 0.08522727272727272, - "Filipino,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese": 0.07954545454545454, - "Spanish,Malay,Indonesian": 0.09659090909090909, - "Spanish,Malay,Vietnamese": 0.09659090909090909, - "Spanish,Chinese,Indonesian": 0.056818181818181816, - "Spanish,Chinese,Vietnamese": 0.0625, - "Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "Malay,Chinese,Indonesian": 0.07954545454545454, - "Malay,Chinese,Vietnamese": 0.06818181818181818, - "Malay,Indonesian,Vietnamese": 0.07386363636363637, - "Chinese,Indonesian,Vietnamese": 0.06818181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.03977272727272727, - "English,Filipino,Spanish,Chinese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Chinese": 0.03977272727272727, - "English,Filipino,Malay,Indonesian": 0.03409090909090909, - "English,Filipino,Malay,Vietnamese": 0.045454545454545456, - "English,Filipino,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.05113636363636364, - "English,Filipino,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Chinese": 0.03409090909090909, - "English,Spanish,Malay,Indonesian": 0.03409090909090909, - "English,Spanish,Malay,Vietnamese": 0.03409090909090909, - "English,Spanish,Chinese,Indonesian": 0.017045454545454544, - "English,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Spanish,Indonesian,Vietnamese": 0.05113636363636364, - "English,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Malay,Chinese,Vietnamese": 0.03409090909090909, - "English,Malay,Indonesian,Vietnamese": 0.03977272727272727, - "English,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Chinese": 0.03409090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.011363636363636364, - "Filipino,Spanish,Malay,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Chinese,Indonesian": 0.011363636363636364, - "Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.017045454545454544, - "Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Malay,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Spanish,Malay,Indonesian,Vietnamese": 0.017045454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.0, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 - } - }, - "AC3_2": 0.2835544283542368, - "AC3_3": 0.142239991375433, - "AC3_4": 0.05683096429290244, - "AC3_5": 0.018820162881409648, - "AC3_6": 0.004827780913013429, - "AC3_7": 0.0 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.22330097087378642 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.26666666666666666 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.35514018691588783 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.35, - "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.4, - "politics": 0.5, - "culture": 0.3, - "film": 0.5, - "law": 0.2, - "geography": 0.3 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.06568549848925372 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.024051741167667114 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.01620495439857982 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.05470205211997378 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.02165029329957093 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.43757292882147025 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.38469789059706827, - "category_acc": { - "high_school_european_history": 0.43902439024390244, - "business_ethics": 0.41414141414141414, - "clinical_knowledge": 0.4090909090909091, - "medical_genetics": 0.41414141414141414, - "high_school_us_history": 0.46798029556650245, - "high_school_physics": 0.3, - "high_school_world_history": 0.5254237288135594, - "virology": 0.30303030303030304, - "high_school_microeconomics": 0.3924050632911392, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.32323232323232326, - "high_school_biology": 0.42071197411003236, - "abstract_algebra": 0.21212121212121213, - "professional_accounting": 0.37722419928825623, - "philosophy": 0.4161290322580645, - "professional_medicine": 0.3763837638376384, - "nutrition": 0.38688524590163936, - "global_facts": 0.26262626262626265, - "machine_learning": 0.35135135135135137, - "security_studies": 0.44672131147540983, - "public_relations": 0.3302752293577982, - "professional_psychology": 0.40425531914893614, - "prehistory": 0.38390092879256965, - "anatomy": 0.23880597014925373, - "human_sexuality": 0.4230769230769231, - "college_medicine": 0.36627906976744184, - "high_school_government_and_politics": 0.5104166666666666, - "college_chemistry": 0.31313131313131315, - "logical_fallacies": 0.5061728395061729, - "high_school_geography": 0.4213197969543147, - "elementary_mathematics": 0.2838196286472148, - "human_aging": 0.45045045045045046, - "college_mathematics": 0.3838383838383838, - "high_school_psychology": 0.5055147058823529, - "formal_logic": 0.248, - "high_school_statistics": 0.3767441860465116, - "international_law": 0.5666666666666667, - "high_school_mathematics": 0.2936802973977695, - "high_school_computer_science": 0.4444444444444444, - "conceptual_physics": 0.37606837606837606, - "miscellaneous": 0.37084398976982097, - "high_school_chemistry": 0.3316831683168317, - "marketing": 0.5321888412017167, - "professional_law": 0.3744292237442922, - "management": 0.4019607843137255, - "college_physics": 0.297029702970297, - "jurisprudence": 0.48598130841121495, - "world_religions": 0.4647058823529412, - "sociology": 0.51, - "us_foreign_policy": 0.48484848484848486, - "high_school_macroeconomics": 0.3676092544987147, - "computer_security": 0.32323232323232326, - "moral_scenarios": 0.23154362416107382, - "moral_disputes": 0.4318840579710145, - "electrical_engineering": 0.3402777777777778, - "astronomy": 0.3576158940397351, - "college_biology": 0.44755244755244755 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.30237741456166417 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.32938978829389787, - "category_acc": { - "computer_network": 0.5, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.35714285714285715, - "college_physics": 0.4583333333333333, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.4523809523809524, - "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.375, - "high_school_chemistry": 0.375, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.24, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.3333333333333333, - "business_administration": 0.34210526315789475, - "marxism": 0.4583333333333333, - "mao_zedong_thought": 0.4827586206896552, - "education_science": 0.3235294117647059, - "teacher_qualification": 0.30612244897959184, - "high_school_politics": 0.375, - "high_school_geography": 0.4583333333333333, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.32142857142857145, - "ideological_and_moral_cultivation": 0.5416666666666666, - "logic": 0.3333333333333333, - "law": 0.1724137931034483, - "chinese_language_and_literature": 0.25, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.125, - "high_school_history": 0.32, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.34615384615384615, - "sports_science": 0.375, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.125, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.2037037037037037, - "fire_engineer": 0.3611111111111111, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.2777777777777778, - "physician": 0.3333333333333333 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.3010752688172043 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.3118632360559489, - "category_acc": { - "agronomy": 0.33136094674556216, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.35365853658536583, - "arts": 0.25625, - "astronomy": 0.23636363636363636, - "business_ethics": 0.3349282296650718, - "chinese_civil_service_exam": 0.35625, - "chinese_driving_rule": 0.3053435114503817, - "chinese_food_culture": 0.23529411764705882, - "chinese_foreign_policy": 0.27102803738317754, - "chinese_history": 0.29411764705882354, - "chinese_literature": 0.3284313725490196, - "chinese_teacher_qualification": 0.3016759776536313, - "clinical_knowledge": 0.2489451476793249, - "college_actuarial_science": 0.3018867924528302, - "college_education": 0.42990654205607476, - "college_engineering_hydrology": 0.3584905660377358, - "college_law": 0.19444444444444445, - "college_mathematics": 0.20952380952380953, - "college_medical_statistics": 0.3584905660377358, - "college_medicine": 0.29304029304029305, - "computer_science": 0.28921568627450983, - "computer_security": 0.30409356725146197, - "conceptual_physics": 0.2789115646258503, - "construction_project_management": 0.31654676258992803, - "economics": 0.3522012578616352, - "education": 0.3374233128834356, - "electrical_engineering": 0.38372093023255816, - "elementary_chinese": 0.32936507936507936, - "elementary_commonsense": 0.29797979797979796, - "elementary_information_and_technology": 0.38235294117647056, - "elementary_mathematics": 0.29130434782608694, - "ethnology": 0.3111111111111111, - "food_science": 0.34965034965034963, - "genetics": 0.3409090909090909, - "global_facts": 0.2751677852348993, - "high_school_biology": 0.33727810650887574, - "high_school_chemistry": 0.20454545454545456, - "high_school_geography": 0.2711864406779661, - "high_school_mathematics": 0.21341463414634146, - "high_school_physics": 0.3090909090909091, - "high_school_politics": 0.32867132867132864, - "human_sexuality": 0.2698412698412698, - "international_law": 0.3081081081081081, - "journalism": 0.36046511627906974, - "jurisprudence": 0.31143552311435524, - "legal_and_moral_basis": 0.43457943925233644, - "logical": 0.3170731707317073, - "machine_learning": 0.29508196721311475, - "management": 0.3142857142857143, - "marketing": 0.39444444444444443, - "marxist_theory": 0.37037037037037035, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.3448275862068966, - "philosophy": 0.42857142857142855, - "professional_accounting": 0.32571428571428573, - "professional_law": 0.2559241706161137, - "professional_medicine": 0.2579787234042553, - "professional_psychology": 0.3146551724137931, - "public_relations": 0.3103448275862069, - "security_study": 0.3037037037037037, - "sociology": 0.3185840707964602, - "sports_science": 0.34545454545454546, - "traditional_chinese_medicine": 0.2972972972972973, - "virology": 0.3609467455621302, - "world_history": 0.3167701863354037, - "world_religions": 0.2875 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.21212121212121213 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.27045454545454545 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.331864406779661 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.462602842183994 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.7359137677609016 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.2043174265115656, - "rouge2": 0.08062096329634066, - "rougeL": 0.1560114584427017, - "avg_rouge": 0.14698328275020264 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.21009684475253432, - "rouge2": 0.07243194122892142, - "rougeL": 0.16000172249843952, - "avg_rouge": 0.14751016949329843 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.5011467889908257 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.35282837967401726 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.616 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.4125 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.506 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.6338028169014085 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.5703971119133574 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.5220588235294118 - } - } - } - }, - "flan_t5_small": { - "model_size": "0.06B", - "model_link": "https://huggingface.co/google/flan-t5-small", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.4859243697478991, - "language_acc": { - "English": 0.661344537815126, - "Chinese": 0.31512605042016806, - "Spanish": 0.5714285714285714, - "Vietnamese": 0.3957983193277311 - }, - "consistency_score_2": 0.4648459383753501, - "consistency_score_3": 0.27415966386554624, - "consistency_score_4": 0.16806722689075632, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.37815126050420167, - "English,Spanish": 0.6739495798319328, - "English,Vietnamese": 0.5, - "Chinese,Spanish": 0.3773109243697479, - "Chinese,Vietnamese": 0.3277310924369748, - "Spanish,Vietnamese": 0.5319327731092437 - }, - "3_combine": { - "English,Chinese,Spanish": 0.27647058823529413, - "English,Chinese,Vietnamese": 0.21008403361344538, - "English,Spanish,Vietnamese": 0.39915966386554624, - "Chinese,Spanish,Vietnamese": 0.21092436974789916 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.16806722689075632 - } - }, - "AC3_2": 0.47515150122225763, - "AC3_3": 0.3505424557421666, - "AC3_4": 0.2497523262055113 - }, - "prompt_2": { - "overall_acc": 0.4869747899159664, - "language_acc": { - "English": 0.6512605042016807, - "Chinese": 0.3235294117647059, - "Spanish": 0.5764705882352941, - "Vietnamese": 0.39663865546218485 - }, - "consistency_score_2": 0.4549019607843137, - "consistency_score_3": 0.2657563025210084, - "consistency_score_4": 0.16470588235294117, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.37142857142857144, - "English,Spanish": 0.6957983193277311, - "English,Vietnamese": 0.48487394957983193, - "Chinese,Spanish": 0.38571428571428573, - "Chinese,Vietnamese": 0.2815126050420168, - "Spanish,Vietnamese": 0.5100840336134453 - }, - "3_combine": { - "English,Chinese,Spanish": 0.28319327731092436, - "English,Chinese,Vietnamese": 0.19327731092436976, - "English,Spanish,Vietnamese": 0.39243697478991596, - "Chinese,Spanish,Vietnamese": 0.19411764705882353 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.16470588235294117 - } - }, - "AC3_2": 0.47039230259602344, - "AC3_3": 0.3438588384939208, - "AC3_4": 0.24615618003580428 - }, - "prompt_3": { - "overall_acc": 0.47436974789915964, - "language_acc": { - "English": 0.6386554621848739, - "Chinese": 0.3243697478991597, - "Spanish": 0.5571428571428572, - "Vietnamese": 0.3773109243697479 - }, - "consistency_score_2": 0.44173669467787113, - "consistency_score_3": 0.2510504201680672, - "consistency_score_4": 0.15210084033613444, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3378151260504202, - "English,Spanish": 0.6714285714285714, - "English,Vietnamese": 0.48739495798319327, - "Chinese,Spanish": 0.34285714285714286, - "Chinese,Vietnamese": 0.30084033613445377, - "Spanish,Vietnamese": 0.5100840336134453 - }, - "3_combine": { - "English,Chinese,Spanish": 0.25126050420168067, - "English,Chinese,Vietnamese": 0.18235294117647058, - "English,Spanish,Vietnamese": 0.38907563025210085, - "Chinese,Spanish,Vietnamese": 0.1815126050420168 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.15210084033613444 - } - }, - "AC3_2": 0.45747200266339455, - "AC3_3": 0.3283358521061986, - "AC3_4": 0.2303445321423002 - }, - "prompt_4": { - "overall_acc": 0.4848739495798319, - "language_acc": { - "English": 0.6487394957983194, - "Chinese": 0.3176470588235294, - "Spanish": 0.5747899159663865, - "Vietnamese": 0.3983193277310924 - }, - "consistency_score_2": 0.4592436974789916, - "consistency_score_3": 0.2703781512605042, - "consistency_score_4": 0.16554621848739495, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.37815126050420167, - "English,Spanish": 0.6966386554621848, - "English,Vietnamese": 0.5042016806722689, - "Chinese,Spanish": 0.37478991596638656, - "Chinese,Vietnamese": 0.2848739495798319, - "Spanish,Vietnamese": 0.5168067226890757 - }, - "3_combine": { - "English,Chinese,Spanish": 0.2823529411764706, - "English,Chinese,Vietnamese": 0.2, - "English,Spanish,Vietnamese": 0.4042016806722689, - "Chinese,Spanish,Vietnamese": 0.1949579831932773 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.16554621848739495 - } - }, - "AC3_2": 0.47171092731163566, - "AC3_3": 0.3471670503625134, - "AC3_4": 0.24682213967064853 - }, - "prompt_5": { - "overall_acc": 0.4915966386554622, - "language_acc": { - "English": 0.6831932773109244, - "Chinese": 0.3142857142857143, - "Spanish": 0.565546218487395, - "Vietnamese": 0.40336134453781514 - }, - "consistency_score_2": 0.4564425770308123, - "consistency_score_3": 0.2611344537815126, - "consistency_score_4": 0.16974789915966387, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3596638655462185, - "English,Spanish": 0.673109243697479, - "English,Vietnamese": 0.48403361344537815, - "Chinese,Spanish": 0.34285714285714286, - "Chinese,Vietnamese": 0.3739495798319328, - "Spanish,Vietnamese": 0.5050420168067227 - }, - "3_combine": { - "English,Chinese,Spanish": 0.2529411764705882, - "English,Chinese,Vietnamese": 0.20756302521008405, - "English,Spanish,Vietnamese": 0.37815126050420167, - "Chinese,Spanish,Vietnamese": 0.20588235294117646 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.16974789915966387 - } - }, - "AC3_2": 0.4733678372607945, - "AC3_3": 0.3410854712629424, - "AC3_4": 0.25235710544698375 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.2704761904761905, - "language_acc": { - "Spanish": 0.2866666666666667, - "English": 0.26666666666666666, - "Chinese": 0.29333333333333333, - "Vietnamese": 0.3, - "Filipino": 0.24, - "Indonesian": 0.26666666666666666, - "Malay": 0.24 - }, - "consistency_score_2": 0.4584126984126983, - "consistency_score_3": 0.26819047619047626, - "consistency_score_4": 0.18000000000000005, - "consistency_score_5": 0.13269841269841268, - "consistency_score_6": 0.10476190476190476, - "consistency_score_7": 0.08666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.47333333333333333, - "Spanish,Chinese": 0.43333333333333335, - "Spanish,Vietnamese": 0.44666666666666666, - "Spanish,Filipino": 0.43333333333333335, - "Spanish,Indonesian": 0.41333333333333333, - "Spanish,Malay": 0.47333333333333333, - "English,Chinese": 0.44666666666666666, - "English,Vietnamese": 0.4, - "English,Filipino": 0.4666666666666667, - "English,Indonesian": 0.4266666666666667, - "English,Malay": 0.43333333333333335, - "Chinese,Vietnamese": 0.56, - "Chinese,Filipino": 0.38666666666666666, - "Chinese,Indonesian": 0.43333333333333335, - "Chinese,Malay": 0.42, - "Vietnamese,Filipino": 0.41333333333333333, - "Vietnamese,Indonesian": 0.52, - "Vietnamese,Malay": 0.5133333333333333, - "Filipino,Indonesian": 0.44, - "Filipino,Malay": 0.47333333333333333, - "Indonesian,Malay": 0.62 - }, - "3_combine": { - "Spanish,English,Chinese": 0.26, - "Spanish,English,Vietnamese": 0.26666666666666666, - "Spanish,English,Filipino": 0.2733333333333333, - "Spanish,English,Indonesian": 0.25333333333333335, - "Spanish,English,Malay": 0.26, - "Spanish,Chinese,Vietnamese": 0.3, - "Spanish,Chinese,Filipino": 0.21333333333333335, - "Spanish,Chinese,Indonesian": 0.24, - "Spanish,Chinese,Malay": 0.24, - "Spanish,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,Vietnamese,Malay": 0.29333333333333333, - "Spanish,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Filipino,Malay": 0.26, - "Spanish,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese": 0.26666666666666666, - "English,Chinese,Filipino": 0.24666666666666667, - "English,Chinese,Indonesian": 0.24, - "English,Chinese,Malay": 0.22, - "English,Vietnamese,Filipino": 0.24, - "English,Vietnamese,Indonesian": 0.23333333333333334, - "English,Vietnamese,Malay": 0.23333333333333334, - "English,Filipino,Indonesian": 0.26, - "English,Filipino,Malay": 0.26666666666666666, - "English,Indonesian,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Chinese,Vietnamese,Indonesian": 0.31333333333333335, - "Chinese,Vietnamese,Malay": 0.31333333333333335, - "Chinese,Filipino,Indonesian": 0.23333333333333334, - "Chinese,Filipino,Malay": 0.24, - "Chinese,Indonesian,Malay": 0.30666666666666664, - "Vietnamese,Filipino,Indonesian": 0.28, - "Vietnamese,Filipino,Malay": 0.28, - "Vietnamese,Indonesian,Malay": 0.38, - "Filipino,Indonesian,Malay": 0.34 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.19333333333333333, - "Spanish,English,Chinese,Filipino": 0.16, - "Spanish,English,Chinese,Indonesian": 0.16, - "Spanish,English,Chinese,Malay": 0.16, - "Spanish,English,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,English,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Malay": 0.17333333333333334, - "Spanish,English,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Filipino,Malay": 0.18, - "Spanish,English,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Malay": 0.2, - "Spanish,Chinese,Filipino,Indonesian": 0.14, - "Spanish,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.18, - "English,Chinese,Vietnamese,Malay": 0.16666666666666666, - "English,Chinese,Filipino,Indonesian": 0.16, - "English,Chinese,Filipino,Malay": 0.16, - "English,Chinese,Indonesian,Malay": 0.17333333333333334, - "English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Vietnamese,Filipino,Malay": 0.16, - "English,Vietnamese,Indonesian,Malay": 0.18, - "English,Filipino,Indonesian,Malay": 0.2, - "Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.24, - "Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,English,Vietnamese,Filipino,Malay": 0.12, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.12, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 - } - }, - "AC3_2": 0.3402156959878354, - "AC3_3": 0.269328483818796, - "AC3_4": 0.21615221982516708, - "AC3_5": 0.17804574423780378, - "AC3_6": 0.1510273144388427, - "AC3_7": 0.1312711110743552 - }, - "prompt_2": { - "overall_acc": 0.259047619047619, - "language_acc": { - "Spanish": 0.2866666666666667, - "English": 0.30666666666666664, - "Chinese": 0.25333333333333335, - "Vietnamese": 0.26, - "Filipino": 0.24, - "Indonesian": 0.24666666666666667, - "Malay": 0.22 - }, - "consistency_score_2": 0.4374603174603175, - "consistency_score_3": 0.2558095238095239, - "consistency_score_4": 0.17942857142857147, - "consistency_score_5": 0.1406349206349206, - "consistency_score_6": 0.11904761904761905, - "consistency_score_7": 0.10666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5333333333333333, - "Spanish,Chinese": 0.24666666666666667, - "Spanish,Vietnamese": 0.44, - "Spanish,Filipino": 0.5066666666666667, - "Spanish,Indonesian": 0.4866666666666667, - "Spanish,Malay": 0.47333333333333333, - "English,Chinese": 0.35333333333333333, - "English,Vietnamese": 0.37333333333333335, - "English,Filipino": 0.46, - "English,Indonesian": 0.4, - "English,Malay": 0.44, - "Chinese,Vietnamese": 0.3333333333333333, - "Chinese,Filipino": 0.2733333333333333, - "Chinese,Indonesian": 0.38666666666666666, - "Chinese,Malay": 0.35333333333333333, - "Vietnamese,Filipino": 0.5066666666666667, - "Vietnamese,Indonesian": 0.49333333333333335, - "Vietnamese,Malay": 0.48, - "Filipino,Indonesian": 0.5, - "Filipino,Malay": 0.47333333333333333, - "Indonesian,Malay": 0.6733333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.19333333333333333, - "Spanish,English,Vietnamese": 0.2733333333333333, - "Spanish,English,Filipino": 0.3333333333333333, - "Spanish,English,Indonesian": 0.29333333333333333, - "Spanish,English,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese": 0.16, - "Spanish,Chinese,Filipino": 0.17333333333333334, - "Spanish,Chinese,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,Vietnamese,Indonesian": 0.29333333333333333, - "Spanish,Vietnamese,Malay": 0.2866666666666667, - "Spanish,Filipino,Indonesian": 0.31333333333333335, - "Spanish,Filipino,Malay": 0.30666666666666664, - "Spanish,Indonesian,Malay": 0.36666666666666664, - "English,Chinese,Vietnamese": 0.17333333333333334, - "English,Chinese,Filipino": 0.18, - "English,Chinese,Indonesian": 0.19333333333333333, - "English,Chinese,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino": 0.2866666666666667, - "English,Vietnamese,Indonesian": 0.23333333333333334, - "English,Vietnamese,Malay": 0.23333333333333334, - "English,Filipino,Indonesian": 0.2733333333333333, - "English,Filipino,Malay": 0.28, - "English,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino": 0.18666666666666668, - "Chinese,Vietnamese,Indonesian": 0.21333333333333335, - "Chinese,Vietnamese,Malay": 0.20666666666666667, - "Chinese,Filipino,Indonesian": 0.18666666666666668, - "Chinese,Filipino,Malay": 0.16666666666666666, - "Chinese,Indonesian,Malay": 0.2733333333333333, - "Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "Vietnamese,Filipino,Malay": 0.3, - "Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Filipino,Indonesian,Malay": 0.36666666666666664 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.12666666666666668, - "Spanish,English,Chinese,Filipino": 0.15333333333333332, - "Spanish,English,Chinese,Indonesian": 0.14666666666666667, - "Spanish,English,Chinese,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,English,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.14, - "Spanish,Chinese,Vietnamese,Indonesian": 0.14, - "Spanish,Chinese,Vietnamese,Malay": 0.13333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.22, - "Spanish,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "English,Chinese,Vietnamese,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian": 0.14, - "English,Chinese,Filipino,Malay": 0.13333333333333333, - "English,Chinese,Indonesian,Malay": 0.17333333333333334, - "English,Vietnamese,Filipino,Indonesian": 0.2, - "English,Vietnamese,Filipino,Malay": 0.2, - "English,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.18, - "Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Malay": 0.18, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - } - }, - "AC3_2": 0.32540348130936003, - "AC3_3": 0.25741838868914546, - "AC3_4": 0.21200943274301018, - "AC3_5": 0.18230038953039382, - "AC3_6": 0.16312822353847933, - "AC3_7": 0.15111111106979166 - }, - "prompt_3": { - "overall_acc": 0.2761904761904762, - "language_acc": { - "Spanish": 0.2866666666666667, - "English": 0.3, - "Chinese": 0.24, - "Vietnamese": 0.3, - "Filipino": 0.28, - "Indonesian": 0.25333333333333335, - "Malay": 0.2733333333333333 - }, - "consistency_score_2": 0.3990476190476191, - "consistency_score_3": 0.21523809523809523, - "consistency_score_4": 0.14342857142857143, - "consistency_score_5": 0.10730158730158729, - "consistency_score_6": 0.08476190476190477, - "consistency_score_7": 0.06666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.48, - "Spanish,Chinese": 0.29333333333333333, - "Spanish,Vietnamese": 0.35333333333333333, - "Spanish,Filipino": 0.48, - "Spanish,Indonesian": 0.4666666666666667, - "Spanish,Malay": 0.46, - "English,Chinese": 0.3, - "English,Vietnamese": 0.28, - "English,Filipino": 0.42, - "English,Indonesian": 0.42, - "English,Malay": 0.4, - "Chinese,Vietnamese": 0.32, - "Chinese,Filipino": 0.30666666666666664, - "Chinese,Indonesian": 0.3466666666666667, - "Chinese,Malay": 0.3, - "Vietnamese,Filipino": 0.36666666666666664, - "Vietnamese,Indonesian": 0.35333333333333333, - "Vietnamese,Malay": 0.38, - "Filipino,Indonesian": 0.5333333333333333, - "Filipino,Malay": 0.48, - "Indonesian,Malay": 0.64 - }, - "3_combine": { - "Spanish,English,Chinese": 0.18, - "Spanish,English,Vietnamese": 0.18, - "Spanish,English,Filipino": 0.2733333333333333, - "Spanish,English,Indonesian": 0.2733333333333333, - "Spanish,English,Malay": 0.28, - "Spanish,Chinese,Vietnamese": 0.14, - "Spanish,Chinese,Filipino": 0.18666666666666668, - "Spanish,Chinese,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Filipino": 0.2, - "Spanish,Vietnamese,Indonesian": 0.21333333333333335, - "Spanish,Vietnamese,Malay": 0.21333333333333335, - "Spanish,Filipino,Indonesian": 0.32, - "Spanish,Filipino,Malay": 0.2733333333333333, - "Spanish,Indonesian,Malay": 0.34, - "English,Chinese,Vietnamese": 0.12, - "English,Chinese,Filipino": 0.16666666666666666, - "English,Chinese,Indonesian": 0.18, - "English,Chinese,Malay": 0.16666666666666666, - "English,Vietnamese,Filipino": 0.17333333333333334, - "English,Vietnamese,Indonesian": 0.16666666666666666, - "English,Vietnamese,Malay": 0.16, - "English,Filipino,Indonesian": 0.26, - "English,Filipino,Malay": 0.23333333333333334, - "English,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Chinese,Vietnamese,Indonesian": 0.14, - "Chinese,Vietnamese,Malay": 0.14666666666666667, - "Chinese,Filipino,Indonesian": 0.20666666666666667, - "Chinese,Filipino,Malay": 0.18666666666666668, - "Chinese,Indonesian,Malay": 0.22666666666666666, - "Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Vietnamese,Filipino,Malay": 0.22666666666666666, - "Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Filipino,Indonesian,Malay": 0.37333333333333335 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.08666666666666667, - "Spanish,English,Chinese,Filipino": 0.13333333333333333, - "Spanish,English,Chinese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino": 0.14, - "Spanish,English,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Malay": 0.13333333333333333, - "Spanish,English,Filipino,Indonesian": 0.2, - "Spanish,English,Filipino,Malay": 0.18666666666666668, - "Spanish,English,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.1, - "Spanish,Chinese,Vietnamese,Indonesian": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Indonesian,Malay": 0.15333333333333332, - "Spanish,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.08666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.08666666666666667, - "English,Chinese,Vietnamese,Malay": 0.09333333333333334, - "English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Filipino,Malay": 0.12, - "English,Chinese,Indonesian,Malay": 0.13333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "English,Vietnamese,Filipino,Malay": 0.13333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "English,Filipino,Indonesian,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.08, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,English,Vietnamese,Filipino,Malay": 0.12, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Filipino,Indonesian,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.08, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.08, - "English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 - } - }, - "AC3_2": 0.3264423399341927, - "AC3_3": 0.241934293047853, - "AC3_4": 0.18880747028766828, - "AC3_5": 0.15455692206628846, - "AC3_6": 0.1297147882540494, - "AC3_7": 0.10740740737608026 - }, - "prompt_4": { - "overall_acc": 0.26666666666666666, - "language_acc": { - "Spanish": 0.2866666666666667, - "English": 0.30666666666666664, - "Chinese": 0.23333333333333334, - "Vietnamese": 0.3, - "Filipino": 0.26666666666666666, - "Indonesian": 0.26, - "Malay": 0.21333333333333335 - }, - "consistency_score_2": 0.4288888888888889, - "consistency_score_3": 0.24285714285714285, - "consistency_score_4": 0.16609523809523805, - "consistency_score_5": 0.1288888888888889, - "consistency_score_6": 0.10952380952380954, - "consistency_score_7": 0.1, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5, - "Spanish,Chinese": 0.24, - "Spanish,Vietnamese": 0.44666666666666666, - "Spanish,Filipino": 0.4533333333333333, - "Spanish,Indonesian": 0.44666666666666666, - "Spanish,Malay": 0.41333333333333333, - "English,Chinese": 0.3333333333333333, - "English,Vietnamese": 0.38, - "English,Filipino": 0.5266666666666666, - "English,Indonesian": 0.4066666666666667, - "English,Malay": 0.43333333333333335, - "Chinese,Vietnamese": 0.3466666666666667, - "Chinese,Filipino": 0.2866666666666667, - "Chinese,Indonesian": 0.38666666666666666, - "Chinese,Malay": 0.36666666666666664, - "Vietnamese,Filipino": 0.47333333333333333, - "Vietnamese,Indonesian": 0.49333333333333335, - "Vietnamese,Malay": 0.4866666666666667, - "Filipino,Indonesian": 0.4866666666666667, - "Filipino,Malay": 0.44, - "Indonesian,Malay": 0.66 - }, - "3_combine": { - "Spanish,English,Chinese": 0.16, - "Spanish,English,Vietnamese": 0.26666666666666666, - "Spanish,English,Filipino": 0.3333333333333333, - "Spanish,English,Indonesian": 0.26, - "Spanish,English,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese": 0.16, - "Spanish,Chinese,Filipino": 0.15333333333333332, - "Spanish,Chinese,Indonesian": 0.16, - "Spanish,Chinese,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino": 0.2733333333333333, - "Spanish,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,Vietnamese,Malay": 0.2733333333333333, - "Spanish,Filipino,Indonesian": 0.26, - "Spanish,Filipino,Malay": 0.23333333333333334, - "Spanish,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese": 0.16, - "English,Chinese,Filipino": 0.18, - "English,Chinese,Indonesian": 0.18, - "English,Chinese,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino": 0.3, - "English,Vietnamese,Indonesian": 0.24666666666666667, - "English,Vietnamese,Malay": 0.24666666666666667, - "English,Filipino,Indonesian": 0.3, - "English,Filipino,Malay": 0.3, - "English,Indonesian,Malay": 0.32666666666666666, - "Chinese,Vietnamese,Filipino": 0.16, - "Chinese,Vietnamese,Indonesian": 0.22, - "Chinese,Vietnamese,Malay": 0.20666666666666667, - "Chinese,Filipino,Indonesian": 0.18666666666666668, - "Chinese,Filipino,Malay": 0.15333333333333332, - "Chinese,Indonesian,Malay": 0.26666666666666666, - "Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Vietnamese,Filipino,Malay": 0.28, - "Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Filipino,Indonesian,Malay": 0.36 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.12666666666666668, - "Spanish,English,Chinese,Filipino": 0.12666666666666668, - "Spanish,English,Chinese,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Malay": 0.12, - "Spanish,English,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,English,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,English,Vietnamese,Malay": 0.17333333333333334, - "Spanish,English,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Filipino,Malay": 0.2, - "Spanish,English,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.13333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Filipino,Malay": 0.10666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,Vietnamese,Filipino,Malay": 0.18, - "Spanish,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.12, - "English,Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian": 0.14, - "English,Chinese,Filipino,Malay": 0.12666666666666668, - "English,Chinese,Indonesian,Malay": 0.16, - "English,Vietnamese,Filipino,Indonesian": 0.22, - "English,Vietnamese,Filipino,Malay": 0.21333333333333335, - "English,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "English,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.12, - "Spanish,English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.1, - "Spanish,English,Chinese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 - } - }, - "AC3_2": 0.3288604898355738, - "AC3_3": 0.2542056074267447, - "AC3_4": 0.2046948356334515, - "AC3_5": 0.17378277149164667, - "AC3_6": 0.15527426156210009, - "AC3_7": 0.14545454541487604 - }, - "prompt_5": { - "overall_acc": 0.2752380952380952, - "language_acc": { - "Spanish": 0.26, - "English": 0.29333333333333333, - "Chinese": 0.2866666666666667, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.2866666666666667, - "Indonesian": 0.23333333333333334, - "Malay": 0.28 - }, - "consistency_score_2": 0.44857142857142857, - "consistency_score_3": 0.2533333333333334, - "consistency_score_4": 0.16761904761904764, - "consistency_score_5": 0.12507936507936507, - "consistency_score_6": 0.10095238095238097, - "consistency_score_7": 0.08666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5066666666666667, - "Spanish,Chinese": 0.37333333333333335, - "Spanish,Vietnamese": 0.41333333333333333, - "Spanish,Filipino": 0.4866666666666667, - "Spanish,Indonesian": 0.49333333333333335, - "Spanish,Malay": 0.5066666666666667, - "English,Chinese": 0.34, - "English,Vietnamese": 0.38, - "English,Filipino": 0.48, - "English,Indonesian": 0.4533333333333333, - "English,Malay": 0.43333333333333335, - "Chinese,Vietnamese": 0.5133333333333333, - "Chinese,Filipino": 0.3, - "Chinese,Indonesian": 0.36, - "Chinese,Malay": 0.3333333333333333, - "Vietnamese,Filipino": 0.43333333333333335, - "Vietnamese,Indonesian": 0.43333333333333335, - "Vietnamese,Malay": 0.49333333333333335, - "Filipino,Indonesian": 0.5066666666666667, - "Filipino,Malay": 0.4866666666666667, - "Indonesian,Malay": 0.6933333333333334 - }, - "3_combine": { - "Spanish,English,Chinese": 0.19333333333333333, - "Spanish,English,Vietnamese": 0.24666666666666667, - "Spanish,English,Filipino": 0.32, - "Spanish,English,Indonesian": 0.32666666666666666, - "Spanish,English,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese": 0.24666666666666667, - "Spanish,Chinese,Filipino": 0.18666666666666668, - "Spanish,Chinese,Indonesian": 0.20666666666666667, - "Spanish,Chinese,Malay": 0.2, - "Spanish,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Malay": 0.28, - "Spanish,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Filipino,Malay": 0.29333333333333333, - "Spanish,Indonesian,Malay": 0.3933333333333333, - "English,Chinese,Vietnamese": 0.20666666666666667, - "English,Chinese,Filipino": 0.18, - "English,Chinese,Indonesian": 0.17333333333333334, - "English,Chinese,Malay": 0.16, - "English,Vietnamese,Filipino": 0.24, - "English,Vietnamese,Indonesian": 0.22, - "English,Vietnamese,Malay": 0.24666666666666667, - "English,Filipino,Indonesian": 0.30666666666666664, - "English,Filipino,Malay": 0.29333333333333333, - "English,Indonesian,Malay": 0.34, - "Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "Chinese,Vietnamese,Malay": 0.24, - "Chinese,Filipino,Indonesian": 0.15333333333333332, - "Chinese,Filipino,Malay": 0.14666666666666667, - "Chinese,Indonesian,Malay": 0.25333333333333335, - "Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Vietnamese,Filipino,Malay": 0.26666666666666666, - "Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Filipino,Indonesian,Malay": 0.38 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.13333333333333333, - "Spanish,English,Chinese,Filipino": 0.12666666666666668, - "Spanish,English,Chinese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,English,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Malay": 0.18666666666666668, - "Spanish,English,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.15333333333333332, - "Spanish,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Filipino,Malay": 0.12666666666666668, - "Spanish,Chinese,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian": 0.11333333333333333, - "English,Chinese,Filipino,Malay": 0.10666666666666667, - "English,Chinese,Indonesian,Malay": 0.13333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "English,Vietnamese,Filipino,Malay": 0.18, - "English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "English,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Chinese,Vietnamese,Filipino,Malay": 0.12, - "Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.1, - "Spanish,English,Chinese,Filipino,Indonesian": 0.1, - "Spanish,English,Chinese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.1, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 - } - }, - "AC3_2": 0.341150375892717, - "AC3_3": 0.2638318317819177, - "AC3_4": 0.20835227849877966, - "AC3_5": 0.17199652576799376, - "AC3_6": 0.1477227244935831, - "AC3_7": 0.13182456136708356 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.27516233766233766, - "language_acc": { - "English": 0.2897727272727273, - "Filipino": 0.2727272727272727, - "Spanish": 0.2897727272727273, - "Malay": 0.2727272727272727, - "Chinese": 0.23863636363636365, - "Indonesian": 0.26704545454545453, - "Vietnamese": 0.29545454545454547 - }, - "consistency_score_2": 0.39015151515151514, - "consistency_score_3": 0.1956168831168831, - "consistency_score_4": 0.11233766233766233, - "consistency_score_5": 0.06899350649350648, - "consistency_score_6": 0.04383116883116883, - "consistency_score_7": 0.028409090909090908, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4318181818181818, - "English,Spanish": 0.5454545454545454, - "English,Malay": 0.4318181818181818, - "English,Chinese": 0.26704545454545453, - "English,Indonesian": 0.39204545454545453, - "English,Vietnamese": 0.3522727272727273, - "Filipino,Spanish": 0.48295454545454547, - "Filipino,Malay": 0.48295454545454547, - "Filipino,Chinese": 0.32386363636363635, - "Filipino,Indonesian": 0.4772727272727273, - "Filipino,Vietnamese": 0.3806818181818182, - "Spanish,Malay": 0.42045454545454547, - "Spanish,Chinese": 0.3068181818181818, - "Spanish,Indonesian": 0.4034090909090909, - "Spanish,Vietnamese": 0.3409090909090909, - "Malay,Chinese": 0.3125, - "Malay,Indonesian": 0.5568181818181818, - "Malay,Vietnamese": 0.3409090909090909, - "Chinese,Indonesian": 0.26136363636363635, - "Chinese,Vietnamese": 0.35795454545454547, - "Indonesian,Vietnamese": 0.32386363636363635 - }, - "3_combine": { - "English,Filipino,Spanish": 0.30113636363636365, - "English,Filipino,Malay": 0.2727272727272727, - "English,Filipino,Chinese": 0.1534090909090909, - "English,Filipino,Indonesian": 0.25, - "English,Filipino,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay": 0.26704545454545453, - "English,Spanish,Chinese": 0.17045454545454544, - "English,Spanish,Indonesian": 0.24431818181818182, - "English,Spanish,Vietnamese": 0.20454545454545456, - "English,Malay,Chinese": 0.14204545454545456, - "English,Malay,Indonesian": 0.2727272727272727, - "English,Malay,Vietnamese": 0.17613636363636365, - "English,Chinese,Indonesian": 0.11931818181818182, - "English,Chinese,Vietnamese": 0.1534090909090909, - "English,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Chinese": 0.1590909090909091, - "Filipino,Spanish,Indonesian": 0.2556818181818182, - "Filipino,Spanish,Vietnamese": 0.1875, - "Filipino,Malay,Chinese": 0.17613636363636365, - "Filipino,Malay,Indonesian": 0.32954545454545453, - "Filipino,Malay,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Malay,Chinese": 0.16477272727272727, - "Spanish,Malay,Indonesian": 0.2897727272727273, - "Spanish,Malay,Vietnamese": 0.1590909090909091, - "Spanish,Chinese,Indonesian": 0.13636363636363635, - "Spanish,Chinese,Vietnamese": 0.14772727272727273, - "Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "Malay,Chinese,Indonesian": 0.18181818181818182, - "Malay,Chinese,Vietnamese": 0.14772727272727273, - "Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Chinese,Indonesian,Vietnamese": 0.13068181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.1875, - "English,Filipino,Spanish,Chinese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese": 0.09659090909090909, - "English,Filipino,Malay,Indonesian": 0.19886363636363635, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese": 0.10795454545454546, - "English,Spanish,Malay,Indonesian": 0.18181818181818182, - "English,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Spanish,Chinese,Indonesian": 0.09090909090909091, - "English,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "English,Spanish,Indonesian,Vietnamese": 0.09090909090909091, - "English,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Chinese": 0.11363636363636363, - "Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian": 0.09090909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Malay,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese,Indonesian": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 - } - }, - "AC3_2": 0.3227198786900791, - "AC3_3": 0.2286693909053012, - "AC3_4": 0.1595411291271043, - "AC3_5": 0.11032452214388068, - "AC3_6": 0.0756171309369375, - "AC3_7": 0.05150097227277732 - }, - "prompt_2": { - "overall_acc": 0.2792207792207792, - "language_acc": { - "English": 0.29545454545454547, - "Filipino": 0.25, - "Spanish": 0.2784090909090909, - "Malay": 0.2840909090909091, - "Chinese": 0.2897727272727273, - "Indonesian": 0.2897727272727273, - "Vietnamese": 0.26704545454545453 - }, - "consistency_score_2": 0.38636363636363635, - "consistency_score_3": 0.1832792207792208, - "consistency_score_4": 0.0956168831168831, - "consistency_score_5": 0.05330086580086579, - "consistency_score_6": 0.03246753246753247, - "consistency_score_7": 0.022727272727272728, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.44886363636363635, - "English,Spanish": 0.5, - "English,Malay": 0.4318181818181818, - "English,Chinese": 0.2784090909090909, - "English,Indonesian": 0.38636363636363635, - "English,Vietnamese": 0.35795454545454547, - "Filipino,Spanish": 0.45454545454545453, - "Filipino,Malay": 0.5511363636363636, - "Filipino,Chinese": 0.19318181818181818, - "Filipino,Indonesian": 0.5454545454545454, - "Filipino,Vietnamese": 0.3806818181818182, - "Spanish,Malay": 0.4772727272727273, - "Spanish,Chinese": 0.24431818181818182, - "Spanish,Indonesian": 0.4147727272727273, - "Spanish,Vietnamese": 0.3181818181818182, - "Malay,Chinese": 0.3068181818181818, - "Malay,Indonesian": 0.6136363636363636, - "Malay,Vietnamese": 0.3352272727272727, - "Chinese,Indonesian": 0.2784090909090909, - "Chinese,Vietnamese": 0.2215909090909091, - "Indonesian,Vietnamese": 0.375 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2556818181818182, - "English,Filipino,Malay": 0.26704545454545453, - "English,Filipino,Chinese": 0.11363636363636363, - "English,Filipino,Indonesian": 0.26704545454545453, - "English,Filipino,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay": 0.26704545454545453, - "English,Spanish,Chinese": 0.125, - "English,Spanish,Indonesian": 0.23295454545454544, - "English,Spanish,Vietnamese": 0.17613636363636365, - "English,Malay,Chinese": 0.1534090909090909, - "English,Malay,Indonesian": 0.2727272727272727, - "English,Malay,Vietnamese": 0.16477272727272727, - "English,Chinese,Indonesian": 0.11363636363636363, - "English,Chinese,Vietnamese": 0.08522727272727272, - "English,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay": 0.30113636363636365, - "Filipino,Spanish,Chinese": 0.09090909090909091, - "Filipino,Spanish,Indonesian": 0.2784090909090909, - "Filipino,Spanish,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese": 0.125, - "Filipino,Malay,Indonesian": 0.39204545454545453, - "Filipino,Malay,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Indonesian": 0.125, - "Filipino,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Indonesian,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Chinese": 0.125, - "Spanish,Malay,Indonesian": 0.3181818181818182, - "Spanish,Malay,Vietnamese": 0.18181818181818182, - "Spanish,Chinese,Indonesian": 0.10227272727272728, - "Spanish,Chinese,Vietnamese": 0.07386363636363637, - "Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "Malay,Chinese,Indonesian": 0.1875, - "Malay,Chinese,Vietnamese": 0.07954545454545454, - "Malay,Indonesian,Vietnamese": 0.2556818181818182, - "Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17045454545454544, - "English,Filipino,Spanish,Chinese": 0.056818181818181816, - "English,Filipino,Spanish,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese": 0.07954545454545454, - "English,Filipino,Malay,Indonesian": 0.19318181818181818, - "English,Filipino,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese": 0.09090909090909091, - "English,Spanish,Malay,Indonesian": 0.17045454545454544, - "English,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Spanish,Chinese,Indonesian": 0.056818181818181816, - "English,Spanish,Chinese,Vietnamese": 0.045454545454545456, - "English,Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Malay,Indonesian,Vietnamese": 0.125, - "English,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese": 0.07386363636363637, - "Filipino,Spanish,Malay,Indonesian": 0.20454545454545456, - "Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Malay,Chinese,Vietnamese": 0.03977272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.0625, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 - } - }, - "AC3_2": 0.3241685143637125, - "AC3_3": 0.22129888358869756, - "AC3_4": 0.14245217749429506, - "AC3_5": 0.08951422862481899, - "AC3_6": 0.058170995652332476, - "AC3_7": 0.04203323556770199 - }, - "prompt_3": { - "overall_acc": 0.2865259740259741, - "language_acc": { - "English": 0.3125, - "Filipino": 0.25, - "Spanish": 0.2840909090909091, - "Malay": 0.2840909090909091, - "Chinese": 0.2897727272727273, - "Indonesian": 0.3068181818181818, - "Vietnamese": 0.2784090909090909 - }, - "consistency_score_2": 0.49296536796536783, - "consistency_score_3": 0.30811688311688307, - "consistency_score_4": 0.21055194805194802, - "consistency_score_5": 0.15097402597402595, - "consistency_score_6": 0.112012987012987, - "consistency_score_7": 0.08522727272727272, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4659090909090909, - "English,Spanish": 0.5454545454545454, - "English,Malay": 0.4659090909090909, - "English,Chinese": 0.32954545454545453, - "English,Indonesian": 0.4602272727272727, - "English,Vietnamese": 0.39204545454545453, - "Filipino,Spanish": 0.5909090909090909, - "Filipino,Malay": 0.6988636363636364, - "Filipino,Chinese": 0.3352272727272727, - "Filipino,Indonesian": 0.6704545454545454, - "Filipino,Vietnamese": 0.48863636363636365, - "Spanish,Malay": 0.6136363636363636, - "Spanish,Chinese": 0.3125, - "Spanish,Indonesian": 0.5454545454545454, - "Spanish,Vietnamese": 0.44886363636363635, - "Malay,Chinese": 0.42045454545454547, - "Malay,Indonesian": 0.7159090909090909, - "Malay,Vietnamese": 0.5284090909090909, - "Chinese,Indonesian": 0.4090909090909091, - "Chinese,Vietnamese": 0.4090909090909091, - "Indonesian,Vietnamese": 0.5056818181818182 - }, - "3_combine": { - "English,Filipino,Spanish": 0.36363636363636365, - "English,Filipino,Malay": 0.375, - "English,Filipino,Chinese": 0.17613636363636365, - "English,Filipino,Indonesian": 0.3522727272727273, - "English,Filipino,Vietnamese": 0.2727272727272727, - "English,Spanish,Malay": 0.3806818181818182, - "English,Spanish,Chinese": 0.18181818181818182, - "English,Spanish,Indonesian": 0.3409090909090909, - "English,Spanish,Vietnamese": 0.26704545454545453, - "English,Malay,Chinese": 0.19886363636363635, - "English,Malay,Indonesian": 0.38636363636363635, - "English,Malay,Vietnamese": 0.2556818181818182, - "English,Chinese,Indonesian": 0.20454545454545456, - "English,Chinese,Vietnamese": 0.17045454545454544, - "English,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Malay": 0.48863636363636365, - "Filipino,Spanish,Chinese": 0.21022727272727273, - "Filipino,Spanish,Indonesian": 0.42613636363636365, - "Filipino,Spanish,Vietnamese": 0.32954545454545453, - "Filipino,Malay,Chinese": 0.2840909090909091, - "Filipino,Malay,Indonesian": 0.5568181818181818, - "Filipino,Malay,Vietnamese": 0.4034090909090909, - "Filipino,Chinese,Indonesian": 0.2784090909090909, - "Filipino,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Indonesian,Vietnamese": 0.36363636363636365, - "Spanish,Malay,Chinese": 0.2556818181818182, - "Spanish,Malay,Indonesian": 0.4715909090909091, - "Spanish,Malay,Vietnamese": 0.3465909090909091, - "Spanish,Chinese,Indonesian": 0.22727272727272727, - "Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Spanish,Indonesian,Vietnamese": 0.3125, - "Malay,Chinese,Indonesian": 0.32386363636363635, - "Malay,Chinese,Vietnamese": 0.26136363636363635, - "Malay,Indonesian,Vietnamese": 0.4034090909090909, - "Chinese,Indonesian,Vietnamese": 0.24431818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.32954545454545453, - "English,Filipino,Spanish,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Indonesian": 0.2897727272727273, - "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, - "English,Filipino,Malay,Chinese": 0.1534090909090909, - "English,Filipino,Malay,Indonesian": 0.32386363636363635, - "English,Filipino,Malay,Vietnamese": 0.22727272727272727, - "English,Filipino,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay,Chinese": 0.1590909090909091, - "English,Spanish,Malay,Indonesian": 0.3125, - "English,Spanish,Malay,Vietnamese": 0.21022727272727273, - "English,Spanish,Chinese,Indonesian": 0.14772727272727273, - "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "English,Malay,Chinese,Indonesian": 0.17613636363636365, - "English,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese": 0.19318181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.39204545454545453, - "Filipino,Spanish,Malay,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Indonesian,Vietnamese": 0.25, - "Filipino,Malay,Chinese,Indonesian": 0.23863636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.32386363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Chinese,Indonesian": 0.20454545454545456, - "Spanish,Malay,Chinese,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Indonesian": 0.2840909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 - } - }, - "AC3_2": 0.3624091111477622, - "AC3_3": 0.296929455970431, - "AC3_4": 0.24273297729387755, - "AC3_5": 0.19775076495973926, - "AC3_6": 0.1610614436075887, - "AC3_7": 0.131376538273513 - }, - "prompt_4": { - "overall_acc": 0.28490259740259744, - "language_acc": { - "English": 0.32386363636363635, - "Filipino": 0.2556818181818182, - "Spanish": 0.32954545454545453, - "Malay": 0.2897727272727273, - "Chinese": 0.2727272727272727, - "Indonesian": 0.2727272727272727, - "Vietnamese": 0.25 - }, - "consistency_score_2": 0.39772727272727276, - "consistency_score_3": 0.1915584415584416, - "consistency_score_4": 0.09805194805194803, - "consistency_score_5": 0.050324675324675314, - "consistency_score_6": 0.024350649350649345, - "consistency_score_7": 0.011363636363636364, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4147727272727273, - "English,Spanish": 0.5, - "English,Malay": 0.4318181818181818, - "English,Chinese": 0.30113636363636365, - "English,Indonesian": 0.3693181818181818, - "English,Vietnamese": 0.32386363636363635, - "Filipino,Spanish": 0.5, - "Filipino,Malay": 0.625, - "Filipino,Chinese": 0.17045454545454544, - "Filipino,Indonesian": 0.5738636363636364, - "Filipino,Vietnamese": 0.4090909090909091, - "Spanish,Malay": 0.4602272727272727, - "Spanish,Chinese": 0.23295454545454544, - "Spanish,Indonesian": 0.4034090909090909, - "Spanish,Vietnamese": 0.3465909090909091, - "Malay,Chinese": 0.24431818181818182, - "Malay,Indonesian": 0.6875, - "Malay,Vietnamese": 0.4431818181818182, - "Chinese,Indonesian": 0.26136363636363635, - "Chinese,Vietnamese": 0.2215909090909091, - "Indonesian,Vietnamese": 0.4318181818181818 - }, - "3_combine": { - "English,Filipino,Spanish": 0.26704545454545453, - "English,Filipino,Malay": 0.2784090909090909, - "English,Filipino,Chinese": 0.09090909090909091, - "English,Filipino,Indonesian": 0.23863636363636365, - "English,Filipino,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay": 0.26136363636363635, - "English,Spanish,Chinese": 0.125, - "English,Spanish,Indonesian": 0.2159090909090909, - "English,Spanish,Vietnamese": 0.1875, - "English,Malay,Chinese": 0.125, - "English,Malay,Indonesian": 0.29545454545454547, - "English,Malay,Vietnamese": 0.18181818181818182, - "English,Chinese,Indonesian": 0.10795454545454546, - "English,Chinese,Vietnamese": 0.07386363636363637, - "English,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay": 0.3465909090909091, - "Filipino,Spanish,Chinese": 0.08522727272727272, - "Filipino,Spanish,Indonesian": 0.2897727272727273, - "Filipino,Spanish,Vietnamese": 0.21022727272727273, - "Filipino,Malay,Chinese": 0.09659090909090909, - "Filipino,Malay,Indonesian": 0.4659090909090909, - "Filipino,Malay,Vietnamese": 0.29545454545454547, - "Filipino,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Chinese,Vietnamese": 0.05113636363636364, - "Filipino,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Chinese": 0.10795454545454546, - "Spanish,Malay,Indonesian": 0.32386363636363635, - "Spanish,Malay,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Indonesian": 0.10795454545454546, - "Spanish,Chinese,Vietnamese": 0.07386363636363637, - "Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "Malay,Chinese,Indonesian": 0.16477272727272727, - "Malay,Chinese,Vietnamese": 0.08522727272727272, - "Malay,Indonesian,Vietnamese": 0.3352272727272727, - "Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.19318181818181818, - "English,Filipino,Spanish,Chinese": 0.045454545454545456, - "English,Filipino,Spanish,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese": 0.045454545454545456, - "English,Filipino,Malay,Indonesian": 0.20454545454545456, - "English,Filipino,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese": 0.06818181818181818, - "English,Spanish,Malay,Indonesian": 0.18181818181818182, - "English,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Spanish,Chinese,Indonesian": 0.0625, - "English,Spanish,Chinese,Vietnamese": 0.03977272727272727, - "English,Spanish,Indonesian,Vietnamese": 0.09659090909090909, - "English,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese": 0.056818181818181816, - "Filipino,Spanish,Malay,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Malay,Chinese,Indonesian,Vietnamese": 0.0625 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 - } - }, - "AC3_2": 0.3319911360447612, - "AC3_3": 0.2290869266538373, - "AC3_4": 0.14589331817400056, - "AC3_5": 0.08553976287502074, - "AC3_6": 0.044866550757554545, - "AC3_7": 0.021855541711178424 - }, - "prompt_5": { - "overall_acc": 0.2930194805194805, - "language_acc": { - "English": 0.35795454545454547, - "Filipino": 0.25, - "Spanish": 0.2897727272727273, - "Malay": 0.26704545454545453, - "Chinese": 0.2727272727272727, - "Indonesian": 0.3181818181818182, - "Vietnamese": 0.29545454545454547 - }, - "consistency_score_2": 0.3576839826839827, - "consistency_score_3": 0.16363636363636358, - "consistency_score_4": 0.0896103896103896, - "consistency_score_5": 0.054653679653679656, - "consistency_score_6": 0.0349025974025974, - "consistency_score_7": 0.022727272727272728, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.45454545454545453, - "English,Spanish": 0.5511363636363636, - "English,Malay": 0.375, - "English,Chinese": 0.2840909090909091, - "English,Indonesian": 0.3977272727272727, - "English,Vietnamese": 0.375, - "Filipino,Spanish": 0.4375, - "Filipino,Malay": 0.4659090909090909, - "Filipino,Chinese": 0.125, - "Filipino,Indonesian": 0.42613636363636365, - "Filipino,Vietnamese": 0.3125, - "Spanish,Malay": 0.4318181818181818, - "Spanish,Chinese": 0.1590909090909091, - "Spanish,Indonesian": 0.4090909090909091, - "Spanish,Vietnamese": 0.3522727272727273, - "Malay,Chinese": 0.14204545454545456, - "Malay,Indonesian": 0.5568181818181818, - "Malay,Vietnamese": 0.3125, - "Chinese,Indonesian": 0.19318181818181818, - "Chinese,Vietnamese": 0.42613636363636365, - "Indonesian,Vietnamese": 0.32386363636363635 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Malay": 0.22727272727272727, - "English,Filipino,Chinese": 0.08522727272727272, - "English,Filipino,Indonesian": 0.22727272727272727, - "English,Filipino,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay": 0.25, - "English,Spanish,Chinese": 0.11931818181818182, - "English,Spanish,Indonesian": 0.26704545454545453, - "English,Spanish,Vietnamese": 0.2159090909090909, - "English,Malay,Chinese": 0.08522727272727272, - "English,Malay,Indonesian": 0.25, - "English,Malay,Vietnamese": 0.16477272727272727, - "English,Chinese,Indonesian": 0.09659090909090909, - "English,Chinese,Vietnamese": 0.1590909090909091, - "English,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay": 0.24431818181818182, - "Filipino,Spanish,Chinese": 0.056818181818181816, - "Filipino,Spanish,Indonesian": 0.23863636363636365, - "Filipino,Spanish,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese": 0.056818181818181816, - "Filipino,Malay,Indonesian": 0.2897727272727273, - "Filipino,Malay,Vietnamese": 0.18181818181818182, - "Filipino,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Indonesian,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Chinese": 0.056818181818181816, - "Spanish,Malay,Indonesian": 0.29545454545454547, - "Spanish,Malay,Vietnamese": 0.18181818181818182, - "Spanish,Chinese,Indonesian": 0.0625, - "Spanish,Chinese,Vietnamese": 0.09090909090909091, - "Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "Malay,Chinese,Indonesian": 0.09659090909090909, - "Malay,Chinese,Vietnamese": 0.09090909090909091, - "Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.16477272727272727, - "English,Filipino,Spanish,Chinese": 0.05113636363636364, - "English,Filipino,Spanish,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese": 0.045454545454545456, - "English,Filipino,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese": 0.056818181818181816, - "English,Spanish,Malay,Indonesian": 0.19318181818181818, - "English,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Spanish,Chinese,Indonesian": 0.05113636363636364, - "English,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "English,Spanish,Indonesian,Vietnamese": 0.125, - "English,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Malay,Chinese": 0.03409090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian": 0.03409090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Chinese,Indonesian": 0.045454545454545456, - "Filipino,Malay,Chinese,Vietnamese": 0.045454545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Malay,Chinese,Indonesian,Vietnamese": 0.0625 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.03409090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 - } - }, - "AC3_2": 0.32213867208916597, - "AC3_3": 0.2099990304294039, - "AC3_4": 0.1372479874454653, - "AC3_5": 0.0921244125259307, - "AC3_6": 0.06237543395294721, - "AC3_7": 0.042182752966308475 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.3592233009708738 - }, - "prompt_2": { - "accuracy": 0.36893203883495146 - }, - "prompt_3": { - "accuracy": 0.33980582524271846 - }, - "prompt_4": { - "accuracy": 0.3300970873786408 - }, - "prompt_5": { - "accuracy": 0.34951456310679613 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.2571428571428571 - }, - "prompt_2": { - "accuracy": 0.2571428571428571 - }, - "prompt_3": { - "accuracy": 0.2571428571428571 - }, - "prompt_4": { - "accuracy": 0.2571428571428571 - }, - "prompt_5": { - "accuracy": 0.2571428571428571 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.2803738317757009 - }, - "prompt_2": { - "accuracy": 0.2897196261682243 - }, - "prompt_3": { - "accuracy": 0.2616822429906542 - }, - "prompt_4": { - "accuracy": 0.27102803738317754 - }, - "prompt_5": { - "accuracy": 0.2616822429906542 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.28, - "category_acc": { - "brand": 0.4, - "demographics": 0.8, - "biology": 0.1, - "history": 0.4666666666666667, - "literature": 0.2, - "politics": 0.2, - "culture": 0.2, - "film": 0.2, - "law": 0.2, - "geography": 0.2 - } - }, - "prompt_2": { - "accuracy": 0.28, - "category_acc": { - "brand": 0.4, - "demographics": 0.6, - "biology": 0.1, - "history": 0.4666666666666667, - "literature": 0.2, - "politics": 0.3, - "culture": 0.2, - "film": 0.2, - "law": 0.2, - "geography": 0.2 - } - }, - "prompt_3": { - "accuracy": 0.23, - "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.2, - "history": 0.3333333333333333, - "literature": 0.2, - "politics": 0.2, - "culture": 0.2, - "film": 0.1, - "law": 0.1, - "geography": 0.3 - } - }, - "prompt_4": { - "accuracy": 0.31, - "category_acc": { - "brand": 0.4, - "demographics": 0.8, - "biology": 0.2, - "history": 0.4666666666666667, - "literature": 0.3, - "politics": 0.2, - "culture": 0.2, - "film": 0.2, - "law": 0.3, - "geography": 0.2 - } - }, - "prompt_5": { - "accuracy": 0.28, - "category_acc": { - "brand": 0.3, - "demographics": 0.8, - "biology": 0.3, - "history": 0.3333333333333333, - "literature": 0.3, - "politics": 0.2, - "culture": 0.2, - "film": 0.2, - "law": 0.2, - "geography": 0.2 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.1778634553271136 - }, - "prompt_2": { - "bleu_score": 0.17587938617494803 - }, - "prompt_3": { - "bleu_score": 0.18016948741331085 - }, - "prompt_4": { - "bleu_score": 0.1755807406460382 - }, - "prompt_5": { - "bleu_score": 0.15871208497539763 - } - }, - "indommlu": { - "prompt_1": { - "accuracy": 0.22377995860871888, - "category_acc": { - "History": 0.15863453815261044, - "Geography": 0.20408163265306123, - "Lampungic": 0.2585034013605442, - "Social science": 0.2637729549248748, - "Balinese": 0.2611464968152866, - "Makassarese": 0.23118279569892472, - "Banjarese": 0.1736111111111111, - "Chemistry": 0.1635036496350365, - "Biology": 0.1727810650887574, - "Science": 0.24045407636738905, - "Christian religion": 0.23383084577114427, - "Art": 0.22129783693843594, - "Islam religion": 0.22190611664295876, - "Hindu religion": 0.21333333333333335, - "Madurese": 0.21694915254237288, - "Sport": 0.19594594594594594, - "Indonesian language": 0.24221668742216687, - "Physics": 0.20202020202020202, - "Minangkabau culture": 0.24623115577889448, - "Dayak language": 0.25688073394495414, - "Sociology": 0.17338709677419356, - "Economy": 0.1987704918032787, - "Sundanese": 0.24373379429559205, - "Javanese": 0.2560483870967742, - "Civic education": 0.22889842632331903 - } - }, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.03374471826935659 - }, - "prompt_2": { - "bleu_score": 0.03354584784772205 - }, - "prompt_3": { - "bleu_score": 0.030709309536872887 - }, - "prompt_4": { - "bleu_score": 0.033406005624105214 - }, - "prompt_5": { - "bleu_score": 0.03408654681827874 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.023547771145113035 - }, - "prompt_2": { - "bleu_score": 0.023013474143517277 - }, - "prompt_3": { - "bleu_score": 0.020874466049663108 - }, - "prompt_4": { - "bleu_score": 0.02411759361984998 - }, - "prompt_5": { - "bleu_score": 0.02376678283625278 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.0061225167958979445 - }, - "prompt_2": { - "bleu_score": 0.008007806300952039 - }, - "prompt_3": { - "bleu_score": 0.008798554334606034 - }, - "prompt_4": { - "bleu_score": 0.008702923563946885 - }, - "prompt_5": { - "bleu_score": 0.00887049961254504 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.028766122560955086 - }, - "prompt_2": { - "bleu_score": 0.03126042463241794 - }, - "prompt_3": { - "bleu_score": 0.030200207989540616 - }, - "prompt_4": { - "bleu_score": 0.03010036911944672 - }, - "prompt_5": { - "bleu_score": 0.03042639392728816 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.29638273045507585 - }, - "prompt_2": { - "accuracy": 0.2893815635939323 - }, - "prompt_3": { - "accuracy": 0.2823803967327888 - }, - "prompt_4": { - "accuracy": 0.2952158693115519 - }, - "prompt_5": { - "accuracy": 0.28588098016336055 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.2888094386843046, - "category_acc": { - "high_school_european_history": 0.3231707317073171, - "business_ethics": 0.29292929292929293, - "clinical_knowledge": 0.29545454545454547, - "medical_genetics": 0.1919191919191919, - "high_school_us_history": 0.3399014778325123, - "high_school_physics": 0.2733333333333333, - "high_school_world_history": 0.3220338983050847, - "virology": 0.26666666666666666, - "high_school_microeconomics": 0.3333333333333333, - "econometrics": 0.24778761061946902, - "college_computer_science": 0.23232323232323232, - "high_school_biology": 0.30097087378640774, - "abstract_algebra": 0.2727272727272727, - "professional_accounting": 0.26334519572953735, - "philosophy": 0.29354838709677417, - "professional_medicine": 0.36900369003690037, - "nutrition": 0.3540983606557377, - "global_facts": 0.1919191919191919, - "machine_learning": 0.18018018018018017, - "security_studies": 0.2336065573770492, - "public_relations": 0.3302752293577982, - "professional_psychology": 0.2700490998363339, - "prehistory": 0.28173374613003094, - "anatomy": 0.2462686567164179, - "human_sexuality": 0.3384615384615385, - "college_medicine": 0.27325581395348836, - "high_school_government_and_politics": 0.328125, - "college_chemistry": 0.30303030303030304, - "logical_fallacies": 0.345679012345679, - "high_school_geography": 0.27918781725888325, - "elementary_mathematics": 0.2572944297082228, - "human_aging": 0.24774774774774774, - "college_mathematics": 0.20202020202020202, - "high_school_psychology": 0.31985294117647056, - "formal_logic": 0.32, - "high_school_statistics": 0.2651162790697674, - "international_law": 0.2833333333333333, - "high_school_mathematics": 0.24535315985130113, - "high_school_computer_science": 0.29292929292929293, - "conceptual_physics": 0.29914529914529914, - "miscellaneous": 0.27621483375959077, - "high_school_chemistry": 0.2623762376237624, - "marketing": 0.41630901287553645, - "professional_law": 0.30006523157208087, - "management": 0.37254901960784315, - "college_physics": 0.19801980198019803, - "jurisprudence": 0.308411214953271, - "world_religions": 0.24705882352941178, - "sociology": 0.325, - "us_foreign_policy": 0.35353535353535354, - "high_school_macroeconomics": 0.30077120822622105, - "computer_security": 0.2222222222222222, - "moral_scenarios": 0.24608501118568232, - "moral_disputes": 0.28695652173913044, - "electrical_engineering": 0.2847222222222222, - "astronomy": 0.33774834437086093, - "college_biology": 0.27972027972027974 - } - }, - "prompt_2": { - "accuracy": 0.29309974973185554, - "category_acc": { - "high_school_european_history": 0.3353658536585366, - "business_ethics": 0.31313131313131315, - "clinical_knowledge": 0.32196969696969696, - "medical_genetics": 0.1919191919191919, - "high_school_us_history": 0.3399014778325123, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.3305084745762712, - "virology": 0.2606060606060606, - "high_school_microeconomics": 0.2869198312236287, - "econometrics": 0.17699115044247787, - "college_computer_science": 0.2828282828282828, - "high_school_biology": 0.2783171521035599, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.25622775800711745, - "philosophy": 0.3225806451612903, - "professional_medicine": 0.33579335793357934, - "nutrition": 0.32459016393442625, - "global_facts": 0.29292929292929293, - "machine_learning": 0.27927927927927926, - "security_studies": 0.22950819672131148, - "public_relations": 0.3119266055045872, - "professional_psychology": 0.2765957446808511, - "prehistory": 0.30340557275541796, - "anatomy": 0.22388059701492538, - "human_sexuality": 0.3076923076923077, - "college_medicine": 0.3023255813953488, - "high_school_government_and_politics": 0.3072916666666667, - "college_chemistry": 0.26262626262626265, - "logical_fallacies": 0.3333333333333333, - "high_school_geography": 0.28426395939086296, - "elementary_mathematics": 0.23342175066312998, - "human_aging": 0.3063063063063063, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.3235294117647059, - "formal_logic": 0.288, - "high_school_statistics": 0.25116279069767444, - "international_law": 0.3333333333333333, - "high_school_mathematics": 0.2862453531598513, - "high_school_computer_science": 0.3333333333333333, - "conceptual_physics": 0.2948717948717949, - "miscellaneous": 0.309462915601023, - "high_school_chemistry": 0.24257425742574257, - "marketing": 0.44206008583690987, - "professional_law": 0.2948467058056099, - "management": 0.3235294117647059, - "college_physics": 0.2079207920792079, - "jurisprudence": 0.32710280373831774, - "world_religions": 0.2529411764705882, - "sociology": 0.355, - "us_foreign_policy": 0.31313131313131315, - "high_school_macroeconomics": 0.29562982005141386, - "computer_security": 0.31313131313131315, - "moral_scenarios": 0.24720357941834453, - "moral_disputes": 0.3217391304347826, - "electrical_engineering": 0.3194444444444444, - "astronomy": 0.2781456953642384, - "college_biology": 0.26573426573426573 - } - }, - "prompt_3": { - "accuracy": 0.28652127279227746, - "category_acc": { - "high_school_european_history": 0.3231707317073171, - "business_ethics": 0.29292929292929293, - "clinical_knowledge": 0.29924242424242425, - "medical_genetics": 0.1717171717171717, - "high_school_us_history": 0.33497536945812806, - "high_school_physics": 0.2866666666666667, - "high_school_world_history": 0.3347457627118644, - "virology": 0.3090909090909091, - "high_school_microeconomics": 0.3037974683544304, - "econometrics": 0.22123893805309736, - "college_computer_science": 0.25252525252525254, - "high_school_biology": 0.3042071197411003, - "abstract_algebra": 0.25252525252525254, - "professional_accounting": 0.2313167259786477, - "philosophy": 0.27741935483870966, - "professional_medicine": 0.35793357933579334, - "nutrition": 0.29508196721311475, - "global_facts": 0.2222222222222222, - "machine_learning": 0.27927927927927926, - "security_studies": 0.2540983606557377, - "public_relations": 0.3119266055045872, - "professional_psychology": 0.2618657937806874, - "prehistory": 0.2848297213622291, - "anatomy": 0.22388059701492538, - "human_sexuality": 0.27692307692307694, - "college_medicine": 0.3081395348837209, - "high_school_government_and_politics": 0.3541666666666667, - "college_chemistry": 0.26262626262626265, - "logical_fallacies": 0.3333333333333333, - "high_school_geography": 0.29441624365482233, - "elementary_mathematics": 0.22015915119363394, - "human_aging": 0.2702702702702703, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.3382352941176471, - "formal_logic": 0.264, - "high_school_statistics": 0.21395348837209302, - "international_law": 0.25833333333333336, - "high_school_mathematics": 0.24535315985130113, - "high_school_computer_science": 0.24242424242424243, - "conceptual_physics": 0.29914529914529914, - "miscellaneous": 0.27877237851662406, - "high_school_chemistry": 0.21782178217821782, - "marketing": 0.43776824034334766, - "professional_law": 0.28636660143509457, - "management": 0.3431372549019608, - "college_physics": 0.2079207920792079, - "jurisprudence": 0.32710280373831774, - "world_religions": 0.2411764705882353, - "sociology": 0.355, - "us_foreign_policy": 0.30303030303030304, - "high_school_macroeconomics": 0.30848329048843187, - "computer_security": 0.25252525252525254, - "moral_scenarios": 0.2505592841163311, - "moral_disputes": 0.33043478260869563, - "electrical_engineering": 0.3333333333333333, - "astronomy": 0.3509933774834437, - "college_biology": 0.27972027972027974 - } - }, - "prompt_4": { - "accuracy": 0.2908115838398284, - "category_acc": { - "high_school_european_history": 0.34146341463414637, - "business_ethics": 0.29292929292929293, - "clinical_knowledge": 0.2727272727272727, - "medical_genetics": 0.1919191919191919, - "high_school_us_history": 0.3694581280788177, - "high_school_physics": 0.26666666666666666, - "high_school_world_history": 0.3050847457627119, - "virology": 0.26666666666666666, - "high_school_microeconomics": 0.28270042194092826, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.2727272727272727, - "high_school_biology": 0.29449838187702265, - "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.24555160142348753, - "philosophy": 0.2903225806451613, - "professional_medicine": 0.35793357933579334, - "nutrition": 0.32131147540983607, - "global_facts": 0.20202020202020202, - "machine_learning": 0.2702702702702703, - "security_studies": 0.29098360655737704, - "public_relations": 0.3211009174311927, - "professional_psychology": 0.28805237315875615, - "prehistory": 0.29411764705882354, - "anatomy": 0.2835820895522388, - "human_sexuality": 0.3230769230769231, - "college_medicine": 0.27906976744186046, - "high_school_government_and_politics": 0.3072916666666667, - "college_chemistry": 0.23232323232323232, - "logical_fallacies": 0.35802469135802467, - "high_school_geography": 0.28426395939086296, - "elementary_mathematics": 0.27055702917771884, - "human_aging": 0.26576576576576577, - "college_mathematics": 0.2727272727272727, - "high_school_psychology": 0.3161764705882353, - "formal_logic": 0.264, - "high_school_statistics": 0.26976744186046514, - "international_law": 0.275, - "high_school_mathematics": 0.22304832713754646, - "high_school_computer_science": 0.30303030303030304, - "conceptual_physics": 0.29914529914529914, - "miscellaneous": 0.2979539641943734, - "high_school_chemistry": 0.22277227722772278, - "marketing": 0.3948497854077253, - "professional_law": 0.29158512720156554, - "management": 0.3431372549019608, - "college_physics": 0.2079207920792079, - "jurisprudence": 0.3364485981308411, - "world_religions": 0.25882352941176473, - "sociology": 0.345, - "us_foreign_policy": 0.30303030303030304, - "high_school_macroeconomics": 0.29562982005141386, - "computer_security": 0.29292929292929293, - "moral_scenarios": 0.24608501118568232, - "moral_disputes": 0.3159420289855073, - "electrical_engineering": 0.3333333333333333, - "astronomy": 0.3576158940397351, - "college_biology": 0.3076923076923077 - } - }, - "prompt_5": { - "accuracy": 0.2986771540936718, - "category_acc": { - "high_school_european_history": 0.3902439024390244, - "business_ethics": 0.2828282828282828, - "clinical_knowledge": 0.3143939393939394, - "medical_genetics": 0.25252525252525254, - "high_school_us_history": 0.35960591133004927, - "high_school_physics": 0.22666666666666666, - "high_school_world_history": 0.3093220338983051, - "virology": 0.3090909090909091, - "high_school_microeconomics": 0.2869198312236287, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.25252525252525254, - "high_school_biology": 0.32362459546925565, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.29537366548042704, - "philosophy": 0.3193548387096774, - "professional_medicine": 0.3025830258302583, - "nutrition": 0.33114754098360655, - "global_facts": 0.3434343434343434, - "machine_learning": 0.3333333333333333, - "security_studies": 0.2459016393442623, - "public_relations": 0.3577981651376147, - "professional_psychology": 0.2864157119476268, - "prehistory": 0.32507739938080493, - "anatomy": 0.26119402985074625, - "human_sexuality": 0.2846153846153846, - "college_medicine": 0.29069767441860467, - "high_school_government_and_politics": 0.2916666666666667, - "college_chemistry": 0.2222222222222222, - "logical_fallacies": 0.36419753086419754, - "high_school_geography": 0.3147208121827411, - "elementary_mathematics": 0.23076923076923078, - "human_aging": 0.2927927927927928, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.32536764705882354, - "formal_logic": 0.296, - "high_school_statistics": 0.21395348837209302, - "international_law": 0.38333333333333336, - "high_school_mathematics": 0.2527881040892193, - "high_school_computer_science": 0.30303030303030304, - "conceptual_physics": 0.33760683760683763, - "miscellaneous": 0.3171355498721228, - "high_school_chemistry": 0.27722772277227725, - "marketing": 0.44635193133047213, - "professional_law": 0.2928897586431833, - "management": 0.23529411764705882, - "college_physics": 0.1782178217821782, - "jurisprudence": 0.35514018691588783, - "world_religions": 0.27647058823529413, - "sociology": 0.36, - "us_foreign_policy": 0.36363636363636365, - "high_school_macroeconomics": 0.2930591259640103, - "computer_security": 0.31313131313131315, - "moral_scenarios": 0.24720357941834453, - "moral_disputes": 0.3130434782608696, - "electrical_engineering": 0.3333333333333333, - "astronomy": 0.33112582781456956, - "college_biology": 0.2727272727272727 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.24665676077265974 - }, - "prompt_2": { - "accuracy": 0.24591381872213966 - }, - "prompt_3": { - "accuracy": 0.24591381872213966 - }, - "prompt_4": { - "accuracy": 0.2436849925705795 - }, - "prompt_5": { - "accuracy": 0.24591381872213966 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.2577833125778331, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.21428571428571427, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.375, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.2, - "business_administration": 0.13157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.20408163265306123, - "high_school_politics": 0.16666666666666666, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.37037037037037035, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.08, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.23076923076923078, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.375, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.3333333333333333, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.2777777777777778, - "physician": 0.2962962962962963 - } - }, - "prompt_2": { - "accuracy": 0.2590286425902864, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.2619047619047619, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.375, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.2, - "business_administration": 0.13157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.20408163265306123, - "high_school_politics": 0.16666666666666666, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.37037037037037035, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.08, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.23076923076923078, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.375, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.3333333333333333, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.2777777777777778, - "physician": 0.2962962962962963 - } - }, - "prompt_3": { - "accuracy": 0.2577833125778331, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.21428571428571427, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.375, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.2, - "business_administration": 0.13157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.20408163265306123, - "high_school_politics": 0.16666666666666666, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.37037037037037035, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.08, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.23076923076923078, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.375, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.3333333333333333, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.2777777777777778, - "physician": 0.2962962962962963 - } - }, - "prompt_4": { - "accuracy": 0.25840597758405975, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.21428571428571427, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.25, - "high_school_chemistry": 0.375, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.2, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.20408163265306123, - "high_school_politics": 0.16666666666666666, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.37037037037037035, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.08, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.23076923076923078, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.375, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.3333333333333333, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.2777777777777778, - "physician": 0.2962962962962963 - } - }, - "prompt_5": { - "accuracy": 0.2577833125778331, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.21428571428571427, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.375, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.2, - "business_administration": 0.13157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.20408163265306123, - "high_school_politics": 0.16666666666666666, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.37037037037037035, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.08, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.23076923076923078, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.375, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.3333333333333333, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.2777777777777778, - "physician": 0.2962962962962963 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.2078853046594982 - }, - "prompt_2": { - "accuracy": 0.2114695340501792 - }, - "prompt_3": { - "accuracy": 0.2078853046594982 - }, - "prompt_4": { - "accuracy": 0.21505376344086022 - }, - "prompt_5": { - "accuracy": 0.2078853046594982 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.24848903470903125, - "category_acc": { - "agronomy": 0.23668639053254437, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.23780487804878048, - "arts": 0.25, - "astronomy": 0.2727272727272727, - "business_ethics": 0.2631578947368421, - "chinese_civil_service_exam": 0.26875, - "chinese_driving_rule": 0.21374045801526717, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.22429906542056074, - "chinese_history": 0.2476780185758514, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.2569832402234637, - "clinical_knowledge": 0.24472573839662448, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.14953271028037382, - "college_engineering_hydrology": 0.27358490566037735, - "college_law": 0.32407407407407407, - "college_mathematics": 0.2857142857142857, - "college_medical_statistics": 0.20754716981132076, - "college_medicine": 0.2271062271062271, - "computer_science": 0.24019607843137256, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.23129251700680273, - "construction_project_management": 0.2446043165467626, - "economics": 0.2578616352201258, - "education": 0.25153374233128833, - "electrical_engineering": 0.25, - "elementary_chinese": 0.20634920634920634, - "elementary_commonsense": 0.2727272727272727, - "elementary_information_and_technology": 0.22268907563025211, - "elementary_mathematics": 0.24347826086956523, - "ethnology": 0.2518518518518518, - "food_science": 0.26573426573426573, - "genetics": 0.26704545454545453, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.18181818181818182, - "high_school_geography": 0.23728813559322035, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.24475524475524477, - "human_sexuality": 0.2698412698412698, - "international_law": 0.25405405405405407, - "journalism": 0.2558139534883721, - "jurisprudence": 0.26034063260340634, - "legal_and_moral_basis": 0.2336448598130841, - "logical": 0.23577235772357724, - "machine_learning": 0.2540983606557377, - "management": 0.24761904761904763, - "marketing": 0.2388888888888889, - "marxist_theory": 0.23809523809523808, - "modern_chinese": 0.25, - "nutrition": 0.27586206896551724, - "philosophy": 0.24761904761904763, - "professional_accounting": 0.25142857142857145, - "professional_law": 0.2559241706161137, - "professional_medicine": 0.25, - "professional_psychology": 0.2543103448275862, - "public_relations": 0.23563218390804597, - "security_study": 0.2222222222222222, - "sociology": 0.25663716814159293, - "sports_science": 0.2727272727272727, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.23076923076923078, - "world_history": 0.2546583850931677, - "world_religions": 0.26875 - } - }, - "prompt_2": { - "accuracy": 0.24866171645657054, - "category_acc": { - "agronomy": 0.23668639053254437, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.23780487804878048, - "arts": 0.25, - "astronomy": 0.2727272727272727, - "business_ethics": 0.2631578947368421, - "chinese_civil_service_exam": 0.26875, - "chinese_driving_rule": 0.21374045801526717, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.22429906542056074, - "chinese_history": 0.25077399380804954, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.2569832402234637, - "clinical_knowledge": 0.24472573839662448, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.14953271028037382, - "college_engineering_hydrology": 0.27358490566037735, - "college_law": 0.32407407407407407, - "college_mathematics": 0.2857142857142857, - "college_medical_statistics": 0.20754716981132076, - "college_medicine": 0.2271062271062271, - "computer_science": 0.24019607843137256, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.23129251700680273, - "construction_project_management": 0.2446043165467626, - "economics": 0.2578616352201258, - "education": 0.25153374233128833, - "electrical_engineering": 0.25, - "elementary_chinese": 0.20634920634920634, - "elementary_commonsense": 0.2727272727272727, - "elementary_information_and_technology": 0.2184873949579832, - "elementary_mathematics": 0.24347826086956523, - "ethnology": 0.2518518518518518, - "food_science": 0.2727272727272727, - "genetics": 0.26704545454545453, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.19696969696969696, - "high_school_geography": 0.23728813559322035, - "high_school_mathematics": 0.25609756097560976, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.24475524475524477, - "human_sexuality": 0.2698412698412698, - "international_law": 0.25405405405405407, - "journalism": 0.2558139534883721, - "jurisprudence": 0.26034063260340634, - "legal_and_moral_basis": 0.2336448598130841, - "logical": 0.23577235772357724, - "machine_learning": 0.2540983606557377, - "management": 0.24761904761904763, - "marketing": 0.2388888888888889, - "marxist_theory": 0.23809523809523808, - "modern_chinese": 0.25, - "nutrition": 0.27586206896551724, - "philosophy": 0.24761904761904763, - "professional_accounting": 0.25142857142857145, - "professional_law": 0.2559241706161137, - "professional_medicine": 0.25, - "professional_psychology": 0.2543103448275862, - "public_relations": 0.23563218390804597, - "security_study": 0.2222222222222222, - "sociology": 0.25663716814159293, - "sports_science": 0.2727272727272727, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.23076923076923078, - "world_history": 0.2546583850931677, - "world_religions": 0.26875 - } - }, - "prompt_3": { - "accuracy": 0.2490070799516491, - "category_acc": { - "agronomy": 0.23668639053254437, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.23780487804878048, - "arts": 0.25, - "astronomy": 0.2727272727272727, - "business_ethics": 0.2631578947368421, - "chinese_civil_service_exam": 0.26875, - "chinese_driving_rule": 0.21374045801526717, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.22429906542056074, - "chinese_history": 0.25077399380804954, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.2569832402234637, - "clinical_knowledge": 0.24472573839662448, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.14953271028037382, - "college_engineering_hydrology": 0.27358490566037735, - "college_law": 0.32407407407407407, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.20754716981132076, - "college_medicine": 0.2271062271062271, - "computer_science": 0.23529411764705882, - "computer_security": 0.25146198830409355, - "conceptual_physics": 0.23129251700680273, - "construction_project_management": 0.2446043165467626, - "economics": 0.2578616352201258, - "education": 0.25153374233128833, - "electrical_engineering": 0.25, - "elementary_chinese": 0.20634920634920634, - "elementary_commonsense": 0.2727272727272727, - "elementary_information_and_technology": 0.22268907563025211, - "elementary_mathematics": 0.24347826086956523, - "ethnology": 0.2518518518518518, - "food_science": 0.26573426573426573, - "genetics": 0.2727272727272727, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.19696969696969696, - "high_school_geography": 0.23728813559322035, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.24475524475524477, - "human_sexuality": 0.2698412698412698, - "international_law": 0.25405405405405407, - "journalism": 0.2558139534883721, - "jurisprudence": 0.26034063260340634, - "legal_and_moral_basis": 0.2336448598130841, - "logical": 0.23577235772357724, - "machine_learning": 0.2540983606557377, - "management": 0.24761904761904763, - "marketing": 0.2388888888888889, - "marxist_theory": 0.23809523809523808, - "modern_chinese": 0.25, - "nutrition": 0.27586206896551724, - "philosophy": 0.24761904761904763, - "professional_accounting": 0.25142857142857145, - "professional_law": 0.2559241706161137, - "professional_medicine": 0.25, - "professional_psychology": 0.2543103448275862, - "public_relations": 0.23563218390804597, - "security_study": 0.2222222222222222, - "sociology": 0.25663716814159293, - "sports_science": 0.2727272727272727, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.23076923076923078, - "world_history": 0.2546583850931677, - "world_religions": 0.26875 - } - }, - "prompt_4": { - "accuracy": 0.24831635296149196, - "category_acc": { - "agronomy": 0.23668639053254437, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.23780487804878048, - "arts": 0.25, - "astronomy": 0.26666666666666666, - "business_ethics": 0.2679425837320574, - "chinese_civil_service_exam": 0.275, - "chinese_driving_rule": 0.22137404580152673, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.22429906542056074, - "chinese_history": 0.2476780185758514, - "chinese_literature": 0.2549019607843137, - "chinese_teacher_qualification": 0.24581005586592178, - "clinical_knowledge": 0.2489451476793249, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.14953271028037382, - "college_engineering_hydrology": 0.27358490566037735, - "college_law": 0.32407407407407407, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.2169811320754717, - "college_medicine": 0.2271062271062271, - "computer_science": 0.23039215686274508, - "computer_security": 0.25146198830409355, - "conceptual_physics": 0.23129251700680273, - "construction_project_management": 0.2446043165467626, - "economics": 0.2578616352201258, - "education": 0.25153374233128833, - "electrical_engineering": 0.23837209302325582, - "elementary_chinese": 0.20238095238095238, - "elementary_commonsense": 0.2727272727272727, - "elementary_information_and_technology": 0.2184873949579832, - "elementary_mathematics": 0.25217391304347825, - "ethnology": 0.24444444444444444, - "food_science": 0.26573426573426573, - "genetics": 0.26704545454545453, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.26627218934911245, - "high_school_chemistry": 0.21212121212121213, - "high_school_geography": 0.23728813559322035, - "high_school_mathematics": 0.25, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.23776223776223776, - "human_sexuality": 0.2698412698412698, - "international_law": 0.2594594594594595, - "journalism": 0.2558139534883721, - "jurisprudence": 0.26034063260340634, - "legal_and_moral_basis": 0.22897196261682243, - "logical": 0.23577235772357724, - "machine_learning": 0.2459016393442623, - "management": 0.2523809523809524, - "marketing": 0.2388888888888889, - "marxist_theory": 0.23809523809523808, - "modern_chinese": 0.25, - "nutrition": 0.2827586206896552, - "philosophy": 0.2571428571428571, - "professional_accounting": 0.24571428571428572, - "professional_law": 0.24644549763033174, - "professional_medicine": 0.24202127659574468, - "professional_psychology": 0.25, - "public_relations": 0.2413793103448276, - "security_study": 0.22962962962962963, - "sociology": 0.25663716814159293, - "sports_science": 0.2727272727272727, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.2485207100591716, - "world_history": 0.2546583850931677, - "world_religions": 0.26875 - } - }, - "prompt_5": { - "accuracy": 0.2487480573303402, - "category_acc": { - "agronomy": 0.23668639053254437, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.23780487804878048, - "arts": 0.25, - "astronomy": 0.2727272727272727, - "business_ethics": 0.2631578947368421, - "chinese_civil_service_exam": 0.26875, - "chinese_driving_rule": 0.21374045801526717, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.22429906542056074, - "chinese_history": 0.2476780185758514, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.2569832402234637, - "clinical_knowledge": 0.24472573839662448, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.14953271028037382, - "college_engineering_hydrology": 0.27358490566037735, - "college_law": 0.32407407407407407, - "college_mathematics": 0.29523809523809524, - "college_medical_statistics": 0.20754716981132076, - "college_medicine": 0.2271062271062271, - "computer_science": 0.24019607843137256, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.23129251700680273, - "construction_project_management": 0.2446043165467626, - "economics": 0.2578616352201258, - "education": 0.25153374233128833, - "electrical_engineering": 0.2558139534883721, - "elementary_chinese": 0.20634920634920634, - "elementary_commonsense": 0.2727272727272727, - "elementary_information_and_technology": 0.2184873949579832, - "elementary_mathematics": 0.24347826086956523, - "ethnology": 0.2518518518518518, - "food_science": 0.26573426573426573, - "genetics": 0.26704545454545453, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.19696969696969696, - "high_school_geography": 0.23728813559322035, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.24475524475524477, - "human_sexuality": 0.2698412698412698, - "international_law": 0.25405405405405407, - "journalism": 0.2558139534883721, - "jurisprudence": 0.26034063260340634, - "legal_and_moral_basis": 0.2336448598130841, - "logical": 0.23577235772357724, - "machine_learning": 0.2540983606557377, - "management": 0.24761904761904763, - "marketing": 0.2388888888888889, - "marxist_theory": 0.23809523809523808, - "modern_chinese": 0.25, - "nutrition": 0.27586206896551724, - "philosophy": 0.24761904761904763, - "professional_accounting": 0.25142857142857145, - "professional_law": 0.2559241706161137, - "professional_medicine": 0.25, - "professional_psychology": 0.2543103448275862, - "public_relations": 0.23563218390804597, - "security_study": 0.2222222222222222, - "sociology": 0.25663716814159293, - "sports_science": 0.2727272727272727, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.23076923076923078, - "world_history": 0.2546583850931677, - "world_religions": 0.26875 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.3939393939393939 - }, - "prompt_2": { - "accuracy": 0.3939393939393939 - }, - "prompt_3": { - "accuracy": 0.3939393939393939 - }, - "prompt_4": { - "accuracy": 0.42424242424242425 - }, - "prompt_5": { - "accuracy": 0.3939393939393939 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.19090909090909092 - }, - "prompt_2": { - "accuracy": 0.20909090909090908 - }, - "prompt_3": { - "accuracy": 0.20454545454545456 - }, - "prompt_4": { - "accuracy": 0.24772727272727274 - }, - "prompt_5": { - "accuracy": 0.2409090909090909 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.34508474576271186 - }, - "prompt_2": { - "accuracy": 0.32440677966101694 - }, - "prompt_3": { - "accuracy": 0.3233898305084746 - }, - "prompt_4": { - "accuracy": 0.3352542372881356 - }, - "prompt_5": { - "accuracy": 0.3352542372881356 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.28347045624532535 - }, - "prompt_2": { - "accuracy": 0.27898279730740466 - }, - "prompt_3": { - "accuracy": 0.27898279730740466 - }, - "prompt_4": { - "accuracy": 0.28272251308900526 - }, - "prompt_5": { - "accuracy": 0.27898279730740466 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.5095541401273885 - }, - "prompt_2": { - "accuracy": 0.5218030377266046 - }, - "prompt_3": { - "accuracy": 0.512983831455169 - }, - "prompt_4": { - "accuracy": 0.5154336109750123 - }, - "prompt_5": { - "accuracy": 0.5080842724154826 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.43481455633608285, - "rouge2": 0.19020306710468923, - "rougeL": 0.3555281762061519, - "avg_rouge": 0.326848599882308 - }, - "prompt_2": { - "rouge1": 0.43319141489410384, - "rouge2": 0.18633011055371998, - "rougeL": 0.3530126019631636, - "avg_rouge": 0.3241780424703291 - }, - "prompt_3": { - "rouge1": 0.4303838979441285, - "rouge2": 0.18417594383860397, - "rougeL": 0.34948463192329804, - "avg_rouge": 0.32134815790201016 - }, - "prompt_4": { - "rouge1": 0.43070227421856316, - "rouge2": 0.18614737183994354, - "rougeL": 0.3529860498921421, - "avg_rouge": 0.3232785653168829 - }, - "prompt_5": { - "rouge1": 0.4308048074328594, - "rouge2": 0.18586059514617964, - "rougeL": 0.3528154914643337, - "avg_rouge": 0.3231602980144576 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.23467138626961317, - "rouge2": 0.06744069122334906, - "rougeL": 0.1779908580361024, - "avg_rouge": 0.16003431184302153 - }, - "prompt_2": { - "rouge1": 0.2483010108284747, - "rouge2": 0.075125300962018, - "rougeL": 0.18996631825453364, - "avg_rouge": 0.17113087668167545 - }, - "prompt_3": { - "rouge1": 0.23828052499310123, - "rouge2": 0.07068930036983678, - "rougeL": 0.18168303338974037, - "avg_rouge": 0.16355095291755947 - }, - "prompt_4": { - "rouge1": 0.2552198528902313, - "rouge2": 0.07694264335968432, - "rougeL": 0.19993614328948328, - "avg_rouge": 0.17736621317979964 - }, - "prompt_5": { - "rouge1": 0.22558914523023577, - "rouge2": 0.0636998106706898, - "rougeL": 0.1683448949296202, - "avg_rouge": 0.15254461694351526 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.856651376146789 - }, - "prompt_2": { - "accuracy": 0.8486238532110092 - }, - "prompt_3": { - "accuracy": 0.8451834862385321 - }, - "prompt_4": { - "accuracy": 0.8577981651376146 - }, - "prompt_5": { - "accuracy": 0.8692660550458715 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.6720997123681688 - }, - "prompt_2": { - "accuracy": 0.6663470757430489 - }, - "prompt_3": { - "accuracy": 0.6816874400767018 - }, - "prompt_4": { - "accuracy": 0.663470757430489 - }, - "prompt_5": { - "accuracy": 0.6519654841802492 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.476 - }, - "prompt_2": { - "accuracy": 0.4805 - }, - "prompt_3": { - "accuracy": 0.4485 - }, - "prompt_4": { - "accuracy": 0.4995 - }, - "prompt_5": { - "accuracy": 0.493 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.362 - }, - "prompt_2": { - "accuracy": 0.3515 - }, - "prompt_3": { - "accuracy": 0.356 - }, - "prompt_4": { - "accuracy": 0.3595 - }, - "prompt_5": { - "accuracy": 0.353 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.4215 - }, - "prompt_2": { - "accuracy": 0.4625 - }, - "prompt_3": { - "accuracy": 0.4275 - }, - "prompt_4": { - "accuracy": 0.4375 - }, - "prompt_5": { - "accuracy": 0.442 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.5774647887323944 - }, - "prompt_2": { - "accuracy": 0.5774647887323944 - }, - "prompt_3": { - "accuracy": 0.5492957746478874 - }, - "prompt_4": { - "accuracy": 0.5633802816901409 - }, - "prompt_5": { - "accuracy": 0.5633802816901409 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.4657039711191336 - }, - "prompt_2": { - "accuracy": 0.47653429602888087 - }, - "prompt_3": { - "accuracy": 0.4259927797833935 - }, - "prompt_4": { - "accuracy": 0.44765342960288806 - }, - "prompt_5": { - "accuracy": 0.4548736462093863 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.6740196078431373 - }, - "prompt_2": { - "accuracy": 0.6666666666666666 - }, - "prompt_3": { - "accuracy": 0.6813725490196079 - }, - "prompt_4": { - "accuracy": 0.5563725490196079 - }, - "prompt_5": { - "accuracy": 0.6176470588235294 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.44327731092436984, - "language_acc": { - "English": 0.5815126050420169, - "Chinese": 0.30756302521008405, - "Spanish": 0.5470588235294118, - "Vietnamese": 0.3369747899159664 - }, - "consistency_score_2": 0.49355742296918764, - "consistency_score_3": 0.2968487394957983, - "consistency_score_4": 0.21008403361344538, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3630252100840336, - "English,Spanish": 0.6781512605042017, - "English,Vietnamese": 0.46554621848739497, - "Chinese,Spanish": 0.4008403361344538, - "Chinese,Vietnamese": 0.5403361344537815, - "Spanish,Vietnamese": 0.5134453781512605 - }, - "3_combine": { - "English,Chinese,Spanish": 0.2773109243697479, - "English,Chinese,Vietnamese": 0.2495798319327731, - "English,Spanish,Vietnamese": 0.3739495798319328, - "Chinese,Spanish,Vietnamese": 0.2865546218487395 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.21008403361344538 - } - }, - "AC3_2": 0.46706809494116786, - "AC3_3": 0.355578109708491, - "AC3_4": 0.285065794763038 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.2676190476190476, - "language_acc": { - "Spanish": 0.24, - "English": 0.2866666666666667, - "Chinese": 0.26666666666666666, - "Vietnamese": 0.30666666666666664, - "Filipino": 0.2733333333333333, - "Indonesian": 0.26, - "Malay": 0.24 - }, - "consistency_score_2": 0.4523809523809523, - "consistency_score_3": 0.2556190476190475, - "consistency_score_4": 0.164, - "consistency_score_5": 0.11365079365079363, - "consistency_score_6": 0.0819047619047619, - "consistency_score_7": 0.06, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.4866666666666667, - "Spanish,Chinese": 0.36, - "Spanish,Vietnamese": 0.4266666666666667, - "Spanish,Filipino": 0.49333333333333335, - "Spanish,Indonesian": 0.4533333333333333, - "Spanish,Malay": 0.5266666666666666, - "English,Chinese": 0.4066666666666667, - "English,Vietnamese": 0.42, - "English,Filipino": 0.5466666666666666, - "English,Indonesian": 0.43333333333333335, - "English,Malay": 0.46, - "Chinese,Vietnamese": 0.37333333333333335, - "Chinese,Filipino": 0.37333333333333335, - "Chinese,Indonesian": 0.3333333333333333, - "Chinese,Malay": 0.38666666666666666, - "Vietnamese,Filipino": 0.5133333333333333, - "Vietnamese,Indonesian": 0.49333333333333335, - "Vietnamese,Malay": 0.44, - "Filipino,Indonesian": 0.4666666666666667, - "Filipino,Malay": 0.52, - "Indonesian,Malay": 0.5866666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.22666666666666666, - "Spanish,English,Vietnamese": 0.26, - "Spanish,English,Filipino": 0.34, - "Spanish,English,Indonesian": 0.28, - "Spanish,English,Malay": 0.32, - "Spanish,Chinese,Vietnamese": 0.16666666666666666, - "Spanish,Chinese,Filipino": 0.18, - "Spanish,Chinese,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,Vietnamese,Indonesian": 0.24666666666666667, - "Spanish,Vietnamese,Malay": 0.26, - "Spanish,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Filipino,Malay": 0.3466666666666667, - "Spanish,Indonesian,Malay": 0.34, - "English,Chinese,Vietnamese": 0.2, - "English,Chinese,Filipino": 0.22666666666666666, - "English,Chinese,Indonesian": 0.2, - "English,Chinese,Malay": 0.22, - "English,Vietnamese,Filipino": 0.29333333333333333, - "English,Vietnamese,Indonesian": 0.23333333333333334, - "English,Vietnamese,Malay": 0.22, - "English,Filipino,Indonesian": 0.29333333333333333, - "English,Filipino,Malay": 0.32666666666666666, - "English,Indonesian,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "Chinese,Vietnamese,Malay": 0.18666666666666668, - "Chinese,Filipino,Indonesian": 0.17333333333333334, - "Chinese,Filipino,Malay": 0.22666666666666666, - "Chinese,Indonesian,Malay": 0.24, - "Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Vietnamese,Filipino,Malay": 0.3, - "Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Filipino,Indonesian,Malay": 0.35333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.11333333333333333, - "Spanish,English,Chinese,Filipino": 0.14, - "Spanish,English,Chinese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Malay": 0.16, - "Spanish,English,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Malay": 0.18, - "Spanish,English,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino": 0.12, - "Spanish,Chinese,Vietnamese,Indonesian": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.1, - "Spanish,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,Vietnamese,Filipino,Malay": 0.2, - "Spanish,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.14, - "English,Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "English,Chinese,Filipino,Indonesian": 0.12666666666666668, - "English,Chinese,Filipino,Malay": 0.16, - "English,Chinese,Indonesian,Malay": 0.16, - "English,Vietnamese,Filipino,Indonesian": 0.18, - "English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.16, - "English,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06 - } - }, - "AC3_2": 0.3362937767232597, - "AC3_3": 0.2614814429094452, - "AC3_4": 0.20337157982931595, - "AC3_5": 0.15954640969607073, - "AC3_6": 0.12542364081010002, - "AC3_7": 0.09802325578403358 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.275974025974026, - "language_acc": { - "English": 0.26704545454545453, - "Filipino": 0.29545454545454547, - "Spanish": 0.3352272727272727, - "Malay": 0.22727272727272727, - "Chinese": 0.25, - "Indonesian": 0.2897727272727273, - "Vietnamese": 0.26704545454545453 - }, - "consistency_score_2": 0.43127705627705637, - "consistency_score_3": 0.22337662337662337, - "consistency_score_4": 0.12483766233766236, - "consistency_score_5": 0.07142857142857142, - "consistency_score_6": 0.04058441558441559, - "consistency_score_7": 0.022727272727272728, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.48295454545454547, - "English,Spanish": 0.5909090909090909, - "English,Malay": 0.3977272727272727, - "English,Chinese": 0.2840909090909091, - "English,Indonesian": 0.4431818181818182, - "English,Vietnamese": 0.3181818181818182, - "Filipino,Spanish": 0.4772727272727273, - "Filipino,Malay": 0.5397727272727273, - "Filipino,Chinese": 0.32954545454545453, - "Filipino,Indonesian": 0.5284090909090909, - "Filipino,Vietnamese": 0.4375, - "Spanish,Malay": 0.44886363636363635, - "Spanish,Chinese": 0.38636363636363635, - "Spanish,Indonesian": 0.48863636363636365, - "Spanish,Vietnamese": 0.3409090909090909, - "Malay,Chinese": 0.3465909090909091, - "Malay,Indonesian": 0.5909090909090909, - "Malay,Vietnamese": 0.42045454545454547, - "Chinese,Indonesian": 0.3806818181818182, - "Chinese,Vietnamese": 0.3409090909090909, - "Indonesian,Vietnamese": 0.48295454545454547 - }, - "3_combine": { - "English,Filipino,Spanish": 0.3352272727272727, - "English,Filipino,Malay": 0.26704545454545453, - "English,Filipino,Chinese": 0.13636363636363635, - "English,Filipino,Indonesian": 0.29545454545454547, - "English,Filipino,Vietnamese": 0.1875, - "English,Spanish,Malay": 0.2840909090909091, - "English,Spanish,Chinese": 0.20454545454545456, - "English,Spanish,Indonesian": 0.3068181818181818, - "English,Spanish,Vietnamese": 0.1875, - "English,Malay,Chinese": 0.125, - "English,Malay,Indonesian": 0.2727272727272727, - "English,Malay,Vietnamese": 0.18181818181818182, - "English,Chinese,Indonesian": 0.14204545454545456, - "English,Chinese,Vietnamese": 0.09090909090909091, - "English,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Malay": 0.2897727272727273, - "Filipino,Spanish,Chinese": 0.17613636363636365, - "Filipino,Spanish,Indonesian": 0.30113636363636365, - "Filipino,Spanish,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese": 0.20454545454545456, - "Filipino,Malay,Indonesian": 0.3693181818181818, - "Filipino,Malay,Vietnamese": 0.26704545454545453, - "Filipino,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Indonesian,Vietnamese": 0.2897727272727273, - "Spanish,Malay,Chinese": 0.17613636363636365, - "Spanish,Malay,Indonesian": 0.3125, - "Spanish,Malay,Vietnamese": 0.19318181818181818, - "Spanish,Chinese,Indonesian": 0.20454545454545456, - "Spanish,Chinese,Vietnamese": 0.13068181818181818, - "Spanish,Indonesian,Vietnamese": 0.23295454545454544, - "Malay,Chinese,Indonesian": 0.22727272727272727, - "Malay,Chinese,Vietnamese": 0.17045454545454544, - "Malay,Indonesian,Vietnamese": 0.29545454545454547, - "Chinese,Indonesian,Vietnamese": 0.20454545454545456 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.20454545454545456, - "English,Filipino,Spanish,Chinese": 0.10795454545454546, - "English,Filipino,Spanish,Indonesian": 0.2215909090909091, - "English,Filipino,Spanish,Vietnamese": 0.125, - "English,Filipino,Malay,Chinese": 0.07954545454545454, - "English,Filipino,Malay,Indonesian": 0.19886363636363635, - "English,Filipino,Malay,Vietnamese": 0.125, - "English,Filipino,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese": 0.09659090909090909, - "English,Spanish,Malay,Indonesian": 0.19318181818181818, - "English,Spanish,Malay,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Indonesian": 0.10795454545454546, - "English,Spanish,Chinese,Vietnamese": 0.0625, - "English,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "English,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese": 0.11363636363636363, - "Filipino,Spanish,Malay,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 - } - }, - "AC3_2": 0.3365742902374893, - "AC3_3": 0.24690524035384456, - "AC3_4": 0.171910916096458, - "AC3_5": 0.11348464616225996, - "AC3_6": 0.07076257074021705, - "AC3_7": 0.041996047416770495 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.3300970873786408 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.26666666666666666 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.2803738317757009 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.26, - "category_acc": { - "brand": 0.4, - "demographics": 0.6, - "biology": 0.1, - "history": 0.26666666666666666, - "literature": 0.2, - "politics": 0.4, - "culture": 0.3, - "film": 0.1, - "law": 0.3, - "geography": 0.1 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.17248866676608138 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.03149974329141327 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.021228910954348285 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.0067501959892748 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.02882120937985038 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.2847141190198366 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.2940293171254916, - "category_acc": { - "high_school_european_history": 0.3597560975609756, - "business_ethics": 0.24242424242424243, - "clinical_knowledge": 0.32575757575757575, - "medical_genetics": 0.1919191919191919, - "high_school_us_history": 0.35960591133004927, - "high_school_physics": 0.24666666666666667, - "high_school_world_history": 0.3389830508474576, - "virology": 0.2606060606060606, - "high_school_microeconomics": 0.29957805907172996, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.32323232323232326, - "high_school_biology": 0.29449838187702265, - "abstract_algebra": 0.25252525252525254, - "professional_accounting": 0.2313167259786477, - "philosophy": 0.34838709677419355, - "professional_medicine": 0.3210332103321033, - "nutrition": 0.35081967213114756, - "global_facts": 0.23232323232323232, - "machine_learning": 0.24324324324324326, - "security_studies": 0.2581967213114754, - "public_relations": 0.29357798165137616, - "professional_psychology": 0.2864157119476268, - "prehistory": 0.3219814241486068, - "anatomy": 0.31343283582089554, - "human_sexuality": 0.27692307692307694, - "college_medicine": 0.3430232558139535, - "high_school_government_and_politics": 0.296875, - "college_chemistry": 0.29292929292929293, - "logical_fallacies": 0.3333333333333333, - "high_school_geography": 0.2639593908629442, - "elementary_mathematics": 0.2493368700265252, - "human_aging": 0.24324324324324326, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.30514705882352944, - "formal_logic": 0.288, - "high_school_statistics": 0.2837209302325581, - "international_law": 0.3333333333333333, - "high_school_mathematics": 0.24907063197026022, - "high_school_computer_science": 0.31313131313131315, - "conceptual_physics": 0.24786324786324787, - "miscellaneous": 0.2979539641943734, - "high_school_chemistry": 0.22277227722772278, - "marketing": 0.40772532188841204, - "professional_law": 0.28245270711024134, - "management": 0.28431372549019607, - "college_physics": 0.2376237623762376, - "jurisprudence": 0.2803738317757009, - "world_religions": 0.2823529411764706, - "sociology": 0.35, - "us_foreign_policy": 0.40404040404040403, - "high_school_macroeconomics": 0.31876606683804626, - "computer_security": 0.25252525252525254, - "moral_scenarios": 0.25838926174496646, - "moral_disputes": 0.336231884057971, - "electrical_engineering": 0.3541666666666667, - "astronomy": 0.3576158940397351, - "college_biology": 0.26573426573426573 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.2451708766716196 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.26774595267745954, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.21428571428571427, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.19047619047619047, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.21666666666666667, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.20408163265306123, - "high_school_politics": 0.4583333333333333, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.20833333333333334, - "logic": 0.4074074074074074, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.21052631578947367, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.08, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.2692307692307692, - "sports_science": 0.2916666666666667, - "plant_protection": 0.1111111111111111, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.35185185185185186, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.2962962962962963, - "physician": 0.25925925925925924 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.2114695340501792 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.25133828354342946, - "category_acc": { - "agronomy": 0.2603550295857988, - "anatomy": 0.25, - "ancient_chinese": 0.23170731707317074, - "arts": 0.25625, - "astronomy": 0.2606060606060606, - "business_ethics": 0.2535885167464115, - "chinese_civil_service_exam": 0.2875, - "chinese_driving_rule": 0.2595419847328244, - "chinese_food_culture": 0.25735294117647056, - "chinese_foreign_policy": 0.24299065420560748, - "chinese_history": 0.25386996904024767, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.24472573839662448, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.1588785046728972, - "college_engineering_hydrology": 0.24528301886792453, - "college_law": 0.32407407407407407, - "college_mathematics": 0.3142857142857143, - "college_medical_statistics": 0.2358490566037736, - "college_medicine": 0.2454212454212454, - "computer_science": 0.20098039215686275, - "computer_security": 0.25146198830409355, - "conceptual_physics": 0.24489795918367346, - "construction_project_management": 0.2589928057553957, - "economics": 0.2578616352201258, - "education": 0.25153374233128833, - "electrical_engineering": 0.23837209302325582, - "elementary_chinese": 0.20238095238095238, - "elementary_commonsense": 0.2777777777777778, - "elementary_information_and_technology": 0.23949579831932774, - "elementary_mathematics": 0.22608695652173913, - "ethnology": 0.24444444444444444, - "food_science": 0.25874125874125875, - "genetics": 0.25, - "global_facts": 0.2550335570469799, - "high_school_biology": 0.26627218934911245, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.23728813559322035, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.26573426573426573, - "human_sexuality": 0.25396825396825395, - "international_law": 0.2702702702702703, - "journalism": 0.2558139534883721, - "jurisprudence": 0.25304136253041365, - "legal_and_moral_basis": 0.2523364485981308, - "logical": 0.24390243902439024, - "machine_learning": 0.23770491803278687, - "management": 0.24761904761904763, - "marketing": 0.2388888888888889, - "marxist_theory": 0.25396825396825395, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.25517241379310346, - "philosophy": 0.2571428571428571, - "professional_accounting": 0.26285714285714284, - "professional_law": 0.26066350710900477, - "professional_medicine": 0.2473404255319149, - "professional_psychology": 0.2543103448275862, - "public_relations": 0.2413793103448276, - "security_study": 0.22962962962962963, - "sociology": 0.25663716814159293, - "sports_science": 0.2787878787878788, - "traditional_chinese_medicine": 0.25405405405405407, - "virology": 0.25443786982248523, - "world_history": 0.2732919254658385, - "world_religions": 0.25625 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.36363636363636365 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.18409090909090908 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3257627118644068 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.28272251308900526 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.49828515433610976 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.43494562646559415, - "rouge2": 0.18609956628432, - "rougeL": 0.3551068299617649, - "avg_rouge": 0.3253840075705597 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.3531392584790787, - "rouge2": 0.11980411381896806, - "rougeL": 0.2857943346980614, - "avg_rouge": 0.2529125689987027 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.5217889908256881 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.5023969319271333 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.495 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.3285 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.495 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.4788732394366197 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.49097472924187724 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.5392156862745098 - } - } - } - }, - "flan_t5_base": { - "model_size": "0.25B", - "model_link": "https://huggingface.co/google/flan-t5-base", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.7216386554621849, - "language_acc": { - "English": 0.9495798319327731, - "Chinese": 0.35294117647058826, - "Spanish": 0.8546218487394958, - "Vietnamese": 0.7294117647058823 - }, - "consistency_score_2": 0.561764705882353, - "consistency_score_3": 0.38613445378151257, - "consistency_score_4": 0.2605042016806723, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.35126050420168065, - "English,Spanish": 0.8630252100840337, - "English,Vietnamese": 0.7260504201680672, - "Chinese,Spanish": 0.35126050420168065, - "Chinese,Vietnamese": 0.35714285714285715, - "Spanish,Vietnamese": 0.7218487394957983 - }, - "3_combine": { - "English,Chinese,Spanish": 0.31512605042016806, - "English,Chinese,Vietnamese": 0.27647058823529413, - "English,Spanish,Vietnamese": 0.6756302521008404, - "Chinese,Spanish,Vietnamese": 0.2773109243697479 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.2605042016806723 - } - }, - "AC3_2": 0.6317439072042977, - "AC3_3": 0.5030805418622274, - "AC3_4": 0.3828158000778606 - }, - "prompt_2": { - "overall_acc": 0.7195378151260504, - "language_acc": { - "English": 0.9470588235294117, - "Chinese": 0.34705882352941175, - "Spanish": 0.8554621848739495, - "Vietnamese": 0.7285714285714285 - }, - "consistency_score_2": 0.5621848739495798, - "consistency_score_3": 0.38844537815126046, - "consistency_score_4": 0.2689075630252101, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.34705882352941175, - "English,Spanish": 0.8571428571428571, - "English,Vietnamese": 0.7243697478991596, - "Chinese,Spanish": 0.3445378151260504, - "Chinese,Vietnamese": 0.3739495798319328, - "Spanish,Vietnamese": 0.7260504201680672 - }, - "3_combine": { - "English,Chinese,Spanish": 0.31092436974789917, - "English,Chinese,Vietnamese": 0.2848739495798319, - "English,Spanish,Vietnamese": 0.673109243697479, - "Chinese,Spanish,Vietnamese": 0.2848739495798319 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.2689075630252101 - } - }, - "AC3_2": 0.6312024891418078, - "AC3_3": 0.5045223435904401, - "AC3_4": 0.3915019780098478 - }, - "prompt_3": { - "overall_acc": 0.7161764705882353, - "language_acc": { - "English": 0.9453781512605042, - "Chinese": 0.3495798319327731, - "Spanish": 0.8495798319327731, - "Vietnamese": 0.7201680672268908 - }, - "consistency_score_2": 0.557843137254902, - "consistency_score_3": 0.37899159663865545, - "consistency_score_4": 0.2571428571428571, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.346218487394958, - "English,Spanish": 0.8579831932773109, - "English,Vietnamese": 0.7142857142857143, - "Chinese,Spanish": 0.34369747899159664, - "Chinese,Vietnamese": 0.3680672268907563, - "Spanish,Vietnamese": 0.7168067226890756 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3058823529411765, - "English,Chinese,Vietnamese": 0.27563025210084036, - "English,Spanish,Vietnamese": 0.6655462184873949, - "Chinese,Spanish,Vietnamese": 0.2689075630252101 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.2571428571428571 - } - }, - "AC3_2": 0.6271710838518202, - "AC3_3": 0.4956770968053732, - "AC3_4": 0.37841571332179447 - }, - "prompt_4": { - "overall_acc": 0.7258403361344538, - "language_acc": { - "English": 0.9487394957983193, - "Chinese": 0.3689075630252101, - "Spanish": 0.8588235294117647, - "Vietnamese": 0.726890756302521 - }, - "consistency_score_2": 0.5683473389355743, - "consistency_score_3": 0.3978991596638656, - "consistency_score_4": 0.2747899159663866, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3663865546218487, - "English,Spanish": 0.8638655462184874, - "English,Vietnamese": 0.726890756302521, - "Chinese,Spanish": 0.3680672268907563, - "Chinese,Vietnamese": 0.36050420168067226, - "Spanish,Vietnamese": 0.7243697478991596 - }, - "3_combine": { - "English,Chinese,Spanish": 0.33277310924369746, - "English,Chinese,Vietnamese": 0.28991596638655465, - "English,Spanish,Vietnamese": 0.6773109243697479, - "Chinese,Spanish,Vietnamese": 0.2915966386554622 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.2747899159663866 - } - }, - "AC3_2": 0.6375109753381358, - "AC3_3": 0.5140181703183097, - "AC3_4": 0.3986559561490596 - }, - "prompt_5": { - "overall_acc": 0.7014705882352941, - "language_acc": { - "English": 0.9403361344537815, - "Chinese": 0.3226890756302521, - "Spanish": 0.8445378151260504, - "Vietnamese": 0.6983193277310924 - }, - "consistency_score_2": 0.5317927170868347, - "consistency_score_3": 0.3464285714285714, - "consistency_score_4": 0.22268907563025211, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.32100840336134456, - "English,Spanish": 0.8436974789915966, - "English,Vietnamese": 0.6882352941176471, - "Chinese,Spanish": 0.32100840336134456, - "Chinese,Vietnamese": 0.32605042016806723, - "Spanish,Vietnamese": 0.6907563025210084 - }, - "3_combine": { - "English,Chinese,Spanish": 0.27899159663865547, - "English,Chinese,Vietnamese": 0.23697478991596638, - "English,Spanish,Vietnamese": 0.6310924369747899, - "Chinese,Spanish,Vietnamese": 0.23865546218487396 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.22268907563025211 - } - }, - "AC3_2": 0.604959133113041, - "AC3_3": 0.46380312746175417, - "AC3_4": 0.33805811477560993 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.3142857142857142, - "language_acc": { - "Spanish": 0.3333333333333333, - "English": 0.41333333333333333, - "Chinese": 0.30666666666666664, - "Vietnamese": 0.3, - "Filipino": 0.2866666666666667, - "Indonesian": 0.2733333333333333, - "Malay": 0.2866666666666667 - }, - "consistency_score_2": 0.4253968253968254, - "consistency_score_3": 0.23695238095238094, - "consistency_score_4": 0.15295238095238092, - "consistency_score_5": 0.10603174603174603, - "consistency_score_6": 0.07523809523809524, - "consistency_score_7": 0.05333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.4866666666666667, - "Spanish,Chinese": 0.24666666666666667, - "Spanish,Vietnamese": 0.43333333333333335, - "Spanish,Filipino": 0.4266666666666667, - "Spanish,Indonesian": 0.4533333333333333, - "Spanish,Malay": 0.41333333333333333, - "English,Chinese": 0.32666666666666666, - "English,Vietnamese": 0.36666666666666664, - "English,Filipino": 0.52, - "English,Indonesian": 0.41333333333333333, - "English,Malay": 0.42, - "Chinese,Vietnamese": 0.36666666666666664, - "Chinese,Filipino": 0.38666666666666666, - "Chinese,Indonesian": 0.31333333333333335, - "Chinese,Malay": 0.37333333333333335, - "Vietnamese,Filipino": 0.44, - "Vietnamese,Indonesian": 0.41333333333333333, - "Vietnamese,Malay": 0.46, - "Filipino,Indonesian": 0.5066666666666667, - "Filipino,Malay": 0.5933333333333334, - "Indonesian,Malay": 0.5733333333333334 - }, - "3_combine": { - "Spanish,English,Chinese": 0.14, - "Spanish,English,Vietnamese": 0.26, - "Spanish,English,Filipino": 0.30666666666666664, - "Spanish,English,Indonesian": 0.26666666666666666, - "Spanish,English,Malay": 0.26, - "Spanish,Chinese,Vietnamese": 0.16666666666666666, - "Spanish,Chinese,Filipino": 0.14, - "Spanish,Chinese,Indonesian": 0.13333333333333333, - "Spanish,Chinese,Malay": 0.14, - "Spanish,Vietnamese,Filipino": 0.2733333333333333, - "Spanish,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,Vietnamese,Malay": 0.2733333333333333, - "Spanish,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Filipino,Malay": 0.30666666666666664, - "Spanish,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Vietnamese": 0.14666666666666667, - "English,Chinese,Filipino": 0.19333333333333333, - "English,Chinese,Indonesian": 0.14, - "English,Chinese,Malay": 0.16, - "English,Vietnamese,Filipino": 0.26666666666666666, - "English,Vietnamese,Indonesian": 0.21333333333333335, - "English,Vietnamese,Malay": 0.24, - "English,Filipino,Indonesian": 0.29333333333333333, - "English,Filipino,Malay": 0.3333333333333333, - "English,Indonesian,Malay": 0.2866666666666667, - "Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "Chinese,Vietnamese,Malay": 0.21333333333333335, - "Chinese,Filipino,Indonesian": 0.19333333333333333, - "Chinese,Filipino,Malay": 0.25333333333333335, - "Chinese,Indonesian,Malay": 0.20666666666666667, - "Vietnamese,Filipino,Indonesian": 0.26, - "Vietnamese,Filipino,Malay": 0.32666666666666666, - "Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Filipino,Indonesian,Malay": 0.38666666666666666 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.10666666666666667, - "Spanish,English,Chinese,Filipino": 0.1, - "Spanish,English,Chinese,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Malay": 0.09333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,English,Vietnamese,Malay": 0.2, - "Spanish,English,Filipino,Indonesian": 0.19333333333333333, - "Spanish,English,Filipino,Malay": 0.22, - "Spanish,English,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.11333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.1, - "Spanish,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.22, - "English,Chinese,Vietnamese,Filipino": 0.12, - "English,Chinese,Vietnamese,Indonesian": 0.09333333333333334, - "English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "English,Chinese,Filipino,Indonesian": 0.11333333333333333, - "English,Chinese,Filipino,Malay": 0.14, - "English,Chinese,Indonesian,Malay": 0.1, - "English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "English,Vietnamese,Filipino,Malay": 0.20666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.18, - "English,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.22 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.08, - "Spanish,English,Chinese,Vietnamese,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.08, - "Spanish,English,Chinese,Indonesian,Malay": 0.06666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Filipino,Indonesian,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.07333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.08, - "English,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.06, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.05333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334 - } - }, - "AC3_2": 0.36149601466602704, - "AC3_3": 0.2701944910162422, - "AC3_4": 0.20576553486947827, - "AC3_5": 0.15856711260793693, - "AC3_6": 0.12141110719900902, - "AC3_7": 0.09119170981975355 - }, - "prompt_2": { - "overall_acc": 0.31428571428571433, - "language_acc": { - "Spanish": 0.3333333333333333, - "English": 0.41333333333333333, - "Chinese": 0.24666666666666667, - "Vietnamese": 0.3, - "Filipino": 0.3, - "Indonesian": 0.3, - "Malay": 0.30666666666666664 - }, - "consistency_score_2": 0.4161904761904762, - "consistency_score_3": 0.22742857142857145, - "consistency_score_4": 0.14419047619047623, - "consistency_score_5": 0.1, - "consistency_score_6": 0.07238095238095237, - "consistency_score_7": 0.05333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.4666666666666667, - "Spanish,Chinese": 0.3, - "Spanish,Vietnamese": 0.44666666666666666, - "Spanish,Filipino": 0.48, - "Spanish,Indonesian": 0.4866666666666667, - "Spanish,Malay": 0.44666666666666666, - "English,Chinese": 0.26666666666666666, - "English,Vietnamese": 0.36666666666666664, - "English,Filipino": 0.5133333333333333, - "English,Indonesian": 0.38, - "English,Malay": 0.38, - "Chinese,Vietnamese": 0.28, - "Chinese,Filipino": 0.2866666666666667, - "Chinese,Indonesian": 0.32, - "Chinese,Malay": 0.30666666666666664, - "Vietnamese,Filipino": 0.41333333333333333, - "Vietnamese,Indonesian": 0.44, - "Vietnamese,Malay": 0.5133333333333333, - "Filipino,Indonesian": 0.48, - "Filipino,Malay": 0.5333333333333333, - "Indonesian,Malay": 0.6333333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.15333333333333332, - "Spanish,English,Vietnamese": 0.26666666666666666, - "Spanish,English,Filipino": 0.32, - "Spanish,English,Indonesian": 0.25333333333333335, - "Spanish,English,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese": 0.13333333333333333, - "Spanish,Chinese,Filipino": 0.16, - "Spanish,Chinese,Indonesian": 0.16, - "Spanish,Chinese,Malay": 0.14, - "Spanish,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,Vietnamese,Malay": 0.30666666666666664, - "Spanish,Filipino,Indonesian": 0.3, - "Spanish,Filipino,Malay": 0.3, - "Spanish,Indonesian,Malay": 0.34, - "English,Chinese,Vietnamese": 0.09333333333333334, - "English,Chinese,Filipino": 0.15333333333333332, - "English,Chinese,Indonesian": 0.11333333333333333, - "English,Chinese,Malay": 0.12666666666666668, - "English,Vietnamese,Filipino": 0.24, - "English,Vietnamese,Indonesian": 0.22, - "English,Vietnamese,Malay": 0.24, - "English,Filipino,Indonesian": 0.28, - "English,Filipino,Malay": 0.3, - "English,Indonesian,Malay": 0.2866666666666667, - "Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Chinese,Vietnamese,Malay": 0.18, - "Chinese,Filipino,Indonesian": 0.17333333333333334, - "Chinese,Filipino,Malay": 0.16666666666666666, - "Chinese,Indonesian,Malay": 0.19333333333333333, - "Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Vietnamese,Filipino,Malay": 0.31333333333333335, - "Vietnamese,Indonesian,Malay": 0.3466666666666667, - "Filipino,Indonesian,Malay": 0.37333333333333335 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.07333333333333333, - "Spanish,English,Chinese,Filipino": 0.09333333333333334, - "Spanish,English,Chinese,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Malay": 0.08, - "Spanish,English,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.18, - "Spanish,English,Vietnamese,Malay": 0.2, - "Spanish,English,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.1, - "Spanish,Chinese,Vietnamese,Malay": 0.10666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.1, - "Spanish,Chinese,Indonesian,Malay": 0.11333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.07333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.08, - "English,Chinese,Vietnamese,Malay": 0.08666666666666667, - "English,Chinese,Filipino,Indonesian": 0.08666666666666667, - "English,Chinese,Filipino,Malay": 0.10666666666666667, - "English,Chinese,Indonesian,Malay": 0.09333333333333334, - "English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "English,Vietnamese,Filipino,Malay": 0.2, - "English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "English,Filipino,Indonesian,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.22 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.06, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.06, - "English,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.08, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.05333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.06, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.06, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334 - } - }, - "AC3_2": 0.35813000553860586, - "AC3_3": 0.2638939119464632, - "AC3_4": 0.1976853225279546, - "AC3_5": 0.15172413789441141, - "AC3_6": 0.11766361714057606, - "AC3_7": 0.09119170981975355 - }, - "prompt_3": { - "overall_acc": 0.3219047619047619, - "language_acc": { - "Spanish": 0.34, - "English": 0.4, - "Chinese": 0.28, - "Vietnamese": 0.32666666666666666, - "Filipino": 0.3, - "Indonesian": 0.2866666666666667, - "Malay": 0.32 - }, - "consistency_score_2": 0.44095238095238093, - "consistency_score_3": 0.25904761904761914, - "consistency_score_4": 0.17523809523809525, - "consistency_score_5": 0.1276190476190476, - "consistency_score_6": 0.09619047619047619, - "consistency_score_7": 0.07333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5066666666666667, - "Spanish,Chinese": 0.34, - "Spanish,Vietnamese": 0.4666666666666667, - "Spanish,Filipino": 0.47333333333333333, - "Spanish,Indonesian": 0.4533333333333333, - "Spanish,Malay": 0.41333333333333333, - "English,Chinese": 0.3466666666666667, - "English,Vietnamese": 0.38, - "English,Filipino": 0.5466666666666666, - "English,Indonesian": 0.4266666666666667, - "English,Malay": 0.44, - "Chinese,Vietnamese": 0.32, - "Chinese,Filipino": 0.35333333333333333, - "Chinese,Indonesian": 0.37333333333333335, - "Chinese,Malay": 0.37333333333333335, - "Vietnamese,Filipino": 0.41333333333333333, - "Vietnamese,Indonesian": 0.46, - "Vietnamese,Malay": 0.5266666666666666, - "Filipino,Indonesian": 0.47333333333333333, - "Filipino,Malay": 0.5266666666666666, - "Indonesian,Malay": 0.6466666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.22666666666666666, - "Spanish,English,Vietnamese": 0.29333333333333333, - "Spanish,English,Filipino": 0.3466666666666667, - "Spanish,English,Indonesian": 0.28, - "Spanish,English,Malay": 0.26, - "Spanish,Chinese,Vietnamese": 0.16, - "Spanish,Chinese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Indonesian": 0.2, - "Spanish,Chinese,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,Vietnamese,Indonesian": 0.3, - "Spanish,Vietnamese,Malay": 0.3, - "Spanish,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Filipino,Malay": 0.29333333333333333, - "Spanish,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese": 0.16666666666666666, - "English,Chinese,Filipino": 0.24, - "English,Chinese,Indonesian": 0.18666666666666668, - "English,Chinese,Malay": 0.20666666666666667, - "English,Vietnamese,Filipino": 0.28, - "English,Vietnamese,Indonesian": 0.25333333333333335, - "English,Vietnamese,Malay": 0.2733333333333333, - "English,Filipino,Indonesian": 0.30666666666666664, - "English,Filipino,Malay": 0.3333333333333333, - "English,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino": 0.16666666666666666, - "Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "Chinese,Vietnamese,Malay": 0.2, - "Chinese,Filipino,Indonesian": 0.20666666666666667, - "Chinese,Filipino,Malay": 0.21333333333333335, - "Chinese,Indonesian,Malay": 0.24, - "Vietnamese,Filipino,Indonesian": 0.26, - "Vietnamese,Filipino,Malay": 0.32, - "Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Filipino,Indonesian,Malay": 0.38 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.12, - "Spanish,English,Chinese,Filipino": 0.16666666666666666, - "Spanish,English,Chinese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino": 0.22, - "Spanish,English,Vietnamese,Indonesian": 0.21333333333333335, - "Spanish,English,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.14, - "Spanish,Chinese,Filipino,Malay": 0.14, - "Spanish,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.14, - "English,Chinese,Vietnamese,Indonesian": 0.12, - "English,Chinese,Vietnamese,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian": 0.15333333333333332, - "English,Chinese,Filipino,Malay": 0.17333333333333334, - "English,Chinese,Indonesian,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "English,Vietnamese,Filipino,Malay": 0.23333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.22, - "English,Filipino,Indonesian,Malay": 0.26, - "Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.24 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,English,Vietnamese,Filipino,Malay": 0.18, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,English,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "English,Chinese,Vietnamese,Filipino,Malay": 0.12, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.08, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 - } - }, - "AC3_2": 0.37213958737170394, - "AC3_3": 0.28707572204270093, - "AC3_4": 0.22693669033931807, - "AC3_5": 0.18277643256628107, - "AC3_6": 0.14812018653578424, - "AC3_7": 0.11945381523082091 - }, - "prompt_4": { - "overall_acc": 0.3304761904761905, - "language_acc": { - "Spanish": 0.3466666666666667, - "English": 0.4066666666666667, - "Chinese": 0.30666666666666664, - "Vietnamese": 0.3333333333333333, - "Filipino": 0.30666666666666664, - "Indonesian": 0.30666666666666664, - "Malay": 0.30666666666666664 - }, - "consistency_score_2": 0.4615873015873015, - "consistency_score_3": 0.28019047619047616, - "consistency_score_4": 0.1998095238095238, - "consistency_score_5": 0.15746031746031747, - "consistency_score_6": 0.13142857142857142, - "consistency_score_7": 0.11333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.49333333333333335, - "Spanish,Chinese": 0.31333333333333335, - "Spanish,Vietnamese": 0.4533333333333333, - "Spanish,Filipino": 0.47333333333333333, - "Spanish,Indonesian": 0.49333333333333335, - "Spanish,Malay": 0.44666666666666666, - "English,Chinese": 0.37333333333333335, - "English,Vietnamese": 0.38, - "English,Filipino": 0.5266666666666666, - "English,Indonesian": 0.42, - "English,Malay": 0.4266666666666667, - "Chinese,Vietnamese": 0.42, - "Chinese,Filipino": 0.38666666666666666, - "Chinese,Indonesian": 0.4066666666666667, - "Chinese,Malay": 0.4533333333333333, - "Vietnamese,Filipino": 0.44666666666666666, - "Vietnamese,Indonesian": 0.4866666666666667, - "Vietnamese,Malay": 0.5333333333333333, - "Filipino,Indonesian": 0.5, - "Filipino,Malay": 0.5933333333333334, - "Indonesian,Malay": 0.6666666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.19333333333333333, - "Spanish,English,Vietnamese": 0.28, - "Spanish,English,Filipino": 0.34, - "Spanish,English,Indonesian": 0.2866666666666667, - "Spanish,English,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,Chinese,Filipino": 0.18666666666666668, - "Spanish,Chinese,Indonesian": 0.20666666666666667, - "Spanish,Chinese,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,Vietnamese,Malay": 0.32, - "Spanish,Filipino,Indonesian": 0.31333333333333335, - "Spanish,Filipino,Malay": 0.31333333333333335, - "Spanish,Indonesian,Malay": 0.36, - "English,Chinese,Vietnamese": 0.18666666666666668, - "English,Chinese,Filipino": 0.22666666666666666, - "English,Chinese,Indonesian": 0.21333333333333335, - "English,Chinese,Malay": 0.20666666666666667, - "English,Vietnamese,Filipino": 0.26, - "English,Vietnamese,Indonesian": 0.24, - "English,Vietnamese,Malay": 0.26666666666666666, - "English,Filipino,Indonesian": 0.30666666666666664, - "English,Filipino,Malay": 0.34, - "English,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino": 0.22666666666666666, - "Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "Chinese,Vietnamese,Malay": 0.3, - "Chinese,Filipino,Indonesian": 0.24, - "Chinese,Filipino,Malay": 0.29333333333333333, - "Chinese,Indonesian,Malay": 0.31333333333333335, - "Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Vietnamese,Filipino,Malay": 0.36, - "Vietnamese,Indonesian,Malay": 0.4, - "Filipino,Indonesian,Malay": 0.42 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.14666666666666667, - "Spanish,English,Chinese,Filipino": 0.14666666666666667, - "Spanish,English,Chinese,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino": 0.22, - "Spanish,English,Vietnamese,Indonesian": 0.22, - "Spanish,English,Vietnamese,Malay": 0.22666666666666666, - "Spanish,English,Filipino,Indonesian": 0.24, - "Spanish,English,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Malay": 0.17333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Indonesian,Malay": 0.18, - "Spanish,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.24, - "Spanish,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino": 0.14, - "English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "English,Chinese,Vietnamese,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian": 0.16666666666666666, - "English,Chinese,Filipino,Malay": 0.18, - "English,Chinese,Indonesian,Malay": 0.18, - "English,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "English,Vietnamese,Filipino,Malay": 0.23333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.22, - "English,Filipino,Indonesian,Malay": 0.2733333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.28 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.2, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "English,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.16, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - } - }, - "AC3_2": 0.3851802652438598, - "AC3_3": 0.3032629295763379, - "AC3_4": 0.24904419809206157, - "AC3_5": 0.21329367656811543, - "AC3_6": 0.18806480113748816, - "AC3_7": 0.16878397707212664 - }, - "prompt_5": { - "overall_acc": 0.29904761904761906, - "language_acc": { - "Spanish": 0.29333333333333333, - "English": 0.37333333333333335, - "Chinese": 0.2733333333333333, - "Vietnamese": 0.29333333333333333, - "Filipino": 0.3, - "Indonesian": 0.26666666666666666, - "Malay": 0.29333333333333333 - }, - "consistency_score_2": 0.44634920634920633, - "consistency_score_3": 0.2615238095238095, - "consistency_score_4": 0.17485714285714285, - "consistency_score_5": 0.12476190476190474, - "consistency_score_6": 0.09142857142857143, - "consistency_score_7": 0.06666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.52, - "Spanish,Chinese": 0.3, - "Spanish,Vietnamese": 0.43333333333333335, - "Spanish,Filipino": 0.44666666666666666, - "Spanish,Indonesian": 0.47333333333333333, - "Spanish,Malay": 0.4533333333333333, - "English,Chinese": 0.2866666666666667, - "English,Vietnamese": 0.35333333333333333, - "English,Filipino": 0.49333333333333335, - "English,Indonesian": 0.3933333333333333, - "English,Malay": 0.43333333333333335, - "Chinese,Vietnamese": 0.43333333333333335, - "Chinese,Filipino": 0.36666666666666664, - "Chinese,Indonesian": 0.4066666666666667, - "Chinese,Malay": 0.44666666666666666, - "Vietnamese,Filipino": 0.44666666666666666, - "Vietnamese,Indonesian": 0.5066666666666667, - "Vietnamese,Malay": 0.5133333333333333, - "Filipino,Indonesian": 0.49333333333333335, - "Filipino,Malay": 0.5133333333333333, - "Indonesian,Malay": 0.66 - }, - "3_combine": { - "Spanish,English,Chinese": 0.16666666666666666, - "Spanish,English,Vietnamese": 0.25333333333333335, - "Spanish,English,Filipino": 0.3, - "Spanish,English,Indonesian": 0.2733333333333333, - "Spanish,English,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese": 0.18666666666666668, - "Spanish,Chinese,Filipino": 0.18, - "Spanish,Chinese,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Filipino": 0.26, - "Spanish,Vietnamese,Indonesian": 0.28, - "Spanish,Vietnamese,Malay": 0.29333333333333333, - "Spanish,Filipino,Indonesian": 0.3, - "Spanish,Filipino,Malay": 0.29333333333333333, - "Spanish,Indonesian,Malay": 0.35333333333333333, - "English,Chinese,Vietnamese": 0.14666666666666667, - "English,Chinese,Filipino": 0.2, - "English,Chinese,Indonesian": 0.17333333333333334, - "English,Chinese,Malay": 0.18, - "English,Vietnamese,Filipino": 0.26, - "English,Vietnamese,Indonesian": 0.24, - "English,Vietnamese,Malay": 0.24666666666666667, - "English,Filipino,Indonesian": 0.26666666666666666, - "English,Filipino,Malay": 0.30666666666666664, - "English,Indonesian,Malay": 0.32666666666666666, - "Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Chinese,Vietnamese,Malay": 0.28, - "Chinese,Filipino,Indonesian": 0.23333333333333334, - "Chinese,Filipino,Malay": 0.24, - "Chinese,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Vietnamese,Filipino,Malay": 0.32666666666666666, - "Vietnamese,Indonesian,Malay": 0.3933333333333333, - "Filipino,Indonesian,Malay": 0.36666666666666664 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.10666666666666667, - "Spanish,English,Chinese,Filipino": 0.13333333333333333, - "Spanish,English,Chinese,Indonesian": 0.12, - "Spanish,English,Chinese,Malay": 0.12, - "Spanish,English,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.18, - "Spanish,English,Vietnamese,Malay": 0.2, - "Spanish,English,Filipino,Indonesian": 0.19333333333333333, - "Spanish,English,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian": 0.14, - "Spanish,Chinese,Vietnamese,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Malay": 0.22, - "Spanish,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.22666666666666666, - "English,Chinese,Vietnamese,Filipino": 0.12, - "English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian": 0.14, - "English,Chinese,Filipino,Malay": 0.14666666666666667, - "English,Chinese,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "English,Vietnamese,Filipino,Malay": 0.20666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "English,Filipino,Indonesian,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.18, - "Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.08, - "Spanish,English,Chinese,Vietnamese,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.12, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.08666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 - } - }, - "AC3_2": 0.3581439116904129, - "AC3_3": 0.2790298205090966, - "AC3_4": 0.22067983458825224, - "AC3_5": 0.17606848577975054, - "AC3_6": 0.14004181181082553, - "AC3_7": 0.10902777774796549 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.27516233766233766, - "language_acc": { - "English": 0.42045454545454547, - "Filipino": 0.26704545454545453, - "Spanish": 0.2897727272727273, - "Malay": 0.24431818181818182, - "Chinese": 0.23295454545454544, - "Indonesian": 0.24431818181818182, - "Vietnamese": 0.22727272727272727 - }, - "consistency_score_2": 0.3528138528138528, - "consistency_score_3": 0.15357142857142855, - "consistency_score_4": 0.07516233766233765, - "consistency_score_5": 0.03896103896103895, - "consistency_score_6": 0.022727272727272724, - "consistency_score_7": 0.017045454545454544, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.35795454545454547, - "English,Spanish": 0.3693181818181818, - "English,Malay": 0.2897727272727273, - "English,Chinese": 0.25, - "English,Indonesian": 0.26136363636363635, - "English,Vietnamese": 0.25, - "Filipino,Spanish": 0.4431818181818182, - "Filipino,Malay": 0.4772727272727273, - "Filipino,Chinese": 0.26136363636363635, - "Filipino,Indonesian": 0.48863636363636365, - "Filipino,Vietnamese": 0.4602272727272727, - "Spanish,Malay": 0.3693181818181818, - "Spanish,Chinese": 0.2556818181818182, - "Spanish,Indonesian": 0.42613636363636365, - "Spanish,Vietnamese": 0.3181818181818182, - "Malay,Chinese": 0.24431818181818182, - "Malay,Indonesian": 0.5454545454545454, - "Malay,Vietnamese": 0.4602272727272727, - "Chinese,Indonesian": 0.26704545454545453, - "Chinese,Vietnamese": 0.18181818181818182, - "Indonesian,Vietnamese": 0.4318181818181818 - }, - "3_combine": { - "English,Filipino,Spanish": 0.19886363636363635, - "English,Filipino,Malay": 0.17613636363636365, - "English,Filipino,Chinese": 0.10227272727272728, - "English,Filipino,Indonesian": 0.1534090909090909, - "English,Filipino,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay": 0.14204545454545456, - "English,Spanish,Chinese": 0.08522727272727272, - "English,Spanish,Indonesian": 0.1534090909090909, - "English,Spanish,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese": 0.09659090909090909, - "English,Malay,Indonesian": 0.14772727272727273, - "English,Malay,Vietnamese": 0.125, - "English,Chinese,Indonesian": 0.07386363636363637, - "English,Chinese,Vietnamese": 0.05113636363636364, - "English,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay": 0.2215909090909091, - "Filipino,Spanish,Chinese": 0.11363636363636363, - "Filipino,Spanish,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Chinese": 0.13068181818181818, - "Filipino,Malay,Indonesian": 0.3125, - "Filipino,Malay,Vietnamese": 0.2784090909090909, - "Filipino,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Indonesian,Vietnamese": 0.26704545454545453, - "Spanish,Malay,Chinese": 0.10227272727272728, - "Spanish,Malay,Indonesian": 0.24431818181818182, - "Spanish,Malay,Vietnamese": 0.17613636363636365, - "Spanish,Chinese,Indonesian": 0.11931818181818182, - "Spanish,Chinese,Vietnamese": 0.07954545454545454, - "Spanish,Indonesian,Vietnamese": 0.1875, - "Malay,Chinese,Indonesian": 0.1534090909090909, - "Malay,Chinese,Vietnamese": 0.07954545454545454, - "Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.10227272727272728, - "English,Filipino,Spanish,Chinese": 0.045454545454545456, - "English,Filipino,Spanish,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese": 0.0625, - "English,Filipino,Malay,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese": 0.045454545454545456, - "English,Spanish,Malay,Indonesian": 0.07954545454545454, - "English,Spanish,Malay,Vietnamese": 0.056818181818181816, - "English,Spanish,Chinese,Indonesian": 0.045454545454545456, - "English,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Spanish,Indonesian,Vietnamese": 0.0625, - "English,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese": 0.0625, - "Filipino,Spanish,Malay,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.0625, - "Filipino,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Malay,Chinese,Indonesian,Vietnamese": 0.0625 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.045454545454545456, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.03409090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544 - } - }, - "AC3_2": 0.309187150585567, - "AC3_3": 0.19712500671552963, - "AC3_4": 0.11807243957488045, - "AC3_5": 0.06825732404954808, - "AC3_6": 0.04198662371639825, - "AC3_7": 0.032102272716286614 - }, - "prompt_2": { - "overall_acc": 0.2743506493506494, - "language_acc": { - "English": 0.4147727272727273, - "Filipino": 0.2556818181818182, - "Spanish": 0.2840909090909091, - "Malay": 0.23863636363636365, - "Chinese": 0.2727272727272727, - "Indonesian": 0.22727272727272727, - "Vietnamese": 0.22727272727272727 - }, - "consistency_score_2": 0.3625541125541125, - "consistency_score_3": 0.16233766233766234, - "consistency_score_4": 0.0801948051948052, - "consistency_score_5": 0.041396103896103896, - "consistency_score_6": 0.021915584415584416, - "consistency_score_7": 0.011363636363636364, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.36363636363636365, - "English,Spanish": 0.375, - "English,Malay": 0.26136363636363635, - "English,Chinese": 0.26136363636363635, - "English,Indonesian": 0.29545454545454547, - "English,Vietnamese": 0.24431818181818182, - "Filipino,Spanish": 0.4090909090909091, - "Filipino,Malay": 0.4772727272727273, - "Filipino,Chinese": 0.22727272727272727, - "Filipino,Indonesian": 0.48295454545454547, - "Filipino,Vietnamese": 0.5738636363636364, - "Spanish,Malay": 0.375, - "Spanish,Chinese": 0.22727272727272727, - "Spanish,Indonesian": 0.45454545454545453, - "Spanish,Vietnamese": 0.3352272727272727, - "Malay,Chinese": 0.21022727272727273, - "Malay,Indonesian": 0.6022727272727273, - "Malay,Vietnamese": 0.5113636363636364, - "Chinese,Indonesian": 0.2215909090909091, - "Chinese,Vietnamese": 0.2215909090909091, - "Indonesian,Vietnamese": 0.48295454545454547 - }, - "3_combine": { - "English,Filipino,Spanish": 0.17613636363636365, - "English,Filipino,Malay": 0.1590909090909091, - "English,Filipino,Chinese": 0.09659090909090909, - "English,Filipino,Indonesian": 0.18181818181818182, - "English,Filipino,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay": 0.13636363636363635, - "English,Spanish,Chinese": 0.09090909090909091, - "English,Spanish,Indonesian": 0.18181818181818182, - "English,Spanish,Vietnamese": 0.11931818181818182, - "English,Malay,Chinese": 0.0625, - "English,Malay,Indonesian": 0.16477272727272727, - "English,Malay,Vietnamese": 0.11931818181818182, - "English,Chinese,Indonesian": 0.0625, - "English,Chinese,Vietnamese": 0.056818181818181816, - "English,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Malay": 0.2159090909090909, - "Filipino,Spanish,Chinese": 0.10227272727272728, - "Filipino,Spanish,Indonesian": 0.25, - "Filipino,Spanish,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese": 0.10227272727272728, - "Filipino,Malay,Indonesian": 0.3352272727272727, - "Filipino,Malay,Vietnamese": 0.3409090909090909, - "Filipino,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Indonesian,Vietnamese": 0.3181818181818182, - "Spanish,Malay,Chinese": 0.08522727272727272, - "Spanish,Malay,Indonesian": 0.2784090909090909, - "Spanish,Malay,Vietnamese": 0.19318181818181818, - "Spanish,Chinese,Indonesian": 0.09659090909090909, - "Spanish,Chinese,Vietnamese": 0.09659090909090909, - "Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "Malay,Chinese,Indonesian": 0.11931818181818182, - "Malay,Chinese,Vietnamese": 0.10795454545454546, - "Malay,Indonesian,Vietnamese": 0.3409090909090909, - "Chinese,Indonesian,Vietnamese": 0.10795454545454546 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.09659090909090909, - "English,Filipino,Spanish,Chinese": 0.05113636363636364, - "English,Filipino,Spanish,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese": 0.045454545454545456, - "English,Filipino,Malay,Indonesian": 0.11363636363636363, - "English,Filipino,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese": 0.022727272727272728, - "English,Spanish,Malay,Indonesian": 0.10227272727272728, - "English,Spanish,Malay,Vietnamese": 0.06818181818181818, - "English,Spanish,Chinese,Indonesian": 0.03409090909090909, - "English,Spanish,Chinese,Vietnamese": 0.03977272727272727, - "English,Spanish,Indonesian,Vietnamese": 0.08522727272727272, - "English,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese": 0.045454545454545456, - "Filipino,Spanish,Malay,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Malay,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Spanish,Malay,Chinese,Indonesian": 0.05113636363636364, - "Spanish,Malay,Chinese,Vietnamese": 0.05113636363636364, - "Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 - } - }, - "AC3_2": 0.31234483438585287, - "AC3_3": 0.2039781779091629, - "AC3_4": 0.12411100800457381, - "AC3_5": 0.07193770237764398, - "AC3_6": 0.040588863177902786, - "AC3_7": 0.021823347099799845 - }, - "prompt_3": { - "overall_acc": 0.27272727272727276, - "language_acc": { - "English": 0.4034090909090909, - "Filipino": 0.2556818181818182, - "Spanish": 0.30113636363636365, - "Malay": 0.23863636363636365, - "Chinese": 0.23863636363636365, - "Indonesian": 0.2215909090909091, - "Vietnamese": 0.25 - }, - "consistency_score_2": 0.3574134199134199, - "consistency_score_3": 0.15665584415584416, - "consistency_score_4": 0.07548701298701298, - "consistency_score_5": 0.03679653679653679, - "consistency_score_6": 0.01704545454545454, - "consistency_score_7": 0.005681818181818182, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3409090909090909, - "English,Spanish": 0.4147727272727273, - "English,Malay": 0.2784090909090909, - "English,Chinese": 0.23295454545454544, - "English,Indonesian": 0.29545454545454547, - "English,Vietnamese": 0.23863636363636365, - "Filipino,Spanish": 0.42045454545454547, - "Filipino,Malay": 0.4943181818181818, - "Filipino,Chinese": 0.23863636363636365, - "Filipino,Indonesian": 0.48295454545454547, - "Filipino,Vietnamese": 0.5397727272727273, - "Spanish,Malay": 0.36363636363636365, - "Spanish,Chinese": 0.2556818181818182, - "Spanish,Indonesian": 0.4318181818181818, - "Spanish,Vietnamese": 0.3125, - "Malay,Chinese": 0.17613636363636365, - "Malay,Indonesian": 0.5738636363636364, - "Malay,Vietnamese": 0.5397727272727273, - "Chinese,Indonesian": 0.17613636363636365, - "Chinese,Vietnamese": 0.19886363636363635, - "Indonesian,Vietnamese": 0.5 - }, - "3_combine": { - "English,Filipino,Spanish": 0.19318181818181818, - "English,Filipino,Malay": 0.17045454545454544, - "English,Filipino,Chinese": 0.07386363636363637, - "English,Filipino,Indonesian": 0.18181818181818182, - "English,Filipino,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay": 0.1590909090909091, - "English,Spanish,Chinese": 0.09090909090909091, - "English,Spanish,Indonesian": 0.18181818181818182, - "English,Spanish,Vietnamese": 0.125, - "English,Malay,Chinese": 0.056818181818181816, - "English,Malay,Indonesian": 0.17613636363636365, - "English,Malay,Vietnamese": 0.13636363636363635, - "English,Chinese,Indonesian": 0.03977272727272727, - "English,Chinese,Vietnamese": 0.045454545454545456, - "English,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay": 0.2215909090909091, - "Filipino,Spanish,Chinese": 0.125, - "Filipino,Spanish,Indonesian": 0.25, - "Filipino,Spanish,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Chinese": 0.09090909090909091, - "Filipino,Malay,Indonesian": 0.3352272727272727, - "Filipino,Malay,Vietnamese": 0.3465909090909091, - "Filipino,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Indonesian,Vietnamese": 0.3181818181818182, - "Spanish,Malay,Chinese": 0.0625, - "Spanish,Malay,Indonesian": 0.2556818181818182, - "Spanish,Malay,Vietnamese": 0.19886363636363635, - "Spanish,Chinese,Indonesian": 0.07954545454545454, - "Spanish,Chinese,Vietnamese": 0.07954545454545454, - "Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "Malay,Chinese,Indonesian": 0.06818181818181818, - "Malay,Chinese,Vietnamese": 0.07954545454545454, - "Malay,Indonesian,Vietnamese": 0.3522727272727273, - "Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.11363636363636363, - "English,Filipino,Spanish,Chinese": 0.045454545454545456, - "English,Filipino,Spanish,Indonesian": 0.125, - "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese": 0.03409090909090909, - "English,Filipino,Malay,Indonesian": 0.11363636363636363, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese": 0.022727272727272728, - "English,Spanish,Malay,Indonesian": 0.10795454545454546, - "English,Spanish,Malay,Vietnamese": 0.08522727272727272, - "English,Spanish,Chinese,Indonesian": 0.022727272727272728, - "English,Spanish,Chinese,Vietnamese": 0.028409090909090908, - "English,Spanish,Indonesian,Vietnamese": 0.08522727272727272, - "English,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese": 0.03977272727272727, - "Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.05113636363636364, - "Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.05113636363636364, - "Filipino,Malay,Chinese,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Spanish,Malay,Chinese,Indonesian": 0.03409090909090909, - "Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 - } - }, - "AC3_2": 0.30937975716234767, - "AC3_3": 0.19900326511948974, - "AC3_4": 0.11824539093870602, - "AC3_5": 0.0648442466414788, - "AC3_6": 0.03208556148625353, - "AC3_7": 0.01113172541344137 - }, - "prompt_4": { - "overall_acc": 0.27516233766233766, - "language_acc": { - "English": 0.4090909090909091, - "Filipino": 0.2556818181818182, - "Spanish": 0.2784090909090909, - "Malay": 0.2556818181818182, - "Chinese": 0.24431818181818182, - "Indonesian": 0.25, - "Vietnamese": 0.23295454545454544 - }, - "consistency_score_2": 0.3603896103896103, - "consistency_score_3": 0.15892857142857142, - "consistency_score_4": 0.07483766233766233, - "consistency_score_5": 0.03490259740259739, - "consistency_score_6": 0.01542207792207792, - "consistency_score_7": 0.005681818181818182, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.35795454545454547, - "English,Spanish": 0.39204545454545453, - "English,Malay": 0.2727272727272727, - "English,Chinese": 0.26704545454545453, - "English,Indonesian": 0.30113636363636365, - "English,Vietnamese": 0.25, - "Filipino,Spanish": 0.3977272727272727, - "Filipino,Malay": 0.4943181818181818, - "Filipino,Chinese": 0.17613636363636365, - "Filipino,Indonesian": 0.4943181818181818, - "Filipino,Vietnamese": 0.5113636363636364, - "Spanish,Malay": 0.4375, - "Spanish,Chinese": 0.2556818181818182, - "Spanish,Indonesian": 0.48863636363636365, - "Spanish,Vietnamese": 0.3522727272727273, - "Malay,Chinese": 0.19886363636363635, - "Malay,Indonesian": 0.5511363636363636, - "Malay,Vietnamese": 0.4943181818181818, - "Chinese,Indonesian": 0.26704545454545453, - "Chinese,Vietnamese": 0.11931818181818182, - "Indonesian,Vietnamese": 0.48863636363636365 - }, - "3_combine": { - "English,Filipino,Spanish": 0.17613636363636365, - "English,Filipino,Malay": 0.16477272727272727, - "English,Filipino,Chinese": 0.08522727272727272, - "English,Filipino,Indonesian": 0.17045454545454544, - "English,Filipino,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay": 0.1590909090909091, - "English,Spanish,Chinese": 0.10227272727272728, - "English,Spanish,Indonesian": 0.19318181818181818, - "English,Spanish,Vietnamese": 0.125, - "English,Malay,Chinese": 0.07386363636363637, - "English,Malay,Indonesian": 0.1534090909090909, - "English,Malay,Vietnamese": 0.11363636363636363, - "English,Chinese,Indonesian": 0.10227272727272728, - "English,Chinese,Vietnamese": 0.03977272727272727, - "English,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Malay": 0.23863636363636365, - "Filipino,Spanish,Chinese": 0.07386363636363637, - "Filipino,Spanish,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Chinese": 0.07386363636363637, - "Filipino,Malay,Indonesian": 0.32954545454545453, - "Filipino,Malay,Vietnamese": 0.3068181818181818, - "Filipino,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Chinese,Vietnamese": 0.0625, - "Filipino,Indonesian,Vietnamese": 0.3068181818181818, - "Spanish,Malay,Chinese": 0.11363636363636363, - "Spanish,Malay,Indonesian": 0.30113636363636365, - "Spanish,Malay,Vietnamese": 0.2215909090909091, - "Spanish,Chinese,Indonesian": 0.14772727272727273, - "Spanish,Chinese,Vietnamese": 0.07954545454545454, - "Spanish,Indonesian,Vietnamese": 0.23295454545454544, - "Malay,Chinese,Indonesian": 0.13068181818181818, - "Malay,Chinese,Vietnamese": 0.045454545454545456, - "Malay,Indonesian,Vietnamese": 0.3181818181818182, - "Chinese,Indonesian,Vietnamese": 0.06818181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.10227272727272728, - "English,Filipino,Spanish,Chinese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese": 0.03977272727272727, - "English,Filipino,Malay,Indonesian": 0.10795454545454546, - "English,Filipino,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese": 0.03977272727272727, - "English,Spanish,Malay,Indonesian": 0.10227272727272728, - "English,Spanish,Malay,Vietnamese": 0.06818181818181818, - "English,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Spanish,Chinese,Vietnamese": 0.028409090909090908, - "English,Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "English,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Chinese": 0.03977272727272727, - "Filipino,Spanish,Malay,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.0625, - "Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "Spanish,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Vietnamese": 0.05113636363636364, - "English,Filipino,Spanish,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.045454545454545456, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 - } - }, - "AC3_2": 0.3120615016675732, - "AC3_3": 0.20148386575121283, - "AC3_4": 0.1176714634750416, - "AC3_5": 0.0619475419668301, - "AC3_6": 0.029207175496006994, - "AC3_7": 0.01113373620202616 - }, - "prompt_5": { - "overall_acc": 0.28084415584415584, - "language_acc": { - "English": 0.4034090909090909, - "Filipino": 0.2784090909090909, - "Spanish": 0.26704545454545453, - "Malay": 0.26136363636363635, - "Chinese": 0.26704545454545453, - "Indonesian": 0.24431818181818182, - "Vietnamese": 0.24431818181818182 - }, - "consistency_score_2": 0.37878787878787884, - "consistency_score_3": 0.1788961038961039, - "consistency_score_4": 0.09334415584415583, - "consistency_score_5": 0.050324675324675314, - "consistency_score_6": 0.025974025974025976, - "consistency_score_7": 0.011363636363636364, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3125, - "English,Spanish": 0.4090909090909091, - "English,Malay": 0.26136363636363635, - "English,Chinese": 0.30113636363636365, - "English,Indonesian": 0.2840909090909091, - "English,Vietnamese": 0.2784090909090909, - "Filipino,Spanish": 0.48863636363636365, - "Filipino,Malay": 0.5511363636363636, - "Filipino,Chinese": 0.2727272727272727, - "Filipino,Indonesian": 0.44886363636363635, - "Filipino,Vietnamese": 0.45454545454545453, - "Spanish,Malay": 0.42045454545454547, - "Spanish,Chinese": 0.2840909090909091, - "Spanish,Indonesian": 0.44886363636363635, - "Spanish,Vietnamese": 0.35795454545454547, - "Malay,Chinese": 0.2840909090909091, - "Malay,Indonesian": 0.5568181818181818, - "Malay,Vietnamese": 0.48295454545454547, - "Chinese,Indonesian": 0.3068181818181818, - "Chinese,Vietnamese": 0.3465909090909091, - "Indonesian,Vietnamese": 0.4034090909090909 - }, - "3_combine": { - "English,Filipino,Spanish": 0.20454545454545456, - "English,Filipino,Malay": 0.17045454545454544, - "English,Filipino,Chinese": 0.09659090909090909, - "English,Filipino,Indonesian": 0.1534090909090909, - "English,Filipino,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay": 0.17613636363636365, - "English,Spanish,Chinese": 0.13636363636363635, - "English,Spanish,Indonesian": 0.1875, - "English,Spanish,Vietnamese": 0.1534090909090909, - "English,Malay,Chinese": 0.10227272727272728, - "English,Malay,Indonesian": 0.14772727272727273, - "English,Malay,Vietnamese": 0.13636363636363635, - "English,Chinese,Indonesian": 0.08522727272727272, - "English,Chinese,Vietnamese": 0.10795454545454546, - "English,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay": 0.29545454545454547, - "Filipino,Spanish,Chinese": 0.13636363636363635, - "Filipino,Spanish,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese": 0.18181818181818182, - "Filipino,Malay,Indonesian": 0.3125, - "Filipino,Malay,Vietnamese": 0.32386363636363635, - "Filipino,Chinese,Indonesian": 0.125, - "Filipino,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Indonesian,Vietnamese": 0.23863636363636365, - "Spanish,Malay,Chinese": 0.13068181818181818, - "Spanish,Malay,Indonesian": 0.26704545454545453, - "Spanish,Malay,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Indonesian": 0.125, - "Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Spanish,Indonesian,Vietnamese": 0.19886363636363635, - "Malay,Chinese,Indonesian": 0.1534090909090909, - "Malay,Chinese,Vietnamese": 0.18181818181818182, - "Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Chinese,Indonesian,Vietnamese": 0.16477272727272727 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.125, - "English,Filipino,Spanish,Chinese": 0.0625, - "English,Filipino,Spanish,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese": 0.06818181818181818, - "English,Filipino,Malay,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese": 0.0625, - "English,Spanish,Malay,Indonesian": 0.10795454545454546, - "English,Spanish,Malay,Vietnamese": 0.07954545454545454, - "English,Spanish,Chinese,Indonesian": 0.045454545454545456, - "English,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "English,Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "English,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese": 0.09090909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Chinese,Indonesian": 0.06818181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.0625, - "English,Filipino,Spanish,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 - } - }, - "AC3_2": 0.32254455957523537, - "AC3_3": 0.21856656756563647, - "AC3_4": 0.14011747467872745, - "AC3_5": 0.08535459635823439, - "AC3_6": 0.04755033324912109, - "AC3_7": 0.021843434335959033 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.4174757281553398 - }, - "prompt_2": { - "accuracy": 0.42718446601941745 - }, - "prompt_3": { - "accuracy": 0.39805825242718446 - }, - "prompt_4": { - "accuracy": 0.3883495145631068 - }, - "prompt_5": { - "accuracy": 0.4077669902912621 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.24761904761904763 - }, - "prompt_2": { - "accuracy": 0.2571428571428571 - }, - "prompt_3": { - "accuracy": 0.24761904761904763 - }, - "prompt_4": { - "accuracy": 0.26666666666666666 - }, - "prompt_5": { - "accuracy": 0.2571428571428571 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.2616822429906542 - }, - "prompt_2": { - "accuracy": 0.2897196261682243 - }, - "prompt_3": { - "accuracy": 0.29906542056074764 - }, - "prompt_4": { - "accuracy": 0.29906542056074764 - }, - "prompt_5": { - "accuracy": 0.308411214953271 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.17, - "category_acc": { - "brand": 0.1, - "demographics": 0.0, - "biology": 0.1, - "history": 0.26666666666666666, - "literature": 0.2, - "politics": 0.2, - "culture": 0.2, - "film": 0.2, - "law": 0.1, - "geography": 0.2 - } - }, - "prompt_2": { - "accuracy": 0.2, - "category_acc": { - "brand": 0.1, - "demographics": 0.0, - "biology": 0.2, - "history": 0.2, - "literature": 0.2, - "politics": 0.2, - "culture": 0.2, - "film": 0.3, - "law": 0.1, - "geography": 0.4 - } - }, - "prompt_3": { - "accuracy": 0.22, - "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.2, - "history": 0.26666666666666666, - "literature": 0.2, - "politics": 0.2, - "culture": 0.2, - "film": 0.2, - "law": 0.2, - "geography": 0.4 - } - }, - "prompt_4": { - "accuracy": 0.2, - "category_acc": { - "brand": 0.1, - "demographics": 0.0, - "biology": 0.1, - "history": 0.3333333333333333, - "literature": 0.2, - "politics": 0.3, - "culture": 0.3, - "film": 0.2, - "law": 0.1, - "geography": 0.2 - } - }, - "prompt_5": { - "accuracy": 0.22, - "category_acc": { - "brand": 0.1, - "demographics": 0.4, - "biology": 0.1, - "history": 0.26666666666666666, - "literature": 0.2, - "politics": 0.2, - "culture": 0.2, - "film": 0.2, - "law": 0.2, - "geography": 0.4 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.1762398612912264 - }, - "prompt_2": { - "bleu_score": 0.17734002042907843 - }, - "prompt_3": { - "bleu_score": 0.1794117113837163 - }, - "prompt_4": { - "bleu_score": 0.1839610854412572 - }, - "prompt_5": { - "bleu_score": 0.16304723755362402 - } - }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.06384522696048195 - }, - "prompt_2": { - "bleu_score": 0.06579044295906208 - }, - "prompt_3": { - "bleu_score": 0.06369532865159085 - }, - "prompt_4": { - "bleu_score": 0.06513210020365194 - }, - "prompt_5": { - "bleu_score": 0.0604037164927284 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.03223805891704692 - }, - "prompt_2": { - "bleu_score": 0.031105516142788385 - }, - "prompt_3": { - "bleu_score": 0.030670443688375845 - }, - "prompt_4": { - "bleu_score": 0.033707993801729 - }, - "prompt_5": { - "bleu_score": 0.031164792684394777 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.008323244414777916 - }, - "prompt_2": { - "bleu_score": 0.01322913125332852 - }, - "prompt_3": { - "bleu_score": 0.013625883358503812 - }, - "prompt_4": { - "bleu_score": 0.012602681533998064 - }, - "prompt_5": { - "bleu_score": 0.011822640872179395 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.05726226454268131 - }, - "prompt_2": { - "bleu_score": 0.057803231348560566 - }, - "prompt_3": { - "bleu_score": 0.056273922014826096 - }, - "prompt_4": { - "bleu_score": 0.060835604539694756 - }, - "prompt_5": { - "bleu_score": 0.056852779946093605 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.338389731621937 - }, - "prompt_2": { - "accuracy": 0.3220536756126021 - }, - "prompt_3": { - "accuracy": 0.32555425904317387 - }, - "prompt_4": { - "accuracy": 0.33255542590431736 - }, - "prompt_5": { - "accuracy": 0.33605600933488916 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.35144797997854843, - "category_acc": { - "high_school_european_history": 0.4878048780487805, - "business_ethics": 0.3939393939393939, - "clinical_knowledge": 0.3787878787878788, - "medical_genetics": 0.45454545454545453, - "high_school_us_history": 0.42857142857142855, - "high_school_physics": 0.26, - "high_school_world_history": 0.4576271186440678, - "virology": 0.36363636363636365, - "high_school_microeconomics": 0.3755274261603376, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.3434343434343434, - "high_school_biology": 0.32038834951456313, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.33451957295373663, - "philosophy": 0.34838709677419355, - "professional_medicine": 0.3025830258302583, - "nutrition": 0.34098360655737703, - "global_facts": 0.30303030303030304, - "machine_learning": 0.27927927927927926, - "security_studies": 0.35655737704918034, - "public_relations": 0.41284403669724773, - "professional_psychology": 0.353518821603928, - "prehistory": 0.3560371517027864, - "anatomy": 0.31343283582089554, - "human_sexuality": 0.3769230769230769, - "college_medicine": 0.32558139534883723, - "high_school_government_and_politics": 0.421875, - "college_chemistry": 0.1919191919191919, - "logical_fallacies": 0.41975308641975306, - "high_school_geography": 0.41116751269035534, - "elementary_mathematics": 0.27055702917771884, - "human_aging": 0.3783783783783784, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.4485294117647059, - "formal_logic": 0.32, - "high_school_statistics": 0.2651162790697674, - "international_law": 0.525, - "high_school_mathematics": 0.2379182156133829, - "high_school_computer_science": 0.32323232323232326, - "conceptual_physics": 0.2863247863247863, - "miscellaneous": 0.40664961636828645, - "high_school_chemistry": 0.2623762376237624, - "marketing": 0.6008583690987125, - "professional_law": 0.33268101761252444, - "management": 0.4411764705882353, - "college_physics": 0.24752475247524752, - "jurisprudence": 0.3925233644859813, - "world_religions": 0.29411764705882354, - "sociology": 0.535, - "us_foreign_policy": 0.5151515151515151, - "high_school_macroeconomics": 0.33161953727506427, - "computer_security": 0.43434343434343436, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.391304347826087, - "electrical_engineering": 0.3194444444444444, - "astronomy": 0.304635761589404, - "college_biology": 0.2727272727272727 - } - }, - "prompt_2": { - "accuracy": 0.350232391848409, - "category_acc": { - "high_school_european_history": 0.5, - "business_ethics": 0.36363636363636365, - "clinical_knowledge": 0.4090909090909091, - "medical_genetics": 0.3838383838383838, - "high_school_us_history": 0.4187192118226601, - "high_school_physics": 0.3, - "high_school_world_history": 0.4533898305084746, - "virology": 0.3515151515151515, - "high_school_microeconomics": 0.38396624472573837, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.31313131313131315, - "high_school_biology": 0.2977346278317152, - "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.33807829181494664, - "philosophy": 0.35161290322580646, - "professional_medicine": 0.31365313653136534, - "nutrition": 0.34098360655737703, - "global_facts": 0.2222222222222222, - "machine_learning": 0.3153153153153153, - "security_studies": 0.3770491803278688, - "public_relations": 0.3853211009174312, - "professional_psychology": 0.36661211129296234, - "prehistory": 0.33126934984520123, - "anatomy": 0.30597014925373134, - "human_sexuality": 0.36153846153846153, - "college_medicine": 0.3953488372093023, - "high_school_government_and_politics": 0.4270833333333333, - "college_chemistry": 0.2727272727272727, - "logical_fallacies": 0.41975308641975306, - "high_school_geography": 0.39593908629441626, - "elementary_mathematics": 0.2625994694960212, - "human_aging": 0.3783783783783784, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.4430147058823529, - "formal_logic": 0.288, - "high_school_statistics": 0.27906976744186046, - "international_law": 0.5166666666666667, - "high_school_mathematics": 0.24907063197026022, - "high_school_computer_science": 0.3434343434343434, - "conceptual_physics": 0.28205128205128205, - "miscellaneous": 0.39258312020460356, - "high_school_chemistry": 0.25742574257425743, - "marketing": 0.5708154506437768, - "professional_law": 0.33659491193737767, - "management": 0.46078431372549017, - "college_physics": 0.3069306930693069, - "jurisprudence": 0.37383177570093457, - "world_religions": 0.29411764705882354, - "sociology": 0.5, - "us_foreign_policy": 0.47474747474747475, - "high_school_macroeconomics": 0.33676092544987146, - "computer_security": 0.40404040404040403, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.37681159420289856, - "electrical_engineering": 0.2847222222222222, - "astronomy": 0.32450331125827814, - "college_biology": 0.27972027972027974 - } - }, - "prompt_3": { - "accuracy": 0.3372184483375045, - "category_acc": { - "high_school_european_history": 0.4878048780487805, - "business_ethics": 0.3838383838383838, - "clinical_knowledge": 0.3787878787878788, - "medical_genetics": 0.37373737373737376, - "high_school_us_history": 0.4039408866995074, - "high_school_physics": 0.23333333333333334, - "high_school_world_history": 0.4533898305084746, - "virology": 0.36363636363636365, - "high_school_microeconomics": 0.3459915611814346, - "econometrics": 0.3008849557522124, - "college_computer_science": 0.32323232323232326, - "high_school_biology": 0.29449838187702265, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.3167259786476868, - "philosophy": 0.34516129032258064, - "professional_medicine": 0.2952029520295203, - "nutrition": 0.36065573770491804, - "global_facts": 0.25252525252525254, - "machine_learning": 0.1891891891891892, - "security_studies": 0.3483606557377049, - "public_relations": 0.41284403669724773, - "professional_psychology": 0.3502454991816694, - "prehistory": 0.33436532507739936, - "anatomy": 0.31343283582089554, - "human_sexuality": 0.3153846153846154, - "college_medicine": 0.3430232558139535, - "high_school_government_and_politics": 0.40625, - "college_chemistry": 0.18181818181818182, - "logical_fallacies": 0.41358024691358025, - "high_school_geography": 0.38578680203045684, - "elementary_mathematics": 0.21750663129973474, - "human_aging": 0.36036036036036034, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.4227941176470588, - "formal_logic": 0.264, - "high_school_statistics": 0.2186046511627907, - "international_law": 0.5, - "high_school_mathematics": 0.22676579925650558, - "high_school_computer_science": 0.3333333333333333, - "conceptual_physics": 0.25213675213675213, - "miscellaneous": 0.3836317135549872, - "high_school_chemistry": 0.2623762376237624, - "marketing": 0.5450643776824035, - "professional_law": 0.33463796477495106, - "management": 0.43137254901960786, - "college_physics": 0.297029702970297, - "jurisprudence": 0.3644859813084112, - "world_religions": 0.2823529411764706, - "sociology": 0.51, - "us_foreign_policy": 0.4444444444444444, - "high_school_macroeconomics": 0.3264781491002571, - "computer_security": 0.35353535353535354, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.3739130434782609, - "electrical_engineering": 0.2777777777777778, - "astronomy": 0.33112582781456956, - "college_biology": 0.3076923076923077 - } - }, - "prompt_4": { - "accuracy": 0.3484447622452628, - "category_acc": { - "high_school_european_history": 0.49390243902439024, - "business_ethics": 0.3434343434343434, - "clinical_knowledge": 0.38636363636363635, - "medical_genetics": 0.40404040404040403, - "high_school_us_history": 0.4236453201970443, - "high_school_physics": 0.26666666666666666, - "high_school_world_history": 0.4703389830508475, - "virology": 0.3515151515151515, - "high_school_microeconomics": 0.37130801687763715, - "econometrics": 0.3008849557522124, - "college_computer_science": 0.3333333333333333, - "high_school_biology": 0.3074433656957929, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.3416370106761566, - "philosophy": 0.34838709677419355, - "professional_medicine": 0.3173431734317343, - "nutrition": 0.3540983606557377, - "global_facts": 0.21212121212121213, - "machine_learning": 0.27927927927927926, - "security_studies": 0.36475409836065575, - "public_relations": 0.42201834862385323, - "professional_psychology": 0.353518821603928, - "prehistory": 0.33436532507739936, - "anatomy": 0.27611940298507465, - "human_sexuality": 0.35384615384615387, - "college_medicine": 0.3430232558139535, - "high_school_government_and_politics": 0.4114583333333333, - "college_chemistry": 0.2222222222222222, - "logical_fallacies": 0.4074074074074074, - "high_school_geography": 0.39593908629441626, - "elementary_mathematics": 0.26525198938992045, - "human_aging": 0.3918918918918919, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.43566176470588236, - "formal_logic": 0.32, - "high_school_statistics": 0.2744186046511628, - "international_law": 0.5, - "high_school_mathematics": 0.25650557620817843, - "high_school_computer_science": 0.3333333333333333, - "conceptual_physics": 0.3034188034188034, - "miscellaneous": 0.3938618925831202, - "high_school_chemistry": 0.27722772277227725, - "marketing": 0.5836909871244635, - "professional_law": 0.33202870189171557, - "management": 0.45098039215686275, - "college_physics": 0.25742574257425743, - "jurisprudence": 0.3644859813084112, - "world_religions": 0.27058823529411763, - "sociology": 0.535, - "us_foreign_policy": 0.48484848484848486, - "high_school_macroeconomics": 0.3393316195372751, - "computer_security": 0.42424242424242425, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.3536231884057971, - "electrical_engineering": 0.3194444444444444, - "astronomy": 0.32450331125827814, - "college_biology": 0.27972027972027974 - } - }, - "prompt_5": { - "accuracy": 0.34537003932785126, - "category_acc": { - "high_school_european_history": 0.47560975609756095, - "business_ethics": 0.37373737373737376, - "clinical_knowledge": 0.35984848484848486, - "medical_genetics": 0.3838383838383838, - "high_school_us_history": 0.43349753694581283, - "high_school_physics": 0.22666666666666666, - "high_school_world_history": 0.4703389830508475, - "virology": 0.3333333333333333, - "high_school_microeconomics": 0.33755274261603374, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.32323232323232326, - "high_school_biology": 0.3074433656957929, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.3487544483985765, - "philosophy": 0.3419354838709677, - "professional_medicine": 0.31365313653136534, - "nutrition": 0.3442622950819672, - "global_facts": 0.30303030303030304, - "machine_learning": 0.22522522522522523, - "security_studies": 0.36065573770491804, - "public_relations": 0.3669724770642202, - "professional_psychology": 0.35515548281505727, - "prehistory": 0.34674922600619196, - "anatomy": 0.29850746268656714, - "human_sexuality": 0.38461538461538464, - "college_medicine": 0.3081395348837209, - "high_school_government_and_politics": 0.4114583333333333, - "college_chemistry": 0.21212121212121213, - "logical_fallacies": 0.42592592592592593, - "high_school_geography": 0.4010152284263959, - "elementary_mathematics": 0.2625994694960212, - "human_aging": 0.4009009009009009, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.4264705882352941, - "formal_logic": 0.344, - "high_school_statistics": 0.26976744186046514, - "international_law": 0.5083333333333333, - "high_school_mathematics": 0.2342007434944238, - "high_school_computer_science": 0.3434343434343434, - "conceptual_physics": 0.28205128205128205, - "miscellaneous": 0.39641943734015345, - "high_school_chemistry": 0.25742574257425743, - "marketing": 0.592274678111588, - "professional_law": 0.32746249184605347, - "management": 0.4411764705882353, - "college_physics": 0.27722772277227725, - "jurisprudence": 0.3925233644859813, - "world_religions": 0.27058823529411763, - "sociology": 0.53, - "us_foreign_policy": 0.46464646464646464, - "high_school_macroeconomics": 0.34190231362467866, - "computer_security": 0.41414141414141414, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.3710144927536232, - "electrical_engineering": 0.2986111111111111, - "astronomy": 0.31125827814569534, - "college_biology": 0.2867132867132867 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.24442793462109955 - }, - "prompt_2": { - "accuracy": 0.24219910846953938 - }, - "prompt_3": { - "accuracy": 0.24219910846953938 - }, - "prompt_4": { - "accuracy": 0.24219910846953938 - }, - "prompt_5": { - "accuracy": 0.24219910846953938 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.26214196762141967, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.16666666666666666, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.21666666666666667, - "business_administration": 0.13157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.0, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.2962962962962963, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.25, - "high_school_history": 0.08, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.3076923076923077, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.3148148148148148, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.3148148148148148, - "physician": 0.2962962962962963 - } - }, - "prompt_2": { - "accuracy": 0.2627646326276463, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.19047619047619047, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.21666666666666667, - "business_administration": 0.13157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.0, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.2962962962962963, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.25, - "high_school_history": 0.08, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.3076923076923077, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.3148148148148148, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.3148148148148148, - "physician": 0.2962962962962963 - } - }, - "prompt_3": { - "accuracy": 0.26214196762141967, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.14285714285714285, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.21666666666666667, - "business_administration": 0.13157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.0, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.2962962962962963, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.25, - "high_school_history": 0.08, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.3076923076923077, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.3148148148148148, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.3148148148148148, - "physician": 0.2962962962962963 - } - }, - "prompt_4": { - "accuracy": 0.2640099626400996, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.16666666666666666, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.375, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.21666666666666667, - "business_administration": 0.13157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.0, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.3333333333333333, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.25, - "high_school_history": 0.04, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.3076923076923077, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.3148148148148148, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.3148148148148148, - "physician": 0.2962962962962963 - } - }, - "prompt_5": { - "accuracy": 0.263387297633873, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.14285714285714285, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.21666666666666667, - "business_administration": 0.13157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.0, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.2962962962962963, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.25, - "high_school_history": 0.08, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.3076923076923077, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.3148148148148148, - "fire_engineer": 0.19444444444444445, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.3148148148148148, - "physician": 0.2962962962962963 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.22580645161290322 - }, - "prompt_2": { - "accuracy": 0.2222222222222222 - }, - "prompt_3": { - "accuracy": 0.21505376344086022 - }, - "prompt_4": { - "accuracy": 0.25448028673835127 - }, - "prompt_5": { - "accuracy": 0.2222222222222222 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.2532377827663616, - "category_acc": { - "agronomy": 0.2485207100591716, - "anatomy": 0.25, - "ancient_chinese": 0.27439024390243905, - "arts": 0.275, - "astronomy": 0.20606060606060606, - "business_ethics": 0.24880382775119617, - "chinese_civil_service_exam": 0.2375, - "chinese_driving_rule": 0.2824427480916031, - "chinese_food_culture": 0.23529411764705882, - "chinese_foreign_policy": 0.308411214953271, - "chinese_history": 0.24148606811145512, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.31843575418994413, - "clinical_knowledge": 0.27848101265822783, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.1588785046728972, - "college_engineering_hydrology": 0.25471698113207547, - "college_law": 0.25, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.2641509433962264, - "college_medicine": 0.22344322344322345, - "computer_science": 0.2107843137254902, - "computer_security": 0.2222222222222222, - "conceptual_physics": 0.2653061224489796, - "construction_project_management": 0.2302158273381295, - "economics": 0.27044025157232704, - "education": 0.25766871165644173, - "electrical_engineering": 0.2441860465116279, - "elementary_chinese": 0.24206349206349206, - "elementary_commonsense": 0.29292929292929293, - "elementary_information_and_technology": 0.2647058823529412, - "elementary_mathematics": 0.2391304347826087, - "ethnology": 0.25925925925925924, - "food_science": 0.24475524475524477, - "genetics": 0.26704545454545453, - "global_facts": 0.2751677852348993, - "high_school_biology": 0.22485207100591717, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.2711864406779661, - "high_school_mathematics": 0.32926829268292684, - "high_school_physics": 0.2, - "high_school_politics": 0.3076923076923077, - "human_sexuality": 0.2619047619047619, - "international_law": 0.2702702702702703, - "journalism": 0.22674418604651161, - "jurisprudence": 0.2384428223844282, - "legal_and_moral_basis": 0.2803738317757009, - "logical": 0.25203252032520324, - "machine_learning": 0.22131147540983606, - "management": 0.26666666666666666, - "marketing": 0.2611111111111111, - "marxist_theory": 0.24338624338624337, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.2482758620689655, - "philosophy": 0.23809523809523808, - "professional_accounting": 0.28, - "professional_law": 0.24170616113744076, - "professional_medicine": 0.2473404255319149, - "professional_psychology": 0.23275862068965517, - "public_relations": 0.26436781609195403, - "security_study": 0.25925925925925924, - "sociology": 0.2831858407079646, - "sports_science": 0.2545454545454545, - "traditional_chinese_medicine": 0.23243243243243245, - "virology": 0.2603550295857988, - "world_history": 0.2422360248447205, - "world_religions": 0.26875 - } - }, - "prompt_2": { - "accuracy": 0.2527197375237437, - "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.24324324324324326, - "ancient_chinese": 0.2621951219512195, - "arts": 0.275, - "astronomy": 0.2, - "business_ethics": 0.24401913875598086, - "chinese_civil_service_exam": 0.25, - "chinese_driving_rule": 0.2824427480916031, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.3177570093457944, - "chinese_history": 0.23529411764705882, - "chinese_literature": 0.24509803921568626, - "chinese_teacher_qualification": 0.3128491620111732, - "clinical_knowledge": 0.2742616033755274, - "college_actuarial_science": 0.20754716981132076, - "college_education": 0.16822429906542055, - "college_engineering_hydrology": 0.24528301886792453, - "college_law": 0.24074074074074073, - "college_mathematics": 0.29523809523809524, - "college_medical_statistics": 0.2641509433962264, - "college_medicine": 0.23076923076923078, - "computer_science": 0.22058823529411764, - "computer_security": 0.22807017543859648, - "conceptual_physics": 0.272108843537415, - "construction_project_management": 0.22302158273381295, - "economics": 0.27044025157232704, - "education": 0.25766871165644173, - "electrical_engineering": 0.23837209302325582, - "elementary_chinese": 0.23809523809523808, - "elementary_commonsense": 0.2828282828282828, - "elementary_information_and_technology": 0.25630252100840334, - "elementary_mathematics": 0.23478260869565218, - "ethnology": 0.25925925925925924, - "food_science": 0.24475524475524477, - "genetics": 0.26704545454545453, - "global_facts": 0.2751677852348993, - "high_school_biology": 0.24260355029585798, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.2711864406779661, - "high_school_mathematics": 0.3170731707317073, - "high_school_physics": 0.2, - "high_school_politics": 0.32167832167832167, - "human_sexuality": 0.2619047619047619, - "international_law": 0.2702702702702703, - "journalism": 0.22674418604651161, - "jurisprudence": 0.2360097323600973, - "legal_and_moral_basis": 0.2803738317757009, - "logical": 0.25203252032520324, - "machine_learning": 0.22131147540983606, - "management": 0.26666666666666666, - "marketing": 0.2611111111111111, - "marxist_theory": 0.24338624338624337, - "modern_chinese": 0.2672413793103448, - "nutrition": 0.2482758620689655, - "philosophy": 0.23809523809523808, - "professional_accounting": 0.28, - "professional_law": 0.23696682464454977, - "professional_medicine": 0.2473404255319149, - "professional_psychology": 0.23275862068965517, - "public_relations": 0.26436781609195403, - "security_study": 0.25925925925925924, - "sociology": 0.2831858407079646, - "sports_science": 0.2545454545454545, - "traditional_chinese_medicine": 0.23243243243243245, - "virology": 0.25443786982248523, - "world_history": 0.2422360248447205, - "world_religions": 0.26875 - } - }, - "prompt_3": { - "accuracy": 0.25341046451390087, - "category_acc": { - "agronomy": 0.25443786982248523, - "anatomy": 0.25, - "ancient_chinese": 0.27439024390243905, - "arts": 0.275, - "astronomy": 0.2, - "business_ethics": 0.24401913875598086, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.2824427480916031, - "chinese_food_culture": 0.23529411764705882, - "chinese_foreign_policy": 0.3177570093457944, - "chinese_history": 0.23529411764705882, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.31843575418994413, - "clinical_knowledge": 0.270042194092827, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.16822429906542055, - "college_engineering_hydrology": 0.25471698113207547, - "college_law": 0.24074074074074073, - "college_mathematics": 0.2857142857142857, - "college_medical_statistics": 0.2641509433962264, - "college_medicine": 0.23809523809523808, - "computer_science": 0.20588235294117646, - "computer_security": 0.2222222222222222, - "conceptual_physics": 0.2585034013605442, - "construction_project_management": 0.2302158273381295, - "economics": 0.27044025157232704, - "education": 0.25766871165644173, - "electrical_engineering": 0.23837209302325582, - "elementary_chinese": 0.24603174603174602, - "elementary_commonsense": 0.2878787878787879, - "elementary_information_and_technology": 0.25210084033613445, - "elementary_mathematics": 0.2391304347826087, - "ethnology": 0.25925925925925924, - "food_science": 0.24475524475524477, - "genetics": 0.26704545454545453, - "global_facts": 0.2751677852348993, - "high_school_biology": 0.23076923076923078, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.2796610169491525, - "high_school_mathematics": 0.3231707317073171, - "high_school_physics": 0.19090909090909092, - "high_school_politics": 0.3076923076923077, - "human_sexuality": 0.2619047619047619, - "international_law": 0.2648648648648649, - "journalism": 0.22674418604651161, - "jurisprudence": 0.23357664233576642, - "legal_and_moral_basis": 0.2803738317757009, - "logical": 0.25203252032520324, - "machine_learning": 0.22131147540983606, - "management": 0.26666666666666666, - "marketing": 0.2611111111111111, - "marxist_theory": 0.24338624338624337, - "modern_chinese": 0.25862068965517243, - "nutrition": 0.25517241379310346, - "philosophy": 0.24761904761904763, - "professional_accounting": 0.28, - "professional_law": 0.23696682464454977, - "professional_medicine": 0.2473404255319149, - "professional_psychology": 0.23275862068965517, - "public_relations": 0.26436781609195403, - "security_study": 0.25925925925925924, - "sociology": 0.2831858407079646, - "sports_science": 0.2545454545454545, - "traditional_chinese_medicine": 0.22702702702702704, - "virology": 0.26627218934911245, - "world_history": 0.2484472049689441, - "world_religions": 0.2625 - } - }, - "prompt_4": { - "accuracy": 0.25176998791227767, - "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.23648648648648649, - "ancient_chinese": 0.2621951219512195, - "arts": 0.275, - "astronomy": 0.21818181818181817, - "business_ethics": 0.24401913875598086, - "chinese_civil_service_exam": 0.25, - "chinese_driving_rule": 0.33587786259541985, - "chinese_food_culture": 0.27941176470588236, - "chinese_foreign_policy": 0.2897196261682243, - "chinese_history": 0.23219814241486067, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.2737430167597765, - "clinical_knowledge": 0.2869198312236287, - "college_actuarial_science": 0.19811320754716982, - "college_education": 0.14953271028037382, - "college_engineering_hydrology": 0.24528301886792453, - "college_law": 0.25925925925925924, - "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.24528301886792453, - "college_medicine": 0.24175824175824176, - "computer_science": 0.2107843137254902, - "computer_security": 0.23391812865497075, - "conceptual_physics": 0.3129251700680272, - "construction_project_management": 0.22302158273381295, - "economics": 0.27044025157232704, - "education": 0.2085889570552147, - "electrical_engineering": 0.22093023255813954, - "elementary_chinese": 0.23015873015873015, - "elementary_commonsense": 0.26262626262626265, - "elementary_information_and_technology": 0.2647058823529412, - "elementary_mathematics": 0.23478260869565218, - "ethnology": 0.3037037037037037, - "food_science": 0.2867132867132867, - "genetics": 0.2556818181818182, - "global_facts": 0.28187919463087246, - "high_school_biology": 0.24260355029585798, - "high_school_chemistry": 0.2196969696969697, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.3353658536585366, - "high_school_physics": 0.2, - "high_school_politics": 0.2867132867132867, - "human_sexuality": 0.23015873015873015, - "international_law": 0.2756756756756757, - "journalism": 0.2441860465116279, - "jurisprudence": 0.24330900243309003, - "legal_and_moral_basis": 0.24766355140186916, - "logical": 0.2682926829268293, - "machine_learning": 0.21311475409836064, - "management": 0.21904761904761905, - "marketing": 0.25555555555555554, - "marxist_theory": 0.25396825396825395, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.2827586206896552, - "philosophy": 0.24761904761904763, - "professional_accounting": 0.2342857142857143, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.23138297872340424, - "professional_psychology": 0.2629310344827586, - "public_relations": 0.25862068965517243, - "security_study": 0.26666666666666666, - "sociology": 0.252212389380531, - "sports_science": 0.24242424242424243, - "traditional_chinese_medicine": 0.25405405405405407, - "virology": 0.2603550295857988, - "world_history": 0.2546583850931677, - "world_religions": 0.28125 - } - }, - "prompt_5": { - "accuracy": 0.2522016922811259, - "category_acc": { - "agronomy": 0.2485207100591716, - "anatomy": 0.24324324324324326, - "ancient_chinese": 0.2682926829268293, - "arts": 0.275, - "astronomy": 0.2, - "business_ethics": 0.24401913875598086, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.2824427480916031, - "chinese_food_culture": 0.23529411764705882, - "chinese_foreign_policy": 0.308411214953271, - "chinese_history": 0.23529411764705882, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.3128491620111732, - "clinical_knowledge": 0.270042194092827, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.17757009345794392, - "college_engineering_hydrology": 0.2641509433962264, - "college_law": 0.24074074074074073, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.2641509433962264, - "college_medicine": 0.22344322344322345, - "computer_science": 0.21568627450980393, - "computer_security": 0.23976608187134502, - "conceptual_physics": 0.272108843537415, - "construction_project_management": 0.23741007194244604, - "economics": 0.27044025157232704, - "education": 0.25766871165644173, - "electrical_engineering": 0.23255813953488372, - "elementary_chinese": 0.23412698412698413, - "elementary_commonsense": 0.2878787878787879, - "elementary_information_and_technology": 0.25210084033613445, - "elementary_mathematics": 0.23043478260869565, - "ethnology": 0.25925925925925924, - "food_science": 0.24475524475524477, - "genetics": 0.26704545454545453, - "global_facts": 0.2751677852348993, - "high_school_biology": 0.21893491124260356, - "high_school_chemistry": 0.2196969696969697, - "high_school_geography": 0.2711864406779661, - "high_school_mathematics": 0.32926829268292684, - "high_school_physics": 0.19090909090909092, - "high_school_politics": 0.3146853146853147, - "human_sexuality": 0.2619047619047619, - "international_law": 0.2648648648648649, - "journalism": 0.22674418604651161, - "jurisprudence": 0.2384428223844282, - "legal_and_moral_basis": 0.2803738317757009, - "logical": 0.25203252032520324, - "machine_learning": 0.21311475409836064, - "management": 0.26666666666666666, - "marketing": 0.2611111111111111, - "marxist_theory": 0.24338624338624337, - "modern_chinese": 0.25, - "nutrition": 0.25517241379310346, - "philosophy": 0.23809523809523808, - "professional_accounting": 0.2857142857142857, - "professional_law": 0.23696682464454977, - "professional_medicine": 0.2473404255319149, - "professional_psychology": 0.23275862068965517, - "public_relations": 0.26436781609195403, - "security_study": 0.26666666666666666, - "sociology": 0.2831858407079646, - "sports_science": 0.2545454545454545, - "traditional_chinese_medicine": 0.22702702702702704, - "virology": 0.2603550295857988, - "world_history": 0.2422360248447205, - "world_religions": 0.26875 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.24242424242424243 - }, - "prompt_2": { - "accuracy": 0.21212121212121213 - }, - "prompt_3": { - "accuracy": 0.21212121212121213 - }, - "prompt_4": { - "accuracy": 0.18181818181818182 - }, - "prompt_5": { - "accuracy": 0.2727272727272727 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.15454545454545454 - }, - "prompt_2": { - "accuracy": 0.15227272727272728 - }, - "prompt_3": { - "accuracy": 0.13636363636363635 - }, - "prompt_4": { - "accuracy": 0.22954545454545455 - }, - "prompt_5": { - "accuracy": 0.29545454545454547 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.34711864406779663 - }, - "prompt_2": { - "accuracy": 0.3383050847457627 - }, - "prompt_3": { - "accuracy": 0.33559322033898303 - }, - "prompt_4": { - "accuracy": 0.34711864406779663 - }, - "prompt_5": { - "accuracy": 0.34711864406779663 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.27860882572924456 - }, - "prompt_2": { - "accuracy": 0.2819745699326851 - }, - "prompt_3": { - "accuracy": 0.281226626776365 - }, - "prompt_4": { - "accuracy": 0.27786088257292446 - }, - "prompt_5": { - "accuracy": 0.28047868362004486 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.7339539441450269 - }, - "prompt_2": { - "accuracy": 0.732484076433121 - }, - "prompt_3": { - "accuracy": 0.7359137677609016 - }, - "prompt_4": { - "accuracy": 0.7275845173934345 - }, - "prompt_5": { - "accuracy": 0.7329740323370897 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.488468788604541, - "rouge2": 0.2397705965965483, - "rougeL": 0.40646134652351057, - "avg_rouge": 0.3782335772415333 - }, - "prompt_2": { - "rouge1": 0.4850453424830358, - "rouge2": 0.23758721809330177, - "rougeL": 0.40319572164503803, - "avg_rouge": 0.37527609407379187 - }, - "prompt_3": { - "rouge1": 0.4898628798073896, - "rouge2": 0.24128432853326126, - "rougeL": 0.4066689491337958, - "avg_rouge": 0.37927205249148227 - }, - "prompt_4": { - "rouge1": 0.4868273667099931, - "rouge2": 0.2394466220970154, - "rougeL": 0.40597189702486663, - "avg_rouge": 0.37741529527729173 - }, - "prompt_5": { - "rouge1": 0.4886148780007011, - "rouge2": 0.24015176921813242, - "rougeL": 0.4051863127372784, - "avg_rouge": 0.3779843199853706 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.2655814561363983, - "rouge2": 0.08397129926679955, - "rougeL": 0.2043448967502899, - "avg_rouge": 0.18463255071782925 - }, - "prompt_2": { - "rouge1": 0.27405978806051756, - "rouge2": 0.08832292551988413, - "rougeL": 0.20952981504822002, - "avg_rouge": 0.19063750954287392 - }, - "prompt_3": { - "rouge1": 0.2338172850567672, - "rouge2": 0.06835029402704514, - "rougeL": 0.17486193016228904, - "avg_rouge": 0.15900983641536712 - }, - "prompt_4": { - "rouge1": 0.23672067704298078, - "rouge2": 0.06983068377097468, - "rougeL": 0.17726851663330187, - "avg_rouge": 0.1612732924824191 - }, - "prompt_5": { - "rouge1": 0.2704288857228005, - "rouge2": 0.08732643284919703, - "rougeL": 0.2061517654200443, - "avg_rouge": 0.18796902799734727 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.9094036697247706 - }, - "prompt_2": { - "accuracy": 0.9071100917431193 - }, - "prompt_3": { - "accuracy": 0.9036697247706422 - }, - "prompt_4": { - "accuracy": 0.908256880733945 - }, - "prompt_5": { - "accuracy": 0.9139908256880734 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.5071907957813998 - }, - "prompt_2": { - "accuracy": 0.6864813039309684 - }, - "prompt_3": { - "accuracy": 0.6596356663470757 - }, - "prompt_4": { - "accuracy": 0.6049856184084372 - }, - "prompt_5": { - "accuracy": 0.5225311601150527 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.809 - }, - "prompt_2": { - "accuracy": 0.8175 - }, - "prompt_3": { - "accuracy": 0.806 - }, - "prompt_4": { - "accuracy": 0.8175 - }, - "prompt_5": { - "accuracy": 0.812 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.5655 - }, - "prompt_2": { - "accuracy": 0.56 - }, - "prompt_3": { - "accuracy": 0.5735 - }, - "prompt_4": { - "accuracy": 0.559 - }, - "prompt_5": { - "accuracy": 0.5425 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.831 - }, - "prompt_2": { - "accuracy": 0.7555 - }, - "prompt_3": { - "accuracy": 0.8115 - }, - "prompt_4": { - "accuracy": 0.787 - }, - "prompt_5": { - "accuracy": 0.784 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.4647887323943662 - }, - "prompt_2": { - "accuracy": 0.43661971830985913 - }, - "prompt_3": { - "accuracy": 0.43661971830985913 - }, - "prompt_4": { - "accuracy": 0.4647887323943662 - }, - "prompt_5": { - "accuracy": 0.43661971830985913 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.7942238267148014 - }, - "prompt_2": { - "accuracy": 0.779783393501805 - }, - "prompt_3": { - "accuracy": 0.7472924187725631 - }, - "prompt_4": { - "accuracy": 0.7653429602888087 - }, - "prompt_5": { - "accuracy": 0.7581227436823105 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.49019607843137253 - }, - "prompt_2": { - "accuracy": 0.5245098039215687 - }, - "prompt_3": { - "accuracy": 0.4264705882352941 - }, - "prompt_4": { - "accuracy": 0.5465686274509803 - }, - "prompt_5": { - "accuracy": 0.5955882352941176 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.719327731092437, - "language_acc": { - "English": 0.9420168067226891, - "Chinese": 0.3773109243697479, - "Spanish": 0.853781512605042, - "Vietnamese": 0.704201680672269 - }, - "consistency_score_2": 0.5627450980392156, - "consistency_score_3": 0.3957983193277311, - "consistency_score_4": 0.28319327731092436, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.37899159663865545, - "English,Spanish": 0.8554621848739495, - "English,Vietnamese": 0.6983193277310924, - "Chinese,Spanish": 0.3815126050420168, - "Chinese,Vietnamese": 0.36554621848739494, - "Spanish,Vietnamese": 0.6966386554621848 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3420168067226891, - "English,Chinese,Vietnamese": 0.2949579831932773, - "English,Spanish,Vietnamese": 0.6487394957983194, - "Chinese,Spanish,Vietnamese": 0.29747899159663865 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.28319327731092436 - } - }, - "AC3_2": 0.6314745080410694, - "AC3_3": 0.5106305370220878, - "AC3_4": 0.40639303496055473 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.3104761904761905, - "language_acc": { - "Spanish": 0.32666666666666666, - "English": 0.4066666666666667, - "Chinese": 0.31333333333333335, - "Vietnamese": 0.3, - "Filipino": 0.28, - "Indonesian": 0.29333333333333333, - "Malay": 0.25333333333333335 - }, - "consistency_score_2": 0.4942857142857144, - "consistency_score_3": 0.31523809523809526, - "consistency_score_4": 0.22552380952380952, - "consistency_score_5": 0.17206349206349206, - "consistency_score_6": 0.13714285714285715, - "consistency_score_7": 0.11333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5266666666666666, - "Spanish,Chinese": 0.43333333333333335, - "Spanish,Vietnamese": 0.4533333333333333, - "Spanish,Filipino": 0.5066666666666667, - "Spanish,Indonesian": 0.5266666666666666, - "Spanish,Malay": 0.5066666666666667, - "English,Chinese": 0.4533333333333333, - "English,Vietnamese": 0.4066666666666667, - "English,Filipino": 0.54, - "English,Indonesian": 0.48, - "English,Malay": 0.4266666666666667, - "Chinese,Vietnamese": 0.41333333333333333, - "Chinese,Filipino": 0.4666666666666667, - "Chinese,Indonesian": 0.5, - "Chinese,Malay": 0.5533333333333333, - "Vietnamese,Filipino": 0.4866666666666667, - "Vietnamese,Indonesian": 0.4866666666666667, - "Vietnamese,Malay": 0.4866666666666667, - "Filipino,Indonesian": 0.5266666666666666, - "Filipino,Malay": 0.56, - "Indonesian,Malay": 0.64 - }, - "3_combine": { - "Spanish,English,Chinese": 0.28, - "Spanish,English,Vietnamese": 0.28, - "Spanish,English,Filipino": 0.35333333333333333, - "Spanish,English,Indonesian": 0.3333333333333333, - "Spanish,English,Malay": 0.32666666666666666, - "Spanish,Chinese,Vietnamese": 0.24666666666666667, - "Spanish,Chinese,Filipino": 0.2733333333333333, - "Spanish,Chinese,Indonesian": 0.30666666666666664, - "Spanish,Chinese,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Filipino": 0.32666666666666666, - "Spanish,Vietnamese,Indonesian": 0.32, - "Spanish,Vietnamese,Malay": 0.32, - "Spanish,Filipino,Indonesian": 0.34, - "Spanish,Filipino,Malay": 0.36, - "Spanish,Indonesian,Malay": 0.38, - "English,Chinese,Vietnamese": 0.22666666666666666, - "English,Chinese,Filipino": 0.2866666666666667, - "English,Chinese,Indonesian": 0.2866666666666667, - "English,Chinese,Malay": 0.2866666666666667, - "English,Vietnamese,Filipino": 0.2733333333333333, - "English,Vietnamese,Indonesian": 0.28, - "English,Vietnamese,Malay": 0.25333333333333335, - "English,Filipino,Indonesian": 0.32666666666666666, - "English,Filipino,Malay": 0.3333333333333333, - "English,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino": 0.25333333333333335, - "Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Malay": 0.32, - "Chinese,Filipino,Indonesian": 0.32, - "Chinese,Filipino,Malay": 0.37333333333333335, - "Chinese,Indonesian,Malay": 0.4066666666666667, - "Vietnamese,Filipino,Indonesian": 0.32, - "Vietnamese,Filipino,Malay": 0.32666666666666666, - "Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Filipino,Indonesian,Malay": 0.4066666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.16666666666666666, - "Spanish,English,Chinese,Filipino": 0.18666666666666668, - "Spanish,English,Chinese,Indonesian": 0.20666666666666667, - "Spanish,English,Chinese,Malay": 0.22, - "Spanish,English,Vietnamese,Filipino": 0.22666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.22, - "Spanish,English,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Indonesian,Malay": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.18, - "Spanish,Chinese,Vietnamese,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.22, - "Spanish,Chinese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.25333333333333335, - "Spanish,Chinese,Indonesian,Malay": 0.26, - "Spanish,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,Vietnamese,Filipino,Malay": 0.24, - "Spanish,Vietnamese,Indonesian,Malay": 0.26, - "Spanish,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino": 0.15333333333333332, - "English,Chinese,Vietnamese,Indonesian": 0.18, - "English,Chinese,Vietnamese,Malay": 0.18, - "English,Chinese,Filipino,Indonesian": 0.22, - "English,Chinese,Filipino,Malay": 0.23333333333333334, - "English,Chinese,Indonesian,Malay": 0.23333333333333334, - "English,Vietnamese,Filipino,Indonesian": 0.2, - "English,Vietnamese,Filipino,Malay": 0.2, - "English,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "English,Filipino,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.22, - "Chinese,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16, - "Spanish,English,Chinese,Filipino,Malay": 0.18, - "Spanish,English,Chinese,Indonesian,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,English,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.2, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "English,Chinese,Filipino,Indonesian,Malay": 0.2, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - } - }, - "AC3_2": 0.3813896871884495, - "AC3_3": 0.3128390229260122, - "AC3_4": 0.26126781055808584, - "AC3_5": 0.2214185463200287, - "AC3_6": 0.19024924007907806, - "AC3_7": 0.16605243441774795 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.262987012987013, - "language_acc": { - "English": 0.42613636363636365, - "Filipino": 0.2215909090909091, - "Spanish": 0.24431818181818182, - "Malay": 0.2556818181818182, - "Chinese": 0.22727272727272727, - "Indonesian": 0.24431818181818182, - "Vietnamese": 0.2215909090909091 - }, - "consistency_score_2": 0.35930735930735924, - "consistency_score_3": 0.16071428571428573, - "consistency_score_4": 0.07873376623376625, - "consistency_score_5": 0.037608225108225105, - "consistency_score_6": 0.01461038961038961, - "consistency_score_7": 0.0, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.32386363636363635, - "English,Spanish": 0.375, - "English,Malay": 0.32386363636363635, - "English,Chinese": 0.2556818181818182, - "English,Indonesian": 0.26704545454545453, - "English,Vietnamese": 0.23295454545454544, - "Filipino,Spanish": 0.4659090909090909, - "Filipino,Malay": 0.5113636363636364, - "Filipino,Chinese": 0.20454545454545456, - "Filipino,Indonesian": 0.44886363636363635, - "Filipino,Vietnamese": 0.4431818181818182, - "Spanish,Malay": 0.4943181818181818, - "Spanish,Chinese": 0.3181818181818182, - "Spanish,Indonesian": 0.48863636363636365, - "Spanish,Vietnamese": 0.32386363636363635, - "Malay,Chinese": 0.26704545454545453, - "Malay,Indonesian": 0.5454545454545454, - "Malay,Vietnamese": 0.42613636363636365, - "Chinese,Indonesian": 0.2840909090909091, - "Chinese,Vietnamese": 0.11363636363636363, - "Indonesian,Vietnamese": 0.4318181818181818 - }, - "3_combine": { - "English,Filipino,Spanish": 0.18181818181818182, - "English,Filipino,Malay": 0.1875, - "English,Filipino,Chinese": 0.06818181818181818, - "English,Filipino,Indonesian": 0.13068181818181818, - "English,Filipino,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay": 0.1875, - "English,Spanish,Chinese": 0.11931818181818182, - "English,Spanish,Indonesian": 0.18181818181818182, - "English,Spanish,Vietnamese": 0.11363636363636363, - "English,Malay,Chinese": 0.09090909090909091, - "English,Malay,Indonesian": 0.17613636363636365, - "English,Malay,Vietnamese": 0.11931818181818182, - "English,Chinese,Indonesian": 0.10795454545454546, - "English,Chinese,Vietnamese": 0.028409090909090908, - "English,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay": 0.29545454545454547, - "Filipino,Spanish,Chinese": 0.11363636363636363, - "Filipino,Spanish,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Chinese": 0.10795454545454546, - "Filipino,Malay,Indonesian": 0.32954545454545453, - "Filipino,Malay,Vietnamese": 0.2784090909090909, - "Filipino,Chinese,Indonesian": 0.11931818181818182, - "Filipino,Chinese,Vietnamese": 0.056818181818181816, - "Filipino,Indonesian,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Chinese": 0.13636363636363635, - "Spanish,Malay,Indonesian": 0.32954545454545453, - "Spanish,Malay,Vietnamese": 0.22727272727272727, - "Spanish,Chinese,Indonesian": 0.17045454545454544, - "Spanish,Chinese,Vietnamese": 0.056818181818181816, - "Spanish,Indonesian,Vietnamese": 0.2215909090909091, - "Malay,Chinese,Indonesian": 0.1590909090909091, - "Malay,Chinese,Vietnamese": 0.05113636363636364, - "Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Chinese,Indonesian,Vietnamese": 0.06818181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.125, - "English,Filipino,Spanish,Chinese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian": 0.09659090909090909, - "English,Filipino,Spanish,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese": 0.05113636363636364, - "English,Filipino,Malay,Indonesian": 0.11363636363636363, - "English,Filipino,Malay,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese": 0.0625, - "English,Spanish,Malay,Indonesian": 0.11931818181818182, - "English,Spanish,Malay,Vietnamese": 0.06818181818181818, - "English,Spanish,Chinese,Indonesian": 0.07954545454545454, - "English,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Spanish,Indonesian,Vietnamese": 0.06818181818181818, - "English,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Chinese": 0.07386363636363637, - "Filipino,Spanish,Malay,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Spanish,Chinese,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Vietnamese": 0.05113636363636364, - "English,Filipino,Spanish,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.045454545454545456, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0 - } - }, - "AC3_2": 0.3036928288614181, - "AC3_3": 0.1995073891154748, - "AC3_4": 0.1211864145001196, - "AC3_5": 0.06580593122058302, - "AC3_6": 0.027682843462344852, - "AC3_7": 0.0 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.4368932038834951 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.2571428571428571 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.3177570093457944 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.2, - "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.2, - "history": 0.2, - "literature": 0.2, - "politics": 0.2, - "culture": 0.1, - "film": 0.3, - "law": 0.2, - "geography": 0.3 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.17968757977528557 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.0650861503777802 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.03220531989518922 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.00917571250226757 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.05669801375257065 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.3348891481913652 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.351662495530926, - "category_acc": { - "high_school_european_history": 0.5, - "business_ethics": 0.42424242424242425, - "clinical_knowledge": 0.3939393939393939, - "medical_genetics": 0.35353535353535354, - "high_school_us_history": 0.4236453201970443, - "high_school_physics": 0.23333333333333334, - "high_school_world_history": 0.4364406779661017, - "virology": 0.3515151515151515, - "high_school_microeconomics": 0.379746835443038, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.3333333333333333, - "high_school_biology": 0.3300970873786408, - "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.3309608540925267, - "philosophy": 0.3548387096774194, - "professional_medicine": 0.33210332103321033, - "nutrition": 0.35081967213114756, - "global_facts": 0.24242424242424243, - "machine_learning": 0.2702702702702703, - "security_studies": 0.4098360655737705, - "public_relations": 0.3853211009174312, - "professional_psychology": 0.3502454991816694, - "prehistory": 0.34365325077399383, - "anatomy": 0.34328358208955223, - "human_sexuality": 0.36923076923076925, - "college_medicine": 0.36627906976744184, - "high_school_government_and_politics": 0.4375, - "college_chemistry": 0.30303030303030304, - "logical_fallacies": 0.3950617283950617, - "high_school_geography": 0.41116751269035534, - "elementary_mathematics": 0.27055702917771884, - "human_aging": 0.3783783783783784, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.42463235294117646, - "formal_logic": 0.328, - "high_school_statistics": 0.3023255813953488, - "international_law": 0.5333333333333333, - "high_school_mathematics": 0.2379182156133829, - "high_school_computer_science": 0.3434343434343434, - "conceptual_physics": 0.31196581196581197, - "miscellaneous": 0.391304347826087, - "high_school_chemistry": 0.28217821782178215, - "marketing": 0.5665236051502146, - "professional_law": 0.33072407045009783, - "management": 0.5, - "college_physics": 0.21782178217821782, - "jurisprudence": 0.37383177570093457, - "world_religions": 0.28823529411764703, - "sociology": 0.495, - "us_foreign_policy": 0.5050505050505051, - "high_school_macroeconomics": 0.35218508997429304, - "computer_security": 0.37373737373737376, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.4, - "electrical_engineering": 0.3055555555555556, - "astronomy": 0.32450331125827814, - "college_biology": 0.2867132867132867 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.24665676077265974 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.26214196762141967, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.375, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.16666666666666666, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.16666666666666666, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.26666666666666666, - "business_administration": 0.13157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.20689655172413793, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.0, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.34615384615384615, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.2962962962962963, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.04, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.28846153846153844, - "sports_science": 0.2916666666666667, - "plant_protection": 0.1111111111111111, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.3148148148148148, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.2962962962962963, - "physician": 0.2962962962962963 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.2222222222222222 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.24866171645657054, - "category_acc": { - "agronomy": 0.2603550295857988, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.23780487804878048, - "arts": 0.25, - "astronomy": 0.22424242424242424, - "business_ethics": 0.2583732057416268, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.25190839694656486, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.2523364485981308, - "chinese_history": 0.25386996904024767, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.24472573839662448, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.1588785046728972, - "college_engineering_hydrology": 0.24528301886792453, - "college_law": 0.3333333333333333, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.22641509433962265, - "college_medicine": 0.25274725274725274, - "computer_science": 0.24509803921568626, - "computer_security": 0.23976608187134502, - "conceptual_physics": 0.25170068027210885, - "construction_project_management": 0.2446043165467626, - "economics": 0.24528301886792453, - "education": 0.25153374233128833, - "electrical_engineering": 0.2558139534883721, - "elementary_chinese": 0.20238095238095238, - "elementary_commonsense": 0.2777777777777778, - "elementary_information_and_technology": 0.23949579831932774, - "elementary_mathematics": 0.23043478260869565, - "ethnology": 0.24444444444444444, - "food_science": 0.26573426573426573, - "genetics": 0.25, - "global_facts": 0.2684563758389262, - "high_school_biology": 0.23076923076923078, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.27439024390243905, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.23776223776223776, - "human_sexuality": 0.24603174603174602, - "international_law": 0.23243243243243245, - "journalism": 0.2616279069767442, - "jurisprudence": 0.25304136253041365, - "legal_and_moral_basis": 0.2523364485981308, - "logical": 0.23577235772357724, - "machine_learning": 0.22950819672131148, - "management": 0.24285714285714285, - "marketing": 0.24444444444444444, - "marxist_theory": 0.25396825396825395, - "modern_chinese": 0.25, - "nutrition": 0.2620689655172414, - "philosophy": 0.2571428571428571, - "professional_accounting": 0.25142857142857145, - "professional_law": 0.24644549763033174, - "professional_medicine": 0.26595744680851063, - "professional_psychology": 0.25, - "public_relations": 0.2413793103448276, - "security_study": 0.24444444444444444, - "sociology": 0.252212389380531, - "sports_science": 0.24848484848484848, - "traditional_chinese_medicine": 0.25405405405405407, - "virology": 0.2485207100591716, - "world_history": 0.2546583850931677, - "world_religions": 0.25 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.2727272727272727 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.2409090909090909 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3257627118644068 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.2879581151832461 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.7349338559529642 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.4867433787733231, - "rouge2": 0.24015490146669383, - "rougeL": 0.4043127786869806, - "avg_rouge": 0.3770703529756658 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.379681452311921, - "rouge2": 0.14092947864352295, - "rougeL": 0.3106447743194101, - "avg_rouge": 0.27708523509161803 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.9197247706422018 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.4669223394055609 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.8015 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.616 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.73 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.4225352112676056 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.7653429602888087 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.6495098039215687 - } - } - } - }, - "flan_t5_large": { - "model_size": "0.78B", - "model_link": "https://huggingface.co/google/flan-t5-large", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.7676470588235293, - "language_acc": { - "English": 0.9705882352941176, - "Chinese": 0.37563025210084033, - "Spanish": 0.9319327731092437, - "Vietnamese": 0.7924369747899159 - }, - "consistency_score_2": 0.6124649859943977, - "consistency_score_3": 0.44621848739495795, - "consistency_score_4": 0.32184873949579834, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3764705882352941, - "English,Spanish": 0.9361344537815126, - "English,Vietnamese": 0.788235294117647, - "Chinese,Spanish": 0.38487394957983195, - "Chinese,Vietnamese": 0.4092436974789916, - "Spanish,Vietnamese": 0.7798319327731092 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3638655462184874, - "English,Chinese,Vietnamese": 0.32941176470588235, - "English,Spanish,Vietnamese": 0.761344537815126, - "Chinese,Spanish,Vietnamese": 0.3302521008403361 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.32184873949579834 - } - }, - "AC3_2": 0.6813315583503264, - "AC3_3": 0.564376030739652, - "AC3_4": 0.45354234246421293 - }, - "prompt_2": { - "overall_acc": 0.7487394957983193, - "language_acc": { - "English": 0.9638655462184874, - "Chinese": 0.35798319327731093, - "Spanish": 0.9176470588235294, - "Vietnamese": 0.7554621848739496 - }, - "consistency_score_2": 0.596078431372549, - "consistency_score_3": 0.42626050420168066, - "consistency_score_4": 0.3042016806722689, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3621848739495798, - "English,Spanish": 0.9210084033613445, - "English,Vietnamese": 0.7563025210084033, - "Chinese,Spanish": 0.37478991596638656, - "Chinese,Vietnamese": 0.40756302521008403, - "Spanish,Vietnamese": 0.7546218487394958 - }, - "3_combine": { - "English,Chinese,Spanish": 0.346218487394958, - "English,Chinese,Vietnamese": 0.3134453781512605, - "English,Spanish,Vietnamese": 0.7277310924369748, - "Chinese,Spanish,Vietnamese": 0.3176470588235294 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3042016806722689 - } - }, - "AC3_2": 0.6637440728766278, - "AC3_3": 0.5432477871787939, - "AC3_4": 0.43263160008499596 - }, - "prompt_3": { - "overall_acc": 0.7565126050420168, - "language_acc": { - "English": 0.965546218487395, - "Chinese": 0.36722689075630255, - "Spanish": 0.9210084033613445, - "Vietnamese": 0.7722689075630252 - }, - "consistency_score_2": 0.6050420168067226, - "consistency_score_3": 0.43718487394957983, - "consistency_score_4": 0.3142857142857143, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.37058823529411766, - "English,Spanish": 0.926890756302521, - "English,Vietnamese": 0.7714285714285715, - "Chinese,Spanish": 0.3865546218487395, - "Chinese,Vietnamese": 0.4092436974789916, - "Spanish,Vietnamese": 0.7655462184873949 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3596638655462185, - "English,Chinese,Vietnamese": 0.32016806722689073, - "English,Spanish,Vietnamese": 0.746218487394958, - "Chinese,Spanish,Vietnamese": 0.3226890756302521 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3142857142857143 - } - }, - "AC3_2": 0.6723518908689982, - "AC3_3": 0.554136828919589, - "AC3_4": 0.44408195292806185 - }, - "prompt_4": { - "overall_acc": 0.759453781512605, - "language_acc": { - "English": 0.9680672268907563, - "Chinese": 0.36050420168067226, - "Spanish": 0.926890756302521, - "Vietnamese": 0.7823529411764706 - }, - "consistency_score_2": 0.6053221288515406, - "consistency_score_3": 0.43676470588235294, - "consistency_score_4": 0.31176470588235294, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.36554621848739494, - "English,Spanish": 0.9277310924369748, - "English,Vietnamese": 0.7815126050420168, - "Chinese,Spanish": 0.37478991596638656, - "Chinese,Vietnamese": 0.4042016806722689, - "Spanish,Vietnamese": 0.7781512605042017 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3521008403361345, - "English,Chinese,Vietnamese": 0.31848739495798317, - "English,Spanish,Vietnamese": 0.7546218487394958, - "Chinese,Spanish,Vietnamese": 0.32184873949579834 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.31176470588235294 - } - }, - "AC3_2": 0.6736844873430086, - "AC3_3": 0.5545853219644238, - "AC3_4": 0.4420589965324557 - }, - "prompt_5": { - "overall_acc": 0.7497899159663866, - "language_acc": { - "English": 0.9647058823529412, - "Chinese": 0.36134453781512604, - "Spanish": 0.9168067226890756, - "Vietnamese": 0.7563025210084033 - }, - "consistency_score_2": 0.6015406162464986, - "consistency_score_3": 0.43130252100840333, - "consistency_score_4": 0.3092436974789916, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.36722689075630255, - "English,Spanish": 0.9184873949579831, - "English,Vietnamese": 0.7563025210084033, - "Chinese,Spanish": 0.3815126050420168, - "Chinese,Vietnamese": 0.4294117647058823, - "Spanish,Vietnamese": 0.7563025210084033 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3521008403361345, - "English,Chinese,Vietnamese": 0.31848739495798317, - "English,Spanish,Vietnamese": 0.7302521008403361, - "Chinese,Spanish,Vietnamese": 0.3243697478991597 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3092436974789916 - } - }, - "AC3_2": 0.6675333344742334, - "AC3_3": 0.547605370810623, - "AC3_4": 0.43788564022944026 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.3171428571428571, - "language_acc": { - "Spanish": 0.34, - "English": 0.44666666666666666, - "Chinese": 0.28, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.26666666666666666, - "Indonesian": 0.35333333333333333, - "Malay": 0.24666666666666667 - }, - "consistency_score_2": 0.5466666666666666, - "consistency_score_3": 0.38209523809523804, - "consistency_score_4": 0.29695238095238097, - "consistency_score_5": 0.24412698412698414, - "consistency_score_6": 0.20761904761904762, - "consistency_score_7": 0.18, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.54, - "Spanish,Chinese": 0.5066666666666667, - "Spanish,Vietnamese": 0.5466666666666666, - "Spanish,Filipino": 0.54, - "Spanish,Indonesian": 0.5066666666666667, - "Spanish,Malay": 0.4533333333333333, - "English,Chinese": 0.44, - "English,Vietnamese": 0.4666666666666667, - "English,Filipino": 0.5, - "English,Indonesian": 0.47333333333333333, - "English,Malay": 0.4866666666666667, - "Chinese,Vietnamese": 0.7, - "Chinese,Filipino": 0.56, - "Chinese,Indonesian": 0.5933333333333334, - "Chinese,Malay": 0.6066666666666667, - "Vietnamese,Filipino": 0.5333333333333333, - "Vietnamese,Indonesian": 0.5866666666666667, - "Vietnamese,Malay": 0.62, - "Filipino,Indonesian": 0.5866666666666667, - "Filipino,Malay": 0.6266666666666667, - "Indonesian,Malay": 0.6066666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.32666666666666666, - "Spanish,English,Vietnamese": 0.34, - "Spanish,English,Filipino": 0.37333333333333335, - "Spanish,English,Indonesian": 0.32666666666666666, - "Spanish,English,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese": 0.42, - "Spanish,Chinese,Filipino": 0.36666666666666664, - "Spanish,Chinese,Indonesian": 0.38666666666666666, - "Spanish,Chinese,Malay": 0.36, - "Spanish,Vietnamese,Filipino": 0.36666666666666664, - "Spanish,Vietnamese,Indonesian": 0.37333333333333335, - "Spanish,Vietnamese,Malay": 0.36, - "Spanish,Filipino,Indonesian": 0.37333333333333335, - "Spanish,Filipino,Malay": 0.4, - "Spanish,Indonesian,Malay": 0.34, - "English,Chinese,Vietnamese": 0.35333333333333333, - "English,Chinese,Filipino": 0.32666666666666666, - "English,Chinese,Indonesian": 0.34, - "English,Chinese,Malay": 0.3333333333333333, - "English,Vietnamese,Filipino": 0.31333333333333335, - "English,Vietnamese,Indonesian": 0.32666666666666666, - "English,Vietnamese,Malay": 0.3466666666666667, - "English,Filipino,Indonesian": 0.3466666666666667, - "English,Filipino,Malay": 0.37333333333333335, - "English,Indonesian,Malay": 0.36, - "Chinese,Vietnamese,Filipino": 0.44, - "Chinese,Vietnamese,Indonesian": 0.48, - "Chinese,Vietnamese,Malay": 0.5066666666666667, - "Chinese,Filipino,Indonesian": 0.43333333333333335, - "Chinese,Filipino,Malay": 0.46, - "Chinese,Indonesian,Malay": 0.4533333333333333, - "Vietnamese,Filipino,Indonesian": 0.4, - "Vietnamese,Filipino,Malay": 0.4533333333333333, - "Vietnamese,Indonesian,Malay": 0.44, - "Filipino,Indonesian,Malay": 0.4666666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.26666666666666666, - "Spanish,English,Chinese,Filipino": 0.26, - "Spanish,English,Chinese,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.24, - "Spanish,English,Vietnamese,Malay": 0.24, - "Spanish,English,Filipino,Indonesian": 0.2733333333333333, - "Spanish,English,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Indonesian": 0.34, - "Spanish,Chinese,Vietnamese,Malay": 0.32, - "Spanish,Chinese,Filipino,Indonesian": 0.3, - "Spanish,Chinese,Filipino,Malay": 0.32666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.30666666666666664, - "Spanish,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.32666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino": 0.26, - "English,Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "English,Chinese,Vietnamese,Malay": 0.28, - "English,Chinese,Filipino,Indonesian": 0.28, - "English,Chinese,Filipino,Malay": 0.3, - "English,Chinese,Indonesian,Malay": 0.28, - "English,Vietnamese,Filipino,Indonesian": 0.26, - "English,Vietnamese,Filipino,Malay": 0.2866666666666667, - "English,Vietnamese,Indonesian,Malay": 0.28, - "English,Filipino,Indonesian,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.38666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.38, - "Chinese,Filipino,Indonesian,Malay": 0.38666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.36666666666666664 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Chinese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.24, - "Spanish,English,Chinese,Indonesian,Malay": 0.22, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.2, - "Spanish,English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.28, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.24, - "English,Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.32666666666666666 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18 - } - }, - "AC3_2": 0.40141124581902066, - "AC3_3": 0.3466023270703286, - "AC3_4": 0.30671570360124156, - "AC3_5": 0.2758855849540782, - "AC3_6": 0.2509515166750196, - "AC3_7": 0.22965517236759808 - }, - "prompt_2": { - "overall_acc": 0.3142857142857142, - "language_acc": { - "Spanish": 0.3466666666666667, - "English": 0.4666666666666667, - "Chinese": 0.26, - "Vietnamese": 0.28, - "Filipino": 0.30666666666666664, - "Indonesian": 0.30666666666666664, - "Malay": 0.23333333333333334 - }, - "consistency_score_2": 0.5390476190476189, - "consistency_score_3": 0.36990476190476185, - "consistency_score_4": 0.2826666666666666, - "consistency_score_5": 0.22888888888888892, - "consistency_score_6": 0.1904761904761905, - "consistency_score_7": 0.16, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6, - "Spanish,Chinese": 0.49333333333333335, - "Spanish,Vietnamese": 0.5133333333333333, - "Spanish,Filipino": 0.5266666666666666, - "Spanish,Indonesian": 0.48, - "Spanish,Malay": 0.46, - "English,Chinese": 0.46, - "English,Vietnamese": 0.47333333333333333, - "English,Filipino": 0.5, - "English,Indonesian": 0.5, - "English,Malay": 0.4533333333333333, - "Chinese,Vietnamese": 0.6266666666666667, - "Chinese,Filipino": 0.5266666666666666, - "Chinese,Indonesian": 0.5666666666666667, - "Chinese,Malay": 0.6333333333333333, - "Vietnamese,Filipino": 0.48, - "Vietnamese,Indonesian": 0.54, - "Vietnamese,Malay": 0.6, - "Filipino,Indonesian": 0.6133333333333333, - "Filipino,Malay": 0.62, - "Indonesian,Malay": 0.6533333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.34, - "Spanish,English,Vietnamese": 0.3333333333333333, - "Spanish,English,Filipino": 0.38, - "Spanish,English,Indonesian": 0.3333333333333333, - "Spanish,English,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese": 0.36666666666666664, - "Spanish,Chinese,Filipino": 0.3466666666666667, - "Spanish,Chinese,Indonesian": 0.34, - "Spanish,Chinese,Malay": 0.36, - "Spanish,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian": 0.32, - "Spanish,Vietnamese,Malay": 0.3466666666666667, - "Spanish,Filipino,Indonesian": 0.38666666666666666, - "Spanish,Filipino,Malay": 0.38, - "Spanish,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese": 0.3333333333333333, - "English,Chinese,Filipino": 0.32666666666666666, - "English,Chinese,Indonesian": 0.3333333333333333, - "English,Chinese,Malay": 0.3333333333333333, - "English,Vietnamese,Filipino": 0.31333333333333335, - "English,Vietnamese,Indonesian": 0.32, - "English,Vietnamese,Malay": 0.32666666666666666, - "English,Filipino,Indonesian": 0.38666666666666666, - "English,Filipino,Malay": 0.36, - "English,Indonesian,Malay": 0.36, - "Chinese,Vietnamese,Filipino": 0.38, - "Chinese,Vietnamese,Indonesian": 0.4, - "Chinese,Vietnamese,Malay": 0.47333333333333333, - "Chinese,Filipino,Indonesian": 0.4266666666666667, - "Chinese,Filipino,Malay": 0.44666666666666666, - "Chinese,Indonesian,Malay": 0.47333333333333333, - "Vietnamese,Filipino,Indonesian": 0.37333333333333335, - "Vietnamese,Filipino,Malay": 0.4066666666666667, - "Vietnamese,Indonesian,Malay": 0.44666666666666666, - "Filipino,Indonesian,Malay": 0.5 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.25333333333333335, - "Spanish,English,Chinese,Filipino": 0.26, - "Spanish,English,Chinese,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Malay": 0.24666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.24666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.22, - "Spanish,English,Vietnamese,Malay": 0.24, - "Spanish,English,Filipino,Indonesian": 0.29333333333333333, - "Spanish,English,Filipino,Malay": 0.28, - "Spanish,English,Indonesian,Malay": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.2866666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.30666666666666664, - "Spanish,Chinese,Indonesian,Malay": 0.3, - "Spanish,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.28, - "Spanish,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino": 0.26, - "English,Chinese,Vietnamese,Indonesian": 0.26, - "English,Chinese,Vietnamese,Malay": 0.26666666666666666, - "English,Chinese,Filipino,Indonesian": 0.29333333333333333, - "English,Chinese,Filipino,Malay": 0.2866666666666667, - "English,Chinese,Indonesian,Malay": 0.29333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "English,Vietnamese,Filipino,Malay": 0.28, - "English,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "English,Filipino,Indonesian,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Chinese,Vietnamese,Filipino,Malay": 0.32666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Chinese,Filipino,Indonesian,Malay": 0.38666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.3333333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.2, - "Spanish,English,Chinese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.22, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.2, - "Spanish,English,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.24, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.24, - "English,Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.18, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16 - } - }, - "AC3_2": 0.39706632648408097, - "AC3_3": 0.33983455292519515, - "AC3_4": 0.297638800205405, - "AC3_5": 0.26487434244100794, - "AC3_6": 0.23719676545166046, - "AC3_7": 0.21204819272637537 - }, - "prompt_3": { - "overall_acc": 0.30571428571428566, - "language_acc": { - "Spanish": 0.34, - "English": 0.44, - "Chinese": 0.26666666666666666, - "Vietnamese": 0.24666666666666667, - "Filipino": 0.29333333333333333, - "Indonesian": 0.30666666666666664, - "Malay": 0.24666666666666667 - }, - "consistency_score_2": 0.5695238095238094, - "consistency_score_3": 0.40419047619047616, - "consistency_score_4": 0.3173333333333333, - "consistency_score_5": 0.26571428571428574, - "consistency_score_6": 0.23142857142857146, - "consistency_score_7": 0.20666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.58, - "Spanish,Chinese": 0.5333333333333333, - "Spanish,Vietnamese": 0.5533333333333333, - "Spanish,Filipino": 0.5533333333333333, - "Spanish,Indonesian": 0.5066666666666667, - "Spanish,Malay": 0.5, - "English,Chinese": 0.4866666666666667, - "English,Vietnamese": 0.48, - "English,Filipino": 0.49333333333333335, - "English,Indonesian": 0.48, - "English,Malay": 0.5066666666666667, - "Chinese,Vietnamese": 0.6733333333333333, - "Chinese,Filipino": 0.6, - "Chinese,Indonesian": 0.6133333333333333, - "Chinese,Malay": 0.6533333333333333, - "Vietnamese,Filipino": 0.5533333333333333, - "Vietnamese,Indonesian": 0.5866666666666667, - "Vietnamese,Malay": 0.6066666666666667, - "Filipino,Indonesian": 0.62, - "Filipino,Malay": 0.6733333333333333, - "Indonesian,Malay": 0.7066666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.36, - "Spanish,English,Vietnamese": 0.3466666666666667, - "Spanish,English,Filipino": 0.37333333333333335, - "Spanish,English,Indonesian": 0.3466666666666667, - "Spanish,English,Malay": 0.35333333333333333, - "Spanish,Chinese,Vietnamese": 0.4266666666666667, - "Spanish,Chinese,Filipino": 0.4, - "Spanish,Chinese,Indonesian": 0.38666666666666666, - "Spanish,Chinese,Malay": 0.4, - "Spanish,Vietnamese,Filipino": 0.38666666666666666, - "Spanish,Vietnamese,Indonesian": 0.36, - "Spanish,Vietnamese,Malay": 0.38, - "Spanish,Filipino,Indonesian": 0.38666666666666666, - "Spanish,Filipino,Malay": 0.41333333333333333, - "Spanish,Indonesian,Malay": 0.38666666666666666, - "English,Chinese,Vietnamese": 0.36666666666666664, - "English,Chinese,Filipino": 0.3466666666666667, - "English,Chinese,Indonesian": 0.35333333333333333, - "English,Chinese,Malay": 0.37333333333333335, - "English,Vietnamese,Filipino": 0.32666666666666666, - "English,Vietnamese,Indonesian": 0.32666666666666666, - "English,Vietnamese,Malay": 0.36, - "English,Filipino,Indonesian": 0.36666666666666664, - "English,Filipino,Malay": 0.38, - "English,Indonesian,Malay": 0.3933333333333333, - "Chinese,Vietnamese,Filipino": 0.4666666666666667, - "Chinese,Vietnamese,Indonesian": 0.47333333333333333, - "Chinese,Vietnamese,Malay": 0.5066666666666667, - "Chinese,Filipino,Indonesian": 0.47333333333333333, - "Chinese,Filipino,Malay": 0.5, - "Chinese,Indonesian,Malay": 0.5133333333333333, - "Vietnamese,Filipino,Indonesian": 0.4266666666666667, - "Vietnamese,Filipino,Malay": 0.4666666666666667, - "Vietnamese,Indonesian,Malay": 0.48, - "Filipino,Indonesian,Malay": 0.54 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.28, - "Spanish,English,Chinese,Filipino": 0.2733333333333333, - "Spanish,English,Chinese,Indonesian": 0.28, - "Spanish,English,Chinese,Malay": 0.28, - "Spanish,English,Vietnamese,Filipino": 0.26, - "Spanish,English,Vietnamese,Indonesian": 0.24, - "Spanish,English,Vietnamese,Malay": 0.26666666666666666, - "Spanish,English,Filipino,Indonesian": 0.28, - "Spanish,English,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.34, - "Spanish,Chinese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.3466666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Vietnamese,Filipino,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.3, - "Spanish,Filipino,Indonesian,Malay": 0.3333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.28, - "English,Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "English,Chinese,Vietnamese,Malay": 0.31333333333333335, - "English,Chinese,Filipino,Indonesian": 0.30666666666666664, - "English,Chinese,Filipino,Malay": 0.31333333333333335, - "English,Chinese,Indonesian,Malay": 0.32666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.28, - "English,Vietnamese,Filipino,Malay": 0.30666666666666664, - "English,Vietnamese,Indonesian,Malay": 0.3, - "English,Filipino,Indonesian,Malay": 0.32666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.4066666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.41333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.43333333333333335, - "Vietnamese,Filipino,Indonesian,Malay": 0.3933333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.22666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.24, - "Spanish,English,Chinese,Filipino,Indonesian": 0.24, - "Spanish,English,Chinese,Filipino,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.26, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,English,Vietnamese,Filipino,Malay": 0.24, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.3, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.3, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "English,Chinese,Vietnamese,Filipino,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.28, - "English,Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.36 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.22, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 - } - }, - "AC3_2": 0.39786102902340614, - "AC3_3": 0.348122197045523, - "AC3_4": 0.3114154692254683, - "AC3_5": 0.2843142856645307, - "AC3_6": 0.26343465040688335, - "AC3_7": 0.24661710032361556 - }, - "prompt_4": { - "overall_acc": 0.31523809523809526, - "language_acc": { - "Spanish": 0.34, - "English": 0.4533333333333333, - "Chinese": 0.2733333333333333, - "Vietnamese": 0.28, - "Filipino": 0.29333333333333333, - "Indonesian": 0.32666666666666666, - "Malay": 0.24 - }, - "consistency_score_2": 0.5653968253968253, - "consistency_score_3": 0.407047619047619, - "consistency_score_4": 0.3251428571428572, - "consistency_score_5": 0.2761904761904762, - "consistency_score_6": 0.24380952380952378, - "consistency_score_7": 0.22, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5933333333333334, - "Spanish,Chinese": 0.52, - "Spanish,Vietnamese": 0.5533333333333333, - "Spanish,Filipino": 0.5866666666666667, - "Spanish,Indonesian": 0.5733333333333334, - "Spanish,Malay": 0.5066666666666667, - "English,Chinese": 0.48, - "English,Vietnamese": 0.4666666666666667, - "English,Filipino": 0.54, - "English,Indonesian": 0.5066666666666667, - "English,Malay": 0.48, - "Chinese,Vietnamese": 0.6466666666666666, - "Chinese,Filipino": 0.6, - "Chinese,Indonesian": 0.6, - "Chinese,Malay": 0.64, - "Vietnamese,Filipino": 0.5266666666666666, - "Vietnamese,Indonesian": 0.5333333333333333, - "Vietnamese,Malay": 0.5666666666666667, - "Filipino,Indonesian": 0.62, - "Filipino,Malay": 0.6733333333333333, - "Indonesian,Malay": 0.66 - }, - "3_combine": { - "Spanish,English,Chinese": 0.3933333333333333, - "Spanish,English,Vietnamese": 0.37333333333333335, - "Spanish,English,Filipino": 0.44, - "Spanish,English,Indonesian": 0.38666666666666666, - "Spanish,English,Malay": 0.36, - "Spanish,Chinese,Vietnamese": 0.4, - "Spanish,Chinese,Filipino": 0.42, - "Spanish,Chinese,Indonesian": 0.4066666666666667, - "Spanish,Chinese,Malay": 0.38666666666666666, - "Spanish,Vietnamese,Filipino": 0.4066666666666667, - "Spanish,Vietnamese,Indonesian": 0.36666666666666664, - "Spanish,Vietnamese,Malay": 0.36666666666666664, - "Spanish,Filipino,Indonesian": 0.44, - "Spanish,Filipino,Malay": 0.44, - "Spanish,Indonesian,Malay": 0.4066666666666667, - "English,Chinese,Vietnamese": 0.3466666666666667, - "English,Chinese,Filipino": 0.38, - "English,Chinese,Indonesian": 0.38, - "English,Chinese,Malay": 0.36666666666666664, - "English,Vietnamese,Filipino": 0.3333333333333333, - "English,Vietnamese,Indonesian": 0.31333333333333335, - "English,Vietnamese,Malay": 0.32666666666666666, - "English,Filipino,Indonesian": 0.4, - "English,Filipino,Malay": 0.4066666666666667, - "English,Indonesian,Malay": 0.4, - "Chinese,Vietnamese,Filipino": 0.44, - "Chinese,Vietnamese,Indonesian": 0.4266666666666667, - "Chinese,Vietnamese,Malay": 0.48, - "Chinese,Filipino,Indonesian": 0.46, - "Chinese,Filipino,Malay": 0.5, - "Chinese,Indonesian,Malay": 0.49333333333333335, - "Vietnamese,Filipino,Indonesian": 0.4, - "Vietnamese,Filipino,Malay": 0.44, - "Vietnamese,Indonesian,Malay": 0.44, - "Filipino,Indonesian,Malay": 0.52 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.29333333333333333, - "Spanish,English,Chinese,Filipino": 0.32666666666666666, - "Spanish,English,Chinese,Indonesian": 0.32, - "Spanish,English,Chinese,Malay": 0.29333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.3, - "Spanish,English,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,English,Vietnamese,Malay": 0.26666666666666666, - "Spanish,English,Filipino,Indonesian": 0.3466666666666667, - "Spanish,English,Filipino,Malay": 0.3333333333333333, - "Spanish,English,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Filipino": 0.34, - "Spanish,Chinese,Vietnamese,Indonesian": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Malay": 0.32, - "Spanish,Chinese,Filipino,Indonesian": 0.36, - "Spanish,Chinese,Filipino,Malay": 0.35333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.3466666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.3466666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Spanish,Filipino,Indonesian,Malay": 0.36666666666666664, - "English,Chinese,Vietnamese,Filipino": 0.29333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.28, - "English,Chinese,Vietnamese,Malay": 0.2866666666666667, - "English,Chinese,Filipino,Indonesian": 0.3333333333333333, - "English,Chinese,Filipino,Malay": 0.32666666666666666, - "English,Chinese,Indonesian,Malay": 0.3333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.28, - "English,Vietnamese,Filipino,Malay": 0.29333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "English,Filipino,Indonesian,Malay": 0.3466666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.38666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.42, - "Vietnamese,Filipino,Indonesian,Malay": 0.36666666666666664 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.26, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.24, - "Spanish,English,Chinese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.26, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.28, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.32, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.3, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.26, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22 - } - }, - "AC3_2": 0.40478662404851745, - "AC3_3": 0.35530791636631925, - "AC3_4": 0.32011387774384215, - "AC3_5": 0.29442527408563707, - "AC3_6": 0.274960655422545, - "AC3_7": 0.25914590742489263 - }, - "prompt_5": { - "overall_acc": 0.31523809523809526, - "language_acc": { - "Spanish": 0.3466666666666667, - "English": 0.5, - "Chinese": 0.26, - "Vietnamese": 0.25333333333333335, - "Filipino": 0.30666666666666664, - "Indonesian": 0.28, - "Malay": 0.26 - }, - "consistency_score_2": 0.527936507936508, - "consistency_score_3": 0.3500952380952381, - "consistency_score_4": 0.26076190476190475, - "consistency_score_5": 0.2085714285714286, - "consistency_score_6": 0.17523809523809522, - "consistency_score_7": 0.15333333333333332, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.56, - "Spanish,Chinese": 0.4666666666666667, - "Spanish,Vietnamese": 0.48, - "Spanish,Filipino": 0.48, - "Spanish,Indonesian": 0.48, - "Spanish,Malay": 0.42, - "English,Chinese": 0.4666666666666667, - "English,Vietnamese": 0.44666666666666666, - "English,Filipino": 0.47333333333333333, - "English,Indonesian": 0.48, - "English,Malay": 0.4866666666666667, - "Chinese,Vietnamese": 0.6266666666666667, - "Chinese,Filipino": 0.5466666666666666, - "Chinese,Indonesian": 0.5533333333333333, - "Chinese,Malay": 0.64, - "Vietnamese,Filipino": 0.5, - "Vietnamese,Indonesian": 0.5133333333333333, - "Vietnamese,Malay": 0.6, - "Filipino,Indonesian": 0.5866666666666667, - "Filipino,Malay": 0.6066666666666667, - "Indonesian,Malay": 0.6733333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.32666666666666666, - "Spanish,English,Vietnamese": 0.30666666666666664, - "Spanish,English,Filipino": 0.3333333333333333, - "Spanish,English,Indonesian": 0.29333333333333333, - "Spanish,English,Malay": 0.3, - "Spanish,Chinese,Vietnamese": 0.3466666666666667, - "Spanish,Chinese,Filipino": 0.32, - "Spanish,Chinese,Indonesian": 0.30666666666666664, - "Spanish,Chinese,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,Vietnamese,Malay": 0.31333333333333335, - "Spanish,Filipino,Indonesian": 0.34, - "Spanish,Filipino,Malay": 0.3333333333333333, - "Spanish,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese": 0.32666666666666666, - "English,Chinese,Filipino": 0.31333333333333335, - "English,Chinese,Indonesian": 0.32, - "English,Chinese,Malay": 0.35333333333333333, - "English,Vietnamese,Filipino": 0.3, - "English,Vietnamese,Indonesian": 0.29333333333333333, - "English,Vietnamese,Malay": 0.3333333333333333, - "English,Filipino,Indonesian": 0.3333333333333333, - "English,Filipino,Malay": 0.34, - "English,Indonesian,Malay": 0.36666666666666664, - "Chinese,Vietnamese,Filipino": 0.4, - "Chinese,Vietnamese,Indonesian": 0.38666666666666666, - "Chinese,Vietnamese,Malay": 0.48, - "Chinese,Filipino,Indonesian": 0.4, - "Chinese,Filipino,Malay": 0.44, - "Chinese,Indonesian,Malay": 0.4666666666666667, - "Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "Vietnamese,Filipino,Malay": 0.4, - "Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Filipino,Indonesian,Malay": 0.4666666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.24, - "Spanish,English,Chinese,Filipino": 0.24, - "Spanish,English,Chinese,Indonesian": 0.22, - "Spanish,English,Chinese,Malay": 0.24, - "Spanish,English,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,English,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Vietnamese,Malay": 0.22, - "Spanish,English,Filipino,Indonesian": 0.24, - "Spanish,English,Filipino,Malay": 0.24666666666666667, - "Spanish,English,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.28, - "Spanish,Chinese,Filipino,Indonesian": 0.26, - "Spanish,Chinese,Filipino,Malay": 0.2733333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.26, - "Spanish,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino": 0.26, - "English,Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "English,Chinese,Vietnamese,Malay": 0.28, - "English,Chinese,Filipino,Indonesian": 0.25333333333333335, - "English,Chinese,Filipino,Malay": 0.2733333333333333, - "English,Chinese,Indonesian,Malay": 0.28, - "English,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Vietnamese,Filipino,Malay": 0.26, - "English,Vietnamese,Indonesian,Malay": 0.26, - "English,Filipino,Indonesian,Malay": 0.26, - "Chinese,Vietnamese,Filipino,Indonesian": 0.3, - "Chinese,Vietnamese,Filipino,Malay": 0.34, - "Chinese,Vietnamese,Indonesian,Malay": 0.3466666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.35333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.31333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,English,Chinese,Vietnamese,Malay": 0.2, - "Spanish,English,Chinese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.22, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.18, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - } - }, - "AC3_2": 0.3947597532520917, - "AC3_3": 0.3317535751277033, - "AC3_4": 0.2854239102553099, - "AC3_5": 0.2510441557962293, - "AC3_6": 0.2252575126678971, - "AC3_7": 0.2063143630996009 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.3003246753246754, - "language_acc": { - "English": 0.5113636363636364, - "Filipino": 0.2556818181818182, - "Spanish": 0.32954545454545453, - "Malay": 0.24431818181818182, - "Chinese": 0.24431818181818182, - "Indonesian": 0.26136363636363635, - "Vietnamese": 0.2556818181818182 - }, - "consistency_score_2": 0.449404761904762, - "consistency_score_3": 0.24902597402597404, - "consistency_score_4": 0.14853896103896097, - "consistency_score_5": 0.09063852813852814, - "consistency_score_6": 0.0551948051948052, - "consistency_score_7": 0.03409090909090909, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3181818181818182, - "English,Spanish": 0.4602272727272727, - "English,Malay": 0.32386363636363635, - "English,Chinese": 0.2840909090909091, - "English,Indonesian": 0.3352272727272727, - "English,Vietnamese": 0.3409090909090909, - "Filipino,Spanish": 0.4147727272727273, - "Filipino,Malay": 0.5511363636363636, - "Filipino,Chinese": 0.5056818181818182, - "Filipino,Indonesian": 0.5909090909090909, - "Filipino,Vietnamese": 0.5170454545454546, - "Spanish,Malay": 0.42613636363636365, - "Spanish,Chinese": 0.3522727272727273, - "Spanish,Indonesian": 0.4659090909090909, - "Spanish,Vietnamese": 0.4090909090909091, - "Malay,Chinese": 0.44886363636363635, - "Malay,Indonesian": 0.6193181818181818, - "Malay,Vietnamese": 0.5113636363636364, - "Chinese,Indonesian": 0.48295454545454547, - "Chinese,Vietnamese": 0.5738636363636364, - "Indonesian,Vietnamese": 0.5056818181818182 - }, - "3_combine": { - "English,Filipino,Spanish": 0.1875, - "English,Filipino,Malay": 0.17045454545454544, - "English,Filipino,Chinese": 0.14204545454545456, - "English,Filipino,Indonesian": 0.19318181818181818, - "English,Filipino,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay": 0.19886363636363635, - "English,Spanish,Chinese": 0.14772727272727273, - "English,Spanish,Indonesian": 0.22727272727272727, - "English,Spanish,Vietnamese": 0.19886363636363635, - "English,Malay,Chinese": 0.14772727272727273, - "English,Malay,Indonesian": 0.23863636363636365, - "English,Malay,Vietnamese": 0.19318181818181818, - "English,Chinese,Indonesian": 0.1590909090909091, - "English,Chinese,Vietnamese": 0.17045454545454544, - "English,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Malay": 0.2784090909090909, - "Filipino,Spanish,Chinese": 0.2215909090909091, - "Filipino,Spanish,Indonesian": 0.30113636363636365, - "Filipino,Spanish,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese": 0.3181818181818182, - "Filipino,Malay,Indonesian": 0.4431818181818182, - "Filipino,Malay,Vietnamese": 0.32386363636363635, - "Filipino,Chinese,Indonesian": 0.3522727272727273, - "Filipino,Chinese,Vietnamese": 0.3522727272727273, - "Filipino,Indonesian,Vietnamese": 0.35795454545454547, - "Spanish,Malay,Chinese": 0.19886363636363635, - "Spanish,Malay,Indonesian": 0.3125, - "Spanish,Malay,Vietnamese": 0.2215909090909091, - "Spanish,Chinese,Indonesian": 0.2215909090909091, - "Spanish,Chinese,Vietnamese": 0.25, - "Spanish,Indonesian,Vietnamese": 0.25, - "Malay,Chinese,Indonesian": 0.3409090909090909, - "Malay,Chinese,Vietnamese": 0.3181818181818182, - "Malay,Indonesian,Vietnamese": 0.36363636363636365, - "Chinese,Indonesian,Vietnamese": 0.3465909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.11363636363636363, - "English,Filipino,Spanish,Chinese": 0.07954545454545454, - "English,Filipino,Spanish,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese": 0.09090909090909091, - "English,Filipino,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Vietnamese": 0.09659090909090909, - "English,Filipino,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian": 0.1590909090909091, - "English,Spanish,Malay,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Indonesian": 0.09090909090909091, - "English,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "English,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "English,Malay,Chinese,Indonesian": 0.125, - "English,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese": 0.1534090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.23863636363636365, - "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.26136363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Malay,Chinese,Indonesian,Vietnamese": 0.25 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Vietnamese": 0.0625, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - } - }, - "AC3_2": 0.3600427900741609, - "AC3_3": 0.272280172496566, - "AC3_4": 0.19876822991743745, - "AC3_5": 0.13925088748516634, - "AC3_6": 0.09325149733485677, - "AC3_7": 0.06123124446536174 - }, - "prompt_2": { - "overall_acc": 0.30113636363636365, - "language_acc": { - "English": 0.5113636363636364, - "Filipino": 0.26704545454545453, - "Spanish": 0.3181818181818182, - "Malay": 0.23295454545454544, - "Chinese": 0.24431818181818182, - "Indonesian": 0.24431818181818182, - "Vietnamese": 0.2897727272727273 - }, - "consistency_score_2": 0.43966450216450215, - "consistency_score_3": 0.24464285714285713, - "consistency_score_4": 0.15194805194805194, - "consistency_score_5": 0.09929653679653679, - "consistency_score_6": 0.06655844155844157, - "consistency_score_7": 0.045454545454545456, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3181818181818182, - "English,Spanish": 0.4943181818181818, - "English,Malay": 0.2727272727272727, - "English,Chinese": 0.2556818181818182, - "English,Indonesian": 0.3181818181818182, - "English,Vietnamese": 0.35795454545454547, - "Filipino,Spanish": 0.4090909090909091, - "Filipino,Malay": 0.5625, - "Filipino,Chinese": 0.4431818181818182, - "Filipino,Indonesian": 0.5227272727272727, - "Filipino,Vietnamese": 0.4772727272727273, - "Spanish,Malay": 0.45454545454545453, - "Spanish,Chinese": 0.3693181818181818, - "Spanish,Indonesian": 0.4375, - "Spanish,Vietnamese": 0.4431818181818182, - "Malay,Chinese": 0.4318181818181818, - "Malay,Indonesian": 0.625, - "Malay,Vietnamese": 0.5113636363636364, - "Chinese,Indonesian": 0.48295454545454547, - "Chinese,Vietnamese": 0.5568181818181818, - "Indonesian,Vietnamese": 0.48863636363636365 - }, - "3_combine": { - "English,Filipino,Spanish": 0.19318181818181818, - "English,Filipino,Malay": 0.1590909090909091, - "English,Filipino,Chinese": 0.13068181818181818, - "English,Filipino,Indonesian": 0.17613636363636365, - "English,Filipino,Vietnamese": 0.1875, - "English,Spanish,Malay": 0.19886363636363635, - "English,Spanish,Chinese": 0.14772727272727273, - "English,Spanish,Indonesian": 0.2215909090909091, - "English,Spanish,Vietnamese": 0.24431818181818182, - "English,Malay,Chinese": 0.125, - "English,Malay,Indonesian": 0.19886363636363635, - "English,Malay,Vietnamese": 0.19886363636363635, - "English,Chinese,Indonesian": 0.1534090909090909, - "English,Chinese,Vietnamese": 0.17045454545454544, - "English,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay": 0.2784090909090909, - "Filipino,Spanish,Chinese": 0.2159090909090909, - "Filipino,Spanish,Indonesian": 0.2784090909090909, - "Filipino,Spanish,Vietnamese": 0.25, - "Filipino,Malay,Chinese": 0.2840909090909091, - "Filipino,Malay,Indonesian": 0.4147727272727273, - "Filipino,Malay,Vietnamese": 0.32954545454545453, - "Filipino,Chinese,Indonesian": 0.2897727272727273, - "Filipino,Chinese,Vietnamese": 0.29545454545454547, - "Filipino,Indonesian,Vietnamese": 0.3068181818181818, - "Spanish,Malay,Chinese": 0.22727272727272727, - "Spanish,Malay,Indonesian": 0.3352272727272727, - "Spanish,Malay,Vietnamese": 0.2840909090909091, - "Spanish,Chinese,Indonesian": 0.2215909090909091, - "Spanish,Chinese,Vietnamese": 0.26136363636363635, - "Spanish,Indonesian,Vietnamese": 0.2784090909090909, - "Malay,Chinese,Indonesian": 0.32386363636363635, - "Malay,Chinese,Vietnamese": 0.3068181818181818, - "Malay,Indonesian,Vietnamese": 0.36363636363636365, - "Chinese,Indonesian,Vietnamese": 0.3181818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.11931818181818182, - "English,Filipino,Spanish,Chinese": 0.09090909090909091, - "English,Filipino,Spanish,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese": 0.07954545454545454, - "English,Filipino,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Vietnamese": 0.125, - "English,Filipino,Chinese,Indonesian": 0.10227272727272728, - "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese": 0.09090909090909091, - "English,Spanish,Malay,Indonesian": 0.1590909090909091, - "English,Spanish,Malay,Vietnamese": 0.14204545454545456, - "English,Spanish,Chinese,Indonesian": 0.10795454545454546, - "English,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "English,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese": 0.14772727272727273, - "Filipino,Spanish,Malay,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "Filipino,Malay,Chinese,Indonesian": 0.20454545454545456, - "Filipino,Malay,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.1875, - "Spanish,Malay,Chinese,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - } - }, - "AC3_2": 0.3574482036837359, - "AC3_3": 0.2699657941206725, - "AC3_4": 0.2019803914736126, - "AC3_5": 0.14934735868506224, - "AC3_6": 0.10902067024929867, - "AC3_7": 0.07898658716051893 - }, - "prompt_3": { - "overall_acc": 0.29058441558441556, - "language_acc": { - "English": 0.45454545454545453, - "Filipino": 0.26136363636363635, - "Spanish": 0.32386363636363635, - "Malay": 0.23295454545454544, - "Chinese": 0.2556818181818182, - "Indonesian": 0.25, - "Vietnamese": 0.2556818181818182 - }, - "consistency_score_2": 0.45021645021645024, - "consistency_score_3": 0.25649350649350655, - "consistency_score_4": 0.16087662337662337, - "consistency_score_5": 0.10470779220779221, - "consistency_score_6": 0.0689935064935065, - "consistency_score_7": 0.045454545454545456, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.29545454545454547, - "English,Spanish": 0.48295454545454547, - "English,Malay": 0.2840909090909091, - "English,Chinese": 0.3068181818181818, - "English,Indonesian": 0.3465909090909091, - "English,Vietnamese": 0.3352272727272727, - "Filipino,Spanish": 0.45454545454545453, - "Filipino,Malay": 0.5681818181818182, - "Filipino,Chinese": 0.4715909090909091, - "Filipino,Indonesian": 0.5625, - "Filipino,Vietnamese": 0.4659090909090909, - "Spanish,Malay": 0.48295454545454547, - "Spanish,Chinese": 0.36363636363636365, - "Spanish,Indonesian": 0.44886363636363635, - "Spanish,Vietnamese": 0.4090909090909091, - "Malay,Chinese": 0.4659090909090909, - "Malay,Indonesian": 0.6647727272727273, - "Malay,Vietnamese": 0.5227272727272727, - "Chinese,Indonesian": 0.4772727272727273, - "Chinese,Vietnamese": 0.5681818181818182, - "Indonesian,Vietnamese": 0.4772727272727273 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2159090909090909, - "English,Filipino,Malay": 0.17613636363636365, - "English,Filipino,Chinese": 0.1534090909090909, - "English,Filipino,Indonesian": 0.19318181818181818, - "English,Filipino,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay": 0.2215909090909091, - "English,Spanish,Chinese": 0.17613636363636365, - "English,Spanish,Indonesian": 0.22727272727272727, - "English,Spanish,Vietnamese": 0.2215909090909091, - "English,Malay,Chinese": 0.14772727272727273, - "English,Malay,Indonesian": 0.22727272727272727, - "English,Malay,Vietnamese": 0.17045454545454544, - "English,Chinese,Indonesian": 0.18181818181818182, - "English,Chinese,Vietnamese": 0.19318181818181818, - "English,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay": 0.32386363636363635, - "Filipino,Spanish,Chinese": 0.23863636363636365, - "Filipino,Spanish,Indonesian": 0.3068181818181818, - "Filipino,Spanish,Vietnamese": 0.24431818181818182, - "Filipino,Malay,Chinese": 0.3125, - "Filipino,Malay,Indonesian": 0.44886363636363635, - "Filipino,Malay,Vietnamese": 0.32386363636363635, - "Filipino,Chinese,Indonesian": 0.3181818181818182, - "Filipino,Chinese,Vietnamese": 0.3068181818181818, - "Filipino,Indonesian,Vietnamese": 0.3125, - "Spanish,Malay,Chinese": 0.23863636363636365, - "Spanish,Malay,Indonesian": 0.35795454545454547, - "Spanish,Malay,Vietnamese": 0.2727272727272727, - "Spanish,Chinese,Indonesian": 0.21022727272727273, - "Spanish,Chinese,Vietnamese": 0.2556818181818182, - "Spanish,Indonesian,Vietnamese": 0.25, - "Malay,Chinese,Indonesian": 0.3522727272727273, - "Malay,Chinese,Vietnamese": 0.3409090909090909, - "Malay,Indonesian,Vietnamese": 0.375, - "Chinese,Indonesian,Vietnamese": 0.3181818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.14204545454545456, - "English,Filipino,Spanish,Chinese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese": 0.09659090909090909, - "English,Filipino,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese": 0.10227272727272728, - "English,Spanish,Malay,Indonesian": 0.17613636363636365, - "English,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian": 0.10227272727272728, - "English,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "English,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "English,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Chinese": 0.18181818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.1875, - "Filipino,Spanish,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.2556818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.1875, - "Spanish,Malay,Chinese,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.1534090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0625, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - } - }, - "AC3_2": 0.35320121801840465, - "AC3_3": 0.27247678132905273, - "AC3_4": 0.20709755899343385, - "AC3_5": 0.15394410513719364, - "AC3_6": 0.11151094951605531, - "AC3_7": 0.07861220902359788 - }, - "prompt_4": { - "overall_acc": 0.29464285714285715, - "language_acc": { - "English": 0.4943181818181818, - "Filipino": 0.2556818181818182, - "Spanish": 0.32954545454545453, - "Malay": 0.23863636363636365, - "Chinese": 0.26136363636363635, - "Indonesian": 0.22727272727272727, - "Vietnamese": 0.2556818181818182 - }, - "consistency_score_2": 0.43262987012987025, - "consistency_score_3": 0.2347402597402598, - "consistency_score_4": 0.13993506493506494, - "consistency_score_5": 0.08685064935064934, - "consistency_score_6": 0.056006493506493504, - "consistency_score_7": 0.03977272727272727, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.32954545454545453, - "English,Spanish": 0.4659090909090909, - "English,Malay": 0.3125, - "English,Chinese": 0.30113636363636365, - "English,Indonesian": 0.30113636363636365, - "English,Vietnamese": 0.36363636363636365, - "Filipino,Spanish": 0.4090909090909091, - "Filipino,Malay": 0.5568181818181818, - "Filipino,Chinese": 0.4772727272727273, - "Filipino,Indonesian": 0.5738636363636364, - "Filipino,Vietnamese": 0.48295454545454547, - "Spanish,Malay": 0.42045454545454547, - "Spanish,Chinese": 0.3181818181818182, - "Spanish,Indonesian": 0.3977272727272727, - "Spanish,Vietnamese": 0.375, - "Malay,Chinese": 0.42613636363636365, - "Malay,Indonesian": 0.625, - "Malay,Vietnamese": 0.48295454545454547, - "Chinese,Indonesian": 0.45454545454545453, - "Chinese,Vietnamese": 0.5170454545454546, - "Indonesian,Vietnamese": 0.4943181818181818 - }, - "3_combine": { - "English,Filipino,Spanish": 0.19886363636363635, - "English,Filipino,Malay": 0.17613636363636365, - "English,Filipino,Chinese": 0.1590909090909091, - "English,Filipino,Indonesian": 0.19318181818181818, - "English,Filipino,Vietnamese": 0.1875, - "English,Spanish,Malay": 0.19318181818181818, - "English,Spanish,Chinese": 0.13636363636363635, - "English,Spanish,Indonesian": 0.18181818181818182, - "English,Spanish,Vietnamese": 0.19886363636363635, - "English,Malay,Chinese": 0.14772727272727273, - "English,Malay,Indonesian": 0.20454545454545456, - "English,Malay,Vietnamese": 0.20454545454545456, - "English,Chinese,Indonesian": 0.14772727272727273, - "English,Chinese,Vietnamese": 0.1875, - "English,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay": 0.2727272727272727, - "Filipino,Spanish,Chinese": 0.19886363636363635, - "Filipino,Spanish,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Chinese": 0.2897727272727273, - "Filipino,Malay,Indonesian": 0.4375, - "Filipino,Malay,Vietnamese": 0.3181818181818182, - "Filipino,Chinese,Indonesian": 0.3125, - "Filipino,Chinese,Vietnamese": 0.29545454545454547, - "Filipino,Indonesian,Vietnamese": 0.3409090909090909, - "Spanish,Malay,Chinese": 0.1875, - "Spanish,Malay,Indonesian": 0.2784090909090909, - "Spanish,Malay,Vietnamese": 0.22727272727272727, - "Spanish,Chinese,Indonesian": 0.17613636363636365, - "Spanish,Chinese,Vietnamese": 0.19886363636363635, - "Spanish,Indonesian,Vietnamese": 0.22727272727272727, - "Malay,Chinese,Indonesian": 0.30113636363636365, - "Malay,Chinese,Vietnamese": 0.2840909090909091, - "Malay,Indonesian,Vietnamese": 0.35795454545454547, - "Chinese,Indonesian,Vietnamese": 0.3125 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.11931818181818182, - "English,Filipino,Spanish,Chinese": 0.08522727272727272, - "English,Filipino,Spanish,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese": 0.09090909090909091, - "English,Filipino,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian": 0.14204545454545456, - "English,Spanish,Malay,Vietnamese": 0.11931818181818182, - "English,Spanish,Chinese,Indonesian": 0.07386363636363637, - "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Malay,Chinese,Vietnamese": 0.11931818181818182, - "English,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Chinese,Vietnamese": 0.125, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian": 0.23295454545454544, - "Filipino,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Indonesian": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Indonesian": 0.0625, - "English,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727 - } - }, - "AC3_2": 0.3505460777579224, - "AC3_3": 0.2613024049972955, - "AC3_4": 0.18975132073119122, - "AC3_5": 0.13415653491924115, - "AC3_6": 0.09412202378268159, - "AC3_7": 0.07008495143535323 - }, - "prompt_5": { - "overall_acc": 0.30519480519480524, - "language_acc": { - "English": 0.5284090909090909, - "Filipino": 0.2556818181818182, - "Spanish": 0.3352272727272727, - "Malay": 0.24431818181818182, - "Chinese": 0.26136363636363635, - "Indonesian": 0.2215909090909091, - "Vietnamese": 0.2897727272727273 - }, - "consistency_score_2": 0.42911255411255406, - "consistency_score_3": 0.23295454545454541, - "consistency_score_4": 0.1413961038961039, - "consistency_score_5": 0.08982683982683981, - "consistency_score_6": 0.05844155844155845, - "consistency_score_7": 0.03977272727272727, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.4772727272727273, - "English,Malay": 0.2784090909090909, - "English,Chinese": 0.2840909090909091, - "English,Indonesian": 0.3352272727272727, - "English,Vietnamese": 0.3806818181818182, - "Filipino,Spanish": 0.38636363636363635, - "Filipino,Malay": 0.5454545454545454, - "Filipino,Chinese": 0.4375, - "Filipino,Indonesian": 0.5, - "Filipino,Vietnamese": 0.45454545454545453, - "Spanish,Malay": 0.4318181818181818, - "Spanish,Chinese": 0.3125, - "Spanish,Indonesian": 0.4090909090909091, - "Spanish,Vietnamese": 0.4147727272727273, - "Malay,Chinese": 0.4147727272727273, - "Malay,Indonesian": 0.6647727272727273, - "Malay,Vietnamese": 0.4715909090909091, - "Chinese,Indonesian": 0.4431818181818182, - "Chinese,Vietnamese": 0.5625, - "Indonesian,Vietnamese": 0.4715909090909091 - }, - "3_combine": { - "English,Filipino,Spanish": 0.19318181818181818, - "English,Filipino,Malay": 0.17045454545454544, - "English,Filipino,Chinese": 0.14772727272727273, - "English,Filipino,Indonesian": 0.19318181818181818, - "English,Filipino,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay": 0.20454545454545456, - "English,Spanish,Chinese": 0.14204545454545456, - "English,Spanish,Indonesian": 0.21022727272727273, - "English,Spanish,Vietnamese": 0.23295454545454544, - "English,Malay,Chinese": 0.13636363636363635, - "English,Malay,Indonesian": 0.22727272727272727, - "English,Malay,Vietnamese": 0.18181818181818182, - "English,Chinese,Indonesian": 0.1590909090909091, - "English,Chinese,Vietnamese": 0.19318181818181818, - "English,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Malay": 0.26136363636363635, - "Filipino,Spanish,Chinese": 0.1875, - "Filipino,Spanish,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Vietnamese": 0.21022727272727273, - "Filipino,Malay,Chinese": 0.2784090909090909, - "Filipino,Malay,Indonesian": 0.4090909090909091, - "Filipino,Malay,Vietnamese": 0.2897727272727273, - "Filipino,Chinese,Indonesian": 0.26704545454545453, - "Filipino,Chinese,Vietnamese": 0.30113636363636365, - "Filipino,Indonesian,Vietnamese": 0.29545454545454547, - "Spanish,Malay,Chinese": 0.17613636363636365, - "Spanish,Malay,Indonesian": 0.3125, - "Spanish,Malay,Vietnamese": 0.23863636363636365, - "Spanish,Chinese,Indonesian": 0.16477272727272727, - "Spanish,Chinese,Vietnamese": 0.23295454545454544, - "Spanish,Indonesian,Vietnamese": 0.23863636363636365, - "Malay,Chinese,Indonesian": 0.30113636363636365, - "Malay,Chinese,Vietnamese": 0.29545454545454547, - "Malay,Indonesian,Vietnamese": 0.35795454545454547, - "Chinese,Indonesian,Vietnamese": 0.3068181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.11931818181818182, - "English,Filipino,Spanish,Chinese": 0.09090909090909091, - "English,Filipino,Spanish,Indonesian": 0.125, - "English,Filipino,Spanish,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese": 0.09090909090909091, - "English,Filipino,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian": 0.16477272727272727, - "English,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian": 0.08522727272727272, - "English,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "English,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Chinese": 0.125, - "Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.21022727272727273, - "Filipino,Malay,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese,Indonesian": 0.125, - "Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.1875, - "Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.05113636363636364, - "English,Filipino,Spanish,Malay,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Indonesian": 0.0625, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.0625, - "English,Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727 - } - }, - "AC3_2": 0.35669783417320805, - "AC3_3": 0.26422597005922405, - "AC3_4": 0.19325676137052145, - "AC3_5": 0.1388009250671574, - "AC3_6": 0.09809833021421034, - "AC3_7": 0.07037433153040186 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.4368932038834951 - }, - "prompt_2": { - "accuracy": 0.4368932038834951 - }, - "prompt_3": { - "accuracy": 0.44660194174757284 - }, - "prompt_4": { - "accuracy": 0.44660194174757284 - }, - "prompt_5": { - "accuracy": 0.42718446601941745 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.20952380952380953 - }, - "prompt_2": { - "accuracy": 0.23809523809523808 - }, - "prompt_3": { - "accuracy": 0.24761904761904763 - }, - "prompt_4": { - "accuracy": 0.24761904761904763 - }, - "prompt_5": { - "accuracy": 0.21904761904761905 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.38317757009345793 - }, - "prompt_2": { - "accuracy": 0.3644859813084112 - }, - "prompt_3": { - "accuracy": 0.38317757009345793 - }, - "prompt_4": { - "accuracy": 0.40186915887850466 - }, - "prompt_5": { - "accuracy": 0.40186915887850466 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.24, - "category_acc": { - "brand": 0.1, - "demographics": 0.0, - "biology": 0.3, - "history": 0.13333333333333333, - "literature": 0.5, - "politics": 0.4, - "culture": 0.1, - "film": 0.1, - "law": 0.3, - "geography": 0.4 - } - }, - "prompt_2": { - "accuracy": 0.24, - "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.3, - "history": 0.13333333333333333, - "literature": 0.5, - "politics": 0.3, - "culture": 0.1, - "film": 0.1, - "law": 0.3, - "geography": 0.4 - } - }, - "prompt_3": { - "accuracy": 0.23, - "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.3, - "history": 0.2, - "literature": 0.3, - "politics": 0.2, - "culture": 0.1, - "film": 0.1, - "law": 0.3, - "geography": 0.5 - } - }, - "prompt_4": { - "accuracy": 0.26, - "category_acc": { - "brand": 0.1, - "demographics": 0.4, - "biology": 0.3, - "history": 0.2, - "literature": 0.4, - "politics": 0.4, - "culture": 0.1, - "film": 0.1, - "law": 0.3, - "geography": 0.4 - } - }, - "prompt_5": { - "accuracy": 0.25, - "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.3, - "history": 0.2, - "literature": 0.3, - "politics": 0.2, - "culture": 0.1, - "film": 0.1, - "law": 0.3, - "geography": 0.7 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.17858137998030377 - }, - "prompt_2": { - "bleu_score": 0.17980302230654807 - }, - "prompt_3": { - "bleu_score": 0.18216833917677464 - }, - "prompt_4": { - "bleu_score": 0.18530270379921296 - }, - "prompt_5": { - "bleu_score": 0.16536410084591763 - } - }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.12727876041975192 - }, - "prompt_2": { - "bleu_score": 0.12736747783713767 - }, - "prompt_3": { - "bleu_score": 0.12583225634571862 - }, - "prompt_4": { - "bleu_score": 0.12853331243136193 - }, - "prompt_5": { - "bleu_score": 0.11768978068874901 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.03668860167770236 - }, - "prompt_2": { - "bleu_score": 0.03618786209438715 - }, - "prompt_3": { - "bleu_score": 0.03710046510973257 - }, - "prompt_4": { - "bleu_score": 0.03652197947072188 - }, - "prompt_5": { - "bleu_score": 0.03551228284442551 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.014386014759031592 - }, - "prompt_2": { - "bleu_score": 0.016014292013673933 - }, - "prompt_3": { - "bleu_score": 0.015055163809418064 - }, - "prompt_4": { - "bleu_score": 0.014725706605220303 - }, - "prompt_5": { - "bleu_score": 0.012050434975626145 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.11834634299984877 - }, - "prompt_2": { - "bleu_score": 0.11788291617465893 - }, - "prompt_3": { - "bleu_score": 0.1166194754868489 - }, - "prompt_4": { - "bleu_score": 0.11828979591628382 - }, - "prompt_5": { - "bleu_score": 0.11258991824927955 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.4574095682613769 - }, - "prompt_2": { - "accuracy": 0.43990665110851807 - }, - "prompt_3": { - "accuracy": 0.4364060676779463 - }, - "prompt_4": { - "accuracy": 0.4364060676779463 - }, - "prompt_5": { - "accuracy": 0.43990665110851807 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.41344297461565965, - "category_acc": { - "high_school_european_history": 0.5426829268292683, - "business_ethics": 0.5656565656565656, - "clinical_knowledge": 0.44696969696969696, - "medical_genetics": 0.43434343434343436, - "high_school_us_history": 0.5369458128078818, - "high_school_physics": 0.31333333333333335, - "high_school_world_history": 0.576271186440678, - "virology": 0.34545454545454546, - "high_school_microeconomics": 0.459915611814346, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.30303030303030304, - "high_school_biology": 0.42394822006472493, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.3416370106761566, - "philosophy": 0.4161290322580645, - "professional_medicine": 0.4132841328413284, - "nutrition": 0.46557377049180326, - "global_facts": 0.30303030303030304, - "machine_learning": 0.24324324324324326, - "security_studies": 0.5573770491803278, - "public_relations": 0.47706422018348627, - "professional_psychology": 0.37152209492635024, - "prehistory": 0.4055727554179567, - "anatomy": 0.3880597014925373, - "human_sexuality": 0.4, - "college_medicine": 0.4186046511627907, - "high_school_government_and_politics": 0.5416666666666666, - "college_chemistry": 0.36363636363636365, - "logical_fallacies": 0.5246913580246914, - "high_school_geography": 0.5482233502538071, - "elementary_mathematics": 0.27851458885941643, - "human_aging": 0.45495495495495497, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.5661764705882353, - "formal_logic": 0.36, - "high_school_statistics": 0.33488372093023255, - "international_law": 0.5583333333333333, - "high_school_mathematics": 0.2825278810408922, - "high_school_computer_science": 0.36363636363636365, - "conceptual_physics": 0.358974358974359, - "miscellaneous": 0.5140664961636828, - "high_school_chemistry": 0.28217821782178215, - "marketing": 0.7167381974248928, - "professional_law": 0.33724722765818654, - "management": 0.6372549019607843, - "college_physics": 0.31683168316831684, - "jurisprudence": 0.4766355140186916, - "world_religions": 0.38235294117647056, - "sociology": 0.565, - "us_foreign_policy": 0.5757575757575758, - "high_school_macroeconomics": 0.43958868894601544, - "computer_security": 0.47474747474747475, - "moral_scenarios": 0.23937360178970918, - "moral_disputes": 0.48985507246376814, - "electrical_engineering": 0.4027777777777778, - "astronomy": 0.3841059602649007, - "college_biology": 0.36363636363636365 - } - }, - "prompt_2": { - "accuracy": 0.41344297461565965, - "category_acc": { - "high_school_european_history": 0.573170731707317, - "business_ethics": 0.5050505050505051, - "clinical_knowledge": 0.4431818181818182, - "medical_genetics": 0.4444444444444444, - "high_school_us_history": 0.5467980295566502, - "high_school_physics": 0.30666666666666664, - "high_school_world_history": 0.5889830508474576, - "virology": 0.36363636363636365, - "high_school_microeconomics": 0.4345991561181435, - "econometrics": 0.24778761061946902, - "college_computer_science": 0.31313131313131315, - "high_school_biology": 0.43042071197411, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.37722419928825623, - "philosophy": 0.4161290322580645, - "professional_medicine": 0.39114391143911437, - "nutrition": 0.4557377049180328, - "global_facts": 0.29292929292929293, - "machine_learning": 0.3153153153153153, - "security_studies": 0.5573770491803278, - "public_relations": 0.46788990825688076, - "professional_psychology": 0.37479541734860883, - "prehistory": 0.39009287925696595, - "anatomy": 0.417910447761194, - "human_sexuality": 0.4230769230769231, - "college_medicine": 0.43023255813953487, - "high_school_government_and_politics": 0.5260416666666666, - "college_chemistry": 0.3333333333333333, - "logical_fallacies": 0.5308641975308642, - "high_school_geography": 0.5076142131979695, - "elementary_mathematics": 0.2838196286472148, - "human_aging": 0.40540540540540543, - "college_mathematics": 0.2727272727272727, - "high_school_psychology": 0.5496323529411765, - "formal_logic": 0.304, - "high_school_statistics": 0.33488372093023255, - "international_law": 0.525, - "high_school_mathematics": 0.2899628252788104, - "high_school_computer_science": 0.42424242424242425, - "conceptual_physics": 0.32905982905982906, - "miscellaneous": 0.5025575447570333, - "high_school_chemistry": 0.27722772277227725, - "marketing": 0.703862660944206, - "professional_law": 0.3339856490541422, - "management": 0.6078431372549019, - "college_physics": 0.33663366336633666, - "jurisprudence": 0.4672897196261682, - "world_religions": 0.38235294117647056, - "sociology": 0.575, - "us_foreign_policy": 0.6161616161616161, - "high_school_macroeconomics": 0.4241645244215938, - "computer_security": 0.5050505050505051, - "moral_scenarios": 0.27293064876957496, - "moral_disputes": 0.5043478260869565, - "electrical_engineering": 0.4027777777777778, - "astronomy": 0.40397350993377484, - "college_biology": 0.34265734265734266 - } - }, - "prompt_3": { - "accuracy": 0.41444404719342154, - "category_acc": { - "high_school_european_history": 0.5792682926829268, - "business_ethics": 0.5555555555555556, - "clinical_knowledge": 0.4583333333333333, - "medical_genetics": 0.40404040404040403, - "high_school_us_history": 0.5467980295566502, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.5932203389830508, - "virology": 0.3696969696969697, - "high_school_microeconomics": 0.45147679324894513, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.31313131313131315, - "high_school_biology": 0.42394822006472493, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.3736654804270463, - "philosophy": 0.43548387096774194, - "professional_medicine": 0.4022140221402214, - "nutrition": 0.46885245901639344, - "global_facts": 0.2828282828282828, - "machine_learning": 0.32432432432432434, - "security_studies": 0.5655737704918032, - "public_relations": 0.47706422018348627, - "professional_psychology": 0.3698854337152209, - "prehistory": 0.39628482972136225, - "anatomy": 0.3880597014925373, - "human_sexuality": 0.4153846153846154, - "college_medicine": 0.42441860465116277, - "high_school_government_and_politics": 0.5260416666666666, - "college_chemistry": 0.3333333333333333, - "logical_fallacies": 0.5246913580246914, - "high_school_geography": 0.5126903553299492, - "elementary_mathematics": 0.28116710875331563, - "human_aging": 0.4189189189189189, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.5514705882352942, - "formal_logic": 0.312, - "high_school_statistics": 0.3209302325581395, - "international_law": 0.5333333333333333, - "high_school_mathematics": 0.2862453531598513, - "high_school_computer_science": 0.3939393939393939, - "conceptual_physics": 0.33760683760683763, - "miscellaneous": 0.5, - "high_school_chemistry": 0.31683168316831684, - "marketing": 0.6995708154506438, - "professional_law": 0.33920417482061316, - "management": 0.6470588235294118, - "college_physics": 0.3465346534653465, - "jurisprudence": 0.4485981308411215, - "world_religions": 0.40588235294117647, - "sociology": 0.585, - "us_foreign_policy": 0.6161616161616161, - "high_school_macroeconomics": 0.442159383033419, - "computer_security": 0.5050505050505051, - "moral_scenarios": 0.24049217002237136, - "moral_disputes": 0.5072463768115942, - "electrical_engineering": 0.4097222222222222, - "astronomy": 0.3973509933774834, - "college_biology": 0.3146853146853147 - } - }, - "prompt_4": { - "accuracy": 0.4140865212727923, - "category_acc": { - "high_school_european_history": 0.5487804878048781, - "business_ethics": 0.5050505050505051, - "clinical_knowledge": 0.45454545454545453, - "medical_genetics": 0.40404040404040403, - "high_school_us_history": 0.5517241379310345, - "high_school_physics": 0.3, - "high_school_world_history": 0.5889830508474576, - "virology": 0.3696969696969697, - "high_school_microeconomics": 0.45569620253164556, - "econometrics": 0.23893805309734514, - "college_computer_science": 0.3333333333333333, - "high_school_biology": 0.44660194174757284, - "abstract_algebra": 0.2727272727272727, - "professional_accounting": 0.36298932384341637, - "philosophy": 0.41935483870967744, - "professional_medicine": 0.4059040590405904, - "nutrition": 0.46557377049180326, - "global_facts": 0.2828282828282828, - "machine_learning": 0.2702702702702703, - "security_studies": 0.5614754098360656, - "public_relations": 0.44036697247706424, - "professional_psychology": 0.3617021276595745, - "prehistory": 0.3993808049535604, - "anatomy": 0.40298507462686567, - "human_sexuality": 0.4230769230769231, - "college_medicine": 0.4011627906976744, - "high_school_government_and_politics": 0.546875, - "college_chemistry": 0.3333333333333333, - "logical_fallacies": 0.5308641975308642, - "high_school_geography": 0.5279187817258884, - "elementary_mathematics": 0.2864721485411141, - "human_aging": 0.4144144144144144, - "college_mathematics": 0.25252525252525254, - "high_school_psychology": 0.5551470588235294, - "formal_logic": 0.304, - "high_school_statistics": 0.34418604651162793, - "international_law": 0.5333333333333333, - "high_school_mathematics": 0.2936802973977695, - "high_school_computer_science": 0.37373737373737376, - "conceptual_physics": 0.33760683760683763, - "miscellaneous": 0.5127877237851662, - "high_school_chemistry": 0.2871287128712871, - "marketing": 0.6995708154506438, - "professional_law": 0.3359425962165688, - "management": 0.6176470588235294, - "college_physics": 0.3465346534653465, - "jurisprudence": 0.4672897196261682, - "world_religions": 0.38235294117647056, - "sociology": 0.59, - "us_foreign_policy": 0.6262626262626263, - "high_school_macroeconomics": 0.42930591259640105, - "computer_security": 0.48484848484848486, - "moral_scenarios": 0.25727069351230425, - "moral_disputes": 0.5072463768115942, - "electrical_engineering": 0.4097222222222222, - "astronomy": 0.4105960264900662, - "college_biology": 0.34265734265734266 - } - }, - "prompt_5": { - "accuracy": 0.4200929567393636, - "category_acc": { - "high_school_european_history": 0.573170731707317, - "business_ethics": 0.5252525252525253, - "clinical_knowledge": 0.45075757575757575, - "medical_genetics": 0.43434343434343436, - "high_school_us_history": 0.5467980295566502, - "high_school_physics": 0.31333333333333335, - "high_school_world_history": 0.597457627118644, - "virology": 0.38181818181818183, - "high_school_microeconomics": 0.4345991561181435, - "econometrics": 0.23893805309734514, - "college_computer_science": 0.29292929292929293, - "high_school_biology": 0.44660194174757284, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.3736654804270463, - "philosophy": 0.43548387096774194, - "professional_medicine": 0.3800738007380074, - "nutrition": 0.4721311475409836, - "global_facts": 0.29292929292929293, - "machine_learning": 0.3153153153153153, - "security_studies": 0.5614754098360656, - "public_relations": 0.46788990825688076, - "professional_psychology": 0.3862520458265139, - "prehistory": 0.4179566563467492, - "anatomy": 0.417910447761194, - "human_sexuality": 0.4153846153846154, - "college_medicine": 0.436046511627907, - "high_school_government_and_politics": 0.5364583333333334, - "college_chemistry": 0.32323232323232326, - "logical_fallacies": 0.5555555555555556, - "high_school_geography": 0.5532994923857868, - "elementary_mathematics": 0.2864721485411141, - "human_aging": 0.42792792792792794, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.5588235294117647, - "formal_logic": 0.304, - "high_school_statistics": 0.32558139534883723, - "international_law": 0.5666666666666667, - "high_school_mathematics": 0.275092936802974, - "high_school_computer_science": 0.3939393939393939, - "conceptual_physics": 0.3333333333333333, - "miscellaneous": 0.5089514066496164, - "high_school_chemistry": 0.30198019801980197, - "marketing": 0.7339055793991416, - "professional_law": 0.33724722765818654, - "management": 0.6078431372549019, - "college_physics": 0.3564356435643564, - "jurisprudence": 0.5046728971962616, - "world_religions": 0.3764705882352941, - "sociology": 0.595, - "us_foreign_policy": 0.6161616161616161, - "high_school_macroeconomics": 0.42930591259640105, - "computer_security": 0.5151515151515151, - "moral_scenarios": 0.2751677852348993, - "moral_disputes": 0.5014492753623189, - "electrical_engineering": 0.4097222222222222, - "astronomy": 0.3841059602649007, - "college_biology": 0.34965034965034963 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.2578008915304606 - }, - "prompt_2": { - "accuracy": 0.25037147102526003 - }, - "prompt_3": { - "accuracy": 0.2540861812778603 - }, - "prompt_4": { - "accuracy": 0.2540861812778603 - }, - "prompt_5": { - "accuracy": 0.25037147102526003 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.2559153175591532, - "category_acc": { - "computer_network": 0.375, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.2619047619047619, - "college_physics": 0.08333333333333333, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.1724137931034483, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.25, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.19230769230769232, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.3, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.3673469387755102, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.07142857142857142, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.25925925925925924, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.25, - "art_studies": 0.47368421052631576, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.25, - "high_school_history": 0.16, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.21153846153846154, - "sports_science": 0.16666666666666666, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.16666666666666666, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.18518518518518517, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.25925925925925924, - "physician": 0.2962962962962963 - } - }, - "prompt_2": { - "accuracy": 0.26214196762141967, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.125, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.2857142857142857, - "college_physics": 0.125, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.5, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.3333333333333333, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.32653061224489793, - "high_school_politics": 0.16666666666666666, - "high_school_geography": 0.125, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.14285714285714285, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.2962962962962963, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.25, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.20588235294117646, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.12, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.21153846153846154, - "sports_science": 0.20833333333333334, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.20833333333333334, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.16666666666666666, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.25925925925925924, - "physician": 0.2777777777777778 - } - }, - "prompt_3": { - "accuracy": 0.24283935242839352, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.08333333333333333, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.23809523809523808, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.19047619047619047, - "metrology_engineer": 0.1724137931034483, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.4166666666666667, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.19230769230769232, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.2833333333333333, - "business_administration": 0.18421052631578946, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.3448275862068966, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.2857142857142857, - "high_school_politics": 0.125, - "high_school_geography": 0.25, - "middle_school_politics": 0.19230769230769232, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.07142857142857142, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.2962962962962963, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.25, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.20588235294117646, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.25, - "high_school_history": 0.12, - "middle_school_history": 0.25925925925925924, - "civil_servant": 0.23076923076923078, - "sports_science": 0.20833333333333334, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.16666666666666666, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.16666666666666666, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.25925925925925924, - "physician": 0.2962962962962963 - } - }, - "prompt_4": { - "accuracy": 0.24097135740971357, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.08333333333333333, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.23809523809523808, - "college_physics": 0.125, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.13043478260869565, - "discrete_mathematics": 0.42857142857142855, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.10344827586206896, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.25, - "high_school_chemistry": 0.25, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.08333333333333333, - "middle_school_biology": 0.19230769230769232, - "middle_school_physics": 0.125, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.38333333333333336, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.27586206896551724, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.30612244897959184, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.125, - "middle_school_politics": 0.11538461538461539, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.10714285714285714, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.2222222222222222, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.25, - "high_school_history": 0.2, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.19230769230769232, - "sports_science": 0.125, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.16666666666666666, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.12962962962962962, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.2777777777777778, - "physician": 0.3148148148148148 - } - }, - "prompt_5": { - "accuracy": 0.25155666251556663, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.125, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.30952380952380953, - "college_physics": 0.125, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.19230769230769232, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.3333333333333333, - "business_administration": 0.18421052631578946, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.2857142857142857, - "high_school_politics": 0.25, - "high_school_geography": 0.125, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.2962962962962963, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.25, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.23529411764705882, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.16, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.25, - "sports_science": 0.16666666666666666, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.16666666666666666, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.2037037037037037, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.2222222222222222, - "tax_accountant": 0.2777777777777778, - "physician": 0.25925925925925924 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.2939068100358423 - }, - "prompt_2": { - "accuracy": 0.2903225806451613 - }, - "prompt_3": { - "accuracy": 0.27956989247311825 - }, - "prompt_4": { - "accuracy": 0.26881720430107525 - }, - "prompt_5": { - "accuracy": 0.27956989247311825 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.2532377827663616, - "category_acc": { - "agronomy": 0.22485207100591717, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.25609756097560976, - "arts": 0.25625, - "astronomy": 0.20606060606060606, - "business_ethics": 0.2727272727272727, - "chinese_civil_service_exam": 0.23125, - "chinese_driving_rule": 0.2824427480916031, - "chinese_food_culture": 0.23529411764705882, - "chinese_foreign_policy": 0.205607476635514, - "chinese_history": 0.28792569659442724, - "chinese_literature": 0.2647058823529412, - "chinese_teacher_qualification": 0.27932960893854747, - "clinical_knowledge": 0.22362869198312235, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.2897196261682243, - "college_engineering_hydrology": 0.25471698113207547, - "college_law": 0.2037037037037037, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.2169811320754717, - "college_medicine": 0.23076923076923078, - "computer_science": 0.27941176470588236, - "computer_security": 0.22807017543859648, - "conceptual_physics": 0.23129251700680273, - "construction_project_management": 0.31654676258992803, - "economics": 0.25157232704402516, - "education": 0.26380368098159507, - "electrical_engineering": 0.27325581395348836, - "elementary_chinese": 0.28174603174603174, - "elementary_commonsense": 0.22727272727272727, - "elementary_information_and_technology": 0.29411764705882354, - "elementary_mathematics": 0.2565217391304348, - "ethnology": 0.2814814814814815, - "food_science": 0.26573426573426573, - "genetics": 0.2784090909090909, - "global_facts": 0.2550335570469799, - "high_school_biology": 0.23076923076923078, - "high_school_chemistry": 0.21212121212121213, - "high_school_geography": 0.22033898305084745, - "high_school_mathematics": 0.21341463414634146, - "high_school_physics": 0.22727272727272727, - "high_school_politics": 0.23076923076923078, - "human_sexuality": 0.29365079365079366, - "international_law": 0.2648648648648649, - "journalism": 0.2558139534883721, - "jurisprudence": 0.25304136253041365, - "legal_and_moral_basis": 0.21962616822429906, - "logical": 0.2764227642276423, - "machine_learning": 0.2540983606557377, - "management": 0.24761904761904763, - "marketing": 0.2388888888888889, - "marxist_theory": 0.2275132275132275, - "modern_chinese": 0.22413793103448276, - "nutrition": 0.2482758620689655, - "philosophy": 0.2761904761904762, - "professional_accounting": 0.26285714285714284, - "professional_law": 0.2796208530805687, - "professional_medicine": 0.2393617021276596, - "professional_psychology": 0.2974137931034483, - "public_relations": 0.2471264367816092, - "security_study": 0.2518518518518518, - "sociology": 0.25663716814159293, - "sports_science": 0.26666666666666666, - "traditional_chinese_medicine": 0.2756756756756757, - "virology": 0.21893491124260356, - "world_history": 0.2546583850931677, - "world_religions": 0.25 - } - }, - "prompt_2": { - "accuracy": 0.24589880849594198, - "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.21951219512195122, - "arts": 0.275, - "astronomy": 0.2545454545454545, - "business_ethics": 0.2535885167464115, - "chinese_civil_service_exam": 0.23125, - "chinese_driving_rule": 0.25190839694656486, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.2616822429906542, - "chinese_history": 0.2476780185758514, - "chinese_literature": 0.2549019607843137, - "chinese_teacher_qualification": 0.2346368715083799, - "clinical_knowledge": 0.24472573839662448, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.2523364485981308, - "college_engineering_hydrology": 0.2641509433962264, - "college_law": 0.21296296296296297, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.2169811320754717, - "college_medicine": 0.2454212454212454, - "computer_science": 0.21568627450980393, - "computer_security": 0.19883040935672514, - "conceptual_physics": 0.2653061224489796, - "construction_project_management": 0.2446043165467626, - "economics": 0.24528301886792453, - "education": 0.26380368098159507, - "electrical_engineering": 0.2441860465116279, - "elementary_chinese": 0.2857142857142857, - "elementary_commonsense": 0.25252525252525254, - "elementary_information_and_technology": 0.25630252100840334, - "elementary_mathematics": 0.23043478260869565, - "ethnology": 0.2740740740740741, - "food_science": 0.2727272727272727, - "genetics": 0.24431818181818182, - "global_facts": 0.24161073825503357, - "high_school_biology": 0.25443786982248523, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.211864406779661, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.23776223776223776, - "human_sexuality": 0.2619047619047619, - "international_law": 0.2648648648648649, - "journalism": 0.26744186046511625, - "jurisprudence": 0.25304136253041365, - "legal_and_moral_basis": 0.26635514018691586, - "logical": 0.23577235772357724, - "machine_learning": 0.26229508196721313, - "management": 0.2, - "marketing": 0.18888888888888888, - "marxist_theory": 0.2275132275132275, - "modern_chinese": 0.23275862068965517, - "nutrition": 0.2482758620689655, - "philosophy": 0.3142857142857143, - "professional_accounting": 0.24571428571428572, - "professional_law": 0.26540284360189575, - "professional_medicine": 0.22606382978723405, - "professional_psychology": 0.2413793103448276, - "public_relations": 0.25862068965517243, - "security_study": 0.2222222222222222, - "sociology": 0.24336283185840707, - "sports_science": 0.24242424242424243, - "traditional_chinese_medicine": 0.2810810810810811, - "virology": 0.22485207100591717, - "world_history": 0.22981366459627328, - "world_religions": 0.225 - } - }, - "prompt_3": { - "accuracy": 0.25392850975651876, - "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.2621951219512195, - "arts": 0.2625, - "astronomy": 0.21818181818181817, - "business_ethics": 0.24401913875598086, - "chinese_civil_service_exam": 0.26875, - "chinese_driving_rule": 0.2748091603053435, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.2336448598130841, - "chinese_history": 0.25077399380804954, - "chinese_literature": 0.28431372549019607, - "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.26582278481012656, - "college_actuarial_science": 0.22641509433962265, - "college_education": 0.27102803738317754, - "college_engineering_hydrology": 0.24528301886792453, - "college_law": 0.24074074074074073, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.22641509433962265, - "college_medicine": 0.2454212454212454, - "computer_science": 0.2696078431372549, - "computer_security": 0.22807017543859648, - "conceptual_physics": 0.24489795918367346, - "construction_project_management": 0.2733812949640288, - "economics": 0.24528301886792453, - "education": 0.25153374233128833, - "electrical_engineering": 0.26744186046511625, - "elementary_chinese": 0.2857142857142857, - "elementary_commonsense": 0.25252525252525254, - "elementary_information_and_technology": 0.2815126050420168, - "elementary_mathematics": 0.29130434782608694, - "ethnology": 0.2814814814814815, - "food_science": 0.24475524475524477, - "genetics": 0.25, - "global_facts": 0.2684563758389262, - "high_school_biology": 0.23668639053254437, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.3305084745762712, - "high_school_mathematics": 0.22560975609756098, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.22377622377622378, - "human_sexuality": 0.2777777777777778, - "international_law": 0.24864864864864866, - "journalism": 0.2616279069767442, - "jurisprudence": 0.25304136253041365, - "legal_and_moral_basis": 0.2570093457943925, - "logical": 0.2926829268292683, - "machine_learning": 0.22131147540983606, - "management": 0.22380952380952382, - "marketing": 0.22777777777777777, - "marxist_theory": 0.2275132275132275, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.23448275862068965, - "philosophy": 0.2571428571428571, - "professional_accounting": 0.24571428571428572, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.2393617021276596, - "professional_psychology": 0.2629310344827586, - "public_relations": 0.25287356321839083, - "security_study": 0.24444444444444444, - "sociology": 0.26991150442477874, - "sports_science": 0.22424242424242424, - "traditional_chinese_medicine": 0.2864864864864865, - "virology": 0.2485207100591716, - "world_history": 0.2422360248447205, - "world_religions": 0.25 - } - }, - "prompt_4": { - "accuracy": 0.25375582800897944, - "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.25, - "arts": 0.26875, - "astronomy": 0.24848484848484848, - "business_ethics": 0.24880382775119617, - "chinese_civil_service_exam": 0.2375, - "chinese_driving_rule": 0.2595419847328244, - "chinese_food_culture": 0.25735294117647056, - "chinese_foreign_policy": 0.19626168224299065, - "chinese_history": 0.23839009287925697, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.26256983240223464, - "clinical_knowledge": 0.270042194092827, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.2616822429906542, - "college_engineering_hydrology": 0.33962264150943394, - "college_law": 0.23148148148148148, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.2358490566037736, - "college_medicine": 0.2564102564102564, - "computer_science": 0.2647058823529412, - "computer_security": 0.23391812865497075, - "conceptual_physics": 0.25170068027210885, - "construction_project_management": 0.2517985611510791, - "economics": 0.2389937106918239, - "education": 0.24539877300613497, - "electrical_engineering": 0.2616279069767442, - "elementary_chinese": 0.2857142857142857, - "elementary_commonsense": 0.23737373737373738, - "elementary_information_and_technology": 0.2689075630252101, - "elementary_mathematics": 0.28695652173913044, - "ethnology": 0.2740740740740741, - "food_science": 0.27972027972027974, - "genetics": 0.25, - "global_facts": 0.2483221476510067, - "high_school_biology": 0.23668639053254437, - "high_school_chemistry": 0.25, - "high_school_geography": 0.2711864406779661, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.25874125874125875, - "human_sexuality": 0.24603174603174602, - "international_law": 0.24324324324324326, - "journalism": 0.26744186046511625, - "jurisprudence": 0.25304136253041365, - "legal_and_moral_basis": 0.2570093457943925, - "logical": 0.2601626016260163, - "machine_learning": 0.22131147540983606, - "management": 0.2571428571428571, - "marketing": 0.25, - "marxist_theory": 0.23809523809523808, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.2896551724137931, - "philosophy": 0.26666666666666666, - "professional_accounting": 0.24571428571428572, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.22872340425531915, - "professional_psychology": 0.23706896551724138, - "public_relations": 0.2471264367816092, - "security_study": 0.22962962962962963, - "sociology": 0.2610619469026549, - "sports_science": 0.24848484848484848, - "traditional_chinese_medicine": 0.2648648648648649, - "virology": 0.24260355029585798, - "world_history": 0.2981366459627329, - "world_religions": 0.275 - } - }, - "prompt_5": { - "accuracy": 0.24607149024348127, - "category_acc": { - "agronomy": 0.21893491124260356, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.22560975609756098, - "arts": 0.25625, - "astronomy": 0.26666666666666666, - "business_ethics": 0.2583732057416268, - "chinese_civil_service_exam": 0.275, - "chinese_driving_rule": 0.2748091603053435, - "chinese_food_culture": 0.22058823529411764, - "chinese_foreign_policy": 0.22429906542056074, - "chinese_history": 0.25696594427244585, - "chinese_literature": 0.25980392156862747, - "chinese_teacher_qualification": 0.24022346368715083, - "clinical_knowledge": 0.2109704641350211, - "college_actuarial_science": 0.22641509433962265, - "college_education": 0.24299065420560748, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.17592592592592593, - "college_mathematics": 0.21904761904761905, - "college_medical_statistics": 0.19811320754716982, - "college_medicine": 0.2490842490842491, - "computer_science": 0.27450980392156865, - "computer_security": 0.21637426900584794, - "conceptual_physics": 0.272108843537415, - "construction_project_management": 0.26618705035971224, - "economics": 0.24528301886792453, - "education": 0.26380368098159507, - "electrical_engineering": 0.23837209302325582, - "elementary_chinese": 0.2857142857142857, - "elementary_commonsense": 0.23737373737373738, - "elementary_information_and_technology": 0.23949579831932774, - "elementary_mathematics": 0.20869565217391303, - "ethnology": 0.2962962962962963, - "food_science": 0.2727272727272727, - "genetics": 0.2215909090909091, - "global_facts": 0.2348993288590604, - "high_school_biology": 0.25443786982248523, - "high_school_chemistry": 0.25, - "high_school_geography": 0.2796610169491525, - "high_school_mathematics": 0.24390243902439024, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.24475524475524477, - "human_sexuality": 0.24603174603174602, - "international_law": 0.2756756756756757, - "journalism": 0.2616279069767442, - "jurisprudence": 0.24817518248175183, - "legal_and_moral_basis": 0.21495327102803738, - "logical": 0.24390243902439024, - "machine_learning": 0.2540983606557377, - "management": 0.21428571428571427, - "marketing": 0.22777777777777777, - "marxist_theory": 0.2275132275132275, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.23448275862068965, - "philosophy": 0.3047619047619048, - "professional_accounting": 0.24571428571428572, - "professional_law": 0.2559241706161137, - "professional_medicine": 0.24468085106382978, - "professional_psychology": 0.25862068965517243, - "public_relations": 0.2471264367816092, - "security_study": 0.21481481481481482, - "sociology": 0.23893805309734514, - "sports_science": 0.22424242424242424, - "traditional_chinese_medicine": 0.2810810810810811, - "virology": 0.21301775147928995, - "world_history": 0.22981366459627328, - "world_religions": 0.25 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.30303030303030304 - }, - "prompt_2": { - "accuracy": 0.24242424242424243 - }, - "prompt_3": { - "accuracy": 0.2727272727272727 - }, - "prompt_4": { - "accuracy": 0.3333333333333333 - }, - "prompt_5": { - "accuracy": 0.3939393939393939 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.2727272727272727 - }, - "prompt_2": { - "accuracy": 0.30454545454545456 - }, - "prompt_3": { - "accuracy": 0.23863636363636365 - }, - "prompt_4": { - "accuracy": 0.325 - }, - "prompt_5": { - "accuracy": 0.2840909090909091 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3345762711864407 - }, - "prompt_2": { - "accuracy": 0.32610169491525426 - }, - "prompt_3": { - "accuracy": 0.32677966101694916 - }, - "prompt_4": { - "accuracy": 0.3328813559322034 - }, - "prompt_5": { - "accuracy": 0.3335593220338983 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.28721017202692595 - }, - "prompt_2": { - "accuracy": 0.275243081525804 - }, - "prompt_3": { - "accuracy": 0.2677636499626028 - }, - "prompt_4": { - "accuracy": 0.268511593118923 - }, - "prompt_5": { - "accuracy": 0.27673896783844426 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.875061244487996 - }, - "prompt_2": { - "accuracy": 0.8682018618324351 - }, - "prompt_3": { - "accuracy": 0.8740813326800588 - }, - "prompt_4": { - "accuracy": 0.8726114649681529 - }, - "prompt_5": { - "accuracy": 0.8716315531602156 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.5184941938237266, - "rouge2": 0.26911215829849966, - "rougeL": 0.43302146061673596, - "avg_rouge": 0.40687593757965407 - }, - "prompt_2": { - "rouge1": 0.5187461086357531, - "rouge2": 0.26932154972696193, - "rougeL": 0.4314584130951696, - "avg_rouge": 0.40650869048596155 - }, - "prompt_3": { - "rouge1": 0.5163406671436445, - "rouge2": 0.2635412399924256, - "rougeL": 0.42801157617364916, - "avg_rouge": 0.40263116110323977 - }, - "prompt_4": { - "rouge1": 0.5187156796981305, - "rouge2": 0.26840154921197373, - "rougeL": 0.4319044858007276, - "avg_rouge": 0.4063405715702773 - }, - "prompt_5": { - "rouge1": 0.5174655546068478, - "rouge2": 0.26816881208607707, - "rougeL": 0.4310352587579342, - "avg_rouge": 0.40555654181695305 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.24421750699601816, - "rouge2": 0.07467360540993599, - "rougeL": 0.18426802878245258, - "avg_rouge": 0.1677197137294689 - }, - "prompt_2": { - "rouge1": 0.24390178370568294, - "rouge2": 0.07415199984827352, - "rougeL": 0.18592737078559401, - "avg_rouge": 0.1679937181131835 - }, - "prompt_3": { - "rouge1": 0.23272166113088152, - "rouge2": 0.06643160430178606, - "rougeL": 0.17368266596292145, - "avg_rouge": 0.157611977131863 - }, - "prompt_4": { - "rouge1": 0.23318330611333593, - "rouge2": 0.06579962056867525, - "rougeL": 0.17388243614817955, - "avg_rouge": 0.1576217876100636 - }, - "prompt_5": { - "rouge1": 0.24293948172938584, - "rouge2": 0.07043795906654217, - "rougeL": 0.18542877396721455, - "avg_rouge": 0.16626873825438085 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.9529816513761468 - }, - "prompt_2": { - "accuracy": 0.9506880733944955 - }, - "prompt_3": { - "accuracy": 0.9495412844036697 - }, - "prompt_4": { - "accuracy": 0.9529816513761468 - }, - "prompt_5": { - "accuracy": 0.948394495412844 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.7171620325982742 - }, - "prompt_2": { - "accuracy": 0.7277085330776606 - }, - "prompt_3": { - "accuracy": 0.7229146692233941 - }, - "prompt_4": { - "accuracy": 0.713326941514861 - }, - "prompt_5": { - "accuracy": 0.7248322147651006 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.824 - }, - "prompt_2": { - "accuracy": 0.8525 - }, - "prompt_3": { - "accuracy": 0.821 - }, - "prompt_4": { - "accuracy": 0.8275 - }, - "prompt_5": { - "accuracy": 0.8465 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.703 - }, - "prompt_2": { - "accuracy": 0.694 - }, - "prompt_3": { - "accuracy": 0.709 - }, - "prompt_4": { - "accuracy": 0.705 - }, - "prompt_5": { - "accuracy": 0.7 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.8255 - }, - "prompt_2": { - "accuracy": 0.8055 - }, - "prompt_3": { - "accuracy": 0.764 - }, - "prompt_4": { - "accuracy": 0.868 - }, - "prompt_5": { - "accuracy": 0.895 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.5915492957746479 - }, - "prompt_2": { - "accuracy": 0.5915492957746479 - }, - "prompt_3": { - "accuracy": 0.5774647887323944 - }, - "prompt_4": { - "accuracy": 0.5633802816901409 - }, - "prompt_5": { - "accuracy": 0.5633802816901409 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.8483754512635379 - }, - "prompt_2": { - "accuracy": 0.8267148014440433 - }, - "prompt_3": { - "accuracy": 0.8411552346570397 - }, - "prompt_4": { - "accuracy": 0.8122743682310469 - }, - "prompt_5": { - "accuracy": 0.8158844765342961 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.7696078431372549 - }, - "prompt_2": { - "accuracy": 0.8137254901960784 - }, - "prompt_3": { - "accuracy": 0.7647058823529411 - }, - "prompt_4": { - "accuracy": 0.8137254901960784 - }, - "prompt_5": { - "accuracy": 0.803921568627451 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.7466386554621849, - "language_acc": { - "English": 0.9672268907563025, - "Chinese": 0.36050420168067226, - "Spanish": 0.9142857142857143, - "Vietnamese": 0.7445378151260504 - }, - "consistency_score_2": 0.5992997198879552, - "consistency_score_3": 0.4273109243697479, - "consistency_score_4": 0.30756302521008405, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3689075630252101, - "English,Spanish": 0.9201680672268907, - "English,Vietnamese": 0.7420168067226891, - "Chinese,Spanish": 0.38487394957983195, - "Chinese,Vietnamese": 0.43445378151260505, - "Spanish,Vietnamese": 0.7453781512605042 - }, - "3_combine": { - "English,Chinese,Spanish": 0.35294117647058826, - "English,Chinese,Vietnamese": 0.31512605042016806, - "English,Spanish,Vietnamese": 0.7176470588235294, - "Chinese,Spanish,Vietnamese": 0.3235294117647059 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.30756302521008405 - } - }, - "AC3_2": 0.6649046423486613, - "AC3_3": 0.5435443897926701, - "AC3_4": 0.4356632090446197 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.3238095238095238, - "language_acc": { - "Spanish": 0.36666666666666664, - "English": 0.42, - "Chinese": 0.28, - "Vietnamese": 0.2733333333333333, - "Filipino": 0.32666666666666666, - "Indonesian": 0.3, - "Malay": 0.3 - }, - "consistency_score_2": 0.6057142857142859, - "consistency_score_3": 0.45219047619047614, - "consistency_score_4": 0.36057142857142854, - "consistency_score_5": 0.2968253968253968, - "consistency_score_6": 0.24761904761904763, - "consistency_score_7": 0.20666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5866666666666667, - "Spanish,Chinese": 0.5666666666666667, - "Spanish,Vietnamese": 0.54, - "Spanish,Filipino": 0.56, - "Spanish,Indonesian": 0.5333333333333333, - "Spanish,Malay": 0.5333333333333333, - "English,Chinese": 0.5333333333333333, - "English,Vietnamese": 0.49333333333333335, - "English,Filipino": 0.5733333333333334, - "English,Indonesian": 0.5333333333333333, - "English,Malay": 0.5733333333333334, - "Chinese,Vietnamese": 0.7333333333333333, - "Chinese,Filipino": 0.6666666666666666, - "Chinese,Indonesian": 0.6933333333333334, - "Chinese,Malay": 0.7466666666666667, - "Vietnamese,Filipino": 0.56, - "Vietnamese,Indonesian": 0.6066666666666667, - "Vietnamese,Malay": 0.64, - "Filipino,Indonesian": 0.6533333333333333, - "Filipino,Malay": 0.7066666666666667, - "Indonesian,Malay": 0.6866666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.4, - "Spanish,English,Vietnamese": 0.35333333333333333, - "Spanish,English,Filipino": 0.42, - "Spanish,English,Indonesian": 0.38666666666666666, - "Spanish,English,Malay": 0.41333333333333333, - "Spanish,Chinese,Vietnamese": 0.4666666666666667, - "Spanish,Chinese,Filipino": 0.43333333333333335, - "Spanish,Chinese,Indonesian": 0.43333333333333335, - "Spanish,Chinese,Malay": 0.4533333333333333, - "Spanish,Vietnamese,Filipino": 0.38666666666666666, - "Spanish,Vietnamese,Indonesian": 0.38666666666666666, - "Spanish,Vietnamese,Malay": 0.4066666666666667, - "Spanish,Filipino,Indonesian": 0.42, - "Spanish,Filipino,Malay": 0.44, - "Spanish,Indonesian,Malay": 0.4266666666666667, - "English,Chinese,Vietnamese": 0.42, - "English,Chinese,Filipino": 0.42, - "English,Chinese,Indonesian": 0.43333333333333335, - "English,Chinese,Malay": 0.47333333333333333, - "English,Vietnamese,Filipino": 0.35333333333333333, - "English,Vietnamese,Indonesian": 0.37333333333333335, - "English,Vietnamese,Malay": 0.42, - "English,Filipino,Indonesian": 0.42, - "English,Filipino,Malay": 0.4666666666666667, - "English,Indonesian,Malay": 0.44666666666666666, - "Chinese,Vietnamese,Filipino": 0.52, - "Chinese,Vietnamese,Indonesian": 0.5533333333333333, - "Chinese,Vietnamese,Malay": 0.5933333333333334, - "Chinese,Filipino,Indonesian": 0.5466666666666666, - "Chinese,Filipino,Malay": 0.58, - "Chinese,Indonesian,Malay": 0.5866666666666667, - "Vietnamese,Filipino,Indonesian": 0.4533333333333333, - "Vietnamese,Filipino,Malay": 0.48, - "Vietnamese,Indonesian,Malay": 0.5066666666666667, - "Filipino,Indonesian,Malay": 0.5533333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.31333333333333335, - "Spanish,English,Chinese,Filipino": 0.32, - "Spanish,English,Chinese,Indonesian": 0.3333333333333333, - "Spanish,English,Chinese,Malay": 0.35333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.2733333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,English,Vietnamese,Malay": 0.30666666666666664, - "Spanish,English,Filipino,Indonesian": 0.32666666666666666, - "Spanish,English,Filipino,Malay": 0.36, - "Spanish,English,Indonesian,Malay": 0.34, - "Spanish,Chinese,Vietnamese,Filipino": 0.36, - "Spanish,Chinese,Vietnamese,Indonesian": 0.36666666666666664, - "Spanish,Chinese,Vietnamese,Malay": 0.38, - "Spanish,Chinese,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.38, - "Spanish,Chinese,Indonesian,Malay": 0.38, - "Spanish,Vietnamese,Filipino,Indonesian": 0.32, - "Spanish,Vietnamese,Filipino,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.34, - "Spanish,Filipino,Indonesian,Malay": 0.36, - "English,Chinese,Vietnamese,Filipino": 0.3333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.3466666666666667, - "English,Chinese,Vietnamese,Malay": 0.3933333333333333, - "English,Chinese,Filipino,Indonesian": 0.36, - "English,Chinese,Filipino,Malay": 0.38666666666666666, - "English,Chinese,Indonesian,Malay": 0.38666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "English,Vietnamese,Filipino,Malay": 0.3333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.3466666666666667, - "English,Filipino,Indonesian,Malay": 0.37333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.44, - "Chinese,Vietnamese,Filipino,Malay": 0.4533333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.48, - "Chinese,Filipino,Indonesian,Malay": 0.4866666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.4066666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.2866666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.3, - "Spanish,English,Chinese,Indonesian,Malay": 0.3, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,English,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.32, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.32666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3933333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667 - } - }, - "AC3_2": 0.42201405147684723, - "AC3_3": 0.3773803679061105, - "AC3_4": 0.3412031330263639, - "AC3_5": 0.3097308488113782, - "AC3_6": 0.2806349205858096, - "AC3_7": 0.2523040095275429 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.29464285714285715, - "language_acc": { - "English": 0.44886363636363635, - "Filipino": 0.26136363636363635, - "Spanish": 0.2840909090909091, - "Malay": 0.2727272727272727, - "Chinese": 0.26704545454545453, - "Indonesian": 0.24431818181818182, - "Vietnamese": 0.2840909090909091 - }, - "consistency_score_2": 0.44128787878787873, - "consistency_score_3": 0.2457792207792207, - "consistency_score_4": 0.1551948051948052, - "consistency_score_5": 0.10876623376623377, - "consistency_score_6": 0.0836038961038961, - "consistency_score_7": 0.06818181818181818, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.32386363636363635, - "English,Spanish": 0.4772727272727273, - "English,Malay": 0.30113636363636365, - "English,Chinese": 0.30113636363636365, - "English,Indonesian": 0.3409090909090909, - "English,Vietnamese": 0.3125, - "Filipino,Spanish": 0.5, - "Filipino,Malay": 0.4772727272727273, - "Filipino,Chinese": 0.5170454545454546, - "Filipino,Indonesian": 0.5340909090909091, - "Filipino,Vietnamese": 0.4772727272727273, - "Spanish,Malay": 0.4375, - "Spanish,Chinese": 0.45454545454545453, - "Spanish,Indonesian": 0.45454545454545453, - "Spanish,Vietnamese": 0.4147727272727273, - "Malay,Chinese": 0.4375, - "Malay,Indonesian": 0.5511363636363636, - "Malay,Vietnamese": 0.39204545454545453, - "Chinese,Indonesian": 0.4943181818181818, - "Chinese,Vietnamese": 0.6363636363636364, - "Indonesian,Vietnamese": 0.4318181818181818 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2215909090909091, - "English,Filipino,Malay": 0.1590909090909091, - "English,Filipino,Chinese": 0.17045454545454544, - "English,Filipino,Indonesian": 0.19318181818181818, - "English,Filipino,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay": 0.19886363636363635, - "English,Spanish,Chinese": 0.19886363636363635, - "English,Spanish,Indonesian": 0.2159090909090909, - "English,Spanish,Vietnamese": 0.19318181818181818, - "English,Malay,Chinese": 0.14772727272727273, - "English,Malay,Indonesian": 0.19886363636363635, - "English,Malay,Vietnamese": 0.14204545454545456, - "English,Chinese,Indonesian": 0.17613636363636365, - "English,Chinese,Vietnamese": 0.21022727272727273, - "English,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay": 0.2897727272727273, - "Filipino,Spanish,Chinese": 0.30113636363636365, - "Filipino,Spanish,Indonesian": 0.29545454545454547, - "Filipino,Spanish,Vietnamese": 0.2727272727272727, - "Filipino,Malay,Chinese": 0.30113636363636365, - "Filipino,Malay,Indonesian": 0.3465909090909091, - "Filipino,Malay,Vietnamese": 0.2556818181818182, - "Filipino,Chinese,Indonesian": 0.3465909090909091, - "Filipino,Chinese,Vietnamese": 0.3693181818181818, - "Filipino,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Chinese": 0.25, - "Spanish,Malay,Indonesian": 0.29545454545454547, - "Spanish,Malay,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Indonesian": 0.26704545454545453, - "Spanish,Chinese,Vietnamese": 0.3352272727272727, - "Spanish,Indonesian,Vietnamese": 0.22727272727272727, - "Malay,Chinese,Indonesian": 0.30113636363636365, - "Malay,Chinese,Vietnamese": 0.3068181818181818, - "Malay,Indonesian,Vietnamese": 0.25, - "Chinese,Indonesian,Vietnamese": 0.35795454545454547 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.13068181818181818, - "English,Filipino,Spanish,Chinese": 0.125, - "English,Filipino,Spanish,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Vietnamese": 0.125, - "English,Filipino,Malay,Chinese": 0.11363636363636363, - "English,Filipino,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Vietnamese": 0.09090909090909091, - "English,Filipino,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Chinese,Vietnamese": 0.125, - "English,Filipino,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese": 0.10795454545454546, - "English,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Spanish,Malay,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian": 0.125, - "English,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "English,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "English,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese": 0.19886363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.1875, - "Filipino,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.18181818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818 - } - }, - "AC3_2": 0.35335477936375004, - "AC3_3": 0.26800197394517333, - "AC3_4": 0.20330463469213542, - "AC3_5": 0.15888186256481182, - "AC3_6": 0.13024984668536863, - "AC3_7": 0.11073825500303588 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.4077669902912621 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.26666666666666666 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.38317757009345793 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.19, - "category_acc": { - "brand": 0.0, - "demographics": 0.2, - "biology": 0.3, - "history": 0.2, - "literature": 0.3, - "politics": 0.2, - "culture": 0.0, - "film": 0.1, - "law": 0.2, - "geography": 0.4 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.18644213060844367 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.1279956080786735 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.036744241432472964 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.015015534973935476 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.12019491878080271 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.4235705950991832 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.4060779406506972, - "category_acc": { - "high_school_european_history": 0.5426829268292683, - "business_ethics": 0.5454545454545454, - "clinical_knowledge": 0.45454545454545453, - "medical_genetics": 0.3434343434343434, - "high_school_us_history": 0.5566502463054187, - "high_school_physics": 0.32666666666666666, - "high_school_world_history": 0.6186440677966102, - "virology": 0.3575757575757576, - "high_school_microeconomics": 0.45147679324894513, - "econometrics": 0.24778761061946902, - "college_computer_science": 0.32323232323232326, - "high_school_biology": 0.4368932038834951, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.3416370106761566, - "philosophy": 0.4032258064516129, - "professional_medicine": 0.4317343173431734, - "nutrition": 0.43934426229508194, - "global_facts": 0.24242424242424243, - "machine_learning": 0.1981981981981982, - "security_studies": 0.5655737704918032, - "public_relations": 0.5137614678899083, - "professional_psychology": 0.3518821603927987, - "prehistory": 0.3715170278637771, - "anatomy": 0.3805970149253731, - "human_sexuality": 0.43846153846153846, - "college_medicine": 0.42441860465116277, - "high_school_government_and_politics": 0.546875, - "college_chemistry": 0.3939393939393939, - "logical_fallacies": 0.5061728395061729, - "high_school_geography": 0.5532994923857868, - "elementary_mathematics": 0.29973474801061006, - "human_aging": 0.4099099099099099, - "college_mathematics": 0.25252525252525254, - "high_school_psychology": 0.5514705882352942, - "formal_logic": 0.368, - "high_school_statistics": 0.35348837209302325, - "international_law": 0.55, - "high_school_mathematics": 0.24535315985130113, - "high_school_computer_science": 0.36363636363636365, - "conceptual_physics": 0.3333333333333333, - "miscellaneous": 0.49104859335038364, - "high_school_chemistry": 0.2871287128712871, - "marketing": 0.6566523605150214, - "professional_law": 0.34181343770384864, - "management": 0.6176470588235294, - "college_physics": 0.31683168316831684, - "jurisprudence": 0.4672897196261682, - "world_religions": 0.3352941176470588, - "sociology": 0.575, - "us_foreign_policy": 0.5353535353535354, - "high_school_macroeconomics": 0.4318766066838046, - "computer_security": 0.47474747474747475, - "moral_scenarios": 0.21700223713646533, - "moral_disputes": 0.4985507246376812, - "electrical_engineering": 0.3819444444444444, - "astronomy": 0.36423841059602646, - "college_biology": 0.35664335664335667 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.25482912332838037 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.2727272727272727, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.19047619047619047, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.4166666666666667, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.21666666666666667, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.24489795918367346, - "high_school_politics": 0.125, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.4074074074074074, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.04, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.2692307692307692, - "sports_science": 0.2916666666666667, - "plant_protection": 0.1111111111111111, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.2962962962962963, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.3148148148148148, - "physician": 0.3333333333333333 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.2222222222222222 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.2514246244171991, - "category_acc": { - "agronomy": 0.2603550295857988, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.23780487804878048, - "arts": 0.25, - "astronomy": 0.24848484848484848, - "business_ethics": 0.2583732057416268, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.2616822429906542, - "chinese_history": 0.24148606811145512, - "chinese_literature": 0.2549019607843137, - "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.22784810126582278, - "college_actuarial_science": 0.3113207547169811, - "college_education": 0.1588785046728972, - "college_engineering_hydrology": 0.2641509433962264, - "college_law": 0.32407407407407407, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.2358490566037736, - "college_medicine": 0.24175824175824176, - "computer_science": 0.2647058823529412, - "computer_security": 0.25146198830409355, - "conceptual_physics": 0.23809523809523808, - "construction_project_management": 0.2446043165467626, - "economics": 0.2578616352201258, - "education": 0.25153374233128833, - "electrical_engineering": 0.23837209302325582, - "elementary_chinese": 0.20634920634920634, - "elementary_commonsense": 0.2878787878787879, - "elementary_information_and_technology": 0.25210084033613445, - "elementary_mathematics": 0.2391304347826087, - "ethnology": 0.2518518518518518, - "food_science": 0.2517482517482518, - "genetics": 0.22727272727272727, - "global_facts": 0.2483221476510067, - "high_school_biology": 0.2485207100591716, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.2682926829268293, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.23776223776223776, - "human_sexuality": 0.25396825396825395, - "international_law": 0.24864864864864866, - "journalism": 0.2616279069767442, - "jurisprudence": 0.26763990267639903, - "legal_and_moral_basis": 0.2523364485981308, - "logical": 0.24390243902439024, - "machine_learning": 0.21311475409836064, - "management": 0.24761904761904763, - "marketing": 0.24444444444444444, - "marxist_theory": 0.25925925925925924, - "modern_chinese": 0.2672413793103448, - "nutrition": 0.25517241379310346, - "philosophy": 0.2571428571428571, - "professional_accounting": 0.24571428571428572, - "professional_law": 0.27014218009478674, - "professional_medicine": 0.2526595744680851, - "professional_psychology": 0.25, - "public_relations": 0.2413793103448276, - "security_study": 0.25925925925925924, - "sociology": 0.252212389380531, - "sports_science": 0.24242424242424243, - "traditional_chinese_medicine": 0.2594594594594595, - "virology": 0.2603550295857988, - "world_history": 0.2608695652173913, - "world_religions": 0.25625 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.3333333333333333 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.3 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.32949152542372884 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.29020194465220644 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.8755512003919648 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.5206549569130728, - "rouge2": 0.2722531410921692, - "rougeL": 0.4351072633647337, - "avg_rouge": 0.40933845378999195 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.39446957126978227, - "rouge2": 0.1541250966260402, - "rougeL": 0.31752650066594423, - "avg_rouge": 0.28870705618725556 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.948394495412844 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.7181208053691275 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.851 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.7035 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.843 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.6197183098591549 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.8050541516245487 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.7965686274509803 - } - } - } - }, - "flan_t5_xl": { - "model_size": "3B", - "model_link": "https://huggingface.co/google/flan-t5-xl", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.7733193277310924, - "language_acc": { - "English": 0.9697478991596639, - "Chinese": 0.3563025210084034, - "Spanish": 0.9563025210084034, - "Vietnamese": 0.8109243697478992 - }, - "consistency_score_2": 0.6148459383753502, - "consistency_score_3": 0.4455882352941177, - "consistency_score_4": 0.3142857142857143, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3621848739495798, - "English,Spanish": 0.9537815126050421, - "English,Vietnamese": 0.8142857142857143, - "Chinese,Spanish": 0.3621848739495798, - "Chinese,Vietnamese": 0.38907563025210085, - "Spanish,Vietnamese": 0.807563025210084 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3495798319327731, - "English,Chinese,Vietnamese": 0.32100840336134456, - "English,Spanish,Vietnamese": 0.7915966386554621, - "Chinese,Spanish,Vietnamese": 0.32016806722689073 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3142857142857143 - } - }, - "AC3_2": 0.6850369466770796, - "AC3_3": 0.5653947928242272, - "AC3_4": 0.4469328623447341 - }, - "prompt_2": { - "overall_acc": 0.7762605042016806, - "language_acc": { - "English": 0.9697478991596639, - "Chinese": 0.36470588235294116, - "Spanish": 0.9588235294117647, - "Vietnamese": 0.8117647058823529 - }, - "consistency_score_2": 0.6179271708683474, - "consistency_score_3": 0.45210084033613446, - "consistency_score_4": 0.3226890756302521, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.37142857142857144, - "English,Spanish": 0.9529411764705882, - "English,Vietnamese": 0.8117647058823529, - "Chinese,Spanish": 0.37058823529411766, - "Chinese,Vietnamese": 0.39327731092436974, - "Spanish,Vietnamese": 0.807563025210084 - }, - "3_combine": { - "English,Chinese,Spanish": 0.35798319327731093, - "English,Chinese,Vietnamese": 0.3302521008403361, - "English,Spanish,Vietnamese": 0.7907563025210084, - "Chinese,Spanish,Vietnamese": 0.32941176470588235 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3226890756302521 - } - }, - "AC3_2": 0.6881031381369556, - "AC3_3": 0.5714084504552837, - "AC3_4": 0.4558731157886766 - }, - "prompt_3": { - "overall_acc": 0.774579831932773, - "language_acc": { - "English": 0.9697478991596639, - "Chinese": 0.3621848739495798, - "Spanish": 0.9554621848739496, - "Vietnamese": 0.8109243697478992 - }, - "consistency_score_2": 0.6170868347338936, - "consistency_score_3": 0.44915966386554623, - "consistency_score_4": 0.31848739495798317, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3689075630252101, - "English,Spanish": 0.9529411764705882, - "English,Vietnamese": 0.8117647058823529, - "Chinese,Spanish": 0.36722689075630255, - "Chinese,Vietnamese": 0.39327731092436974, - "Spanish,Vietnamese": 0.8084033613445378 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3546218487394958, - "English,Chinese,Vietnamese": 0.32605042016806723, - "English,Spanish,Vietnamese": 0.7907563025210084, - "Chinese,Spanish,Vietnamese": 0.32521008403361346 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.31848739495798317 - } - }, - "AC3_2": 0.6869217006492361, - "AC3_3": 0.5686014353774907, - "AC3_4": 0.4513792139545092 - }, - "prompt_4": { - "overall_acc": 0.7731092436974789, - "language_acc": { - "English": 0.9680672268907563, - "Chinese": 0.36554621848739494, - "Spanish": 0.9571428571428572, - "Vietnamese": 0.8016806722689076 - }, - "consistency_score_2": 0.6168067226890757, - "consistency_score_3": 0.4504201680672269, - "consistency_score_4": 0.32184873949579834, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3722689075630252, - "English,Spanish": 0.9529411764705882, - "English,Vietnamese": 0.8033613445378152, - "Chinese,Spanish": 0.37058823529411766, - "Chinese,Vietnamese": 0.4, - "Spanish,Vietnamese": 0.8016806722689076 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3588235294117647, - "English,Chinese,Vietnamese": 0.32941176470588235, - "English,Spanish,Vietnamese": 0.7840336134453781, - "Chinese,Spanish,Vietnamese": 0.32941176470588235 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.32184873949579834 - } - }, - "AC3_2": 0.6861695100763271, - "AC3_3": 0.5692123002582151, - "AC3_4": 0.45449092910067884 - }, - "prompt_5": { - "overall_acc": 0.7752100840336134, - "language_acc": { - "English": 0.9722689075630252, - "Chinese": 0.3563025210084034, - "Spanish": 0.957983193277311, - "Vietnamese": 0.8142857142857143 - }, - "consistency_score_2": 0.6134453781512605, - "consistency_score_3": 0.44264705882352945, - "consistency_score_4": 0.30756302521008405, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3596638655462185, - "English,Spanish": 0.9554621848739496, - "English,Vietnamese": 0.8168067226890756, - "Chinese,Spanish": 0.35798319327731093, - "Chinese,Vietnamese": 0.3764705882352941, - "Spanish,Vietnamese": 0.8142857142857143 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3445378151260504, - "English,Chinese,Vietnamese": 0.3134453781512605, - "English,Spanish,Vietnamese": 0.7983193277310925, - "Chinese,Spanish,Vietnamese": 0.3142857142857143 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.30756302521008405 - } - }, - "AC3_2": 0.6849057322889995, - "AC3_3": 0.5635216998187702, - "AC3_4": 0.44039874385735167 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.3609523809523809, - "language_acc": { - "Spanish": 0.44666666666666666, - "English": 0.5666666666666667, - "Chinese": 0.26666666666666666, - "Vietnamese": 0.24666666666666667, - "Filipino": 0.2866666666666667, - "Indonesian": 0.37333333333333335, - "Malay": 0.34 - }, - "consistency_score_2": 0.5142857142857143, - "consistency_score_3": 0.3352380952380952, - "consistency_score_4": 0.24247619047619046, - "consistency_score_5": 0.1850793650793651, - "consistency_score_6": 0.1466666666666667, - "consistency_score_7": 0.12, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5933333333333334, - "Spanish,Chinese": 0.4266666666666667, - "Spanish,Vietnamese": 0.44, - "Spanish,Filipino": 0.49333333333333335, - "Spanish,Indonesian": 0.5466666666666666, - "Spanish,Malay": 0.5, - "English,Chinese": 0.34, - "English,Vietnamese": 0.36, - "English,Filipino": 0.52, - "English,Indonesian": 0.48, - "English,Malay": 0.47333333333333333, - "Chinese,Vietnamese": 0.66, - "Chinese,Filipino": 0.5066666666666667, - "Chinese,Indonesian": 0.5666666666666667, - "Chinese,Malay": 0.5533333333333333, - "Vietnamese,Filipino": 0.5066666666666667, - "Vietnamese,Indonesian": 0.5266666666666666, - "Vietnamese,Malay": 0.5266666666666666, - "Filipino,Indonesian": 0.5266666666666666, - "Filipino,Malay": 0.5866666666666667, - "Indonesian,Malay": 0.6666666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.26, - "Spanish,English,Vietnamese": 0.2733333333333333, - "Spanish,English,Filipino": 0.37333333333333335, - "Spanish,English,Indonesian": 0.36, - "Spanish,English,Malay": 0.35333333333333333, - "Spanish,Chinese,Vietnamese": 0.32, - "Spanish,Chinese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,Vietnamese,Malay": 0.31333333333333335, - "Spanish,Filipino,Indonesian": 0.34, - "Spanish,Filipino,Malay": 0.36, - "Spanish,Indonesian,Malay": 0.41333333333333333, - "English,Chinese,Vietnamese": 0.25333333333333335, - "English,Chinese,Filipino": 0.26666666666666666, - "English,Chinese,Indonesian": 0.2866666666666667, - "English,Chinese,Malay": 0.26, - "English,Vietnamese,Filipino": 0.26, - "English,Vietnamese,Indonesian": 0.26666666666666666, - "English,Vietnamese,Malay": 0.26, - "English,Filipino,Indonesian": 0.32666666666666666, - "English,Filipino,Malay": 0.35333333333333333, - "English,Indonesian,Malay": 0.36, - "Chinese,Vietnamese,Filipino": 0.38, - "Chinese,Vietnamese,Indonesian": 0.4266666666666667, - "Chinese,Vietnamese,Malay": 0.42, - "Chinese,Filipino,Indonesian": 0.35333333333333333, - "Chinese,Filipino,Malay": 0.38666666666666666, - "Chinese,Indonesian,Malay": 0.44, - "Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "Vietnamese,Filipino,Malay": 0.37333333333333335, - "Vietnamese,Indonesian,Malay": 0.4, - "Filipino,Indonesian,Malay": 0.4533333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.2, - "Spanish,English,Chinese,Filipino": 0.2, - "Spanish,English,Chinese,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.22, - "Spanish,English,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Filipino,Indonesian": 0.26, - "Spanish,English,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.2, - "Spanish,Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.23333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.22666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,Vietnamese,Filipino,Malay": 0.24, - "Spanish,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.3, - "English,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "English,Chinese,Vietnamese,Malay": 0.19333333333333333, - "English,Chinese,Filipino,Indonesian": 0.22666666666666666, - "English,Chinese,Filipino,Malay": 0.22666666666666666, - "English,Chinese,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "English,Vietnamese,Filipino,Malay": 0.21333333333333335, - "English,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "English,Filipino,Indonesian,Malay": 0.28, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.31333333333333335, - "Vietnamese,Filipino,Indonesian,Malay": 0.3 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Vietnamese,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.16, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - } - }, - "AC3_2": 0.4241877817018843, - "AC3_3": 0.3476203504158359, - "AC3_4": 0.29008357378550126, - "AC3_5": 0.2446921372752315, - "AC3_6": 0.20858036268561425, - "AC3_7": 0.18011881184373765 - }, - "prompt_2": { - "overall_acc": 0.3580952380952381, - "language_acc": { - "Spanish": 0.4533333333333333, - "English": 0.5066666666666667, - "Chinese": 0.2866666666666667, - "Vietnamese": 0.24666666666666667, - "Filipino": 0.2733333333333333, - "Indonesian": 0.38, - "Malay": 0.36 - }, - "consistency_score_2": 0.5292063492063492, - "consistency_score_3": 0.3466666666666666, - "consistency_score_4": 0.2472380952380952, - "consistency_score_5": 0.18412698412698414, - "consistency_score_6": 0.13999999999999999, - "consistency_score_7": 0.10666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6266666666666667, - "Spanish,Chinese": 0.43333333333333335, - "Spanish,Vietnamese": 0.44666666666666666, - "Spanish,Filipino": 0.5066666666666667, - "Spanish,Indonesian": 0.5466666666666666, - "Spanish,Malay": 0.52, - "English,Chinese": 0.37333333333333335, - "English,Vietnamese": 0.36, - "English,Filipino": 0.4866666666666667, - "English,Indonesian": 0.52, - "English,Malay": 0.5, - "Chinese,Vietnamese": 0.64, - "Chinese,Filipino": 0.5533333333333333, - "Chinese,Indonesian": 0.58, - "Chinese,Malay": 0.5866666666666667, - "Vietnamese,Filipino": 0.52, - "Vietnamese,Indonesian": 0.54, - "Vietnamese,Malay": 0.52, - "Filipino,Indonesian": 0.5666666666666667, - "Filipino,Malay": 0.62, - "Indonesian,Malay": 0.6666666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.2733333333333333, - "Spanish,English,Vietnamese": 0.28, - "Spanish,English,Filipino": 0.36666666666666664, - "Spanish,English,Indonesian": 0.3933333333333333, - "Spanish,English,Malay": 0.38666666666666666, - "Spanish,Chinese,Vietnamese": 0.31333333333333335, - "Spanish,Chinese,Filipino": 0.2866666666666667, - "Spanish,Chinese,Indonesian": 0.31333333333333335, - "Spanish,Chinese,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,Vietnamese,Indonesian": 0.32666666666666666, - "Spanish,Vietnamese,Malay": 0.30666666666666664, - "Spanish,Filipino,Indonesian": 0.3466666666666667, - "Spanish,Filipino,Malay": 0.38, - "Spanish,Indonesian,Malay": 0.41333333333333333, - "English,Chinese,Vietnamese": 0.25333333333333335, - "English,Chinese,Filipino": 0.2733333333333333, - "English,Chinese,Indonesian": 0.31333333333333335, - "English,Chinese,Malay": 0.29333333333333333, - "English,Vietnamese,Filipino": 0.24666666666666667, - "English,Vietnamese,Indonesian": 0.2866666666666667, - "English,Vietnamese,Malay": 0.26, - "English,Filipino,Indonesian": 0.3466666666666667, - "English,Filipino,Malay": 0.35333333333333333, - "English,Indonesian,Malay": 0.38666666666666666, - "Chinese,Vietnamese,Filipino": 0.4, - "Chinese,Vietnamese,Indonesian": 0.4266666666666667, - "Chinese,Vietnamese,Malay": 0.41333333333333333, - "Chinese,Filipino,Indonesian": 0.3933333333333333, - "Chinese,Filipino,Malay": 0.4266666666666667, - "Chinese,Indonesian,Malay": 0.46, - "Vietnamese,Filipino,Indonesian": 0.36, - "Vietnamese,Filipino,Malay": 0.37333333333333335, - "Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Filipino,Indonesian,Malay": 0.47333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.19333333333333333, - "Spanish,English,Chinese,Filipino": 0.2, - "Spanish,English,Chinese,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Filipino,Malay": 0.28, - "Spanish,English,Indonesian,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.23333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.23333333333333334, - "Spanish,Chinese,Indonesian,Malay": 0.26666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "English,Chinese,Vietnamese,Indonesian": 0.23333333333333334, - "English,Chinese,Vietnamese,Malay": 0.2, - "English,Chinese,Filipino,Indonesian": 0.24, - "English,Chinese,Filipino,Malay": 0.22666666666666666, - "English,Chinese,Indonesian,Malay": 0.25333333333333335, - "English,Vietnamese,Filipino,Indonesian": 0.22, - "English,Vietnamese,Filipino,Malay": 0.2, - "English,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "English,Filipino,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.35333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "English,Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - } - }, - "AC3_2": 0.42715188682467653, - "AC3_3": 0.35228828823830144, - "AC3_4": 0.29251580803702554, - "AC3_5": 0.24320285486760382, - "AC3_6": 0.2013001911641749, - "AC3_7": 0.16437158466408672 - }, - "prompt_3": { - "overall_acc": 0.3580952380952381, - "language_acc": { - "Spanish": 0.44, - "English": 0.5266666666666666, - "Chinese": 0.26666666666666666, - "Vietnamese": 0.25333333333333335, - "Filipino": 0.3, - "Indonesian": 0.3933333333333333, - "Malay": 0.32666666666666666 - }, - "consistency_score_2": 0.5409523809523809, - "consistency_score_3": 0.36266666666666675, - "consistency_score_4": 0.2651428571428572, - "consistency_score_5": 0.2041269841269841, - "consistency_score_6": 0.16285714285714287, - "consistency_score_7": 0.13333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5933333333333334, - "Spanish,Chinese": 0.43333333333333335, - "Spanish,Vietnamese": 0.44666666666666666, - "Spanish,Filipino": 0.52, - "Spanish,Indonesian": 0.5466666666666666, - "Spanish,Malay": 0.5133333333333333, - "English,Chinese": 0.38, - "English,Vietnamese": 0.38666666666666666, - "English,Filipino": 0.5066666666666667, - "English,Indonesian": 0.5333333333333333, - "English,Malay": 0.4866666666666667, - "Chinese,Vietnamese": 0.66, - "Chinese,Filipino": 0.54, - "Chinese,Indonesian": 0.6, - "Chinese,Malay": 0.5866666666666667, - "Vietnamese,Filipino": 0.5333333333333333, - "Vietnamese,Indonesian": 0.5666666666666667, - "Vietnamese,Malay": 0.56, - "Filipino,Indonesian": 0.6, - "Filipino,Malay": 0.6666666666666666, - "Indonesian,Malay": 0.7 - }, - "3_combine": { - "Spanish,English,Chinese": 0.2733333333333333, - "Spanish,English,Vietnamese": 0.2733333333333333, - "Spanish,English,Filipino": 0.37333333333333335, - "Spanish,English,Indonesian": 0.38666666666666666, - "Spanish,English,Malay": 0.36, - "Spanish,Chinese,Vietnamese": 0.32666666666666666, - "Spanish,Chinese,Filipino": 0.2866666666666667, - "Spanish,Chinese,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Malay": 0.30666666666666664, - "Spanish,Vietnamese,Filipino": 0.29333333333333333, - "Spanish,Vietnamese,Indonesian": 0.3333333333333333, - "Spanish,Vietnamese,Malay": 0.32, - "Spanish,Filipino,Indonesian": 0.37333333333333335, - "Spanish,Filipino,Malay": 0.3933333333333333, - "Spanish,Indonesian,Malay": 0.42, - "English,Chinese,Vietnamese": 0.2866666666666667, - "English,Chinese,Filipino": 0.2866666666666667, - "English,Chinese,Indonesian": 0.3333333333333333, - "English,Chinese,Malay": 0.2866666666666667, - "English,Vietnamese,Filipino": 0.28, - "English,Vietnamese,Indonesian": 0.30666666666666664, - "English,Vietnamese,Malay": 0.28, - "English,Filipino,Indonesian": 0.37333333333333335, - "English,Filipino,Malay": 0.38, - "English,Indonesian,Malay": 0.4066666666666667, - "Chinese,Vietnamese,Filipino": 0.4, - "Chinese,Vietnamese,Indonesian": 0.46, - "Chinese,Vietnamese,Malay": 0.44666666666666666, - "Chinese,Filipino,Indonesian": 0.41333333333333333, - "Chinese,Filipino,Malay": 0.44, - "Chinese,Indonesian,Malay": 0.48, - "Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "Vietnamese,Filipino,Malay": 0.42, - "Vietnamese,Indonesian,Malay": 0.4533333333333333, - "Filipino,Indonesian,Malay": 0.5266666666666666 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,English,Chinese,Filipino": 0.20666666666666667, - "Spanish,English,Chinese,Indonesian": 0.24666666666666667, - "Spanish,English,Chinese,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.23333333333333334, - "Spanish,English,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Filipino,Malay": 0.2866666666666667, - "Spanish,English,Indonesian,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian": 0.26, - "Spanish,Chinese,Vietnamese,Malay": 0.24, - "Spanish,Chinese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.24, - "Spanish,Chinese,Indonesian,Malay": 0.28, - "Spanish,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.3333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.21333333333333335, - "English,Chinese,Vietnamese,Indonesian": 0.26, - "English,Chinese,Vietnamese,Malay": 0.22, - "English,Chinese,Filipino,Indonesian": 0.26666666666666666, - "English,Chinese,Filipino,Malay": 0.24, - "English,Chinese,Indonesian,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "English,Vietnamese,Filipino,Malay": 0.23333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.26, - "English,Filipino,Indonesian,Malay": 0.32666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Chinese,Vietnamese,Filipino,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian,Malay": 0.3466666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian": 0.2, - "Spanish,English,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Malay": 0.18, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.2, - "English,Chinese,Vietnamese,Filipino,Malay": 0.18, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 - } - }, - "AC3_2": 0.4309281678293888, - "AC3_3": 0.36036645520017824, - "AC3_4": 0.3046873907998723, - "AC3_5": 0.2600285014729346, - "AC3_6": 0.22389135540230867, - "AC3_7": 0.19431524543849527 - }, - "prompt_4": { - "overall_acc": 0.3590476190476191, - "language_acc": { - "Spanish": 0.4533333333333333, - "English": 0.5266666666666666, - "Chinese": 0.26, - "Vietnamese": 0.24, - "Filipino": 0.29333333333333333, - "Indonesian": 0.38666666666666666, - "Malay": 0.35333333333333333 - }, - "consistency_score_2": 0.533015873015873, - "consistency_score_3": 0.35276190476190467, - "consistency_score_4": 0.2563809523809524, - "consistency_score_5": 0.19587301587301592, - "consistency_score_6": 0.15333333333333332, - "consistency_score_7": 0.12, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6, - "Spanish,Chinese": 0.44, - "Spanish,Vietnamese": 0.42, - "Spanish,Filipino": 0.4866666666666667, - "Spanish,Indonesian": 0.5333333333333333, - "Spanish,Malay": 0.49333333333333335, - "English,Chinese": 0.37333333333333335, - "English,Vietnamese": 0.36, - "English,Filipino": 0.48, - "English,Indonesian": 0.5133333333333333, - "English,Malay": 0.5133333333333333, - "Chinese,Vietnamese": 0.68, - "Chinese,Filipino": 0.5533333333333333, - "Chinese,Indonesian": 0.5866666666666667, - "Chinese,Malay": 0.5733333333333334, - "Vietnamese,Filipino": 0.5333333333333333, - "Vietnamese,Indonesian": 0.5733333333333334, - "Vietnamese,Malay": 0.5466666666666666, - "Filipino,Indonesian": 0.58, - "Filipino,Malay": 0.66, - "Indonesian,Malay": 0.6933333333333334 - }, - "3_combine": { - "Spanish,English,Chinese": 0.2733333333333333, - "Spanish,English,Vietnamese": 0.26, - "Spanish,English,Filipino": 0.35333333333333333, - "Spanish,English,Indonesian": 0.37333333333333335, - "Spanish,English,Malay": 0.37333333333333335, - "Spanish,Chinese,Vietnamese": 0.31333333333333335, - "Spanish,Chinese,Filipino": 0.28, - "Spanish,Chinese,Indonesian": 0.32, - "Spanish,Chinese,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,Vietnamese,Indonesian": 0.32, - "Spanish,Vietnamese,Malay": 0.3, - "Spanish,Filipino,Indonesian": 0.34, - "Spanish,Filipino,Malay": 0.37333333333333335, - "Spanish,Indonesian,Malay": 0.4066666666666667, - "English,Chinese,Vietnamese": 0.26666666666666666, - "English,Chinese,Filipino": 0.28, - "English,Chinese,Indonesian": 0.31333333333333335, - "English,Chinese,Malay": 0.2866666666666667, - "English,Vietnamese,Filipino": 0.26, - "English,Vietnamese,Indonesian": 0.29333333333333333, - "English,Vietnamese,Malay": 0.2733333333333333, - "English,Filipino,Indonesian": 0.3466666666666667, - "English,Filipino,Malay": 0.37333333333333335, - "English,Indonesian,Malay": 0.4066666666666667, - "Chinese,Vietnamese,Filipino": 0.41333333333333333, - "Chinese,Vietnamese,Indonesian": 0.4666666666666667, - "Chinese,Vietnamese,Malay": 0.44, - "Chinese,Filipino,Indonesian": 0.4066666666666667, - "Chinese,Filipino,Malay": 0.44, - "Chinese,Indonesian,Malay": 0.4666666666666667, - "Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "Vietnamese,Filipino,Malay": 0.42, - "Vietnamese,Indonesian,Malay": 0.43333333333333335, - "Filipino,Indonesian,Malay": 0.5066666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.2, - "Spanish,English,Chinese,Filipino": 0.2, - "Spanish,English,Chinese,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino": 0.2, - "Spanish,English,Vietnamese,Indonesian": 0.22, - "Spanish,English,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Filipino,Malay": 0.28, - "Spanish,English,Indonesian,Malay": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.22666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.22666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "English,Chinese,Vietnamese,Malay": 0.21333333333333335, - "English,Chinese,Filipino,Indonesian": 0.24666666666666667, - "English,Chinese,Filipino,Malay": 0.24, - "English,Chinese,Indonesian,Malay": 0.26, - "English,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "English,Filipino,Indonesian,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.32, - "Chinese,Vietnamese,Filipino,Malay": 0.34, - "Chinese,Vietnamese,Indonesian,Malay": 0.36, - "Chinese,Filipino,Indonesian,Malay": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian,Malay": 0.3466666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,English,Chinese,Vietnamese,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.2, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.18, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.18, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.22, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - } - }, - "AC3_2": 0.4290682934614763, - "AC3_3": 0.35587700850027976, - "AC3_4": 0.299150786233097, - "AC3_5": 0.2534695433781002, - "AC3_6": 0.21489467158135575, - "AC3_7": 0.17988071566821576 - }, - "prompt_5": { - "overall_acc": 0.36571428571428577, - "language_acc": { - "Spanish": 0.4533333333333333, - "English": 0.54, - "Chinese": 0.26666666666666666, - "Vietnamese": 0.26, - "Filipino": 0.31333333333333335, - "Indonesian": 0.37333333333333335, - "Malay": 0.35333333333333333 - }, - "consistency_score_2": 0.4847619047619047, - "consistency_score_3": 0.2954285714285713, - "consistency_score_4": 0.19428571428571428, - "consistency_score_5": 0.13206349206349208, - "consistency_score_6": 0.09238095238095238, - "consistency_score_7": 0.06666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6, - "Spanish,Chinese": 0.3333333333333333, - "Spanish,Vietnamese": 0.43333333333333335, - "Spanish,Filipino": 0.5, - "Spanish,Indonesian": 0.5133333333333333, - "Spanish,Malay": 0.49333333333333335, - "English,Chinese": 0.32, - "English,Vietnamese": 0.3466666666666667, - "English,Filipino": 0.5066666666666667, - "English,Indonesian": 0.4866666666666667, - "English,Malay": 0.47333333333333333, - "Chinese,Vietnamese": 0.4666666666666667, - "Chinese,Filipino": 0.3933333333333333, - "Chinese,Indonesian": 0.49333333333333335, - "Chinese,Malay": 0.4266666666666667, - "Vietnamese,Filipino": 0.52, - "Vietnamese,Indonesian": 0.52, - "Vietnamese,Malay": 0.5133333333333333, - "Filipino,Indonesian": 0.5533333333333333, - "Filipino,Malay": 0.6133333333333333, - "Indonesian,Malay": 0.6733333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.22, - "Spanish,English,Vietnamese": 0.26, - "Spanish,English,Filipino": 0.36666666666666664, - "Spanish,English,Indonesian": 0.3466666666666667, - "Spanish,English,Malay": 0.36, - "Spanish,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,Chinese,Filipino": 0.18666666666666668, - "Spanish,Chinese,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Malay": 0.20666666666666667, - "Spanish,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,Vietnamese,Indonesian": 0.30666666666666664, - "Spanish,Vietnamese,Malay": 0.3, - "Spanish,Filipino,Indonesian": 0.32, - "Spanish,Filipino,Malay": 0.36666666666666664, - "Spanish,Indonesian,Malay": 0.38666666666666666, - "English,Chinese,Vietnamese": 0.18, - "English,Chinese,Filipino": 0.21333333333333335, - "English,Chinese,Indonesian": 0.26, - "English,Chinese,Malay": 0.21333333333333335, - "English,Vietnamese,Filipino": 0.25333333333333335, - "English,Vietnamese,Indonesian": 0.25333333333333335, - "English,Vietnamese,Malay": 0.24, - "English,Filipino,Indonesian": 0.3333333333333333, - "English,Filipino,Malay": 0.3466666666666667, - "English,Indonesian,Malay": 0.36, - "Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Chinese,Vietnamese,Indonesian": 0.30666666666666664, - "Chinese,Vietnamese,Malay": 0.26, - "Chinese,Filipino,Indonesian": 0.29333333333333333, - "Chinese,Filipino,Malay": 0.28, - "Chinese,Indonesian,Malay": 0.34, - "Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Vietnamese,Filipino,Malay": 0.37333333333333335, - "Vietnamese,Indonesian,Malay": 0.38666666666666666, - "Filipino,Indonesian,Malay": 0.4666666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.12666666666666668, - "Spanish,English,Chinese,Filipino": 0.14666666666666667, - "Spanish,English,Chinese,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino": 0.2, - "Spanish,English,Vietnamese,Indonesian": 0.2, - "Spanish,English,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Filipino,Indonesian": 0.24, - "Spanish,English,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Malay": 0.13333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.14, - "Spanish,Chinese,Indonesian,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.29333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.14, - "English,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "English,Chinese,Vietnamese,Malay": 0.12, - "English,Chinese,Filipino,Indonesian": 0.18, - "English,Chinese,Filipino,Malay": 0.16, - "English,Chinese,Indonesian,Malay": 0.18, - "English,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "English,Filipino,Indonesian,Malay": 0.28, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2, - "Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.3 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.12, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "English,Chinese,Filipino,Indonesian,Malay": 0.14, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.08, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 - } - }, - "AC3_2": 0.41690609497577574, - "AC3_3": 0.3268354117297971, - "AC3_4": 0.25376093289929197, - "AC3_5": 0.19405247809512705, - "AC3_6": 0.14750222747002847, - "AC3_7": 0.11277533037039338 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.3490259740259741, - "language_acc": { - "English": 0.5227272727272727, - "Filipino": 0.30113636363636365, - "Spanish": 0.42045454545454547, - "Malay": 0.3352272727272727, - "Chinese": 0.24431818181818182, - "Indonesian": 0.35795454545454547, - "Vietnamese": 0.26136363636363635 - }, - "consistency_score_2": 0.41801948051948057, - "consistency_score_3": 0.22029220779220776, - "consistency_score_4": 0.13506493506493505, - "consistency_score_5": 0.09145021645021643, - "consistency_score_6": 0.06655844155844155, - "consistency_score_7": 0.05113636363636364, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.6022727272727273, - "English,Malay": 0.3409090909090909, - "English,Chinese": 0.2897727272727273, - "English,Indonesian": 0.38636363636363635, - "English,Vietnamese": 0.30113636363636365, - "Filipino,Spanish": 0.4090909090909091, - "Filipino,Malay": 0.42613636363636365, - "Filipino,Chinese": 0.39204545454545453, - "Filipino,Indonesian": 0.4659090909090909, - "Filipino,Vietnamese": 0.4431818181818182, - "Spanish,Malay": 0.4090909090909091, - "Spanish,Chinese": 0.29545454545454547, - "Spanish,Indonesian": 0.4602272727272727, - "Spanish,Vietnamese": 0.35795454545454547, - "Malay,Chinese": 0.5, - "Malay,Indonesian": 0.5568181818181818, - "Malay,Vietnamese": 0.4715909090909091, - "Chinese,Indonesian": 0.39204545454545453, - "Chinese,Vietnamese": 0.5170454545454546, - "Indonesian,Vietnamese": 0.42613636363636365 - }, - "3_combine": { - "English,Filipino,Spanish": 0.23863636363636365, - "English,Filipino,Malay": 0.17613636363636365, - "English,Filipino,Chinese": 0.14772727272727273, - "English,Filipino,Indonesian": 0.20454545454545456, - "English,Filipino,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay": 0.24431818181818182, - "English,Spanish,Chinese": 0.17613636363636365, - "English,Spanish,Indonesian": 0.2840909090909091, - "English,Spanish,Vietnamese": 0.2159090909090909, - "English,Malay,Chinese": 0.16477272727272727, - "English,Malay,Indonesian": 0.23295454545454544, - "English,Malay,Vietnamese": 0.16477272727272727, - "English,Chinese,Indonesian": 0.17613636363636365, - "English,Chinese,Vietnamese": 0.17045454545454544, - "English,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay": 0.2215909090909091, - "Filipino,Spanish,Chinese": 0.17045454545454544, - "Filipino,Spanish,Indonesian": 0.26136363636363635, - "Filipino,Spanish,Vietnamese": 0.19886363636363635, - "Filipino,Malay,Chinese": 0.23863636363636365, - "Filipino,Malay,Indonesian": 0.2840909090909091, - "Filipino,Malay,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian": 0.23863636363636365, - "Filipino,Chinese,Vietnamese": 0.25, - "Filipino,Indonesian,Vietnamese": 0.23863636363636365, - "Spanish,Malay,Chinese": 0.19318181818181818, - "Spanish,Malay,Indonesian": 0.26704545454545453, - "Spanish,Malay,Vietnamese": 0.2159090909090909, - "Spanish,Chinese,Indonesian": 0.1875, - "Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Spanish,Indonesian,Vietnamese": 0.21022727272727273, - "Malay,Chinese,Indonesian": 0.30113636363636365, - "Malay,Chinese,Vietnamese": 0.3125, - "Malay,Indonesian,Vietnamese": 0.29545454545454547, - "Chinese,Indonesian,Vietnamese": 0.25 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.1534090909090909, - "English,Filipino,Spanish,Chinese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese": 0.10227272727272728, - "English,Filipino,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Malay,Vietnamese": 0.09659090909090909, - "English,Filipino,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese": 0.11931818181818182, - "English,Spanish,Malay,Indonesian": 0.16477272727272727, - "English,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian": 0.125, - "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "English,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "English,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese": 0.125, - "Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.0625, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 - } - }, - "AC3_2": 0.38041984465596357, - "AC3_3": 0.270104503376136, - "AC3_4": 0.1947616434135639, - "AC3_5": 0.14492724716707206, - "AC3_6": 0.11179738227829356, - "AC3_7": 0.08920339293363665 - }, - "prompt_2": { - "overall_acc": 0.3530844155844156, - "language_acc": { - "English": 0.5340909090909091, - "Filipino": 0.3125, - "Spanish": 0.4375, - "Malay": 0.3522727272727273, - "Chinese": 0.24431818181818182, - "Indonesian": 0.32386363636363635, - "Vietnamese": 0.26704545454545453 - }, - "consistency_score_2": 0.42234848484848486, - "consistency_score_3": 0.22126623376623378, - "consistency_score_4": 0.13620129870129868, - "consistency_score_5": 0.09388528138528139, - "consistency_score_6": 0.07061688311688312, - "consistency_score_7": 0.056818181818181816, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3465909090909091, - "English,Spanish": 0.6306818181818182, - "English,Malay": 0.3522727272727273, - "English,Chinese": 0.29545454545454547, - "English,Indonesian": 0.38636363636363635, - "English,Vietnamese": 0.29545454545454547, - "Filipino,Spanish": 0.45454545454545453, - "Filipino,Malay": 0.48295454545454547, - "Filipino,Chinese": 0.42613636363636365, - "Filipino,Indonesian": 0.4943181818181818, - "Filipino,Vietnamese": 0.4318181818181818, - "Spanish,Malay": 0.3977272727272727, - "Spanish,Chinese": 0.29545454545454547, - "Spanish,Indonesian": 0.44886363636363635, - "Spanish,Vietnamese": 0.3522727272727273, - "Malay,Chinese": 0.4772727272727273, - "Malay,Indonesian": 0.6022727272727273, - "Malay,Vietnamese": 0.4602272727272727, - "Chinese,Indonesian": 0.3806818181818182, - "Chinese,Vietnamese": 0.5, - "Indonesian,Vietnamese": 0.35795454545454547 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2840909090909091, - "English,Filipino,Malay": 0.1875, - "English,Filipino,Chinese": 0.1590909090909091, - "English,Filipino,Indonesian": 0.22727272727272727, - "English,Filipino,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay": 0.2556818181818182, - "English,Spanish,Chinese": 0.19886363636363635, - "English,Spanish,Indonesian": 0.2897727272727273, - "English,Spanish,Vietnamese": 0.2159090909090909, - "English,Malay,Chinese": 0.17045454545454544, - "English,Malay,Indonesian": 0.2556818181818182, - "English,Malay,Vietnamese": 0.14772727272727273, - "English,Chinese,Indonesian": 0.18181818181818182, - "English,Chinese,Vietnamese": 0.1534090909090909, - "English,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay": 0.25, - "Filipino,Spanish,Chinese": 0.19886363636363635, - "Filipino,Spanish,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Vietnamese": 0.21022727272727273, - "Filipino,Malay,Chinese": 0.2727272727272727, - "Filipino,Malay,Indonesian": 0.3352272727272727, - "Filipino,Malay,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian": 0.24431818181818182, - "Filipino,Chinese,Vietnamese": 0.24431818181818182, - "Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese": 0.1875, - "Spanish,Malay,Indonesian": 0.2897727272727273, - "Spanish,Malay,Vietnamese": 0.1875, - "Spanish,Chinese,Indonesian": 0.18181818181818182, - "Spanish,Chinese,Vietnamese": 0.17045454545454544, - "Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "Malay,Chinese,Indonesian": 0.2897727272727273, - "Malay,Chinese,Vietnamese": 0.2897727272727273, - "Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Chinese,Indonesian,Vietnamese": 0.2159090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.16477272727272727, - "English,Filipino,Spanish,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Indonesian": 0.18181818181818182, - "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese": 0.125, - "English,Filipino,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Vietnamese": 0.09090909090909091, - "English,Filipino,Chinese,Indonesian": 0.125, - "English,Filipino,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese": 0.13636363636363635, - "English,Spanish,Malay,Indonesian": 0.19318181818181818, - "English,Spanish,Malay,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Indonesian": 0.13636363636363635, - "English,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "English,Malay,Chinese,Indonesian": 0.14204545454545456, - "English,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese": 0.1590909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.19318181818181818, - "Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.14772727272727273, - "Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Malay,Chinese,Indonesian,Vietnamese": 0.1875 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 - } - }, - "AC3_2": 0.3846230095297937, - "AC3_3": 0.2720486480258719, - "AC3_4": 0.19657453524537472, - "AC3_5": 0.14832965155009792, - "AC3_6": 0.11769480516702742, - "AC3_7": 0.09788478845496786 - }, - "prompt_3": { - "overall_acc": 0.3530844155844156, - "language_acc": { - "English": 0.5227272727272727, - "Filipino": 0.3125, - "Spanish": 0.4375, - "Malay": 0.3465909090909091, - "Chinese": 0.25, - "Indonesian": 0.3352272727272727, - "Vietnamese": 0.26704545454545453 - }, - "consistency_score_2": 0.4320887445887446, - "consistency_score_3": 0.2349025974025974, - "consistency_score_4": 0.1529220779220779, - "consistency_score_5": 0.1136363636363636, - "consistency_score_6": 0.09253246753246751, - "consistency_score_7": 0.07954545454545454, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3352272727272727, - "English,Spanish": 0.6420454545454546, - "English,Malay": 0.3806818181818182, - "English,Chinese": 0.32954545454545453, - "English,Indonesian": 0.39204545454545453, - "English,Vietnamese": 0.3181818181818182, - "Filipino,Spanish": 0.4375, - "Filipino,Malay": 0.4715909090909091, - "Filipino,Chinese": 0.4034090909090909, - "Filipino,Indonesian": 0.48863636363636365, - "Filipino,Vietnamese": 0.4772727272727273, - "Spanish,Malay": 0.4431818181818182, - "Spanish,Chinese": 0.3125, - "Spanish,Indonesian": 0.4431818181818182, - "Spanish,Vietnamese": 0.3693181818181818, - "Malay,Chinese": 0.48863636363636365, - "Malay,Indonesian": 0.5909090909090909, - "Malay,Vietnamese": 0.4659090909090909, - "Chinese,Indonesian": 0.38636363636363635, - "Chinese,Vietnamese": 0.5113636363636364, - "Indonesian,Vietnamese": 0.38636363636363635 - }, - "3_combine": { - "English,Filipino,Spanish": 0.26704545454545453, - "English,Filipino,Malay": 0.19886363636363635, - "English,Filipino,Chinese": 0.16477272727272727, - "English,Filipino,Indonesian": 0.22727272727272727, - "English,Filipino,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay": 0.2840909090909091, - "English,Spanish,Chinese": 0.2215909090909091, - "English,Spanish,Indonesian": 0.29545454545454547, - "English,Spanish,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese": 0.19886363636363635, - "English,Malay,Indonesian": 0.26136363636363635, - "English,Malay,Vietnamese": 0.17045454545454544, - "English,Chinese,Indonesian": 0.19318181818181818, - "English,Chinese,Vietnamese": 0.17613636363636365, - "English,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay": 0.26136363636363635, - "Filipino,Spanish,Chinese": 0.19318181818181818, - "Filipino,Spanish,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Vietnamese": 0.23295454545454544, - "Filipino,Malay,Chinese": 0.26704545454545453, - "Filipino,Malay,Indonesian": 0.32386363636363635, - "Filipino,Malay,Vietnamese": 0.2727272727272727, - "Filipino,Chinese,Indonesian": 0.23863636363636365, - "Filipino,Chinese,Vietnamese": 0.26704545454545453, - "Filipino,Indonesian,Vietnamese": 0.25, - "Spanish,Malay,Chinese": 0.21022727272727273, - "Spanish,Malay,Indonesian": 0.29545454545454547, - "Spanish,Malay,Vietnamese": 0.2215909090909091, - "Spanish,Chinese,Indonesian": 0.19886363636363635, - "Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Spanish,Indonesian,Vietnamese": 0.19886363636363635, - "Malay,Chinese,Indonesian": 0.3068181818181818, - "Malay,Chinese,Vietnamese": 0.3125, - "Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Chinese,Indonesian,Vietnamese": 0.2215909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.14204545454545456, - "English,Filipino,Spanish,Indonesian": 0.18181818181818182, - "English,Filipino,Spanish,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay,Chinese": 0.1534090909090909, - "English,Spanish,Malay,Indonesian": 0.19886363636363635, - "English,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Spanish,Chinese,Indonesian": 0.14772727272727273, - "English,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "English,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "English,Malay,Chinese,Indonesian": 0.16477272727272727, - "English,Malay,Chinese,Vietnamese": 0.125, - "English,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese": 0.1590909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.20454545454545456, - "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.19886363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.17045454545454544, - "Spanish,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Indonesian": 0.125, - "English,Filipino,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - } - }, - "AC3_2": 0.38861186189972174, - "AC3_3": 0.28211659263597866, - "AC3_4": 0.21341387193762462, - "AC3_5": 0.17193675885644127, - "AC3_6": 0.14663615070160502, - "AC3_7": 0.12983967249258763 - }, - "prompt_4": { - "overall_acc": 0.35227272727272724, - "language_acc": { - "English": 0.5340909090909091, - "Filipino": 0.30113636363636365, - "Spanish": 0.42613636363636365, - "Malay": 0.3522727272727273, - "Chinese": 0.24431818181818182, - "Indonesian": 0.3352272727272727, - "Vietnamese": 0.2727272727272727 - }, - "consistency_score_2": 0.4323593073593073, - "consistency_score_3": 0.2357142857142857, - "consistency_score_4": 0.15275974025974023, - "consistency_score_5": 0.11282467532467534, - "consistency_score_6": 0.09172077922077922, - "consistency_score_7": 0.07954545454545454, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3409090909090909, - "English,Spanish": 0.6363636363636364, - "English,Malay": 0.3693181818181818, - "English,Chinese": 0.30113636363636365, - "English,Indonesian": 0.38636363636363635, - "English,Vietnamese": 0.3068181818181818, - "Filipino,Spanish": 0.4602272727272727, - "Filipino,Malay": 0.4659090909090909, - "Filipino,Chinese": 0.42045454545454547, - "Filipino,Indonesian": 0.4943181818181818, - "Filipino,Vietnamese": 0.4659090909090909, - "Spanish,Malay": 0.4090909090909091, - "Spanish,Chinese": 0.3068181818181818, - "Spanish,Indonesian": 0.45454545454545453, - "Spanish,Vietnamese": 0.36363636363636365, - "Malay,Chinese": 0.5, - "Malay,Indonesian": 0.5965909090909091, - "Malay,Vietnamese": 0.48295454545454547, - "Chinese,Indonesian": 0.3806818181818182, - "Chinese,Vietnamese": 0.5397727272727273, - "Indonesian,Vietnamese": 0.3977272727272727 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2727272727272727, - "English,Filipino,Malay": 0.19318181818181818, - "English,Filipino,Chinese": 0.16477272727272727, - "English,Filipino,Indonesian": 0.22727272727272727, - "English,Filipino,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay": 0.26704545454545453, - "English,Spanish,Chinese": 0.2159090909090909, - "English,Spanish,Indonesian": 0.2897727272727273, - "English,Spanish,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese": 0.19318181818181818, - "English,Malay,Indonesian": 0.25, - "English,Malay,Vietnamese": 0.17613636363636365, - "English,Chinese,Indonesian": 0.18181818181818182, - "English,Chinese,Vietnamese": 0.17045454545454544, - "English,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay": 0.2556818181818182, - "Filipino,Spanish,Chinese": 0.20454545454545456, - "Filipino,Spanish,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese": 0.2784090909090909, - "Filipino,Malay,Indonesian": 0.32954545454545453, - "Filipino,Malay,Vietnamese": 0.2727272727272727, - "Filipino,Chinese,Indonesian": 0.25, - "Filipino,Chinese,Vietnamese": 0.2784090909090909, - "Filipino,Indonesian,Vietnamese": 0.2556818181818182, - "Spanish,Malay,Chinese": 0.19886363636363635, - "Spanish,Malay,Indonesian": 0.2897727272727273, - "Spanish,Malay,Vietnamese": 0.2215909090909091, - "Spanish,Chinese,Indonesian": 0.19318181818181818, - "Spanish,Chinese,Vietnamese": 0.19886363636363635, - "Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "Malay,Chinese,Indonesian": 0.30113636363636365, - "Malay,Chinese,Vietnamese": 0.3352272727272727, - "Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Chinese,Indonesian,Vietnamese": 0.24431818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17045454545454544, - "English,Filipino,Spanish,Chinese": 0.14772727272727273, - "English,Filipino,Spanish,Indonesian": 0.18181818181818182, - "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese": 0.13068181818181818, - "English,Filipino,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.13068181818181818, - "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Spanish,Malay,Indonesian": 0.19318181818181818, - "English,Spanish,Malay,Vietnamese": 0.14204545454545456, - "English,Spanish,Chinese,Indonesian": 0.14772727272727273, - "English,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "English,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "English,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Malay,Chinese,Vietnamese": 0.125, - "English,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Chinese": 0.1590909090909091, - "Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.1590909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.20454545454545456, - "Filipino,Malay,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.1875, - "Filipino,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "Spanish,Malay,Chinese,Vietnamese": 0.14772727272727273, - "Spanish,Malay,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Spanish,Malay,Chinese,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - } - }, - "AC3_2": 0.3882288400759127, - "AC3_3": 0.2824406404820587, - "AC3_4": 0.21310744860044953, - "AC3_5": 0.1709106774184328, - "AC3_6": 0.14554595310000934, - "AC3_7": 0.1297846889651599 - }, - "prompt_5": { - "overall_acc": 0.3538961038961039, - "language_acc": { - "English": 0.5284090909090909, - "Filipino": 0.3125, - "Spanish": 0.4318181818181818, - "Malay": 0.3522727272727273, - "Chinese": 0.26136363636363635, - "Indonesian": 0.32954545454545453, - "Vietnamese": 0.26136363636363635 - }, - "consistency_score_2": 0.4218073593073592, - "consistency_score_3": 0.22532467532467532, - "consistency_score_4": 0.14301948051948052, - "consistency_score_5": 0.10173160173160174, - "consistency_score_6": 0.07792207792207792, - "consistency_score_7": 0.0625, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.32386363636363635, - "English,Spanish": 0.6363636363636364, - "English,Malay": 0.36363636363636365, - "English,Chinese": 0.32954545454545453, - "English,Indonesian": 0.39204545454545453, - "English,Vietnamese": 0.3068181818181818, - "Filipino,Spanish": 0.4431818181818182, - "Filipino,Malay": 0.5, - "Filipino,Chinese": 0.38636363636363635, - "Filipino,Indonesian": 0.4602272727272727, - "Filipino,Vietnamese": 0.4147727272727273, - "Spanish,Malay": 0.4034090909090909, - "Spanish,Chinese": 0.32954545454545453, - "Spanish,Indonesian": 0.4431818181818182, - "Spanish,Vietnamese": 0.3352272727272727, - "Malay,Chinese": 0.48863636363636365, - "Malay,Indonesian": 0.5795454545454546, - "Malay,Vietnamese": 0.4772727272727273, - "Chinese,Indonesian": 0.36363636363636365, - "Chinese,Vietnamese": 0.4602272727272727, - "Indonesian,Vietnamese": 0.42045454545454547 - }, - "3_combine": { - "English,Filipino,Spanish": 0.26136363636363635, - "English,Filipino,Malay": 0.20454545454545456, - "English,Filipino,Chinese": 0.16477272727272727, - "English,Filipino,Indonesian": 0.21022727272727273, - "English,Filipino,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay": 0.2840909090909091, - "English,Spanish,Chinese": 0.23295454545454544, - "English,Spanish,Indonesian": 0.3068181818181818, - "English,Spanish,Vietnamese": 0.22727272727272727, - "English,Malay,Chinese": 0.19318181818181818, - "English,Malay,Indonesian": 0.2556818181818182, - "English,Malay,Vietnamese": 0.17045454545454544, - "English,Chinese,Indonesian": 0.18181818181818182, - "English,Chinese,Vietnamese": 0.1590909090909091, - "English,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay": 0.2556818181818182, - "Filipino,Spanish,Chinese": 0.20454545454545456, - "Filipino,Spanish,Indonesian": 0.2556818181818182, - "Filipino,Spanish,Vietnamese": 0.1875, - "Filipino,Malay,Chinese": 0.2727272727272727, - "Filipino,Malay,Indonesian": 0.3181818181818182, - "Filipino,Malay,Vietnamese": 0.26136363636363635, - "Filipino,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Chinese,Vietnamese": 0.21022727272727273, - "Filipino,Indonesian,Vietnamese": 0.23863636363636365, - "Spanish,Malay,Chinese": 0.20454545454545456, - "Spanish,Malay,Indonesian": 0.2840909090909091, - "Spanish,Malay,Vietnamese": 0.20454545454545456, - "Spanish,Chinese,Indonesian": 0.19318181818181818, - "Spanish,Chinese,Vietnamese": 0.16477272727272727, - "Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "Malay,Chinese,Indonesian": 0.2897727272727273, - "Malay,Chinese,Vietnamese": 0.2840909090909091, - "Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Chinese,Indonesian,Vietnamese": 0.21022727272727273 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.1534090909090909, - "English,Filipino,Spanish,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Vietnamese": 0.11931818181818182, - "English,Filipino,Malay,Chinese": 0.13068181818181818, - "English,Filipino,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Indonesian": 0.11931818181818182, - "English,Filipino,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese": 0.1534090909090909, - "English,Spanish,Malay,Indonesian": 0.21022727272727273, - "English,Spanish,Malay,Vietnamese": 0.14204545454545456, - "English,Spanish,Chinese,Indonesian": 0.1534090909090909, - "English,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "English,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Malay,Chinese,Vietnamese": 0.11363636363636363, - "English,Malay,Indonesian,Vietnamese": 0.125, - "English,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese": 0.16477272727272727, - "Filipino,Spanish,Malay,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.14772727272727273, - "Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.18181818181818182, - "Filipino,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.125, - "Spanish,Malay,Chinese,Indonesian": 0.1590909090909091, - "Spanish,Malay,Chinese,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Malay,Chinese,Indonesian,Vietnamese": 0.1875 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - } - }, - "AC3_2": 0.3848789856316011, - "AC3_3": 0.2753406906518987, - "AC3_4": 0.20371281768685004, - "AC3_5": 0.15803436465051107, - "AC3_6": 0.12772190212841467, - "AC3_7": 0.1062378167386189 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.6116504854368932 - }, - "prompt_2": { - "accuracy": 0.6310679611650486 - }, - "prompt_3": { - "accuracy": 0.5533980582524272 - }, - "prompt_4": { - "accuracy": 0.6019417475728155 - }, - "prompt_5": { - "accuracy": 0.5533980582524272 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.26666666666666666 - }, - "prompt_2": { - "accuracy": 0.26666666666666666 - }, - "prompt_3": { - "accuracy": 0.2571428571428571 - }, - "prompt_4": { - "accuracy": 0.24761904761904763 - }, - "prompt_5": { - "accuracy": 0.2761904761904762 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.5607476635514018 - }, - "prompt_2": { - "accuracy": 0.5233644859813084 - }, - "prompt_3": { - "accuracy": 0.5794392523364486 - }, - "prompt_4": { - "accuracy": 0.5607476635514018 - }, - "prompt_5": { - "accuracy": 0.5794392523364486 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.3, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.4, - "politics": 0.2, - "culture": 0.3, - "film": 0.1, - "law": 0.3, - "geography": 0.5 - } - }, - "prompt_2": { - "accuracy": 0.28, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.4, - "politics": 0.2, - "culture": 0.3, - "film": 0.1, - "law": 0.2, - "geography": 0.5 - } - }, - "prompt_3": { - "accuracy": 0.32, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.4, - "history": 0.3333333333333333, - "literature": 0.4, - "politics": 0.2, - "culture": 0.4, - "film": 0.2, - "law": 0.3, - "geography": 0.4 - } - }, - "prompt_4": { - "accuracy": 0.28, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.4, - "history": 0.2, - "literature": 0.5, - "politics": 0.2, - "culture": 0.3, - "film": 0.1, - "law": 0.2, - "geography": 0.4 - } - }, - "prompt_5": { - "accuracy": 0.32, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.4, - "politics": 0.2, - "culture": 0.4, - "film": 0.3, - "law": 0.3, - "geography": 0.4 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.17566885121708076 - }, - "prompt_2": { - "bleu_score": 0.1713078295205753 - }, - "prompt_3": { - "bleu_score": 0.1784941784513121 - }, - "prompt_4": { - "bleu_score": 0.18529617182140173 - }, - "prompt_5": { - "bleu_score": 0.16808729137211453 - } - }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.23523149987793235 - }, - "prompt_2": { - "bleu_score": 0.23084218493303246 - }, - "prompt_3": { - "bleu_score": 0.23167153634087087 - }, - "prompt_4": { - "bleu_score": 0.23577272330022211 - }, - "prompt_5": { - "bleu_score": 0.22745760692553665 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.05727827737401459 - }, - "prompt_2": { - "bleu_score": 0.055968964984837354 - }, - "prompt_3": { - "bleu_score": 0.056131306283731884 - }, - "prompt_4": { - "bleu_score": 0.05710157634197056 - }, - "prompt_5": { - "bleu_score": 0.05434656958085618 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.015581243187674091 - }, - "prompt_2": { - "bleu_score": 0.016213193684516073 - }, - "prompt_3": { - "bleu_score": 0.015660144542147494 - }, - "prompt_4": { - "bleu_score": 0.01834230498909322 - }, - "prompt_5": { - "bleu_score": 0.014695045075112684 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.21967698896302396 - }, - "prompt_2": { - "bleu_score": 0.2172023212871701 - }, - "prompt_3": { - "bleu_score": 0.21745382063841664 - }, - "prompt_4": { - "bleu_score": 0.22183752238405993 - }, - "prompt_5": { - "bleu_score": 0.2079553349376303 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.5040840140023337 - }, - "prompt_2": { - "accuracy": 0.5040840140023337 - }, - "prompt_3": { - "accuracy": 0.5122520420070011 - }, - "prompt_4": { - "accuracy": 0.5099183197199533 - }, - "prompt_5": { - "accuracy": 0.5134189031505251 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.489095459420808, - "category_acc": { - "high_school_european_history": 0.7012195121951219, - "business_ethics": 0.6464646464646465, - "clinical_knowledge": 0.5606060606060606, - "medical_genetics": 0.46464646464646464, - "high_school_us_history": 0.6847290640394089, - "high_school_physics": 0.2733333333333333, - "high_school_world_history": 0.6694915254237288, - "virology": 0.44242424242424244, - "high_school_microeconomics": 0.510548523206751, - "econometrics": 0.30973451327433627, - "college_computer_science": 0.3939393939393939, - "high_school_biology": 0.5825242718446602, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.3736654804270463, - "philosophy": 0.5225806451612903, - "professional_medicine": 0.4575645756457565, - "nutrition": 0.4786885245901639, - "global_facts": 0.31313131313131315, - "machine_learning": 0.35135135135135137, - "security_studies": 0.569672131147541, - "public_relations": 0.5871559633027523, - "professional_psychology": 0.48117839607201307, - "prehistory": 0.47678018575851394, - "anatomy": 0.4552238805970149, - "human_sexuality": 0.5769230769230769, - "college_medicine": 0.47093023255813954, - "high_school_government_and_politics": 0.6875, - "college_chemistry": 0.40404040404040403, - "logical_fallacies": 0.5555555555555556, - "high_school_geography": 0.6700507614213198, - "elementary_mathematics": 0.34748010610079577, - "human_aging": 0.536036036036036, - "college_mathematics": 0.25252525252525254, - "high_school_psychology": 0.6911764705882353, - "formal_logic": 0.352, - "high_school_statistics": 0.40930232558139534, - "international_law": 0.65, - "high_school_mathematics": 0.2862453531598513, - "high_school_computer_science": 0.494949494949495, - "conceptual_physics": 0.4444444444444444, - "miscellaneous": 0.659846547314578, - "high_school_chemistry": 0.30198019801980197, - "marketing": 0.7811158798283262, - "professional_law": 0.3842139595564253, - "management": 0.6764705882352942, - "college_physics": 0.3465346534653465, - "jurisprudence": 0.5794392523364486, - "world_religions": 0.5764705882352941, - "sociology": 0.73, - "us_foreign_policy": 0.7272727272727273, - "high_school_macroeconomics": 0.5269922879177378, - "computer_security": 0.5858585858585859, - "moral_scenarios": 0.23042505592841164, - "moral_disputes": 0.5797101449275363, - "electrical_engineering": 0.4027777777777778, - "astronomy": 0.5099337748344371, - "college_biology": 0.45454545454545453 - } - }, - "prompt_2": { - "accuracy": 0.48873793350017875, - "category_acc": { - "high_school_european_history": 0.7073170731707317, - "business_ethics": 0.6262626262626263, - "clinical_knowledge": 0.5681818181818182, - "medical_genetics": 0.494949494949495, - "high_school_us_history": 0.6847290640394089, - "high_school_physics": 0.3, - "high_school_world_history": 0.6864406779661016, - "virology": 0.4303030303030303, - "high_school_microeconomics": 0.510548523206751, - "econometrics": 0.336283185840708, - "college_computer_science": 0.3939393939393939, - "high_school_biology": 0.5728155339805825, - "abstract_algebra": 0.25252525252525254, - "professional_accounting": 0.37722419928825623, - "philosophy": 0.5225806451612903, - "professional_medicine": 0.44649446494464945, - "nutrition": 0.4721311475409836, - "global_facts": 0.3333333333333333, - "machine_learning": 0.36036036036036034, - "security_studies": 0.5860655737704918, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.46317512274959083, - "prehistory": 0.49226006191950467, - "anatomy": 0.44776119402985076, - "human_sexuality": 0.5615384615384615, - "college_medicine": 0.48255813953488375, - "high_school_government_and_politics": 0.6770833333333334, - "college_chemistry": 0.43434343434343436, - "logical_fallacies": 0.5802469135802469, - "high_school_geography": 0.6700507614213198, - "elementary_mathematics": 0.3209549071618037, - "human_aging": 0.5135135135135135, - "college_mathematics": 0.2727272727272727, - "high_school_psychology": 0.6930147058823529, - "formal_logic": 0.384, - "high_school_statistics": 0.4186046511627907, - "international_law": 0.65, - "high_school_mathematics": 0.3048327137546468, - "high_school_computer_science": 0.46464646464646464, - "conceptual_physics": 0.42735042735042733, - "miscellaneous": 0.649616368286445, - "high_school_chemistry": 0.3069306930693069, - "marketing": 0.7467811158798283, - "professional_law": 0.3835616438356164, - "management": 0.6764705882352942, - "college_physics": 0.31683168316831684, - "jurisprudence": 0.5887850467289719, - "world_religions": 0.5941176470588235, - "sociology": 0.74, - "us_foreign_policy": 0.7171717171717171, - "high_school_macroeconomics": 0.5244215938303342, - "computer_security": 0.5959595959595959, - "moral_scenarios": 0.21923937360178972, - "moral_disputes": 0.5855072463768116, - "electrical_engineering": 0.4166666666666667, - "astronomy": 0.543046357615894, - "college_biology": 0.4965034965034965 - } - }, - "prompt_3": { - "accuracy": 0.48838040757954954, - "category_acc": { - "high_school_european_history": 0.7073170731707317, - "business_ethics": 0.6161616161616161, - "clinical_knowledge": 0.5492424242424242, - "medical_genetics": 0.494949494949495, - "high_school_us_history": 0.6847290640394089, - "high_school_physics": 0.2866666666666667, - "high_school_world_history": 0.6694915254237288, - "virology": 0.44242424242424244, - "high_school_microeconomics": 0.5147679324894515, - "econometrics": 0.336283185840708, - "college_computer_science": 0.41414141414141414, - "high_school_biology": 0.56957928802589, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.3879003558718861, - "philosophy": 0.5225806451612903, - "professional_medicine": 0.45018450184501846, - "nutrition": 0.4819672131147541, - "global_facts": 0.3434343434343434, - "machine_learning": 0.32432432432432434, - "security_studies": 0.5901639344262295, - "public_relations": 0.5871559633027523, - "professional_psychology": 0.4664484451718494, - "prehistory": 0.47368421052631576, - "anatomy": 0.44776119402985076, - "human_sexuality": 0.5769230769230769, - "college_medicine": 0.47674418604651164, - "high_school_government_and_politics": 0.6875, - "college_chemistry": 0.42424242424242425, - "logical_fallacies": 0.5864197530864198, - "high_school_geography": 0.6751269035532995, - "elementary_mathematics": 0.33421750663129973, - "human_aging": 0.5225225225225225, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.6911764705882353, - "formal_logic": 0.384, - "high_school_statistics": 0.413953488372093, - "international_law": 0.65, - "high_school_mathematics": 0.30111524163568776, - "high_school_computer_science": 0.4444444444444444, - "conceptual_physics": 0.43162393162393164, - "miscellaneous": 0.6547314578005116, - "high_school_chemistry": 0.30198019801980197, - "marketing": 0.759656652360515, - "professional_law": 0.3796477495107632, - "management": 0.6764705882352942, - "college_physics": 0.32673267326732675, - "jurisprudence": 0.5887850467289719, - "world_religions": 0.5764705882352941, - "sociology": 0.735, - "us_foreign_policy": 0.7070707070707071, - "high_school_macroeconomics": 0.532133676092545, - "computer_security": 0.6161616161616161, - "moral_scenarios": 0.2225950782997763, - "moral_disputes": 0.5855072463768116, - "electrical_engineering": 0.4166666666666667, - "astronomy": 0.5364238410596026, - "college_biology": 0.46853146853146854 - } - }, - "prompt_4": { - "accuracy": 0.4864497676081516, - "category_acc": { - "high_school_european_history": 0.7073170731707317, - "business_ethics": 0.6363636363636364, - "clinical_knowledge": 0.553030303030303, - "medical_genetics": 0.48484848484848486, - "high_school_us_history": 0.6748768472906403, - "high_school_physics": 0.3, - "high_school_world_history": 0.6610169491525424, - "virology": 0.4484848484848485, - "high_school_microeconomics": 0.5189873417721519, - "econometrics": 0.336283185840708, - "college_computer_science": 0.45454545454545453, - "high_school_biology": 0.5631067961165048, - "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.3807829181494662, - "philosophy": 0.5225806451612903, - "professional_medicine": 0.44280442804428044, - "nutrition": 0.47540983606557374, - "global_facts": 0.3434343434343434, - "machine_learning": 0.36036036036036034, - "security_studies": 0.5778688524590164, - "public_relations": 0.5779816513761468, - "professional_psychology": 0.46317512274959083, - "prehistory": 0.47678018575851394, - "anatomy": 0.44029850746268656, - "human_sexuality": 0.5461538461538461, - "college_medicine": 0.48255813953488375, - "high_school_government_and_politics": 0.6770833333333334, - "college_chemistry": 0.42424242424242425, - "logical_fallacies": 0.5555555555555556, - "high_school_geography": 0.6802030456852792, - "elementary_mathematics": 0.3183023872679045, - "human_aging": 0.509009009009009, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.6875, - "formal_logic": 0.368, - "high_school_statistics": 0.4186046511627907, - "international_law": 0.6583333333333333, - "high_school_mathematics": 0.2899628252788104, - "high_school_computer_science": 0.47474747474747475, - "conceptual_physics": 0.43162393162393164, - "miscellaneous": 0.6483375959079284, - "high_school_chemistry": 0.297029702970297, - "marketing": 0.7424892703862661, - "professional_law": 0.3822570123939987, - "management": 0.6862745098039216, - "college_physics": 0.3465346534653465, - "jurisprudence": 0.5794392523364486, - "world_religions": 0.5764705882352941, - "sociology": 0.745, - "us_foreign_policy": 0.6767676767676768, - "high_school_macroeconomics": 0.5167095115681234, - "computer_security": 0.6060606060606061, - "moral_scenarios": 0.22818791946308725, - "moral_disputes": 0.5855072463768116, - "electrical_engineering": 0.4236111111111111, - "astronomy": 0.5298013245033113, - "college_biology": 0.4755244755244755 - } - }, - "prompt_5": { - "accuracy": 0.486735788344655, - "category_acc": { - "high_school_european_history": 0.7134146341463414, - "business_ethics": 0.6262626262626263, - "clinical_knowledge": 0.5643939393939394, - "medical_genetics": 0.5050505050505051, - "high_school_us_history": 0.6650246305418719, - "high_school_physics": 0.3, - "high_school_world_history": 0.6779661016949152, - "virology": 0.45454545454545453, - "high_school_microeconomics": 0.5189873417721519, - "econometrics": 0.30973451327433627, - "college_computer_science": 0.40404040404040403, - "high_school_biology": 0.5598705501618123, - "abstract_algebra": 0.2727272727272727, - "professional_accounting": 0.3665480427046263, - "philosophy": 0.5, - "professional_medicine": 0.44649446494464945, - "nutrition": 0.4786885245901639, - "global_facts": 0.31313131313131315, - "machine_learning": 0.34234234234234234, - "security_studies": 0.5860655737704918, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.46153846153846156, - "prehistory": 0.47678018575851394, - "anatomy": 0.44029850746268656, - "human_sexuality": 0.5538461538461539, - "college_medicine": 0.4883720930232558, - "high_school_government_and_politics": 0.6770833333333334, - "college_chemistry": 0.41414141414141414, - "logical_fallacies": 0.5555555555555556, - "high_school_geography": 0.6700507614213198, - "elementary_mathematics": 0.32625994694960214, - "human_aging": 0.5180180180180181, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.6893382352941176, - "formal_logic": 0.352, - "high_school_statistics": 0.4232558139534884, - "international_law": 0.6666666666666666, - "high_school_mathematics": 0.31226765799256506, - "high_school_computer_science": 0.47474747474747475, - "conceptual_physics": 0.42735042735042733, - "miscellaneous": 0.6521739130434783, - "high_school_chemistry": 0.3217821782178218, - "marketing": 0.759656652360515, - "professional_law": 0.3770384866275277, - "management": 0.696078431372549, - "college_physics": 0.32673267326732675, - "jurisprudence": 0.5887850467289719, - "world_religions": 0.5764705882352941, - "sociology": 0.74, - "us_foreign_policy": 0.6767676767676768, - "high_school_macroeconomics": 0.5372750642673522, - "computer_security": 0.6161616161616161, - "moral_scenarios": 0.22706935123042504, - "moral_disputes": 0.5768115942028985, - "electrical_engineering": 0.4166666666666667, - "astronomy": 0.5165562913907285, - "college_biology": 0.4755244755244755 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.24665676077265974 - }, - "prompt_2": { - "accuracy": 0.25185735512630014 - }, - "prompt_3": { - "accuracy": 0.25037147102526003 - }, - "prompt_4": { - "accuracy": 0.24294205052005943 - }, - "prompt_5": { - "accuracy": 0.2511144130757801 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.2671232876712329, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.2619047619047619, - "college_physics": 0.041666666666666664, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.19047619047619047, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.43478260869565216, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.25, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.23333333333333334, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.125, - "high_school_geography": 0.25, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.3333333333333333, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.08, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.2692307692307692, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.2962962962962963, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.3148148148148148, - "physician": 0.25925925925925924 - } - }, - "prompt_2": { - "accuracy": 0.26899128268991285, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.23809523809523808, - "college_physics": 0.08333333333333333, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.1724137931034483, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.25, - "high_school_chemistry": 0.25, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.10714285714285714, - "college_economics": 0.25, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.24489795918367346, - "high_school_politics": 0.08333333333333333, - "high_school_geography": 0.25, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.3333333333333333, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.25, - "high_school_history": 0.08, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.25, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.3148148148148148, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.3148148148148148, - "physician": 0.2777777777777778 - } - }, - "prompt_3": { - "accuracy": 0.2671232876712329, - "category_acc": { - "computer_network": 0.16666666666666666, - "operating_system": 0.5, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.2857142857142857, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.19047619047619047, - "electrical_engineer": 0.19047619047619047, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.25, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.10714285714285714, - "college_economics": 0.25, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.08333333333333333, - "high_school_geography": 0.25, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.3333333333333333, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.08, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.2692307692307692, - "sports_science": 0.3333333333333333, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.2962962962962963, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.3148148148148148, - "physician": 0.2777777777777778 - } - }, - "prompt_4": { - "accuracy": 0.2714819427148194, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.2857142857142857, - "college_physics": 0.125, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.1724137931034483, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.25, - "high_school_chemistry": 0.25, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.2, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.041666666666666664, - "high_school_geography": 0.25, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.3333333333333333, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.12, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.25, - "sports_science": 0.3333333333333333, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.2962962962962963, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.3148148148148148, - "physician": 0.2777777777777778 - } - }, - "prompt_5": { - "accuracy": 0.2627646326276463, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.2857142857142857, - "college_physics": 0.08333333333333333, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.14285714285714285, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.24, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.23333333333333334, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.24489795918367346, - "high_school_politics": 0.08333333333333333, - "high_school_geography": 0.25, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.3333333333333333, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.25, - "high_school_history": 0.08, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.25, - "sports_science": 0.2916666666666667, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.375, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.2962962962962963, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.3148148148148148, - "physician": 0.2777777777777778 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.2724014336917563 - }, - "prompt_2": { - "accuracy": 0.26523297491039427 - }, - "prompt_3": { - "accuracy": 0.23297491039426524 - }, - "prompt_4": { - "accuracy": 0.23655913978494625 - }, - "prompt_5": { - "accuracy": 0.24014336917562723 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.2518563287860473, - "category_acc": { - "agronomy": 0.21301775147928995, - "anatomy": 0.3108108108108108, - "ancient_chinese": 0.3048780487804878, - "arts": 0.20625, - "astronomy": 0.28484848484848485, - "business_ethics": 0.2631578947368421, - "chinese_civil_service_exam": 0.29375, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.3014705882352941, - "chinese_foreign_policy": 0.2616822429906542, - "chinese_history": 0.23219814241486067, - "chinese_literature": 0.22058823529411764, - "chinese_teacher_qualification": 0.20670391061452514, - "clinical_knowledge": 0.22784810126582278, - "college_actuarial_science": 0.1792452830188679, - "college_education": 0.2897196261682243, - "college_engineering_hydrology": 0.3584905660377358, - "college_law": 0.18518518518518517, - "college_mathematics": 0.3142857142857143, - "college_medical_statistics": 0.3018867924528302, - "college_medicine": 0.2564102564102564, - "computer_science": 0.25980392156862747, - "computer_security": 0.25146198830409355, - "conceptual_physics": 0.23809523809523808, - "construction_project_management": 0.26618705035971224, - "economics": 0.25157232704402516, - "education": 0.2883435582822086, - "electrical_engineering": 0.1511627906976744, - "elementary_chinese": 0.2619047619047619, - "elementary_commonsense": 0.21717171717171718, - "elementary_information_and_technology": 0.25210084033613445, - "elementary_mathematics": 0.29130434782608694, - "ethnology": 0.2740740740740741, - "food_science": 0.2517482517482518, - "genetics": 0.22727272727272727, - "global_facts": 0.2550335570469799, - "high_school_biology": 0.20118343195266272, - "high_school_chemistry": 0.30303030303030304, - "high_school_geography": 0.2033898305084746, - "high_school_mathematics": 0.18902439024390244, - "high_school_physics": 0.32727272727272727, - "high_school_politics": 0.23076923076923078, - "human_sexuality": 0.23809523809523808, - "international_law": 0.21081081081081082, - "journalism": 0.26744186046511625, - "jurisprudence": 0.27007299270072993, - "legal_and_moral_basis": 0.2570093457943925, - "logical": 0.2601626016260163, - "machine_learning": 0.22131147540983606, - "management": 0.24761904761904763, - "marketing": 0.25, - "marxist_theory": 0.2804232804232804, - "modern_chinese": 0.19827586206896552, - "nutrition": 0.2206896551724138, - "philosophy": 0.26666666666666666, - "professional_accounting": 0.26285714285714284, - "professional_law": 0.24170616113744076, - "professional_medicine": 0.22872340425531915, - "professional_psychology": 0.25862068965517243, - "public_relations": 0.3045977011494253, - "security_study": 0.23703703703703705, - "sociology": 0.26991150442477874, - "sports_science": 0.24242424242424243, - "traditional_chinese_medicine": 0.23243243243243245, - "virology": 0.2781065088757396, - "world_history": 0.2857142857142857, - "world_religions": 0.25625 - } - }, - "prompt_2": { - "accuracy": 0.25582800897945085, - "category_acc": { - "agronomy": 0.2781065088757396, - "anatomy": 0.28378378378378377, - "ancient_chinese": 0.25, - "arts": 0.2375, - "astronomy": 0.2909090909090909, - "business_ethics": 0.2631578947368421, - "chinese_civil_service_exam": 0.25, - "chinese_driving_rule": 0.2824427480916031, - "chinese_food_culture": 0.3235294117647059, - "chinese_foreign_policy": 0.2616822429906542, - "chinese_history": 0.23529411764705882, - "chinese_literature": 0.27450980392156865, - "chinese_teacher_qualification": 0.24022346368715083, - "clinical_knowledge": 0.23628691983122363, - "college_actuarial_science": 0.22641509433962265, - "college_education": 0.2523364485981308, - "college_engineering_hydrology": 0.36792452830188677, - "college_law": 0.26851851851851855, - "college_mathematics": 0.37142857142857144, - "college_medical_statistics": 0.19811320754716982, - "college_medicine": 0.28205128205128205, - "computer_science": 0.2696078431372549, - "computer_security": 0.2631578947368421, - "conceptual_physics": 0.2653061224489796, - "construction_project_management": 0.2589928057553957, - "economics": 0.23270440251572327, - "education": 0.26993865030674846, - "electrical_engineering": 0.2558139534883721, - "elementary_chinese": 0.2222222222222222, - "elementary_commonsense": 0.30808080808080807, - "elementary_information_and_technology": 0.226890756302521, - "elementary_mathematics": 0.2608695652173913, - "ethnology": 0.23703703703703705, - "food_science": 0.24475524475524477, - "genetics": 0.26136363636363635, - "global_facts": 0.22818791946308725, - "high_school_biology": 0.1952662721893491, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.2627118644067797, - "high_school_mathematics": 0.21951219512195122, - "high_school_physics": 0.24545454545454545, - "high_school_politics": 0.1888111888111888, - "human_sexuality": 0.23015873015873015, - "international_law": 0.24324324324324326, - "journalism": 0.2441860465116279, - "jurisprudence": 0.26763990267639903, - "legal_and_moral_basis": 0.2570093457943925, - "logical": 0.2764227642276423, - "machine_learning": 0.20491803278688525, - "management": 0.23809523809523808, - "marketing": 0.3333333333333333, - "marxist_theory": 0.2698412698412698, - "modern_chinese": 0.20689655172413793, - "nutrition": 0.22758620689655173, - "philosophy": 0.2761904761904762, - "professional_accounting": 0.2342857142857143, - "professional_law": 0.24170616113744076, - "professional_medicine": 0.2473404255319149, - "professional_psychology": 0.27155172413793105, - "public_relations": 0.28160919540229884, - "security_study": 0.22962962962962963, - "sociology": 0.23008849557522124, - "sports_science": 0.21212121212121213, - "traditional_chinese_medicine": 0.2594594594594595, - "virology": 0.25443786982248523, - "world_history": 0.2919254658385093, - "world_religions": 0.30625 - } - }, - "prompt_3": { - "accuracy": 0.2510792609221205, - "category_acc": { - "agronomy": 0.28402366863905326, - "anatomy": 0.2905405405405405, - "ancient_chinese": 0.27439024390243905, - "arts": 0.26875, - "astronomy": 0.24848484848484848, - "business_ethics": 0.23923444976076555, - "chinese_civil_service_exam": 0.25625, - "chinese_driving_rule": 0.2748091603053435, - "chinese_food_culture": 0.35294117647058826, - "chinese_foreign_policy": 0.22429906542056074, - "chinese_history": 0.24458204334365324, - "chinese_literature": 0.22058823529411764, - "chinese_teacher_qualification": 0.18994413407821228, - "clinical_knowledge": 0.2489451476793249, - "college_actuarial_science": 0.22641509433962265, - "college_education": 0.29906542056074764, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.25, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.25471698113207547, - "college_medicine": 0.20512820512820512, - "computer_science": 0.28921568627450983, - "computer_security": 0.23391812865497075, - "conceptual_physics": 0.20408163265306123, - "construction_project_management": 0.35251798561151076, - "economics": 0.2389937106918239, - "education": 0.22699386503067484, - "electrical_engineering": 0.29651162790697677, - "elementary_chinese": 0.25, - "elementary_commonsense": 0.23737373737373738, - "elementary_information_and_technology": 0.226890756302521, - "elementary_mathematics": 0.26956521739130435, - "ethnology": 0.2740740740740741, - "food_science": 0.20279720279720279, - "genetics": 0.2556818181818182, - "global_facts": 0.18791946308724833, - "high_school_biology": 0.24260355029585798, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.2682926829268293, - "high_school_physics": 0.23636363636363636, - "high_school_politics": 0.2727272727272727, - "human_sexuality": 0.2619047619047619, - "international_law": 0.2594594594594595, - "journalism": 0.21511627906976744, - "jurisprudence": 0.25790754257907544, - "legal_and_moral_basis": 0.26635514018691586, - "logical": 0.21138211382113822, - "machine_learning": 0.3114754098360656, - "management": 0.22857142857142856, - "marketing": 0.23333333333333334, - "marxist_theory": 0.2275132275132275, - "modern_chinese": 0.1896551724137931, - "nutrition": 0.22758620689655173, - "philosophy": 0.26666666666666666, - "professional_accounting": 0.24571428571428572, - "professional_law": 0.27488151658767773, - "professional_medicine": 0.21808510638297873, - "professional_psychology": 0.25, - "public_relations": 0.28735632183908044, - "security_study": 0.24444444444444444, - "sociology": 0.22566371681415928, - "sports_science": 0.22424242424242424, - "traditional_chinese_medicine": 0.2810810810810811, - "virology": 0.2781065088757396, - "world_history": 0.2857142857142857, - "world_religions": 0.3125 - } - }, - "prompt_4": { - "accuracy": 0.2510792609221205, - "category_acc": { - "agronomy": 0.27218934911242604, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.23170731707317074, - "arts": 0.2125, - "astronomy": 0.2545454545454545, - "business_ethics": 0.33014354066985646, - "chinese_civil_service_exam": 0.2625, - "chinese_driving_rule": 0.20610687022900764, - "chinese_food_culture": 0.3088235294117647, - "chinese_foreign_policy": 0.21495327102803738, - "chinese_history": 0.24458204334365324, - "chinese_literature": 0.24509803921568626, - "chinese_teacher_qualification": 0.2346368715083799, - "clinical_knowledge": 0.2616033755274262, - "college_actuarial_science": 0.19811320754716982, - "college_education": 0.2616822429906542, - "college_engineering_hydrology": 0.22641509433962265, - "college_law": 0.25925925925925924, - "college_mathematics": 0.3619047619047619, - "college_medical_statistics": 0.29245283018867924, - "college_medicine": 0.23076923076923078, - "computer_science": 0.27941176470588236, - "computer_security": 0.2573099415204678, - "conceptual_physics": 0.23129251700680273, - "construction_project_management": 0.30935251798561153, - "economics": 0.18867924528301888, - "education": 0.24539877300613497, - "electrical_engineering": 0.2441860465116279, - "elementary_chinese": 0.28174603174603174, - "elementary_commonsense": 0.23737373737373738, - "elementary_information_and_technology": 0.19747899159663865, - "elementary_mathematics": 0.2782608695652174, - "ethnology": 0.3037037037037037, - "food_science": 0.21678321678321677, - "genetics": 0.2159090909090909, - "global_facts": 0.20134228187919462, - "high_school_biology": 0.21893491124260356, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.23728813559322035, - "high_school_mathematics": 0.2073170731707317, - "high_school_physics": 0.22727272727272727, - "high_school_politics": 0.25874125874125875, - "human_sexuality": 0.2777777777777778, - "international_law": 0.25405405405405407, - "journalism": 0.26744186046511625, - "jurisprudence": 0.2798053527980535, - "legal_and_moral_basis": 0.2336448598130841, - "logical": 0.2682926829268293, - "machine_learning": 0.22950819672131148, - "management": 0.26666666666666666, - "marketing": 0.2722222222222222, - "marxist_theory": 0.2328042328042328, - "modern_chinese": 0.29310344827586204, - "nutrition": 0.2827586206896552, - "philosophy": 0.23809523809523808, - "professional_accounting": 0.17714285714285713, - "professional_law": 0.26066350710900477, - "professional_medicine": 0.23404255319148937, - "professional_psychology": 0.2543103448275862, - "public_relations": 0.26436781609195403, - "security_study": 0.2222222222222222, - "sociology": 0.2831858407079646, - "sports_science": 0.23636363636363636, - "traditional_chinese_medicine": 0.2648648648648649, - "virology": 0.23076923076923078, - "world_history": 0.2670807453416149, - "world_religions": 0.2625 - } - }, - "prompt_5": { - "accuracy": 0.25513728198929375, - "category_acc": { - "agronomy": 0.28994082840236685, - "anatomy": 0.21621621621621623, - "ancient_chinese": 0.23780487804878048, - "arts": 0.19375, - "astronomy": 0.28484848484848485, - "business_ethics": 0.24880382775119617, - "chinese_civil_service_exam": 0.3, - "chinese_driving_rule": 0.2748091603053435, - "chinese_food_culture": 0.3382352941176471, - "chinese_foreign_policy": 0.2336448598130841, - "chinese_history": 0.24148606811145512, - "chinese_literature": 0.23039215686274508, - "chinese_teacher_qualification": 0.21787709497206703, - "clinical_knowledge": 0.25738396624472576, - "college_actuarial_science": 0.32075471698113206, - "college_education": 0.2523364485981308, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.24074074074074073, - "college_mathematics": 0.2857142857142857, - "college_medical_statistics": 0.2830188679245283, - "college_medicine": 0.2673992673992674, - "computer_science": 0.29411764705882354, - "computer_security": 0.29239766081871343, - "conceptual_physics": 0.272108843537415, - "construction_project_management": 0.2589928057553957, - "economics": 0.22012578616352202, - "education": 0.26380368098159507, - "electrical_engineering": 0.20930232558139536, - "elementary_chinese": 0.25, - "elementary_commonsense": 0.23737373737373738, - "elementary_information_and_technology": 0.2815126050420168, - "elementary_mathematics": 0.2217391304347826, - "ethnology": 0.24444444444444444, - "food_science": 0.20279720279720279, - "genetics": 0.23863636363636365, - "global_facts": 0.2214765100671141, - "high_school_biology": 0.23076923076923078, - "high_school_chemistry": 0.20454545454545456, - "high_school_geography": 0.2033898305084746, - "high_school_mathematics": 0.23170731707317074, - "high_school_physics": 0.20909090909090908, - "high_school_politics": 0.23076923076923078, - "human_sexuality": 0.2777777777777778, - "international_law": 0.2756756756756757, - "journalism": 0.29069767441860467, - "jurisprudence": 0.26034063260340634, - "legal_and_moral_basis": 0.21495327102803738, - "logical": 0.21951219512195122, - "machine_learning": 0.23770491803278687, - "management": 0.2619047619047619, - "marketing": 0.3055555555555556, - "marxist_theory": 0.2275132275132275, - "modern_chinese": 0.25, - "nutrition": 0.296551724137931, - "philosophy": 0.37142857142857144, - "professional_accounting": 0.26285714285714284, - "professional_law": 0.2796208530805687, - "professional_medicine": 0.23670212765957446, - "professional_psychology": 0.2413793103448276, - "public_relations": 0.29310344827586204, - "security_study": 0.2962962962962963, - "sociology": 0.26991150442477874, - "sports_science": 0.21818181818181817, - "traditional_chinese_medicine": 0.2594594594594595, - "virology": 0.23076923076923078, - "world_history": 0.2546583850931677, - "world_religions": 0.3125 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.45454545454545453 - }, - "prompt_2": { - "accuracy": 0.3939393939393939 - }, - "prompt_3": { - "accuracy": 0.3939393939393939 - }, - "prompt_4": { - "accuracy": 0.48484848484848486 - }, - "prompt_5": { - "accuracy": 0.45454545454545453 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.37272727272727274 - }, - "prompt_2": { - "accuracy": 0.42727272727272725 - }, - "prompt_3": { - "accuracy": 0.4113636363636364 - }, - "prompt_4": { - "accuracy": 0.4431818181818182 - }, - "prompt_5": { - "accuracy": 0.4590909090909091 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.32610169491525426 - }, - "prompt_2": { - "accuracy": 0.32 - }, - "prompt_3": { - "accuracy": 0.3149152542372881 - }, - "prompt_4": { - "accuracy": 0.3257627118644068 - }, - "prompt_5": { - "accuracy": 0.3257627118644068 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.2782348541510845 - }, - "prompt_2": { - "accuracy": 0.281226626776365 - }, - "prompt_3": { - "accuracy": 0.27973074046372476 - }, - "prompt_4": { - "accuracy": 0.281226626776365 - }, - "prompt_5": { - "accuracy": 0.27898279730740466 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.9216070553650172 - }, - "prompt_2": { - "accuracy": 0.924546790788829 - }, - "prompt_3": { - "accuracy": 0.9220970112689858 - }, - "prompt_4": { - "accuracy": 0.9230769230769231 - }, - "prompt_5": { - "accuracy": 0.9240568348848603 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.5285145404113759, - "rouge2": 0.28422589355627753, - "rougeL": 0.4440415112366329, - "avg_rouge": 0.41892731506809544 - }, - "prompt_2": { - "rouge1": 0.5308584879061065, - "rouge2": 0.28861097795815444, - "rougeL": 0.44465233765777523, - "avg_rouge": 0.42137393450734545 - }, - "prompt_3": { - "rouge1": 0.5302765178235336, - "rouge2": 0.2889588042597731, - "rougeL": 0.4448768594943696, - "avg_rouge": 0.4213707271925588 - }, - "prompt_4": { - "rouge1": 0.5289238690020692, - "rouge2": 0.2867564479895972, - "rougeL": 0.44267635238735126, - "avg_rouge": 0.4194522231263392 - }, - "prompt_5": { - "rouge1": 0.5287451693189019, - "rouge2": 0.28651000021735606, - "rougeL": 0.44321091166538357, - "avg_rouge": 0.4194886937338805 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.2348990714241612, - "rouge2": 0.0654692982269716, - "rougeL": 0.17545716448016013, - "avg_rouge": 0.15860851137709764 - }, - "prompt_2": { - "rouge1": 0.23235043239609107, - "rouge2": 0.06516831879832731, - "rougeL": 0.17253374137518865, - "avg_rouge": 0.156684164189869 - }, - "prompt_3": { - "rouge1": 0.23186523593456612, - "rouge2": 0.06496367292313418, - "rougeL": 0.17169264184833946, - "avg_rouge": 0.15617385023534658 - }, - "prompt_4": { - "rouge1": 0.2346739315980681, - "rouge2": 0.06514981932932087, - "rougeL": 0.17494411893913228, - "avg_rouge": 0.15825595662217376 - }, - "prompt_5": { - "rouge1": 0.23476303055745215, - "rouge2": 0.06338707364815505, - "rougeL": 0.18093628091973574, - "avg_rouge": 0.15969546170844764 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.9587155963302753 - }, - "prompt_2": { - "accuracy": 0.9598623853211009 - }, - "prompt_3": { - "accuracy": 0.9541284403669725 - }, - "prompt_4": { - "accuracy": 0.9587155963302753 - }, - "prompt_5": { - "accuracy": 0.9541284403669725 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.8072866730584851 - }, - "prompt_2": { - "accuracy": 0.8072866730584851 - }, - "prompt_3": { - "accuracy": 0.8015340364333653 - }, - "prompt_4": { - "accuracy": 0.8015340364333653 - }, - "prompt_5": { - "accuracy": 0.8034515819750719 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.844 - }, - "prompt_2": { - "accuracy": 0.8665 - }, - "prompt_3": { - "accuracy": 0.8565 - }, - "prompt_4": { - "accuracy": 0.863 - }, - "prompt_5": { - "accuracy": 0.864 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.8905 - }, - "prompt_2": { - "accuracy": 0.8895 - }, - "prompt_3": { - "accuracy": 0.894 - }, - "prompt_4": { - "accuracy": 0.874 - }, - "prompt_5": { - "accuracy": 0.879 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.941 - }, - "prompt_2": { - "accuracy": 0.942 - }, - "prompt_3": { - "accuracy": 0.941 - }, - "prompt_4": { - "accuracy": 0.946 - }, - "prompt_5": { - "accuracy": 0.9435 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.7746478873239436 - }, - "prompt_2": { - "accuracy": 0.7887323943661971 - }, - "prompt_3": { - "accuracy": 0.7746478873239436 - }, - "prompt_4": { - "accuracy": 0.7605633802816901 - }, - "prompt_5": { - "accuracy": 0.7605633802816901 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.9169675090252708 - }, - "prompt_2": { - "accuracy": 0.9133574007220217 - }, - "prompt_3": { - "accuracy": 0.9097472924187726 - }, - "prompt_4": { - "accuracy": 0.8916967509025271 - }, - "prompt_5": { - "accuracy": 0.9025270758122743 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.8088235294117647 - }, - "prompt_2": { - "accuracy": 0.8284313725490197 - }, - "prompt_3": { - "accuracy": 0.8112745098039216 - }, - "prompt_4": { - "accuracy": 0.8333333333333334 - }, - "prompt_5": { - "accuracy": 0.8259803921568627 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.7739495798319327, - "language_acc": { - "English": 0.9747899159663865, - "Chinese": 0.36470588235294116, - "Spanish": 0.9521008403361344, - "Vietnamese": 0.8042016806722689 - }, - "consistency_score_2": 0.6172268907563025, - "consistency_score_3": 0.4485294117647059, - "consistency_score_4": 0.31848739495798317, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3697478991596639, - "English,Spanish": 0.9537815126050421, - "English,Vietnamese": 0.8025210084033614, - "Chinese,Spanish": 0.3773109243697479, - "Chinese,Vietnamese": 0.4008403361344538, - "Spanish,Vietnamese": 0.7991596638655463 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3596638655462185, - "English,Chinese,Vietnamese": 0.3243697478991597, - "English,Spanish,Vietnamese": 0.7823529411764706, - "Chinese,Spanish,Vietnamese": 0.3277310924369748 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.31848739495798317 - } - }, - "AC3_2": 0.68676045466106, - "AC3_3": 0.5679265691032858, - "AC3_4": 0.451272139583772 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.33999999999999997, - "language_acc": { - "Spanish": 0.3933333333333333, - "English": 0.5466666666666666, - "Chinese": 0.2733333333333333, - "Vietnamese": 0.26666666666666666, - "Filipino": 0.28, - "Indonesian": 0.34, - "Malay": 0.28 - }, - "consistency_score_2": 0.6165079365079363, - "consistency_score_3": 0.45942857142857146, - "consistency_score_4": 0.3636190476190476, - "consistency_score_5": 0.29555555555555557, - "consistency_score_6": 0.24285714285714288, - "consistency_score_7": 0.2, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.62, - "Spanish,Chinese": 0.58, - "Spanish,Vietnamese": 0.54, - "Spanish,Filipino": 0.6, - "Spanish,Indonesian": 0.6066666666666667, - "Spanish,Malay": 0.5533333333333333, - "English,Chinese": 0.48, - "English,Vietnamese": 0.4666666666666667, - "English,Filipino": 0.5666666666666667, - "English,Indonesian": 0.5333333333333333, - "English,Malay": 0.5333333333333333, - "Chinese,Vietnamese": 0.78, - "Chinese,Filipino": 0.6666666666666666, - "Chinese,Indonesian": 0.7266666666666667, - "Chinese,Malay": 0.7, - "Vietnamese,Filipino": 0.6533333333333333, - "Vietnamese,Indonesian": 0.68, - "Vietnamese,Malay": 0.68, - "Filipino,Indonesian": 0.6, - "Filipino,Malay": 0.6466666666666666, - "Indonesian,Malay": 0.7333333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.3933333333333333, - "Spanish,English,Vietnamese": 0.35333333333333333, - "Spanish,English,Filipino": 0.44, - "Spanish,English,Indonesian": 0.43333333333333335, - "Spanish,English,Malay": 0.41333333333333333, - "Spanish,Chinese,Vietnamese": 0.47333333333333333, - "Spanish,Chinese,Filipino": 0.4533333333333333, - "Spanish,Chinese,Indonesian": 0.47333333333333333, - "Spanish,Chinese,Malay": 0.44, - "Spanish,Vietnamese,Filipino": 0.4266666666666667, - "Spanish,Vietnamese,Indonesian": 0.44, - "Spanish,Vietnamese,Malay": 0.42, - "Spanish,Filipino,Indonesian": 0.43333333333333335, - "Spanish,Filipino,Malay": 0.44666666666666666, - "Spanish,Indonesian,Malay": 0.4666666666666667, - "English,Chinese,Vietnamese": 0.38666666666666666, - "English,Chinese,Filipino": 0.4, - "English,Chinese,Indonesian": 0.4066666666666667, - "English,Chinese,Malay": 0.4, - "English,Vietnamese,Filipino": 0.38666666666666666, - "English,Vietnamese,Indonesian": 0.38666666666666666, - "English,Vietnamese,Malay": 0.38, - "English,Filipino,Indonesian": 0.4066666666666667, - "English,Filipino,Malay": 0.43333333333333335, - "English,Indonesian,Malay": 0.43333333333333335, - "Chinese,Vietnamese,Filipino": 0.5666666666666667, - "Chinese,Vietnamese,Indonesian": 0.6066666666666667, - "Chinese,Vietnamese,Malay": 0.5933333333333334, - "Chinese,Filipino,Indonesian": 0.52, - "Chinese,Filipino,Malay": 0.5533333333333333, - "Chinese,Indonesian,Malay": 0.5933333333333334, - "Vietnamese,Filipino,Indonesian": 0.5, - "Vietnamese,Filipino,Malay": 0.5333333333333333, - "Vietnamese,Indonesian,Malay": 0.5666666666666667, - "Filipino,Indonesian,Malay": 0.52 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.30666666666666664, - "Spanish,English,Chinese,Filipino": 0.3333333333333333, - "Spanish,English,Chinese,Indonesian": 0.34, - "Spanish,English,Chinese,Malay": 0.3333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.3, - "Spanish,English,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,English,Vietnamese,Malay": 0.30666666666666664, - "Spanish,English,Filipino,Indonesian": 0.3333333333333333, - "Spanish,English,Filipino,Malay": 0.3466666666666667, - "Spanish,English,Indonesian,Malay": 0.35333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.37333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian": 0.38666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.36, - "Spanish,Chinese,Filipino,Indonesian": 0.38, - "Spanish,Chinese,Filipino,Malay": 0.38, - "Spanish,Chinese,Indonesian,Malay": 0.38666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.36, - "Spanish,Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Spanish,Filipino,Indonesian,Malay": 0.37333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.32666666666666666, - "English,Chinese,Vietnamese,Indonesian": 0.34, - "English,Chinese,Vietnamese,Malay": 0.32666666666666666, - "English,Chinese,Filipino,Indonesian": 0.34, - "English,Chinese,Filipino,Malay": 0.36, - "English,Chinese,Indonesian,Malay": 0.3466666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.32, - "English,Vietnamese,Filipino,Malay": 0.34, - "English,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "English,Filipino,Indonesian,Malay": 0.35333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.4533333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.47333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.5066666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.4666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.4533333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.26, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.26, - "Spanish,English,Chinese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.29333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,English,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.32, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.28, - "English,Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.4066666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 - } - }, - "AC3_2": 0.43828742112908703, - "AC3_3": 0.3907934238253118, - "AC3_4": 0.3514131022781559, - "AC3_5": 0.31622377617402075, - "AC3_6": 0.2833333332847222, - "AC3_7": 0.2518518518052126 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.3457792207792208, - "language_acc": { - "English": 0.5284090909090909, - "Filipino": 0.3068181818181818, - "Spanish": 0.4431818181818182, - "Malay": 0.3352272727272727, - "Chinese": 0.26136363636363635, - "Indonesian": 0.2840909090909091, - "Vietnamese": 0.26136363636363635 - }, - "consistency_score_2": 0.4242424242424243, - "consistency_score_3": 0.22353896103896104, - "consistency_score_4": 0.1350649350649351, - "consistency_score_5": 0.09063852813852813, - "consistency_score_6": 0.06574675324675325, - "consistency_score_7": 0.05113636363636364, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.35795454545454547, - "English,Spanish": 0.5511363636363636, - "English,Malay": 0.375, - "English,Chinese": 0.3181818181818182, - "English,Indonesian": 0.3693181818181818, - "English,Vietnamese": 0.3125, - "Filipino,Spanish": 0.38636363636363635, - "Filipino,Malay": 0.5397727272727273, - "Filipino,Chinese": 0.3693181818181818, - "Filipino,Indonesian": 0.4715909090909091, - "Filipino,Vietnamese": 0.44886363636363635, - "Spanish,Malay": 0.4318181818181818, - "Spanish,Chinese": 0.36363636363636365, - "Spanish,Indonesian": 0.45454545454545453, - "Spanish,Vietnamese": 0.42045454545454547, - "Malay,Chinese": 0.4375, - "Malay,Indonesian": 0.5738636363636364, - "Malay,Vietnamese": 0.44886363636363635, - "Chinese,Indonesian": 0.4147727272727273, - "Chinese,Vietnamese": 0.4034090909090909, - "Indonesian,Vietnamese": 0.4602272727272727 - }, - "3_combine": { - "English,Filipino,Spanish": 0.22727272727272727, - "English,Filipino,Malay": 0.24431818181818182, - "English,Filipino,Chinese": 0.1590909090909091, - "English,Filipino,Indonesian": 0.2215909090909091, - "English,Filipino,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay": 0.26704545454545453, - "English,Spanish,Chinese": 0.21022727272727273, - "English,Spanish,Indonesian": 0.25, - "English,Spanish,Vietnamese": 0.2215909090909091, - "English,Malay,Chinese": 0.16477272727272727, - "English,Malay,Indonesian": 0.24431818181818182, - "English,Malay,Vietnamese": 0.17613636363636365, - "English,Chinese,Indonesian": 0.14772727272727273, - "English,Chinese,Vietnamese": 0.13636363636363635, - "English,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay": 0.2556818181818182, - "Filipino,Spanish,Chinese": 0.17613636363636365, - "Filipino,Spanish,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Chinese": 0.2556818181818182, - "Filipino,Malay,Indonesian": 0.3409090909090909, - "Filipino,Malay,Vietnamese": 0.2727272727272727, - "Filipino,Chinese,Indonesian": 0.21022727272727273, - "Filipino,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Indonesian,Vietnamese": 0.2556818181818182, - "Spanish,Malay,Chinese": 0.2215909090909091, - "Spanish,Malay,Indonesian": 0.30113636363636365, - "Spanish,Malay,Vietnamese": 0.22727272727272727, - "Spanish,Chinese,Indonesian": 0.21022727272727273, - "Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Spanish,Indonesian,Vietnamese": 0.22727272727272727, - "Malay,Chinese,Indonesian": 0.2727272727272727, - "Malay,Chinese,Vietnamese": 0.22727272727272727, - "Malay,Indonesian,Vietnamese": 0.30113636363636365, - "Chinese,Indonesian,Vietnamese": 0.23295454545454544 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.1875, - "English,Filipino,Spanish,Chinese": 0.11363636363636363, - "English,Filipino,Spanish,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese": 0.11931818181818182, - "English,Filipino,Malay,Indonesian": 0.16477272727272727, - "English,Filipino,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Chinese,Indonesian": 0.10227272727272728, - "English,Filipino,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese": 0.125, - "English,Spanish,Malay,Indonesian": 0.1875, - "English,Spanish,Malay,Vietnamese": 0.14204545454545456, - "English,Spanish,Chinese,Indonesian": 0.11931818181818182, - "English,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "English,Spanish,Indonesian,Vietnamese": 0.125, - "English,Malay,Chinese,Indonesian": 0.10227272727272728, - "English,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Indonesian,Vietnamese": 0.125, - "Spanish,Malay,Chinese,Indonesian": 0.17613636363636365, - "Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.10795454545454546 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 - } - }, - "AC3_2": 0.38101322425256995, - "AC3_3": 0.2715357781176812, - "AC3_4": 0.1942527424905347, - "AC3_5": 0.14362807059453997, - "AC3_6": 0.11048566815112808, - "AC3_7": 0.08909648631330293 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.5631067961165048 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.2857142857142857 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.4953271028037383 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.19, - "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.3, - "history": 0.26666666666666666, - "literature": 0.2, - "politics": 0.0, - "culture": 0.2, - "film": 0.1, - "law": 0.1, - "geography": 0.4 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.1805298468355187 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.23461712466769424 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.05567840923314876 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.016671301461117446 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.21768962793903313 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.49474912485414235 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.4813728995352163, - "category_acc": { - "high_school_european_history": 0.7195121951219512, - "business_ethics": 0.5858585858585859, - "clinical_knowledge": 0.5378787878787878, - "medical_genetics": 0.41414141414141414, - "high_school_us_history": 0.6847290640394089, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.6864406779661016, - "virology": 0.41818181818181815, - "high_school_microeconomics": 0.5443037974683544, - "econometrics": 0.3274336283185841, - "college_computer_science": 0.42424242424242425, - "high_school_biology": 0.5436893203883495, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.38434163701067614, - "philosophy": 0.5096774193548387, - "professional_medicine": 0.46863468634686345, - "nutrition": 0.4918032786885246, - "global_facts": 0.2727272727272727, - "machine_learning": 0.2882882882882883, - "security_studies": 0.6065573770491803, - "public_relations": 0.5412844036697247, - "professional_psychology": 0.44844517184942717, - "prehistory": 0.47368421052631576, - "anatomy": 0.417910447761194, - "human_sexuality": 0.5384615384615384, - "college_medicine": 0.4418604651162791, - "high_school_government_and_politics": 0.6822916666666666, - "college_chemistry": 0.3939393939393939, - "logical_fallacies": 0.5802469135802469, - "high_school_geography": 0.6598984771573604, - "elementary_mathematics": 0.3421750663129973, - "human_aging": 0.49099099099099097, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.6838235294117647, - "formal_logic": 0.392, - "high_school_statistics": 0.4186046511627907, - "international_law": 0.625, - "high_school_mathematics": 0.2788104089219331, - "high_school_computer_science": 0.45454545454545453, - "conceptual_physics": 0.4188034188034188, - "miscellaneous": 0.629156010230179, - "high_school_chemistry": 0.3217821782178218, - "marketing": 0.7339055793991416, - "professional_law": 0.3868232224396608, - "management": 0.696078431372549, - "college_physics": 0.33663366336633666, - "jurisprudence": 0.5420560747663551, - "world_religions": 0.5294117647058824, - "sociology": 0.695, - "us_foreign_policy": 0.696969696969697, - "high_school_macroeconomics": 0.5372750642673522, - "computer_security": 0.5656565656565656, - "moral_scenarios": 0.22483221476510068, - "moral_disputes": 0.5739130434782609, - "electrical_engineering": 0.4236111111111111, - "astronomy": 0.5165562913907285, - "college_biology": 0.5034965034965035 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.2473997028231798 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.2671232876712329, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.19047619047619047, - "college_physics": 0.125, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.375, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.125, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.23333333333333334, - "business_administration": 0.18421052631578946, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.20408163265306123, - "high_school_politics": 0.125, - "high_school_geography": 0.25, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.37037037037037035, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.04, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.28846153846153844, - "sports_science": 0.3333333333333333, - "plant_protection": 0.1111111111111111, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.3148148148148148, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.2962962962962963, - "physician": 0.2962962962962963 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.21505376344086022 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.2497841478155759, - "category_acc": { - "agronomy": 0.25443786982248523, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.23170731707317074, - "arts": 0.2625, - "astronomy": 0.22424242424242424, - "business_ethics": 0.2631578947368421, - "chinese_civil_service_exam": 0.2375, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.24299065420560748, - "chinese_history": 0.24148606811145512, - "chinese_literature": 0.2549019607843137, - "chinese_teacher_qualification": 0.2569832402234637, - "clinical_knowledge": 0.24472573839662448, - "college_actuarial_science": 0.3490566037735849, - "college_education": 0.1588785046728972, - "college_engineering_hydrology": 0.27358490566037735, - "college_law": 0.2962962962962963, - "college_mathematics": 0.3047619047619048, - "college_medical_statistics": 0.2358490566037736, - "college_medicine": 0.2454212454212454, - "computer_science": 0.23039215686274508, - "computer_security": 0.25146198830409355, - "conceptual_physics": 0.25170068027210885, - "construction_project_management": 0.2517985611510791, - "economics": 0.25157232704402516, - "education": 0.24539877300613497, - "electrical_engineering": 0.2558139534883721, - "elementary_chinese": 0.21428571428571427, - "elementary_commonsense": 0.2777777777777778, - "elementary_information_and_technology": 0.25210084033613445, - "elementary_mathematics": 0.20869565217391303, - "ethnology": 0.24444444444444444, - "food_science": 0.2517482517482518, - "genetics": 0.23295454545454544, - "global_facts": 0.2550335570469799, - "high_school_biology": 0.2603550295857988, - "high_school_chemistry": 0.22727272727272727, - "high_school_geography": 0.211864406779661, - "high_school_mathematics": 0.2682926829268293, - "high_school_physics": 0.3, - "high_school_politics": 0.2517482517482518, - "human_sexuality": 0.2619047619047619, - "international_law": 0.24324324324324326, - "journalism": 0.2616279069767442, - "jurisprudence": 0.25304136253041365, - "legal_and_moral_basis": 0.24766355140186916, - "logical": 0.23577235772357724, - "machine_learning": 0.2540983606557377, - "management": 0.24285714285714285, - "marketing": 0.24444444444444444, - "marxist_theory": 0.25925925925925924, - "modern_chinese": 0.25, - "nutrition": 0.2620689655172414, - "philosophy": 0.26666666666666666, - "professional_accounting": 0.25142857142857145, - "professional_law": 0.26066350710900477, - "professional_medicine": 0.2553191489361702, - "professional_psychology": 0.25, - "public_relations": 0.2471264367816092, - "security_study": 0.23703703703703705, - "sociology": 0.24778761061946902, - "sports_science": 0.23030303030303031, - "traditional_chinese_medicine": 0.25405405405405407, - "virology": 0.25443786982248523, - "world_history": 0.2484472049689441, - "world_religions": 0.25 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.3333333333333333 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.3409090909090909 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.32745762711864407 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.29020194465220644 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.9235668789808917 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.5318107371424013, - "rouge2": 0.28895352204337643, - "rougeL": 0.4474672725784119, - "avg_rouge": 0.4227438439213966 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.402512932516084, - "rouge2": 0.16546262752213603, - "rougeL": 0.32613240657359005, - "avg_rouge": 0.2980359888706034 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.9575688073394495 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.8283796740172579 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.8635 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.8895 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.946 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.7605633802816901 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.9097472924187726 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.8186274509803921 - } - } - } - }, - "flan_ul2": { - "model_size": "20B", - "model_link": "https://huggingface.co/google/flan-t5-ul2", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.7880252100840336, - "language_acc": { - "English": 0.9781512605042016, - "Chinese": 0.3663865546218487, - "Spanish": 0.9621848739495799, - "Vietnamese": 0.8453781512605042 - }, - "consistency_score_2": 0.630952380952381, - "consistency_score_3": 0.4661764705882353, - "consistency_score_4": 0.3310924369747899, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3722689075630252, - "English,Spanish": 0.9663865546218487, - "English,Vietnamese": 0.8445378151260504, - "Chinese,Spanish": 0.37058823529411766, - "Chinese,Vietnamese": 0.3915966386554622, - "Spanish,Vietnamese": 0.8403361344537815 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3621848739495798, - "English,Chinese,Vietnamese": 0.3369747899159664, - "English,Spanish,Vietnamese": 0.8294117647058824, - "Chinese,Spanish,Vietnamese": 0.33613445378151263 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3310924369747899 - } - }, - "AC3_2": 0.7007952566112398, - "AC3_3": 0.5858050053725457, - "AC3_4": 0.4662766025836167 - }, - "prompt_2": { - "overall_acc": 0.790546218487395, - "language_acc": { - "English": 0.9773109243697479, - "Chinese": 0.3739495798319328, - "Spanish": 0.9621848739495799, - "Vietnamese": 0.8487394957983193 - }, - "consistency_score_2": 0.6351540616246499, - "consistency_score_3": 0.4707983193277311, - "consistency_score_4": 0.3369747899159664, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.37815126050420167, - "English,Spanish": 0.9672268907563025, - "English,Vietnamese": 0.846218487394958, - "Chinese,Spanish": 0.37815126050420167, - "Chinese,Vietnamese": 0.4, - "Spanish,Vietnamese": 0.8411764705882353 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3680672268907563, - "English,Chinese,Vietnamese": 0.3420168067226891, - "English,Spanish,Vietnamese": 0.83109243697479, - "Chinese,Spanish,Vietnamese": 0.3420168067226891 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3369747899159664 - } - }, - "AC3_2": 0.7043817673930688, - "AC3_3": 0.5901445954329774, - "AC3_4": 0.4725307003321481 - }, - "prompt_3": { - "overall_acc": 0.792436974789916, - "language_acc": { - "English": 0.9773109243697479, - "Chinese": 0.3773109243697479, - "Spanish": 0.9621848739495799, - "Vietnamese": 0.8529411764705882 - }, - "consistency_score_2": 0.6382352941176471, - "consistency_score_3": 0.47731092436974787, - "consistency_score_4": 0.3445378151260504, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3798319327731092, - "English,Spanish": 0.965546218487395, - "English,Vietnamese": 0.8521008403361344, - "Chinese,Spanish": 0.3831932773109244, - "Chinese,Vietnamese": 0.4, - "Spanish,Vietnamese": 0.8487394957983193 - }, - "3_combine": { - "English,Chinese,Spanish": 0.37142857142857144, - "English,Chinese,Vietnamese": 0.3495798319327731, - "English,Spanish,Vietnamese": 0.8369747899159664, - "Chinese,Spanish,Vietnamese": 0.35126050420168065 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3445378151260504 - } - }, - "AC3_2": 0.7070259997778245, - "AC3_3": 0.5957699558507267, - "AC3_4": 0.48026483316376917 - }, - "prompt_4": { - "overall_acc": 0.7913865546218488, - "language_acc": { - "English": 0.9773109243697479, - "Chinese": 0.3764705882352941, - "Spanish": 0.9630252100840336, - "Vietnamese": 0.8487394957983193 - }, - "consistency_score_2": 0.6355742296918768, - "consistency_score_3": 0.4726890756302521, - "consistency_score_4": 0.33949579831932775, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.37815126050420167, - "English,Spanish": 0.9663865546218487, - "English,Vietnamese": 0.8478991596638655, - "Chinese,Spanish": 0.3815126050420168, - "Chinese,Vietnamese": 0.3957983193277311, - "Spanish,Vietnamese": 0.8436974789915966 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3697478991596639, - "English,Chinese,Vietnamese": 0.34369747899159664, - "English,Spanish,Vietnamese": 0.8327731092436975, - "Chinese,Spanish,Vietnamese": 0.3445378151260504 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.33949579831932775 - } - }, - "AC3_2": 0.7049736829999066, - "AC3_3": 0.5918629708353985, - "AC3_4": 0.47515536769675676 - }, - "prompt_5": { - "overall_acc": 0.7922268907563026, - "language_acc": { - "English": 0.9773109243697479, - "Chinese": 0.3798319327731092, - "Spanish": 0.9613445378151261, - "Vietnamese": 0.8504201680672269 - }, - "consistency_score_2": 0.6379551820728292, - "consistency_score_3": 0.4752100840336134, - "consistency_score_4": 0.3411764705882353, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3840336134453782, - "English,Spanish": 0.9672268907563025, - "English,Vietnamese": 0.8478991596638655, - "Chinese,Spanish": 0.3865546218487395, - "Chinese,Vietnamese": 0.3983193277310924, - "Spanish,Vietnamese": 0.8436974789915966 - }, - "3_combine": { - "English,Chinese,Spanish": 0.37478991596638656, - "English,Chinese,Vietnamese": 0.346218487394958, - "English,Spanish,Vietnamese": 0.8327731092436975, - "Chinese,Spanish,Vietnamese": 0.34705882352941175 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3411764705882353 - } - }, - "AC3_2": 0.706770501325494, - "AC3_3": 0.5940716813357757, - "AC3_4": 0.4769514255698688 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.39142857142857146, - "language_acc": { - "Spanish": 0.41333333333333333, - "English": 0.5933333333333334, - "Chinese": 0.3, - "Vietnamese": 0.30666666666666664, - "Filipino": 0.36, - "Indonesian": 0.4066666666666667, - "Malay": 0.36 - }, - "consistency_score_2": 0.4546031746031745, - "consistency_score_3": 0.2617142857142858, - "consistency_score_4": 0.16952380952380952, - "consistency_score_5": 0.11936507936507935, - "consistency_score_6": 0.09047619047619049, - "consistency_score_7": 0.07333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6466666666666666, - "Spanish,Chinese": 0.42, - "Spanish,Vietnamese": 0.3933333333333333, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.49333333333333335, - "Spanish,Malay": 0.4866666666666667, - "English,Chinese": 0.3933333333333333, - "English,Vietnamese": 0.37333333333333335, - "English,Filipino": 0.5, - "English,Indonesian": 0.5333333333333333, - "English,Malay": 0.4666666666666667, - "Chinese,Vietnamese": 0.4266666666666667, - "Chinese,Filipino": 0.36666666666666664, - "Chinese,Indonesian": 0.41333333333333333, - "Chinese,Malay": 0.36666666666666664, - "Vietnamese,Filipino": 0.4, - "Vietnamese,Indonesian": 0.49333333333333335, - "Vietnamese,Malay": 0.4, - "Filipino,Indonesian": 0.5066666666666667, - "Filipino,Malay": 0.43333333333333335, - "Indonesian,Malay": 0.5933333333333334 - }, - "3_combine": { - "Spanish,English,Chinese": 0.29333333333333333, - "Spanish,English,Vietnamese": 0.2733333333333333, - "Spanish,English,Filipino": 0.34, - "Spanish,English,Indonesian": 0.37333333333333335, - "Spanish,English,Malay": 0.36, - "Spanish,Chinese,Vietnamese": 0.23333333333333334, - "Spanish,Chinese,Filipino": 0.21333333333333335, - "Spanish,Chinese,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,Vietnamese,Indonesian": 0.26, - "Spanish,Vietnamese,Malay": 0.21333333333333335, - "Spanish,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Filipino,Malay": 0.2733333333333333, - "Spanish,Indonesian,Malay": 0.35333333333333333, - "English,Chinese,Vietnamese": 0.2, - "English,Chinese,Filipino": 0.20666666666666667, - "English,Chinese,Indonesian": 0.23333333333333334, - "English,Chinese,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino": 0.23333333333333334, - "English,Vietnamese,Indonesian": 0.26666666666666666, - "English,Vietnamese,Malay": 0.22, - "English,Filipino,Indonesian": 0.3466666666666667, - "English,Filipino,Malay": 0.2866666666666667, - "English,Indonesian,Malay": 0.35333333333333333, - "Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Chinese,Vietnamese,Indonesian": 0.26, - "Chinese,Vietnamese,Malay": 0.20666666666666667, - "Chinese,Filipino,Indonesian": 0.22, - "Chinese,Filipino,Malay": 0.16666666666666666, - "Chinese,Indonesian,Malay": 0.24666666666666667, - "Vietnamese,Filipino,Indonesian": 0.26, - "Vietnamese,Filipino,Malay": 0.21333333333333335, - "Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Filipino,Indonesian,Malay": 0.3333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.16, - "Spanish,English,Chinese,Filipino": 0.16666666666666666, - "Spanish,English,Chinese,Indonesian": 0.18, - "Spanish,English,Chinese,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,English,Vietnamese,Malay": 0.16666666666666666, - "Spanish,English,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Filipino,Malay": 0.22, - "Spanish,English,Indonesian,Malay": 0.2733333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian": 0.14666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "English,Chinese,Vietnamese,Malay": 0.14, - "English,Chinese,Filipino,Indonesian": 0.14666666666666667, - "English,Chinese,Filipino,Malay": 0.12, - "English,Chinese,Indonesian,Malay": 0.16666666666666666, - "English,Vietnamese,Filipino,Indonesian": 0.18, - "English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "English,Filipino,Indonesian,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.16666666666666666, - "Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Filipino,Indonesian,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.07333333333333333 - } - }, - "AC3_2": 0.42065719641235333, - "AC3_3": 0.3136907886033907, - "AC3_4": 0.23658501087220712, - "AC3_5": 0.18294237766106827, - "AC3_6": 0.14697910781817344, - "AC3_7": 0.12352459013735638 - }, - "prompt_2": { - "overall_acc": 0.3885714285714286, - "language_acc": { - "Spanish": 0.4266666666666667, - "English": 0.5733333333333334, - "Chinese": 0.29333333333333333, - "Vietnamese": 0.32, - "Filipino": 0.36, - "Indonesian": 0.38666666666666666, - "Malay": 0.36 - }, - "consistency_score_2": 0.4565079365079366, - "consistency_score_3": 0.26552380952380955, - "consistency_score_4": 0.1754285714285714, - "consistency_score_5": 0.12666666666666662, - "consistency_score_6": 0.09809523809523811, - "consistency_score_7": 0.08, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6133333333333333, - "Spanish,Chinese": 0.4066666666666667, - "Spanish,Vietnamese": 0.4066666666666667, - "Spanish,Filipino": 0.4266666666666667, - "Spanish,Indonesian": 0.49333333333333335, - "Spanish,Malay": 0.4866666666666667, - "English,Chinese": 0.38, - "English,Vietnamese": 0.38, - "English,Filipino": 0.49333333333333335, - "English,Indonesian": 0.47333333333333333, - "English,Malay": 0.4533333333333333, - "Chinese,Vietnamese": 0.41333333333333333, - "Chinese,Filipino": 0.3933333333333333, - "Chinese,Indonesian": 0.4666666666666667, - "Chinese,Malay": 0.44, - "Vietnamese,Filipino": 0.41333333333333333, - "Vietnamese,Indonesian": 0.49333333333333335, - "Vietnamese,Malay": 0.4, - "Filipino,Indonesian": 0.4866666666666667, - "Filipino,Malay": 0.46, - "Indonesian,Malay": 0.6066666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.28, - "Spanish,English,Vietnamese": 0.26, - "Spanish,English,Filipino": 0.32666666666666666, - "Spanish,English,Indonesian": 0.34, - "Spanish,English,Malay": 0.34, - "Spanish,Chinese,Vietnamese": 0.22666666666666666, - "Spanish,Chinese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Indonesian": 0.24, - "Spanish,Chinese,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Filipino": 0.22666666666666666, - "Spanish,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,Vietnamese,Malay": 0.22666666666666666, - "Spanish,Filipino,Indonesian": 0.3, - "Spanish,Filipino,Malay": 0.26666666666666666, - "Spanish,Indonesian,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese": 0.2, - "English,Chinese,Filipino": 0.22666666666666666, - "English,Chinese,Indonesian": 0.23333333333333334, - "English,Chinese,Malay": 0.24666666666666667, - "English,Vietnamese,Filipino": 0.22666666666666666, - "English,Vietnamese,Indonesian": 0.24666666666666667, - "English,Vietnamese,Malay": 0.22666666666666666, - "English,Filipino,Indonesian": 0.30666666666666664, - "English,Filipino,Malay": 0.29333333333333333, - "English,Indonesian,Malay": 0.32666666666666666, - "Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian": 0.2733333333333333, - "Chinese,Vietnamese,Malay": 0.24666666666666667, - "Chinese,Filipino,Indonesian": 0.24666666666666667, - "Chinese,Filipino,Malay": 0.22, - "Chinese,Indonesian,Malay": 0.32666666666666666, - "Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Vietnamese,Filipino,Malay": 0.23333333333333334, - "Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Filipino,Indonesian,Malay": 0.3333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.16, - "Spanish,English,Chinese,Filipino": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino": 0.16, - "Spanish,English,Vietnamese,Indonesian": 0.16, - "Spanish,English,Vietnamese,Malay": 0.16, - "Spanish,English,Filipino,Indonesian": 0.24, - "Spanish,English,Filipino,Malay": 0.22, - "Spanish,English,Indonesian,Malay": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian": 0.16, - "Spanish,Chinese,Vietnamese,Malay": 0.16666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.14, - "English,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "English,Chinese,Vietnamese,Malay": 0.16, - "English,Chinese,Filipino,Indonesian": 0.16, - "English,Chinese,Filipino,Malay": 0.15333333333333332, - "English,Chinese,Indonesian,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Vietnamese,Filipino,Malay": 0.16, - "English,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "English,Filipino,Indonesian,Malay": 0.22, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Chinese,Vietnamese,Filipino,Malay": 0.16, - "Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.18, - "Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,English,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.12, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - } - }, - "AC3_2": 0.4198089513295225, - "AC3_3": 0.31547383304936494, - "AC3_4": 0.24172528581616534, - "AC3_5": 0.1910536043991485, - "AC3_6": 0.15664523340365272, - "AC3_7": 0.1326829268009518 - }, - "prompt_3": { - "overall_acc": 0.38095238095238093, - "language_acc": { - "Spanish": 0.41333333333333333, - "English": 0.6, - "Chinese": 0.2733333333333333, - "Vietnamese": 0.3, - "Filipino": 0.32666666666666666, - "Indonesian": 0.3933333333333333, - "Malay": 0.36 - }, - "consistency_score_2": 0.45968253968253975, - "consistency_score_3": 0.27009523809523805, - "consistency_score_4": 0.17714285714285716, - "consistency_score_5": 0.12571428571428572, - "consistency_score_6": 0.09619047619047619, - "consistency_score_7": 0.08, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6266666666666667, - "Spanish,Chinese": 0.42, - "Spanish,Vietnamese": 0.38666666666666666, - "Spanish,Filipino": 0.44, - "Spanish,Indonesian": 0.5066666666666667, - "Spanish,Malay": 0.4866666666666667, - "English,Chinese": 0.37333333333333335, - "English,Vietnamese": 0.36666666666666664, - "English,Filipino": 0.4866666666666667, - "English,Indonesian": 0.5266666666666666, - "English,Malay": 0.4666666666666667, - "Chinese,Vietnamese": 0.3933333333333333, - "Chinese,Filipino": 0.3933333333333333, - "Chinese,Indonesian": 0.44666666666666666, - "Chinese,Malay": 0.38, - "Vietnamese,Filipino": 0.3933333333333333, - "Vietnamese,Indonesian": 0.5, - "Vietnamese,Malay": 0.4066666666666667, - "Filipino,Indonesian": 0.5466666666666666, - "Filipino,Malay": 0.48, - "Indonesian,Malay": 0.6266666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.28, - "Spanish,English,Vietnamese": 0.25333333333333335, - "Spanish,English,Filipino": 0.32666666666666666, - "Spanish,English,Indonesian": 0.38, - "Spanish,English,Malay": 0.36, - "Spanish,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,Chinese,Filipino": 0.22, - "Spanish,Chinese,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,Vietnamese,Indonesian": 0.26, - "Spanish,Vietnamese,Malay": 0.22666666666666666, - "Spanish,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Filipino,Malay": 0.2866666666666667, - "Spanish,Indonesian,Malay": 0.38, - "English,Chinese,Vietnamese": 0.18666666666666668, - "English,Chinese,Filipino": 0.21333333333333335, - "English,Chinese,Indonesian": 0.23333333333333334, - "English,Chinese,Malay": 0.22, - "English,Vietnamese,Filipino": 0.21333333333333335, - "English,Vietnamese,Indonesian": 0.26666666666666666, - "English,Vietnamese,Malay": 0.22666666666666666, - "English,Filipino,Indonesian": 0.3466666666666667, - "English,Filipino,Malay": 0.3, - "English,Indonesian,Malay": 0.37333333333333335, - "Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Chinese,Vietnamese,Indonesian": 0.2733333333333333, - "Chinese,Vietnamese,Malay": 0.21333333333333335, - "Chinese,Filipino,Indonesian": 0.25333333333333335, - "Chinese,Filipino,Malay": 0.2, - "Chinese,Indonesian,Malay": 0.2866666666666667, - "Vietnamese,Filipino,Indonesian": 0.28, - "Vietnamese,Filipino,Malay": 0.24, - "Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Filipino,Indonesian,Malay": 0.38 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.14666666666666667, - "Spanish,English,Chinese,Filipino": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian": 0.18, - "Spanish,English,Chinese,Malay": 0.18, - "Spanish,English,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.18, - "Spanish,English,Vietnamese,Malay": 0.16, - "Spanish,English,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Filipino,Malay": 0.23333333333333334, - "Spanish,English,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.14, - "Spanish,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Malay": 0.15333333333333332, - "Spanish,Chinese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.12, - "English,Chinese,Vietnamese,Indonesian": 0.14, - "English,Chinese,Vietnamese,Malay": 0.14, - "English,Chinese,Filipino,Indonesian": 0.16666666666666666, - "English,Chinese,Filipino,Malay": 0.14, - "English,Chinese,Indonesian,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino,Indonesian": 0.18, - "English,Vietnamese,Filipino,Malay": 0.16, - "English,Vietnamese,Indonesian,Malay": 0.2, - "English,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Chinese,Vietnamese,Filipino,Malay": 0.14, - "Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Chinese,Filipino,Indonesian,Malay": 0.16, - "Vietnamese,Filipino,Indonesian,Malay": 0.2 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.1, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.14, - "Spanish,English,Chinese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.12, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - } - }, - "AC3_2": 0.41663070056905466, - "AC3_3": 0.31608570866442576, - "AC3_4": 0.241833252028828, - "AC3_5": 0.18904403863078753, - "AC3_6": 0.15359756673902836, - "AC3_7": 0.1322314049299911 - }, - "prompt_4": { - "overall_acc": 0.3885714285714286, - "language_acc": { - "Spanish": 0.41333333333333333, - "English": 0.5666666666666667, - "Chinese": 0.3, - "Vietnamese": 0.3, - "Filipino": 0.36666666666666664, - "Indonesian": 0.4066666666666667, - "Malay": 0.36666666666666664 - }, - "consistency_score_2": 0.45809523809523806, - "consistency_score_3": 0.26838095238095233, - "consistency_score_4": 0.17790476190476187, - "consistency_score_5": 0.12793650793650793, - "consistency_score_6": 0.0980952380952381, - "consistency_score_7": 0.08, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6, - "Spanish,Chinese": 0.4266666666666667, - "Spanish,Vietnamese": 0.4066666666666667, - "Spanish,Filipino": 0.4533333333333333, - "Spanish,Indonesian": 0.5, - "Spanish,Malay": 0.4866666666666667, - "English,Chinese": 0.38, - "English,Vietnamese": 0.35333333333333333, - "English,Filipino": 0.4866666666666667, - "English,Indonesian": 0.49333333333333335, - "English,Malay": 0.44666666666666666, - "Chinese,Vietnamese": 0.4066666666666667, - "Chinese,Filipino": 0.4066666666666667, - "Chinese,Indonesian": 0.44, - "Chinese,Malay": 0.42, - "Vietnamese,Filipino": 0.42, - "Vietnamese,Indonesian": 0.5, - "Vietnamese,Malay": 0.4066666666666667, - "Filipino,Indonesian": 0.5066666666666667, - "Filipino,Malay": 0.4666666666666667, - "Indonesian,Malay": 0.6133333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.28, - "Spanish,English,Vietnamese": 0.24666666666666667, - "Spanish,English,Filipino": 0.32666666666666666, - "Spanish,English,Indonesian": 0.3466666666666667, - "Spanish,English,Malay": 0.3333333333333333, - "Spanish,Chinese,Vietnamese": 0.24, - "Spanish,Chinese,Filipino": 0.23333333333333334, - "Spanish,Chinese,Indonesian": 0.24666666666666667, - "Spanish,Chinese,Malay": 0.26, - "Spanish,Vietnamese,Filipino": 0.24, - "Spanish,Vietnamese,Indonesian": 0.26, - "Spanish,Vietnamese,Malay": 0.22666666666666666, - "Spanish,Filipino,Indonesian": 0.31333333333333335, - "Spanish,Filipino,Malay": 0.2866666666666667, - "Spanish,Indonesian,Malay": 0.36, - "English,Chinese,Vietnamese": 0.19333333333333333, - "English,Chinese,Filipino": 0.22666666666666666, - "English,Chinese,Indonesian": 0.23333333333333334, - "English,Chinese,Malay": 0.24, - "English,Vietnamese,Filipino": 0.22, - "English,Vietnamese,Indonesian": 0.24666666666666667, - "English,Vietnamese,Malay": 0.21333333333333335, - "English,Filipino,Indonesian": 0.32, - "English,Filipino,Malay": 0.29333333333333333, - "English,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Chinese,Vietnamese,Indonesian": 0.2733333333333333, - "Chinese,Vietnamese,Malay": 0.23333333333333334, - "Chinese,Filipino,Indonesian": 0.24, - "Chinese,Filipino,Malay": 0.21333333333333335, - "Chinese,Indonesian,Malay": 0.31333333333333335, - "Vietnamese,Filipino,Indonesian": 0.28, - "Vietnamese,Filipino,Malay": 0.24, - "Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Filipino,Indonesian,Malay": 0.3466666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.15333333333333332, - "Spanish,English,Chinese,Filipino": 0.18, - "Spanish,English,Chinese,Indonesian": 0.18, - "Spanish,English,Chinese,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian": 0.17333333333333334, - "Spanish,English,Vietnamese,Malay": 0.15333333333333332, - "Spanish,English,Filipino,Indonesian": 0.24666666666666667, - "Spanish,English,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.16, - "Spanish,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.16, - "Spanish,Chinese,Filipino,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.16666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.12, - "English,Chinese,Vietnamese,Indonesian": 0.14666666666666667, - "English,Chinese,Vietnamese,Malay": 0.14, - "English,Chinese,Filipino,Indonesian": 0.16, - "English,Chinese,Filipino,Malay": 0.14666666666666667, - "English,Chinese,Indonesian,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Vietnamese,Filipino,Malay": 0.16, - "English,Vietnamese,Indonesian,Malay": 0.18, - "English,Filipino,Indonesian,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.2 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "English,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - } - }, - "AC3_2": 0.4204788686631766, - "AC3_3": 0.31748167165775815, - "AC3_4": 0.24406571232121524, - "AC3_5": 0.19249451221139255, - "AC3_6": 0.15664523340365272, - "AC3_7": 0.1326829268009518 - }, - "prompt_5": { - "overall_acc": 0.3895238095238095, - "language_acc": { - "Spanish": 0.43333333333333335, - "English": 0.5866666666666667, - "Chinese": 0.2866666666666667, - "Vietnamese": 0.3333333333333333, - "Filipino": 0.34, - "Indonesian": 0.3933333333333333, - "Malay": 0.35333333333333333 - }, - "consistency_score_2": 0.45396825396825385, - "consistency_score_3": 0.26533333333333337, - "consistency_score_4": 0.1756190476190477, - "consistency_score_5": 0.12634920634920635, - "consistency_score_6": 0.09714285714285716, - "consistency_score_7": 0.08, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6266666666666667, - "Spanish,Chinese": 0.42, - "Spanish,Vietnamese": 0.4066666666666667, - "Spanish,Filipino": 0.44666666666666666, - "Spanish,Indonesian": 0.46, - "Spanish,Malay": 0.5133333333333333, - "English,Chinese": 0.35333333333333333, - "English,Vietnamese": 0.36, - "English,Filipino": 0.49333333333333335, - "English,Indonesian": 0.4866666666666667, - "English,Malay": 0.4533333333333333, - "Chinese,Vietnamese": 0.4, - "Chinese,Filipino": 0.36666666666666664, - "Chinese,Indonesian": 0.42, - "Chinese,Malay": 0.4066666666666667, - "Vietnamese,Filipino": 0.4, - "Vietnamese,Indonesian": 0.48, - "Vietnamese,Malay": 0.4266666666666667, - "Filipino,Indonesian": 0.54, - "Filipino,Malay": 0.4533333333333333, - "Indonesian,Malay": 0.62 - }, - "3_combine": { - "Spanish,English,Chinese": 0.2733333333333333, - "Spanish,English,Vietnamese": 0.26, - "Spanish,English,Filipino": 0.34, - "Spanish,English,Indonesian": 0.34, - "Spanish,English,Malay": 0.36666666666666664, - "Spanish,Chinese,Vietnamese": 0.24, - "Spanish,Chinese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Filipino": 0.22, - "Spanish,Vietnamese,Indonesian": 0.26, - "Spanish,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Filipino,Indonesian": 0.3, - "Spanish,Filipino,Malay": 0.29333333333333333, - "Spanish,Indonesian,Malay": 0.36, - "English,Chinese,Vietnamese": 0.19333333333333333, - "English,Chinese,Filipino": 0.2, - "English,Chinese,Indonesian": 0.20666666666666667, - "English,Chinese,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino": 0.21333333333333335, - "English,Vietnamese,Indonesian": 0.24666666666666667, - "English,Vietnamese,Malay": 0.23333333333333334, - "English,Filipino,Indonesian": 0.32, - "English,Filipino,Malay": 0.29333333333333333, - "English,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino": 0.2, - "Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Chinese,Vietnamese,Malay": 0.24666666666666667, - "Chinese,Filipino,Indonesian": 0.23333333333333334, - "Chinese,Filipino,Malay": 0.18666666666666668, - "Chinese,Indonesian,Malay": 0.3, - "Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Vietnamese,Filipino,Malay": 0.23333333333333334, - "Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Filipino,Indonesian,Malay": 0.36 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.16, - "Spanish,English,Chinese,Filipino": 0.16, - "Spanish,English,Chinese,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian": 0.17333333333333334, - "Spanish,English,Vietnamese,Malay": 0.17333333333333334, - "Spanish,English,Filipino,Indonesian": 0.24, - "Spanish,English,Filipino,Malay": 0.24, - "Spanish,English,Indonesian,Malay": 0.2733333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.16, - "Spanish,Vietnamese,Indonesian,Malay": 0.2, - "Spanish,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino": 0.12, - "English,Chinese,Vietnamese,Indonesian": 0.14, - "English,Chinese,Vietnamese,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian": 0.13333333333333333, - "English,Chinese,Filipino,Malay": 0.13333333333333333, - "English,Chinese,Indonesian,Malay": 0.18, - "English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "English,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.2, - "Chinese,Filipino,Indonesian,Malay": 0.16666666666666666, - "Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Vietnamese,Filipino,Malay": 0.12, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Filipino,Indonesian,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.12, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.1, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian,Malay": 0.12, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - } - }, - "AC3_2": 0.41928419085661295, - "AC3_3": 0.3156525111016917, - "AC3_4": 0.24209029478759947, - "AC3_5": 0.1908067398897528, - "AC3_6": 0.15550461277207275, - "AC3_7": 0.1327383366857251 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.39123376623376627, - "language_acc": { - "English": 0.6136363636363636, - "Filipino": 0.375, - "Spanish": 0.5, - "Malay": 0.3522727272727273, - "Chinese": 0.25, - "Indonesian": 0.3465909090909091, - "Vietnamese": 0.30113636363636365 - }, - "consistency_score_2": 0.3825757575757576, - "consistency_score_3": 0.18295454545454545, - "consistency_score_4": 0.09756493506493508, - "consistency_score_5": 0.05303030303030302, - "consistency_score_6": 0.025162337662337664, - "consistency_score_7": 0.005681818181818182, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3465909090909091, - "English,Spanish": 0.6534090909090909, - "English,Malay": 0.3977272727272727, - "English,Chinese": 0.2556818181818182, - "English,Indonesian": 0.4034090909090909, - "English,Vietnamese": 0.2784090909090909, - "Filipino,Spanish": 0.4431818181818182, - "Filipino,Malay": 0.48863636363636365, - "Filipino,Chinese": 0.22727272727272727, - "Filipino,Indonesian": 0.44886363636363635, - "Filipino,Vietnamese": 0.4943181818181818, - "Spanish,Malay": 0.42045454545454547, - "Spanish,Chinese": 0.22727272727272727, - "Spanish,Indonesian": 0.5113636363636364, - "Spanish,Vietnamese": 0.39204545454545453, - "Malay,Chinese": 0.2215909090909091, - "Malay,Indonesian": 0.5795454545454546, - "Malay,Vietnamese": 0.4147727272727273, - "Chinese,Indonesian": 0.24431818181818182, - "Chinese,Vietnamese": 0.19886363636363635, - "Indonesian,Vietnamese": 0.38636363636363635 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2840909090909091, - "English,Filipino,Malay": 0.2215909090909091, - "English,Filipino,Chinese": 0.06818181818181818, - "English,Filipino,Indonesian": 0.19318181818181818, - "English,Filipino,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay": 0.29545454545454547, - "English,Spanish,Chinese": 0.14204545454545456, - "English,Spanish,Indonesian": 0.3352272727272727, - "English,Spanish,Vietnamese": 0.24431818181818182, - "English,Malay,Chinese": 0.07954545454545454, - "English,Malay,Indonesian": 0.2556818181818182, - "English,Malay,Vietnamese": 0.1590909090909091, - "English,Chinese,Indonesian": 0.09090909090909091, - "English,Chinese,Vietnamese": 0.056818181818181816, - "English,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay": 0.2556818181818182, - "Filipino,Spanish,Chinese": 0.10227272727272728, - "Filipino,Spanish,Indonesian": 0.26136363636363635, - "Filipino,Spanish,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese": 0.11363636363636363, - "Filipino,Malay,Indonesian": 0.3068181818181818, - "Filipino,Malay,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Indonesian,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Chinese": 0.08522727272727272, - "Spanish,Malay,Indonesian": 0.29545454545454547, - "Spanish,Malay,Vietnamese": 0.2159090909090909, - "Spanish,Chinese,Indonesian": 0.125, - "Spanish,Chinese,Vietnamese": 0.08522727272727272, - "Spanish,Indonesian,Vietnamese": 0.24431818181818182, - "Malay,Chinese,Indonesian": 0.1534090909090909, - "Malay,Chinese,Vietnamese": 0.09659090909090909, - "Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Chinese,Indonesian,Vietnamese": 0.09659090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.18181818181818182, - "English,Filipino,Spanish,Chinese": 0.05113636363636364, - "English,Filipino,Spanish,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese": 0.03977272727272727, - "English,Filipino,Malay,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese": 0.05113636363636364, - "English,Spanish,Malay,Indonesian": 0.2159090909090909, - "English,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Spanish,Chinese,Vietnamese": 0.045454545454545456, - "English,Spanish,Indonesian,Vietnamese": 0.1590909090909091, - "English,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese": 0.05113636363636364, - "Filipino,Spanish,Malay,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian": 0.0625, - "Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Malay,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.0625, - "Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Spanish,Malay,Chinese,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 - } - }, - "AC3_2": 0.3868563254426954, - "AC3_3": 0.24931888858844461, - "AC3_4": 0.1561816628750512, - "AC3_5": 0.09340050933563672, - "AC3_6": 0.04728361306173132, - "AC3_7": 0.011200966719617114 - }, - "prompt_2": { - "overall_acc": 0.3928571428571429, - "language_acc": { - "English": 0.6022727272727273, - "Filipino": 0.38636363636363635, - "Spanish": 0.48863636363636365, - "Malay": 0.3465909090909091, - "Chinese": 0.26136363636363635, - "Indonesian": 0.36363636363636365, - "Vietnamese": 0.30113636363636365 - }, - "consistency_score_2": 0.3831168831168831, - "consistency_score_3": 0.17987012987012987, - "consistency_score_4": 0.09334415584415587, - "consistency_score_5": 0.049783549783549784, - "consistency_score_6": 0.023538961038961043, - "consistency_score_7": 0.005681818181818182, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.36363636363636365, - "English,Spanish": 0.6477272727272727, - "English,Malay": 0.3977272727272727, - "English,Chinese": 0.26136363636363635, - "English,Indonesian": 0.42045454545454547, - "English,Vietnamese": 0.30113636363636365, - "Filipino,Spanish": 0.4431818181818182, - "Filipino,Malay": 0.45454545454545453, - "Filipino,Chinese": 0.2159090909090909, - "Filipino,Indonesian": 0.48295454545454547, - "Filipino,Vietnamese": 0.48295454545454547, - "Spanish,Malay": 0.39204545454545453, - "Spanish,Chinese": 0.25, - "Spanish,Indonesian": 0.5, - "Spanish,Vietnamese": 0.375, - "Malay,Chinese": 0.24431818181818182, - "Malay,Indonesian": 0.5852272727272727, - "Malay,Vietnamese": 0.4147727272727273, - "Chinese,Indonesian": 0.23295454545454544, - "Chinese,Vietnamese": 0.18181818181818182, - "Indonesian,Vietnamese": 0.3977272727272727 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Malay": 0.21022727272727273, - "English,Filipino,Chinese": 0.07386363636363637, - "English,Filipino,Indonesian": 0.2159090909090909, - "English,Filipino,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay": 0.2784090909090909, - "English,Spanish,Chinese": 0.1534090909090909, - "English,Spanish,Indonesian": 0.3352272727272727, - "English,Spanish,Vietnamese": 0.23863636363636365, - "English,Malay,Chinese": 0.08522727272727272, - "English,Malay,Indonesian": 0.2727272727272727, - "English,Malay,Vietnamese": 0.1534090909090909, - "English,Chinese,Indonesian": 0.09090909090909091, - "English,Chinese,Vietnamese": 0.056818181818181816, - "English,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay": 0.2215909090909091, - "Filipino,Spanish,Chinese": 0.10227272727272728, - "Filipino,Spanish,Indonesian": 0.26704545454545453, - "Filipino,Spanish,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese": 0.10795454545454546, - "Filipino,Malay,Indonesian": 0.29545454545454547, - "Filipino,Malay,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Indonesian": 0.10227272727272728, - "Filipino,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Indonesian,Vietnamese": 0.23863636363636365, - "Spanish,Malay,Chinese": 0.09659090909090909, - "Spanish,Malay,Indonesian": 0.2897727272727273, - "Spanish,Malay,Vietnamese": 0.19318181818181818, - "Spanish,Chinese,Indonesian": 0.125, - "Spanish,Chinese,Vietnamese": 0.07386363636363637, - "Spanish,Indonesian,Vietnamese": 0.23295454545454544, - "Malay,Chinese,Indonesian": 0.16477272727272727, - "Malay,Chinese,Vietnamese": 0.09659090909090909, - "Malay,Indonesian,Vietnamese": 0.2556818181818182, - "Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.1590909090909091, - "English,Filipino,Spanish,Chinese": 0.0625, - "English,Filipino,Spanish,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese": 0.03977272727272727, - "English,Filipino,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Malay,Chinese": 0.06818181818181818, - "English,Spanish,Malay,Indonesian": 0.2159090909090909, - "English,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Indonesian": 0.07386363636363637, - "English,Spanish,Chinese,Vietnamese": 0.03977272727272727, - "English,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "English,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese": 0.045454545454545456, - "Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.05113636363636364, - "Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.06818181818181818, - "Filipino,Malay,Chinese,Vietnamese": 0.056818181818181816, - "Filipino,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian": 0.08522727272727272, - "Spanish,Malay,Chinese,Vietnamese": 0.05113636363636364, - "Spanish,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Malay,Chinese,Indonesian,Vietnamese": 0.0625 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 - } - }, - "AC3_2": 0.3879258815997389, - "AC3_3": 0.24676060896242183, - "AC3_4": 0.15084664914617457, - "AC3_5": 0.08836884385010235, - "AC3_6": 0.044416597037509095, - "AC3_7": 0.011201629325091568 - }, - "prompt_3": { - "overall_acc": 0.390422077922078, - "language_acc": { - "English": 0.6079545454545454, - "Filipino": 0.3806818181818182, - "Spanish": 0.5113636363636364, - "Malay": 0.3352272727272727, - "Chinese": 0.24431818181818182, - "Indonesian": 0.36363636363636365, - "Vietnamese": 0.2897727272727273 - }, - "consistency_score_2": 0.39475108225108224, - "consistency_score_3": 0.1949675324675325, - "consistency_score_4": 0.10746753246753246, - "consistency_score_5": 0.06087662337662338, - "consistency_score_6": 0.03165584415584415, - "consistency_score_7": 0.011363636363636364, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.35795454545454547, - "English,Spanish": 0.6534090909090909, - "English,Malay": 0.42613636363636365, - "English,Chinese": 0.26136363636363635, - "English,Indonesian": 0.4318181818181818, - "English,Vietnamese": 0.2784090909090909, - "Filipino,Spanish": 0.42613636363636365, - "Filipino,Malay": 0.4715909090909091, - "Filipino,Chinese": 0.2556818181818182, - "Filipino,Indonesian": 0.4659090909090909, - "Filipino,Vietnamese": 0.4715909090909091, - "Spanish,Malay": 0.4090909090909091, - "Spanish,Chinese": 0.25, - "Spanish,Indonesian": 0.5113636363636364, - "Spanish,Vietnamese": 0.3977272727272727, - "Malay,Chinese": 0.26704545454545453, - "Malay,Indonesian": 0.6022727272727273, - "Malay,Vietnamese": 0.42613636363636365, - "Chinese,Indonesian": 0.26136363636363635, - "Chinese,Vietnamese": 0.23863636363636365, - "Indonesian,Vietnamese": 0.42613636363636365 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2840909090909091, - "English,Filipino,Malay": 0.22727272727272727, - "English,Filipino,Chinese": 0.09090909090909091, - "English,Filipino,Indonesian": 0.20454545454545456, - "English,Filipino,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay": 0.3068181818181818, - "English,Spanish,Chinese": 0.16477272727272727, - "English,Spanish,Indonesian": 0.3522727272727273, - "English,Spanish,Vietnamese": 0.24431818181818182, - "English,Malay,Chinese": 0.11363636363636363, - "English,Malay,Indonesian": 0.2897727272727273, - "English,Malay,Vietnamese": 0.1590909090909091, - "English,Chinese,Indonesian": 0.11931818181818182, - "English,Chinese,Vietnamese": 0.07386363636363637, - "English,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay": 0.23863636363636365, - "Filipino,Spanish,Chinese": 0.11363636363636363, - "Filipino,Spanish,Indonesian": 0.2556818181818182, - "Filipino,Spanish,Vietnamese": 0.22727272727272727, - "Filipino,Malay,Chinese": 0.125, - "Filipino,Malay,Indonesian": 0.3181818181818182, - "Filipino,Malay,Vietnamese": 0.2556818181818182, - "Filipino,Chinese,Indonesian": 0.13068181818181818, - "Filipino,Chinese,Vietnamese": 0.125, - "Filipino,Indonesian,Vietnamese": 0.25, - "Spanish,Malay,Chinese": 0.10227272727272728, - "Spanish,Malay,Indonesian": 0.30113636363636365, - "Spanish,Malay,Vietnamese": 0.2215909090909091, - "Spanish,Chinese,Indonesian": 0.14204545454545456, - "Spanish,Chinese,Vietnamese": 0.09659090909090909, - "Spanish,Indonesian,Vietnamese": 0.2556818181818182, - "Malay,Chinese,Indonesian": 0.16477272727272727, - "Malay,Chinese,Vietnamese": 0.13636363636363635, - "Malay,Indonesian,Vietnamese": 0.2897727272727273, - "Chinese,Indonesian,Vietnamese": 0.125 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Indonesian": 0.17045454545454544, - "English,Filipino,Spanish,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese": 0.056818181818181816, - "English,Filipino,Malay,Indonesian": 0.14772727272727273, - "English,Filipino,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese": 0.08522727272727272, - "English,Spanish,Malay,Indonesian": 0.23863636363636365, - "English,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Spanish,Chinese,Indonesian": 0.09659090909090909, - "English,Spanish,Chinese,Vietnamese": 0.0625, - "English,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "English,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Malay,Chinese,Vietnamese": 0.0625, - "English,Malay,Indonesian,Vietnamese": 0.125, - "English,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Malay,Chinese": 0.056818181818181816, - "Filipino,Spanish,Malay,Indonesian": 0.18181818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Spanish,Chinese,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1590909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Filipino,Malay,Indonesian,Vietnamese": 0.1875, - "Filipino,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Spanish,Malay,Chinese,Indonesian": 0.09090909090909091, - "Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Indonesian": 0.03977272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.045454545454545456, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 - } - }, - "AC3_2": 0.39257464618591587, - "AC3_3": 0.26006484498298155, - "AC3_4": 0.16854216858983018, - "AC3_5": 0.10532969725779691, - "AC3_6": 0.05856331167443668, - "AC3_7": 0.022084481169893685 - }, - "prompt_4": { - "overall_acc": 0.3871753246753246, - "language_acc": { - "English": 0.5965909090909091, - "Filipino": 0.3806818181818182, - "Spanish": 0.48863636363636365, - "Malay": 0.32954545454545453, - "Chinese": 0.25, - "Indonesian": 0.3693181818181818, - "Vietnamese": 0.29545454545454547 - }, - "consistency_score_2": 0.3893398268398268, - "consistency_score_3": 0.18749999999999997, - "consistency_score_4": 0.10048701298701294, - "consistency_score_5": 0.056006493506493504, - "consistency_score_6": 0.02922077922077922, - "consistency_score_7": 0.011363636363636364, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.36363636363636365, - "English,Spanish": 0.6477272727272727, - "English,Malay": 0.42045454545454547, - "English,Chinese": 0.2727272727272727, - "English,Indonesian": 0.42613636363636365, - "English,Vietnamese": 0.2840909090909091, - "Filipino,Spanish": 0.42045454545454547, - "Filipino,Malay": 0.4659090909090909, - "Filipino,Chinese": 0.24431818181818182, - "Filipino,Indonesian": 0.4431818181818182, - "Filipino,Vietnamese": 0.48295454545454547, - "Spanish,Malay": 0.39204545454545453, - "Spanish,Chinese": 0.2556818181818182, - "Spanish,Indonesian": 0.5113636363636364, - "Spanish,Vietnamese": 0.4090909090909091, - "Malay,Chinese": 0.25, - "Malay,Indonesian": 0.5909090909090909, - "Malay,Vietnamese": 0.42613636363636365, - "Chinese,Indonesian": 0.26136363636363635, - "Chinese,Vietnamese": 0.19886363636363635, - "Indonesian,Vietnamese": 0.4090909090909091 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2784090909090909, - "English,Filipino,Malay": 0.2215909090909091, - "English,Filipino,Chinese": 0.08522727272727272, - "English,Filipino,Indonesian": 0.21022727272727273, - "English,Filipino,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay": 0.29545454545454547, - "English,Spanish,Chinese": 0.17045454545454544, - "English,Spanish,Indonesian": 0.3465909090909091, - "English,Spanish,Vietnamese": 0.2556818181818182, - "English,Malay,Chinese": 0.09659090909090909, - "English,Malay,Indonesian": 0.2897727272727273, - "English,Malay,Vietnamese": 0.1590909090909091, - "English,Chinese,Indonesian": 0.10795454545454546, - "English,Chinese,Vietnamese": 0.056818181818181816, - "English,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Malay": 0.22727272727272727, - "Filipino,Spanish,Chinese": 0.10795454545454546, - "Filipino,Spanish,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Chinese": 0.125, - "Filipino,Malay,Indonesian": 0.30113636363636365, - "Filipino,Malay,Vietnamese": 0.25, - "Filipino,Chinese,Indonesian": 0.11931818181818182, - "Filipino,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Indonesian,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Chinese": 0.09659090909090909, - "Spanish,Malay,Indonesian": 0.29545454545454547, - "Spanish,Malay,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Indonesian": 0.13068181818181818, - "Spanish,Chinese,Vietnamese": 0.09090909090909091, - "Spanish,Indonesian,Vietnamese": 0.25, - "Malay,Chinese,Indonesian": 0.16477272727272727, - "Malay,Chinese,Vietnamese": 0.11363636363636363, - "Malay,Indonesian,Vietnamese": 0.2784090909090909, - "Chinese,Indonesian,Vietnamese": 0.10227272727272728 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.16477272727272727, - "English,Filipino,Spanish,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Chinese": 0.05113636363636364, - "English,Filipino,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Chinese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian": 0.23295454545454544, - "English,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Spanish,Chinese,Indonesian": 0.09090909090909091, - "English,Spanish,Chinese,Vietnamese": 0.05113636363636364, - "English,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "English,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Malay,Indonesian,Vietnamese": 0.125, - "English,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese": 0.056818181818181816, - "Filipino,Spanish,Malay,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Malay,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.0625, - "Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Spanish,Malay,Chinese,Vietnamese": 0.0625, - "Spanish,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian": 0.03977272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 - } - }, - "AC3_2": 0.3882545589798207, - "AC3_3": 0.25264830504078206, - "AC3_4": 0.15956160181416837, - "AC3_5": 0.09785749962113326, - "AC3_6": 0.054340396432609434, - "AC3_7": 0.022079244578796186 - }, - "prompt_5": { - "overall_acc": 0.39366883116883117, - "language_acc": { - "English": 0.6022727272727273, - "Filipino": 0.36363636363636365, - "Spanish": 0.5056818181818182, - "Malay": 0.3352272727272727, - "Chinese": 0.2784090909090909, - "Indonesian": 0.3693181818181818, - "Vietnamese": 0.30113636363636365 - }, - "consistency_score_2": 0.3906926406926408, - "consistency_score_3": 0.18928571428571433, - "consistency_score_4": 0.10178571428571427, - "consistency_score_5": 0.05654761904761903, - "consistency_score_6": 0.02922077922077922, - "consistency_score_7": 0.011363636363636364, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3465909090909091, - "English,Spanish": 0.6647727272727273, - "English,Malay": 0.42045454545454547, - "English,Chinese": 0.24431818181818182, - "English,Indonesian": 0.42045454545454547, - "English,Vietnamese": 0.3068181818181818, - "Filipino,Spanish": 0.42045454545454547, - "Filipino,Malay": 0.4602272727272727, - "Filipino,Chinese": 0.25, - "Filipino,Indonesian": 0.4659090909090909, - "Filipino,Vietnamese": 0.48863636363636365, - "Spanish,Malay": 0.42045454545454547, - "Spanish,Chinese": 0.23295454545454544, - "Spanish,Indonesian": 0.48295454545454547, - "Spanish,Vietnamese": 0.4034090909090909, - "Malay,Chinese": 0.2556818181818182, - "Malay,Indonesian": 0.5909090909090909, - "Malay,Vietnamese": 0.4318181818181818, - "Chinese,Indonesian": 0.23295454545454544, - "Chinese,Vietnamese": 0.23295454545454544, - "Indonesian,Vietnamese": 0.4318181818181818 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2727272727272727, - "English,Filipino,Malay": 0.2159090909090909, - "English,Filipino,Chinese": 0.06818181818181818, - "English,Filipino,Indonesian": 0.20454545454545456, - "English,Filipino,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay": 0.3125, - "English,Spanish,Chinese": 0.1534090909090909, - "English,Spanish,Indonesian": 0.3352272727272727, - "English,Spanish,Vietnamese": 0.26136363636363635, - "English,Malay,Chinese": 0.10227272727272728, - "English,Malay,Indonesian": 0.2840909090909091, - "English,Malay,Vietnamese": 0.16477272727272727, - "English,Chinese,Indonesian": 0.08522727272727272, - "English,Chinese,Vietnamese": 0.07386363636363637, - "English,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay": 0.23295454545454544, - "Filipino,Spanish,Chinese": 0.09659090909090909, - "Filipino,Spanish,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese": 0.13068181818181818, - "Filipino,Malay,Indonesian": 0.30113636363636365, - "Filipino,Malay,Vietnamese": 0.26136363636363635, - "Filipino,Chinese,Indonesian": 0.11931818181818182, - "Filipino,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Indonesian,Vietnamese": 0.2556818181818182, - "Spanish,Malay,Chinese": 0.10227272727272728, - "Spanish,Malay,Indonesian": 0.29545454545454547, - "Spanish,Malay,Vietnamese": 0.22727272727272727, - "Spanish,Chinese,Indonesian": 0.10795454545454546, - "Spanish,Chinese,Vietnamese": 0.08522727272727272, - "Spanish,Indonesian,Vietnamese": 0.26136363636363635, - "Malay,Chinese,Indonesian": 0.1590909090909091, - "Malay,Chinese,Vietnamese": 0.125, - "Malay,Indonesian,Vietnamese": 0.2840909090909091, - "Chinese,Indonesian,Vietnamese": 0.10795454545454546 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.17045454545454544, - "English,Filipino,Spanish,Chinese": 0.056818181818181816, - "English,Filipino,Spanish,Indonesian": 0.16477272727272727, - "English,Filipino,Spanish,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Chinese": 0.045454545454545456, - "English,Filipino,Malay,Indonesian": 0.14204545454545456, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Indonesian,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Chinese": 0.07386363636363637, - "English,Spanish,Malay,Indonesian": 0.22727272727272727, - "English,Spanish,Malay,Vietnamese": 0.1534090909090909, - "English,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Spanish,Chinese,Vietnamese": 0.05113636363636364, - "English,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "English,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Malay,Indonesian,Vietnamese": 0.125, - "English,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese": 0.056818181818181816, - "Filipino,Spanish,Malay,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Malay,Chinese,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Spanish,Malay,Chinese,Vietnamese": 0.06818181818181818, - "Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.03409090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10227272727272728, - "English,Filipino,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.11931818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.05113636363636364 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 - } - }, - "AC3_2": 0.3921750894345073, - "AC3_3": 0.2556490431991742, - "AC3_4": 0.16174990635192538, - "AC3_5": 0.09889036742308983, - "AC3_6": 0.054403370129469, - "AC3_7": 0.022089633807626918 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.5631067961165048 - }, - "prompt_2": { - "accuracy": 0.5533980582524272 - }, - "prompt_3": { - "accuracy": 0.5533980582524272 - }, - "prompt_4": { - "accuracy": 0.5728155339805825 - }, - "prompt_5": { - "accuracy": 0.5631067961165048 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.20952380952380953 - }, - "prompt_2": { - "accuracy": 0.2571428571428571 - }, - "prompt_3": { - "accuracy": 0.18095238095238095 - }, - "prompt_4": { - "accuracy": 0.2 - }, - "prompt_5": { - "accuracy": 0.20952380952380953 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.6542056074766355 - }, - "prompt_2": { - "accuracy": 0.6355140186915887 - }, - "prompt_3": { - "accuracy": 0.6728971962616822 - }, - "prompt_4": { - "accuracy": 0.6915887850467289 - }, - "prompt_5": { - "accuracy": 0.6728971962616822 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.32, - "category_acc": { - "brand": 0.2, - "demographics": 0.0, - "biology": 0.4, - "history": 0.13333333333333333, - "literature": 0.3, - "politics": 0.7, - "culture": 0.4, - "film": 0.3, - "law": 0.2, - "geography": 0.5 - } - }, - "prompt_2": { - "accuracy": 0.29, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.4, - "history": 0.06666666666666667, - "literature": 0.3, - "politics": 0.5, - "culture": 0.4, - "film": 0.3, - "law": 0.2, - "geography": 0.4 - } - }, - "prompt_3": { - "accuracy": 0.34, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.4, - "history": 0.13333333333333333, - "literature": 0.3, - "politics": 0.7, - "culture": 0.4, - "film": 0.3, - "law": 0.2, - "geography": 0.5 - } - }, - "prompt_4": { - "accuracy": 0.35, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.5, - "history": 0.2, - "literature": 0.3, - "politics": 0.7, - "culture": 0.4, - "film": 0.3, - "law": 0.2, - "geography": 0.5 - } - }, - "prompt_5": { - "accuracy": 0.35, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.5, - "history": 0.13333333333333333, - "literature": 0.4, - "politics": 0.6, - "culture": 0.4, - "film": 0.3, - "law": 0.2, - "geography": 0.5 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.17143237165650724 - }, - "prompt_2": { - "bleu_score": 0.17307422379138457 - }, - "prompt_3": { - "bleu_score": 0.17719935633819425 - }, - "prompt_4": { - "bleu_score": 0.17445808927858594 - }, - "prompt_5": { - "bleu_score": 0.17399848467748025 - } - }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.2891201682503245 - }, - "prompt_2": { - "bleu_score": 0.28947688738366734 - }, - "prompt_3": { - "bleu_score": 0.2886695188383577 - }, - "prompt_4": { - "bleu_score": 0.29289408869856 - }, - "prompt_5": { - "bleu_score": 0.27469574157293974 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.07160237729066059 - }, - "prompt_2": { - "bleu_score": 0.0712988777030395 - }, - "prompt_3": { - "bleu_score": 0.07060145981772302 - }, - "prompt_4": { - "bleu_score": 0.07153867390013002 - }, - "prompt_5": { - "bleu_score": 0.06618030887167185 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.018204289503351286 - }, - "prompt_2": { - "bleu_score": 0.01861742796466744 - }, - "prompt_3": { - "bleu_score": 0.01940162741768547 - }, - "prompt_4": { - "bleu_score": 0.015578980647901785 - }, - "prompt_5": { - "bleu_score": 0.01868511562736815 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.2715175767360156 - }, - "prompt_2": { - "bleu_score": 0.2689976754066324 - }, - "prompt_3": { - "bleu_score": 0.2695038726309589 - }, - "prompt_4": { - "bleu_score": 0.27141104372994485 - }, - "prompt_5": { - "bleu_score": 0.25975478216624065 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.5519253208868145 - }, - "prompt_2": { - "accuracy": 0.5565927654609102 - }, - "prompt_3": { - "accuracy": 0.5460910151691949 - }, - "prompt_4": { - "accuracy": 0.5472578763127188 - }, - "prompt_5": { - "accuracy": 0.558926487747958 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.5392920986771541, - "category_acc": { - "high_school_european_history": 0.7378048780487805, - "business_ethics": 0.7070707070707071, - "clinical_knowledge": 0.6060606060606061, - "medical_genetics": 0.5252525252525253, - "high_school_us_history": 0.7684729064039408, - "high_school_physics": 0.28, - "high_school_world_history": 0.7754237288135594, - "virology": 0.4909090909090909, - "high_school_microeconomics": 0.5864978902953587, - "econometrics": 0.35398230088495575, - "college_computer_science": 0.4444444444444444, - "high_school_biology": 0.6634304207119741, - "abstract_algebra": 0.20202020202020202, - "professional_accounting": 0.4306049822064057, - "philosophy": 0.5580645161290323, - "professional_medicine": 0.5571955719557196, - "nutrition": 0.6065573770491803, - "global_facts": 0.2828282828282828, - "machine_learning": 0.40540540540540543, - "security_studies": 0.6557377049180327, - "public_relations": 0.5504587155963303, - "professional_psychology": 0.5515548281505729, - "prehistory": 0.5944272445820433, - "anatomy": 0.47761194029850745, - "human_sexuality": 0.6538461538461539, - "college_medicine": 0.48255813953488375, - "high_school_government_and_politics": 0.8333333333333334, - "college_chemistry": 0.3838383838383838, - "logical_fallacies": 0.6666666666666666, - "high_school_geography": 0.7258883248730964, - "elementary_mathematics": 0.3740053050397878, - "human_aging": 0.6036036036036037, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.7794117647058824, - "formal_logic": 0.288, - "high_school_statistics": 0.4, - "international_law": 0.7, - "high_school_mathematics": 0.2379182156133829, - "high_school_computer_science": 0.45454545454545453, - "conceptual_physics": 0.452991452991453, - "miscellaneous": 0.7186700767263428, - "high_school_chemistry": 0.40594059405940597, - "marketing": 0.8369098712446352, - "professional_law": 0.4246575342465753, - "management": 0.7941176470588235, - "college_physics": 0.42574257425742573, - "jurisprudence": 0.616822429906542, - "world_religions": 0.7, - "sociology": 0.68, - "us_foreign_policy": 0.7474747474747475, - "high_school_macroeconomics": 0.5295629820051414, - "computer_security": 0.6363636363636364, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.6405797101449275, - "electrical_engineering": 0.5277777777777778, - "astronomy": 0.5629139072847682, - "college_biology": 0.5104895104895105 - } - }, - "prompt_2": { - "accuracy": 0.5387200572041473, - "category_acc": { - "high_school_european_history": 0.7439024390243902, - "business_ethics": 0.7171717171717171, - "clinical_knowledge": 0.6022727272727273, - "medical_genetics": 0.5252525252525253, - "high_school_us_history": 0.7684729064039408, - "high_school_physics": 0.26, - "high_school_world_history": 0.7627118644067796, - "virology": 0.47878787878787876, - "high_school_microeconomics": 0.5738396624472574, - "econometrics": 0.34513274336283184, - "college_computer_science": 0.45454545454545453, - "high_school_biology": 0.6601941747572816, - "abstract_algebra": 0.1919191919191919, - "professional_accounting": 0.4234875444839858, - "philosophy": 0.5709677419354838, - "professional_medicine": 0.5682656826568265, - "nutrition": 0.6032786885245902, - "global_facts": 0.2727272727272727, - "machine_learning": 0.40540540540540543, - "security_studies": 0.6639344262295082, - "public_relations": 0.5412844036697247, - "professional_psychology": 0.5482815057283142, - "prehistory": 0.5944272445820433, - "anatomy": 0.4925373134328358, - "human_sexuality": 0.6384615384615384, - "college_medicine": 0.48255813953488375, - "high_school_government_and_politics": 0.8333333333333334, - "college_chemistry": 0.40404040404040403, - "logical_fallacies": 0.6604938271604939, - "high_school_geography": 0.7055837563451777, - "elementary_mathematics": 0.3687002652519894, - "human_aging": 0.5900900900900901, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.7738970588235294, - "formal_logic": 0.312, - "high_school_statistics": 0.386046511627907, - "international_law": 0.7, - "high_school_mathematics": 0.24907063197026022, - "high_school_computer_science": 0.43434343434343436, - "conceptual_physics": 0.452991452991453, - "miscellaneous": 0.7186700767263428, - "high_school_chemistry": 0.400990099009901, - "marketing": 0.8326180257510729, - "professional_law": 0.4298760600130463, - "management": 0.803921568627451, - "college_physics": 0.40594059405940597, - "jurisprudence": 0.6448598130841121, - "world_religions": 0.6882352941176471, - "sociology": 0.69, - "us_foreign_policy": 0.7474747474747475, - "high_school_macroeconomics": 0.5398457583547558, - "computer_security": 0.6464646464646465, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.6492753623188405, - "electrical_engineering": 0.4861111111111111, - "astronomy": 0.5695364238410596, - "college_biology": 0.5104895104895105 - } - }, - "prompt_3": { - "accuracy": 0.5382195209152664, - "category_acc": { - "high_school_european_history": 0.75, - "business_ethics": 0.7070707070707071, - "clinical_knowledge": 0.6060606060606061, - "medical_genetics": 0.5151515151515151, - "high_school_us_history": 0.7733990147783252, - "high_school_physics": 0.28, - "high_school_world_history": 0.7669491525423728, - "virology": 0.48484848484848486, - "high_school_microeconomics": 0.569620253164557, - "econometrics": 0.34513274336283184, - "college_computer_science": 0.4444444444444444, - "high_school_biology": 0.6731391585760518, - "abstract_algebra": 0.21212121212121213, - "professional_accounting": 0.4234875444839858, - "philosophy": 0.5741935483870968, - "professional_medicine": 0.5682656826568265, - "nutrition": 0.6098360655737705, - "global_facts": 0.2727272727272727, - "machine_learning": 0.3783783783783784, - "security_studies": 0.6557377049180327, - "public_relations": 0.5779816513761468, - "professional_psychology": 0.5433715220949263, - "prehistory": 0.5820433436532507, - "anatomy": 0.4701492537313433, - "human_sexuality": 0.6384615384615384, - "college_medicine": 0.47093023255813954, - "high_school_government_and_politics": 0.84375, - "college_chemistry": 0.3939393939393939, - "logical_fallacies": 0.6604938271604939, - "high_school_geography": 0.7208121827411168, - "elementary_mathematics": 0.36074270557029176, - "human_aging": 0.5765765765765766, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.78125, - "formal_logic": 0.328, - "high_school_statistics": 0.386046511627907, - "international_law": 0.7, - "high_school_mathematics": 0.241635687732342, - "high_school_computer_science": 0.4444444444444444, - "conceptual_physics": 0.43162393162393164, - "miscellaneous": 0.7071611253196931, - "high_school_chemistry": 0.40594059405940597, - "marketing": 0.8326180257510729, - "professional_law": 0.43118069145466403, - "management": 0.7745098039215687, - "college_physics": 0.42574257425742573, - "jurisprudence": 0.6261682242990654, - "world_religions": 0.6882352941176471, - "sociology": 0.69, - "us_foreign_policy": 0.7575757575757576, - "high_school_macroeconomics": 0.5372750642673522, - "computer_security": 0.6363636363636364, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.6550724637681159, - "electrical_engineering": 0.5069444444444444, - "astronomy": 0.5827814569536424, - "college_biology": 0.4965034965034965 - } - }, - "prompt_4": { - "accuracy": 0.5392205934930282, - "category_acc": { - "high_school_european_history": 0.75, - "business_ethics": 0.7272727272727273, - "clinical_knowledge": 0.6060606060606061, - "medical_genetics": 0.5151515151515151, - "high_school_us_history": 0.7684729064039408, - "high_school_physics": 0.2733333333333333, - "high_school_world_history": 0.7627118644067796, - "virology": 0.48484848484848486, - "high_school_microeconomics": 0.5780590717299579, - "econometrics": 0.336283185840708, - "college_computer_science": 0.46464646464646464, - "high_school_biology": 0.6666666666666666, - "abstract_algebra": 0.18181818181818182, - "professional_accounting": 0.4377224199288256, - "philosophy": 0.5741935483870968, - "professional_medicine": 0.5535055350553506, - "nutrition": 0.6131147540983607, - "global_facts": 0.2727272727272727, - "machine_learning": 0.38738738738738737, - "security_studies": 0.6639344262295082, - "public_relations": 0.5504587155963303, - "professional_psychology": 0.5499181669394435, - "prehistory": 0.5913312693498453, - "anatomy": 0.4701492537313433, - "human_sexuality": 0.6384615384615384, - "college_medicine": 0.47093023255813954, - "high_school_government_and_politics": 0.828125, - "college_chemistry": 0.3939393939393939, - "logical_fallacies": 0.6728395061728395, - "high_school_geography": 0.7309644670050761, - "elementary_mathematics": 0.3687002652519894, - "human_aging": 0.5630630630630631, - "college_mathematics": 0.32323232323232326, - "high_school_psychology": 0.78125, - "formal_logic": 0.32, - "high_school_statistics": 0.4046511627906977, - "international_law": 0.7166666666666667, - "high_school_mathematics": 0.2527881040892193, - "high_school_computer_science": 0.43434343434343436, - "conceptual_physics": 0.4230769230769231, - "miscellaneous": 0.7071611253196931, - "high_school_chemistry": 0.4158415841584158, - "marketing": 0.8240343347639485, - "professional_law": 0.4292237442922374, - "management": 0.7843137254901961, - "college_physics": 0.39603960396039606, - "jurisprudence": 0.6448598130841121, - "world_religions": 0.711764705882353, - "sociology": 0.695, - "us_foreign_policy": 0.7373737373737373, - "high_school_macroeconomics": 0.5449871465295629, - "computer_security": 0.6464646464646465, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.6405797101449275, - "electrical_engineering": 0.5138888888888888, - "astronomy": 0.5827814569536424, - "college_biology": 0.5104895104895105 - } - }, - "prompt_5": { - "accuracy": 0.5432963889882017, - "category_acc": { - "high_school_european_history": 0.75, - "business_ethics": 0.7373737373737373, - "clinical_knowledge": 0.6022727272727273, - "medical_genetics": 0.5252525252525253, - "high_school_us_history": 0.7832512315270936, - "high_school_physics": 0.26, - "high_school_world_history": 0.7627118644067796, - "virology": 0.4666666666666667, - "high_school_microeconomics": 0.5738396624472574, - "econometrics": 0.36283185840707965, - "college_computer_science": 0.45454545454545453, - "high_school_biology": 0.6634304207119741, - "abstract_algebra": 0.1919191919191919, - "professional_accounting": 0.42704626334519574, - "philosophy": 0.5709677419354838, - "professional_medicine": 0.5535055350553506, - "nutrition": 0.6295081967213115, - "global_facts": 0.31313131313131315, - "machine_learning": 0.38738738738738737, - "security_studies": 0.6762295081967213, - "public_relations": 0.5321100917431193, - "professional_psychology": 0.55810147299509, - "prehistory": 0.6068111455108359, - "anatomy": 0.47761194029850745, - "human_sexuality": 0.6538461538461539, - "college_medicine": 0.47674418604651164, - "high_school_government_and_politics": 0.8333333333333334, - "college_chemistry": 0.3838383838383838, - "logical_fallacies": 0.6728395061728395, - "high_school_geography": 0.7360406091370558, - "elementary_mathematics": 0.3713527851458886, - "human_aging": 0.5900900900900901, - "college_mathematics": 0.32323232323232326, - "high_school_psychology": 0.7886029411764706, - "formal_logic": 0.32, - "high_school_statistics": 0.4, - "international_law": 0.7083333333333334, - "high_school_mathematics": 0.25650557620817843, - "high_school_computer_science": 0.46464646464646464, - "conceptual_physics": 0.4230769230769231, - "miscellaneous": 0.7212276214833759, - "high_school_chemistry": 0.40594059405940597, - "marketing": 0.8283261802575107, - "professional_law": 0.43313763861709065, - "management": 0.7941176470588235, - "college_physics": 0.40594059405940597, - "jurisprudence": 0.6822429906542056, - "world_religions": 0.7235294117647059, - "sociology": 0.69, - "us_foreign_policy": 0.7575757575757576, - "high_school_macroeconomics": 0.532133676092545, - "computer_security": 0.6666666666666666, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.6608695652173913, - "electrical_engineering": 0.5, - "astronomy": 0.5761589403973509, - "college_biology": 0.48951048951048953 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.25185735512630014 - }, - "prompt_2": { - "accuracy": 0.25705794947994054 - }, - "prompt_3": { - "accuracy": 0.2555720653789004 - }, - "prompt_4": { - "accuracy": 0.2399702823179792 - }, - "prompt_5": { - "accuracy": 0.2526002971768202 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.2546699875466999, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.35714285714285715, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.16666666666666666, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.16666666666666666, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.26666666666666666, - "business_administration": 0.18421052631578946, - "marxism": 0.25, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.2857142857142857, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.058823529411764705, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.4074074074074074, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.21052631578947367, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.28, - "middle_school_history": 0.18518518518518517, - "civil_servant": 0.21153846153846154, - "sports_science": 0.125, - "plant_protection": 0.1111111111111111, - "basic_medicine": 0.25, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.35294117647058826, - "accountant": 0.2777777777777778, - "fire_engineer": 0.1111111111111111, - "environmental_impact_assessment_engineer": 0.16666666666666666, - "tax_accountant": 0.2037037037037037, - "physician": 0.3148148148148148 - } - }, - "prompt_2": { - "accuracy": 0.25529265255292655, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.2619047619047619, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.41379310344827586, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.19047619047619047, - "metrology_engineer": 0.1724137931034483, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.125, - "high_school_biology": 0.16666666666666666, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.25, - "business_administration": 0.18421052631578946, - "marxism": 0.25, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.2857142857142857, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.058823529411764705, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.4074074074074074, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.21052631578947367, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.28, - "middle_school_history": 0.18518518518518517, - "civil_servant": 0.23076923076923078, - "sports_science": 0.16666666666666666, - "plant_protection": 0.1111111111111111, - "basic_medicine": 0.20833333333333334, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.37254901960784315, - "accountant": 0.2962962962962963, - "fire_engineer": 0.1111111111111111, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.2037037037037037, - "physician": 0.3333333333333333 - } - }, - "prompt_3": { - "accuracy": 0.25529265255292655, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.25, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.2619047619047619, - "college_physics": 0.25, - "college_chemistry": 0.3793103448275862, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.21428571428571427, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.16666666666666666, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.3076923076923077, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.3, - "business_administration": 0.18421052631578946, - "marxism": 0.25, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.2857142857142857, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.058823529411764705, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.4074074074074074, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.23684210526315788, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.28, - "middle_school_history": 0.18518518518518517, - "civil_servant": 0.21153846153846154, - "sports_science": 0.125, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.16666666666666666, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.35294117647058826, - "accountant": 0.2962962962962963, - "fire_engineer": 0.1111111111111111, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.2037037037037037, - "physician": 0.25925925925925924 - } - }, - "prompt_4": { - "accuracy": 0.26650062266500624, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.23809523809523808, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.25, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.19230769230769232, - "middle_school_physics": 0.125, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.2833333333333333, - "business_administration": 0.21052631578947367, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.27586206896551724, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.2857142857142857, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.11764705882352941, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.37037037037037035, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.36, - "middle_school_history": 0.2222222222222222, - "civil_servant": 0.2692307692307692, - "sports_science": 0.16666666666666666, - "plant_protection": 0.18518518518518517, - "basic_medicine": 0.20833333333333334, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.2777777777777778, - "fire_engineer": 0.1388888888888889, - "environmental_impact_assessment_engineer": 0.2222222222222222, - "tax_accountant": 0.25925925925925924, - "physician": 0.2777777777777778 - } - }, - "prompt_5": { - "accuracy": 0.2546699875466999, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.16666666666666666, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.21428571428571427, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.25, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.26666666666666666, - "business_administration": 0.18421052631578946, - "marxism": 0.25, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.2857142857142857, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.058823529411764705, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.4074074074074074, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.21052631578947367, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.32142857142857145, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.28, - "middle_school_history": 0.18518518518518517, - "civil_servant": 0.21153846153846154, - "sports_science": 0.16666666666666666, - "plant_protection": 0.1111111111111111, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.35294117647058826, - "accountant": 0.2962962962962963, - "fire_engineer": 0.1388888888888889, - "environmental_impact_assessment_engineer": 0.2222222222222222, - "tax_accountant": 0.2037037037037037, - "physician": 0.3333333333333333 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.25089605734767023 - }, - "prompt_2": { - "accuracy": 0.23297491039426524 - }, - "prompt_3": { - "accuracy": 0.23297491039426524 - }, - "prompt_4": { - "accuracy": 0.3118279569892473 - }, - "prompt_5": { - "accuracy": 0.25089605734767023 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.2560870316007598, - "category_acc": { - "agronomy": 0.27218934911242604, - "anatomy": 0.25, - "ancient_chinese": 0.2682926829268293, - "arts": 0.24375, - "astronomy": 0.2545454545454545, - "business_ethics": 0.2679425837320574, - "chinese_civil_service_exam": 0.2625, - "chinese_driving_rule": 0.3053435114503817, - "chinese_food_culture": 0.3014705882352941, - "chinese_foreign_policy": 0.21495327102803738, - "chinese_history": 0.2724458204334365, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.2616033755274262, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.3177570093457944, - "college_engineering_hydrology": 0.2830188679245283, - "college_law": 0.25925925925925924, - "college_mathematics": 0.19047619047619047, - "college_medical_statistics": 0.16037735849056603, - "college_medicine": 0.24175824175824176, - "computer_science": 0.3235294117647059, - "computer_security": 0.29239766081871343, - "conceptual_physics": 0.272108843537415, - "construction_project_management": 0.22302158273381295, - "economics": 0.27044025157232704, - "education": 0.22085889570552147, - "electrical_engineering": 0.21511627906976744, - "elementary_chinese": 0.24603174603174602, - "elementary_commonsense": 0.2676767676767677, - "elementary_information_and_technology": 0.23949579831932774, - "elementary_mathematics": 0.2565217391304348, - "ethnology": 0.28888888888888886, - "food_science": 0.2517482517482518, - "genetics": 0.2556818181818182, - "global_facts": 0.2684563758389262, - "high_school_biology": 0.21893491124260356, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.288135593220339, - "high_school_mathematics": 0.22560975609756098, - "high_school_physics": 0.18181818181818182, - "high_school_politics": 0.3006993006993007, - "human_sexuality": 0.2619047619047619, - "international_law": 0.2648648648648649, - "journalism": 0.22093023255813954, - "jurisprudence": 0.24574209245742093, - "legal_and_moral_basis": 0.27102803738317754, - "logical": 0.2601626016260163, - "machine_learning": 0.30327868852459017, - "management": 0.28095238095238095, - "marketing": 0.24444444444444444, - "marxist_theory": 0.24338624338624337, - "modern_chinese": 0.29310344827586204, - "nutrition": 0.3103448275862069, - "philosophy": 0.21904761904761905, - "professional_accounting": 0.2571428571428571, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.25, - "professional_psychology": 0.24568965517241378, - "public_relations": 0.2471264367816092, - "security_study": 0.26666666666666666, - "sociology": 0.2168141592920354, - "sports_science": 0.24848484848484848, - "traditional_chinese_medicine": 0.21081081081081082, - "virology": 0.28402366863905326, - "world_history": 0.2670807453416149, - "world_religions": 0.26875 - } - }, - "prompt_2": { - "accuracy": 0.24693489898117768, - "category_acc": { - "agronomy": 0.2603550295857988, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.2621951219512195, - "arts": 0.2125, - "astronomy": 0.24242424242424243, - "business_ethics": 0.2583732057416268, - "chinese_civil_service_exam": 0.29375, - "chinese_driving_rule": 0.29770992366412213, - "chinese_food_culture": 0.25, - "chinese_foreign_policy": 0.2803738317757009, - "chinese_history": 0.26006191950464397, - "chinese_literature": 0.2549019607843137, - "chinese_teacher_qualification": 0.2569832402234637, - "clinical_knowledge": 0.26582278481012656, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.3177570093457944, - "college_engineering_hydrology": 0.2830188679245283, - "college_law": 0.25925925925925924, - "college_mathematics": 0.17142857142857143, - "college_medical_statistics": 0.1792452830188679, - "college_medicine": 0.22344322344322345, - "computer_science": 0.29411764705882354, - "computer_security": 0.2222222222222222, - "conceptual_physics": 0.2653061224489796, - "construction_project_management": 0.2517985611510791, - "economics": 0.24528301886792453, - "education": 0.2331288343558282, - "electrical_engineering": 0.22674418604651161, - "elementary_chinese": 0.25793650793650796, - "elementary_commonsense": 0.2878787878787879, - "elementary_information_and_technology": 0.2773109243697479, - "elementary_mathematics": 0.27391304347826084, - "ethnology": 0.24444444444444444, - "food_science": 0.26573426573426573, - "genetics": 0.2784090909090909, - "global_facts": 0.21476510067114093, - "high_school_biology": 0.21893491124260356, - "high_school_chemistry": 0.2196969696969697, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.21951219512195122, - "high_school_physics": 0.22727272727272727, - "high_school_politics": 0.2727272727272727, - "human_sexuality": 0.2619047619047619, - "international_law": 0.2810810810810811, - "journalism": 0.22674418604651161, - "jurisprudence": 0.24087591240875914, - "legal_and_moral_basis": 0.2616822429906542, - "logical": 0.2032520325203252, - "machine_learning": 0.2540983606557377, - "management": 0.2523809523809524, - "marketing": 0.24444444444444444, - "marxist_theory": 0.21693121693121692, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.2827586206896552, - "philosophy": 0.17142857142857143, - "professional_accounting": 0.18857142857142858, - "professional_law": 0.27014218009478674, - "professional_medicine": 0.22074468085106383, - "professional_psychology": 0.20689655172413793, - "public_relations": 0.27586206896551724, - "security_study": 0.25925925925925924, - "sociology": 0.23008849557522124, - "sports_science": 0.17575757575757575, - "traditional_chinese_medicine": 0.23243243243243245, - "virology": 0.21893491124260356, - "world_history": 0.2608695652173913, - "world_religions": 0.25625 - } - }, - "prompt_3": { - "accuracy": 0.2544465549991366, - "category_acc": { - "agronomy": 0.2603550295857988, - "anatomy": 0.19594594594594594, - "ancient_chinese": 0.27439024390243905, - "arts": 0.26875, - "astronomy": 0.23030303030303031, - "business_ethics": 0.2535885167464115, - "chinese_civil_service_exam": 0.29375, - "chinese_driving_rule": 0.31297709923664124, - "chinese_food_culture": 0.27941176470588236, - "chinese_foreign_policy": 0.27102803738317754, - "chinese_history": 0.25696594427244585, - "chinese_literature": 0.25980392156862747, - "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.25316455696202533, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.3177570093457944, - "college_engineering_hydrology": 0.2830188679245283, - "college_law": 0.25925925925925924, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.20754716981132076, - "college_medicine": 0.23443223443223443, - "computer_science": 0.29901960784313725, - "computer_security": 0.2631578947368421, - "conceptual_physics": 0.24489795918367346, - "construction_project_management": 0.2589928057553957, - "economics": 0.27044025157232704, - "education": 0.25153374233128833, - "electrical_engineering": 0.22093023255813954, - "elementary_chinese": 0.25, - "elementary_commonsense": 0.23232323232323232, - "elementary_information_and_technology": 0.2605042016806723, - "elementary_mathematics": 0.2782608695652174, - "ethnology": 0.23703703703703705, - "food_science": 0.24475524475524477, - "genetics": 0.2840909090909091, - "global_facts": 0.2483221476510067, - "high_school_biology": 0.22485207100591717, - "high_school_chemistry": 0.22727272727272727, - "high_school_geography": 0.2627118644067797, - "high_school_mathematics": 0.21951219512195122, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.26573426573426573, - "human_sexuality": 0.2777777777777778, - "international_law": 0.3081081081081081, - "journalism": 0.29069767441860467, - "jurisprudence": 0.24330900243309003, - "legal_and_moral_basis": 0.2757009345794392, - "logical": 0.24390243902439024, - "machine_learning": 0.2786885245901639, - "management": 0.24761904761904763, - "marketing": 0.25555555555555554, - "marxist_theory": 0.25925925925925924, - "modern_chinese": 0.29310344827586204, - "nutrition": 0.23448275862068965, - "philosophy": 0.23809523809523808, - "professional_accounting": 0.24, - "professional_law": 0.27488151658767773, - "professional_medicine": 0.22872340425531915, - "professional_psychology": 0.22413793103448276, - "public_relations": 0.2471264367816092, - "security_study": 0.2740740740740741, - "sociology": 0.24336283185840707, - "sports_science": 0.22424242424242424, - "traditional_chinese_medicine": 0.23783783783783785, - "virology": 0.15976331360946747, - "world_history": 0.2546583850931677, - "world_religions": 0.30625 - } - }, - "prompt_4": { - "accuracy": 0.253151441892592, - "category_acc": { - "agronomy": 0.2485207100591716, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.2621951219512195, - "arts": 0.25625, - "astronomy": 0.19393939393939394, - "business_ethics": 0.2631578947368421, - "chinese_civil_service_exam": 0.2375, - "chinese_driving_rule": 0.3053435114503817, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.2803738317757009, - "chinese_history": 0.25077399380804954, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.2346368715083799, - "clinical_knowledge": 0.2489451476793249, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.2897196261682243, - "college_engineering_hydrology": 0.32075471698113206, - "college_law": 0.2222222222222222, - "college_mathematics": 0.2, - "college_medical_statistics": 0.2358490566037736, - "college_medicine": 0.2490842490842491, - "computer_science": 0.29901960784313725, - "computer_security": 0.23976608187134502, - "conceptual_physics": 0.2789115646258503, - "construction_project_management": 0.2733812949640288, - "economics": 0.2389937106918239, - "education": 0.25153374233128833, - "electrical_engineering": 0.2441860465116279, - "elementary_chinese": 0.26587301587301587, - "elementary_commonsense": 0.2676767676767677, - "elementary_information_and_technology": 0.29831932773109243, - "elementary_mathematics": 0.26956521739130435, - "ethnology": 0.24444444444444444, - "food_science": 0.3076923076923077, - "genetics": 0.2215909090909091, - "global_facts": 0.2214765100671141, - "high_school_biology": 0.23668639053254437, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.2288135593220339, - "high_school_mathematics": 0.20121951219512196, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.26573426573426573, - "human_sexuality": 0.24603174603174602, - "international_law": 0.2702702702702703, - "journalism": 0.25, - "jurisprudence": 0.26034063260340634, - "legal_and_moral_basis": 0.2570093457943925, - "logical": 0.24390243902439024, - "machine_learning": 0.22131147540983606, - "management": 0.24285714285714285, - "marketing": 0.25555555555555554, - "marxist_theory": 0.25396825396825395, - "modern_chinese": 0.2672413793103448, - "nutrition": 0.2482758620689655, - "philosophy": 0.20952380952380953, - "professional_accounting": 0.26285714285714284, - "professional_law": 0.23222748815165878, - "professional_medicine": 0.2393617021276596, - "professional_psychology": 0.2543103448275862, - "public_relations": 0.2413793103448276, - "security_study": 0.2740740740740741, - "sociology": 0.25663716814159293, - "sports_science": 0.24242424242424243, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.2603550295857988, - "world_history": 0.2422360248447205, - "world_religions": 0.2625 - } - }, - "prompt_5": { - "accuracy": 0.25194266965981693, - "category_acc": { - "agronomy": 0.22485207100591717, - "anatomy": 0.3108108108108108, - "ancient_chinese": 0.23780487804878048, - "arts": 0.26875, - "astronomy": 0.23636363636363636, - "business_ethics": 0.24880382775119617, - "chinese_civil_service_exam": 0.28125, - "chinese_driving_rule": 0.3053435114503817, - "chinese_food_culture": 0.2867647058823529, - "chinese_foreign_policy": 0.2803738317757009, - "chinese_history": 0.2631578947368421, - "chinese_literature": 0.24509803921568626, - "chinese_teacher_qualification": 0.24581005586592178, - "clinical_knowledge": 0.2489451476793249, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.3364485981308411, - "college_engineering_hydrology": 0.29245283018867924, - "college_law": 0.25, - "college_mathematics": 0.21904761904761905, - "college_medical_statistics": 0.18867924528301888, - "college_medicine": 0.2454212454212454, - "computer_science": 0.31862745098039214, - "computer_security": 0.22807017543859648, - "conceptual_physics": 0.2789115646258503, - "construction_project_management": 0.2589928057553957, - "economics": 0.2893081761006289, - "education": 0.2147239263803681, - "electrical_engineering": 0.23255813953488372, - "elementary_chinese": 0.24206349206349206, - "elementary_commonsense": 0.2828282828282828, - "elementary_information_and_technology": 0.25630252100840334, - "elementary_mathematics": 0.2565217391304348, - "ethnology": 0.26666666666666666, - "food_science": 0.25874125874125875, - "genetics": 0.2840909090909091, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.23668639053254437, - "high_school_chemistry": 0.21212121212121213, - "high_school_geography": 0.3135593220338983, - "high_school_mathematics": 0.21951219512195122, - "high_school_physics": 0.23636363636363636, - "high_school_politics": 0.2727272727272727, - "human_sexuality": 0.2222222222222222, - "international_law": 0.2594594594594595, - "journalism": 0.2558139534883721, - "jurisprudence": 0.25790754257907544, - "legal_and_moral_basis": 0.2616822429906542, - "logical": 0.24390243902439024, - "machine_learning": 0.26229508196721313, - "management": 0.23333333333333334, - "marketing": 0.25555555555555554, - "marxist_theory": 0.2857142857142857, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.25517241379310346, - "philosophy": 0.17142857142857143, - "professional_accounting": 0.21142857142857144, - "professional_law": 0.2559241706161137, - "professional_medicine": 0.20212765957446807, - "professional_psychology": 0.22844827586206898, - "public_relations": 0.25287356321839083, - "security_study": 0.25925925925925924, - "sociology": 0.24336283185840707, - "sports_science": 0.2, - "traditional_chinese_medicine": 0.23783783783783785, - "virology": 0.17751479289940827, - "world_history": 0.2670807453416149, - "world_religions": 0.29375 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.30303030303030304 - }, - "prompt_2": { - "accuracy": 0.2727272727272727 - }, - "prompt_3": { - "accuracy": 0.24242424242424243 - }, - "prompt_4": { - "accuracy": 0.3333333333333333 - }, - "prompt_5": { - "accuracy": 0.3333333333333333 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.4340909090909091 - }, - "prompt_2": { - "accuracy": 0.42727272727272725 - }, - "prompt_3": { - "accuracy": 0.42045454545454547 - }, - "prompt_4": { - "accuracy": 0.4590909090909091 - }, - "prompt_5": { - "accuracy": 0.4590909090909091 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3488135593220339 - }, - "prompt_2": { - "accuracy": 0.3447457627118644 - }, - "prompt_3": { - "accuracy": 0.34508474576271186 - }, - "prompt_4": { - "accuracy": 0.34915254237288135 - }, - "prompt_5": { - "accuracy": 0.3433898305084746 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.2655198204936425 - }, - "prompt_2": { - "accuracy": 0.27486910994764396 - }, - "prompt_3": { - "accuracy": 0.2636499626028422 - }, - "prompt_4": { - "accuracy": 0.26925953627524307 - }, - "prompt_5": { - "accuracy": 0.2696335078534031 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.9299363057324841 - }, - "prompt_2": { - "accuracy": 0.9294463498285155 - }, - "prompt_3": { - "accuracy": 0.9294463498285155 - }, - "prompt_4": { - "accuracy": 0.9284664380205782 - }, - "prompt_5": { - "accuracy": 0.9279764821166095 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.5431501205662483, - "rouge2": 0.2946609489495607, - "rougeL": 0.4528281209356889, - "avg_rouge": 0.4302130634838326 - }, - "prompt_2": { - "rouge1": 0.5425380643522792, - "rouge2": 0.2933854973680719, - "rougeL": 0.4512721576523579, - "avg_rouge": 0.429065239790903 - }, - "prompt_3": { - "rouge1": 0.537255014095881, - "rouge2": 0.29065273763354427, - "rougeL": 0.4491157337441342, - "avg_rouge": 0.4256744951578531 - }, - "prompt_4": { - "rouge1": 0.5398194584794581, - "rouge2": 0.28897160002838995, - "rougeL": 0.4480830945127036, - "avg_rouge": 0.4256247176735172 - }, - "prompt_5": { - "rouge1": 0.5431239394463732, - "rouge2": 0.2923687974398821, - "rougeL": 0.45257371828948567, - "avg_rouge": 0.4293554850585803 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.2924343006937695, - "rouge2": 0.10194868396663816, - "rougeL": 0.22625041900514214, - "avg_rouge": 0.20687780122184996 - }, - "prompt_2": { - "rouge1": 0.30567366347992114, - "rouge2": 0.11171393330944784, - "rougeL": 0.2398131463087714, - "avg_rouge": 0.21906691436604678 - }, - "prompt_3": { - "rouge1": 0.30414722128476235, - "rouge2": 0.11005958811987257, - "rougeL": 0.23777359742807141, - "avg_rouge": 0.21732680227756881 - }, - "prompt_4": { - "rouge1": 0.256774399895642, - "rouge2": 0.08121046836622561, - "rougeL": 0.19498296069223034, - "avg_rouge": 0.1776559429846993 - }, - "prompt_5": { - "rouge1": 0.30310164947058155, - "rouge2": 0.10835066641631358, - "rougeL": 0.23759937031455777, - "avg_rouge": 0.21635056206715098 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.9621559633027523 - }, - "prompt_2": { - "accuracy": 0.9610091743119266 - }, - "prompt_3": { - "accuracy": 0.9610091743119266 - }, - "prompt_4": { - "accuracy": 0.9621559633027523 - }, - "prompt_5": { - "accuracy": 0.9621559633027523 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.8465963566634708 - }, - "prompt_2": { - "accuracy": 0.8485139022051774 - }, - "prompt_3": { - "accuracy": 0.850431447746884 - }, - "prompt_4": { - "accuracy": 0.8437200383509108 - }, - "prompt_5": { - "accuracy": 0.8465963566634708 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.841 - }, - "prompt_2": { - "accuracy": 0.8655 - }, - "prompt_3": { - "accuracy": 0.832 - }, - "prompt_4": { - "accuracy": 0.866 - }, - "prompt_5": { - "accuracy": 0.87 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.888 - }, - "prompt_2": { - "accuracy": 0.888 - }, - "prompt_3": { - "accuracy": 0.888 - }, - "prompt_4": { - "accuracy": 0.8855 - }, - "prompt_5": { - "accuracy": 0.8865 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.948 - }, - "prompt_2": { - "accuracy": 0.951 - }, - "prompt_3": { - "accuracy": 0.9495 - }, - "prompt_4": { - "accuracy": 0.953 - }, - "prompt_5": { - "accuracy": 0.9435 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.8028169014084507 - }, - "prompt_2": { - "accuracy": 0.8028169014084507 - }, - "prompt_3": { - "accuracy": 0.8028169014084507 - }, - "prompt_4": { - "accuracy": 0.8028169014084507 - }, - "prompt_5": { - "accuracy": 0.8028169014084507 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.9205776173285198 - }, - "prompt_2": { - "accuracy": 0.9205776173285198 - }, - "prompt_3": { - "accuracy": 0.9205776173285198 - }, - "prompt_4": { - "accuracy": 0.9169675090252708 - }, - "prompt_5": { - "accuracy": 0.924187725631769 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.8553921568627451 - }, - "prompt_2": { - "accuracy": 0.8774509803921569 - }, - "prompt_3": { - "accuracy": 0.8578431372549019 - }, - "prompt_4": { - "accuracy": 0.8799019607843137 - }, - "prompt_5": { - "accuracy": 0.875 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.7936974789915967, - "language_acc": { - "English": 0.9798319327731092, - "Chinese": 0.3739495798319328, - "Spanish": 0.9621848739495799, - "Vietnamese": 0.8588235294117647 - }, - "consistency_score_2": 0.6376750700280113, - "consistency_score_3": 0.47205882352941175, - "consistency_score_4": 0.3352941176470588, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.3764705882352941, - "English,Spanish": 0.965546218487395, - "English,Vietnamese": 0.8571428571428571, - "Chinese,Spanish": 0.3764705882352941, - "Chinese,Vietnamese": 0.3983193277310924, - "Spanish,Vietnamese": 0.8521008403361344 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3663865546218487, - "English,Chinese,Vietnamese": 0.3420168067226891, - "English,Spanish,Vietnamese": 0.8403361344537815, - "Chinese,Spanish,Vietnamese": 0.33949579831932775 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.3352941176470588 - } - }, - "AC3_2": 0.7071829005080774, - "AC3_3": 0.5920126921670436, - "AC3_4": 0.4714332625077734 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.37333333333333335, - "language_acc": { - "Spanish": 0.4066666666666667, - "English": 0.5733333333333334, - "Chinese": 0.24, - "Vietnamese": 0.34, - "Filipino": 0.32666666666666666, - "Indonesian": 0.38, - "Malay": 0.3466666666666667 - }, - "consistency_score_2": 0.4755555555555556, - "consistency_score_3": 0.29219047619047617, - "consistency_score_4": 0.2051428571428571, - "consistency_score_5": 0.1565079365079365, - "consistency_score_6": 0.12476190476190475, - "consistency_score_7": 0.1, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.62, - "Spanish,Chinese": 0.4533333333333333, - "Spanish,Vietnamese": 0.44666666666666666, - "Spanish,Filipino": 0.52, - "Spanish,Indonesian": 0.52, - "Spanish,Malay": 0.4533333333333333, - "English,Chinese": 0.41333333333333333, - "English,Vietnamese": 0.43333333333333335, - "English,Filipino": 0.5066666666666667, - "English,Indonesian": 0.49333333333333335, - "English,Malay": 0.46, - "Chinese,Vietnamese": 0.43333333333333335, - "Chinese,Filipino": 0.36, - "Chinese,Indonesian": 0.4666666666666667, - "Chinese,Malay": 0.4066666666666667, - "Vietnamese,Filipino": 0.4266666666666667, - "Vietnamese,Indonesian": 0.5066666666666667, - "Vietnamese,Malay": 0.4533333333333333, - "Filipino,Indonesian": 0.4866666666666667, - "Filipino,Malay": 0.48, - "Indonesian,Malay": 0.6466666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.31333333333333335, - "Spanish,English,Vietnamese": 0.30666666666666664, - "Spanish,English,Filipino": 0.38666666666666666, - "Spanish,English,Indonesian": 0.36666666666666664, - "Spanish,English,Malay": 0.3466666666666667, - "Spanish,Chinese,Vietnamese": 0.2866666666666667, - "Spanish,Chinese,Filipino": 0.25333333333333335, - "Spanish,Chinese,Indonesian": 0.3, - "Spanish,Chinese,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,Vietnamese,Indonesian": 0.3, - "Spanish,Vietnamese,Malay": 0.25333333333333335, - "Spanish,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Filipino,Malay": 0.29333333333333333, - "Spanish,Indonesian,Malay": 0.35333333333333333, - "English,Chinese,Vietnamese": 0.24, - "English,Chinese,Filipino": 0.24666666666666667, - "English,Chinese,Indonesian": 0.26666666666666666, - "English,Chinese,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino": 0.2733333333333333, - "English,Vietnamese,Indonesian": 0.29333333333333333, - "English,Vietnamese,Malay": 0.2733333333333333, - "English,Filipino,Indonesian": 0.32, - "English,Filipino,Malay": 0.31333333333333335, - "English,Indonesian,Malay": 0.3466666666666667, - "Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Malay": 0.24666666666666667, - "Chinese,Filipino,Indonesian": 0.24, - "Chinese,Filipino,Malay": 0.21333333333333335, - "Chinese,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Vietnamese,Filipino,Malay": 0.26, - "Vietnamese,Indonesian,Malay": 0.3466666666666667, - "Filipino,Indonesian,Malay": 0.36 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.2, - "Spanish,English,Chinese,Filipino": 0.20666666666666667, - "Spanish,English,Chinese,Indonesian": 0.22, - "Spanish,English,Chinese,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Malay": 0.2, - "Spanish,English,Filipino,Indonesian": 0.26, - "Spanish,English,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.22, - "Spanish,Chinese,Vietnamese,Malay": 0.18666666666666668, - "Spanish,Chinese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Indonesian,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Filipino": 0.16, - "English,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "English,Chinese,Vietnamese,Malay": 0.18666666666666668, - "English,Chinese,Filipino,Indonesian": 0.18, - "English,Chinese,Filipino,Malay": 0.18, - "English,Chinese,Indonesian,Malay": 0.21333333333333335, - "English,Vietnamese,Filipino,Indonesian": 0.2, - "English,Vietnamese,Filipino,Malay": 0.19333333333333333, - "English,Vietnamese,Indonesian,Malay": 0.22, - "English,Filipino,Indonesian,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.16, - "Chinese,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.2, - "Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.14, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16, - "Spanish,English,Chinese,Filipino,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.16, - "Spanish,English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.16, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "English,Chinese,Vietnamese,Filipino,Malay": 0.12666666666666668, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1 - } - }, - "AC3_2": 0.41828970326660636, - "AC3_3": 0.32781530237394496, - "AC3_4": 0.26478761931543837, - "AC3_5": 0.22055522264660982, - "AC3_6": 0.18702358186175116, - "AC3_7": 0.15774647883991272 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.3806818181818182, - "language_acc": { - "English": 0.5852272727272727, - "Filipino": 0.35795454545454547, - "Spanish": 0.5170454545454546, - "Malay": 0.35795454545454547, - "Chinese": 0.22727272727272727, - "Indonesian": 0.32386363636363635, - "Vietnamese": 0.29545454545454547 - }, - "consistency_score_2": 0.3768939393939394, - "consistency_score_3": 0.17889610389610383, - "consistency_score_4": 0.09577922077922077, - "consistency_score_5": 0.05248917748917749, - "consistency_score_6": 0.025162337662337657, - "consistency_score_7": 0.005681818181818182, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.35795454545454547, - "English,Spanish": 0.6647727272727273, - "English,Malay": 0.4431818181818182, - "English,Chinese": 0.2215909090909091, - "English,Indonesian": 0.39204545454545453, - "English,Vietnamese": 0.30113636363636365, - "Filipino,Spanish": 0.4147727272727273, - "Filipino,Malay": 0.4659090909090909, - "Filipino,Chinese": 0.2159090909090909, - "Filipino,Indonesian": 0.4943181818181818, - "Filipino,Vietnamese": 0.48863636363636365, - "Spanish,Malay": 0.4375, - "Spanish,Chinese": 0.2159090909090909, - "Spanish,Indonesian": 0.4375, - "Spanish,Vietnamese": 0.36363636363636365, - "Malay,Chinese": 0.20454545454545456, - "Malay,Indonesian": 0.5738636363636364, - "Malay,Vietnamese": 0.42045454545454547, - "Chinese,Indonesian": 0.19886363636363635, - "Chinese,Vietnamese": 0.1875, - "Indonesian,Vietnamese": 0.4147727272727273 - }, - "3_combine": { - "English,Filipino,Spanish": 0.2727272727272727, - "English,Filipino,Malay": 0.23295454545454544, - "English,Filipino,Chinese": 0.07386363636363637, - "English,Filipino,Indonesian": 0.2159090909090909, - "English,Filipino,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay": 0.32954545454545453, - "English,Spanish,Chinese": 0.11363636363636363, - "English,Spanish,Indonesian": 0.3068181818181818, - "English,Spanish,Vietnamese": 0.24431818181818182, - "English,Malay,Chinese": 0.07954545454545454, - "English,Malay,Indonesian": 0.2840909090909091, - "English,Malay,Vietnamese": 0.19886363636363635, - "English,Chinese,Indonesian": 0.07386363636363637, - "English,Chinese,Vietnamese": 0.056818181818181816, - "English,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay": 0.23863636363636365, - "Filipino,Spanish,Chinese": 0.09659090909090909, - "Filipino,Spanish,Indonesian": 0.24431818181818182, - "Filipino,Spanish,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Chinese": 0.11363636363636363, - "Filipino,Malay,Indonesian": 0.32386363636363635, - "Filipino,Malay,Vietnamese": 0.23863636363636365, - "Filipino,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Chinese,Vietnamese": 0.07954545454545454, - "Filipino,Indonesian,Vietnamese": 0.26704545454545453, - "Spanish,Malay,Chinese": 0.07954545454545454, - "Spanish,Malay,Indonesian": 0.29545454545454547, - "Spanish,Malay,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Indonesian": 0.10227272727272728, - "Spanish,Chinese,Vietnamese": 0.07386363636363637, - "Spanish,Indonesian,Vietnamese": 0.20454545454545456, - "Malay,Chinese,Indonesian": 0.10795454545454546, - "Malay,Chinese,Vietnamese": 0.07954545454545454, - "Malay,Indonesian,Vietnamese": 0.26704545454545453, - "Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.18181818181818182, - "English,Filipino,Spanish,Chinese": 0.045454545454545456, - "English,Filipino,Spanish,Indonesian": 0.17613636363636365, - "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese": 0.05113636363636364, - "English,Filipino,Malay,Indonesian": 0.1590909090909091, - "English,Filipino,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Indonesian": 0.045454545454545456, - "English,Filipino,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Indonesian,Vietnamese": 0.125, - "English,Spanish,Malay,Chinese": 0.045454545454545456, - "English,Spanish,Malay,Indonesian": 0.23295454545454544, - "English,Spanish,Malay,Vietnamese": 0.16477272727272727, - "English,Spanish,Chinese,Indonesian": 0.045454545454545456, - "English,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "English,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "English,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "English,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese": 0.05113636363636364, - "Filipino,Spanish,Malay,Indonesian": 0.1875, - "Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.0625, - "Filipino,Spanish,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Malay,Chinese,Vietnamese": 0.045454545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese,Indonesian": 0.05113636363636364, - "Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "Spanish,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 - } - }, - "AC3_2": 0.3787784090409103, - "AC3_3": 0.24340665132659953, - "AC3_4": 0.15305095242257785, - "AC3_5": 0.0922576798378807, - "AC3_6": 0.04720454544291424, - "AC3_7": 0.0111965240612732 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.5631067961165048 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.3238095238095238 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.6542056074766355 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.32, - "category_acc": { - "brand": 0.2, - "demographics": 0.4, - "biology": 0.4, - "history": 0.2, - "literature": 0.4, - "politics": 0.5, - "culture": 0.4, - "film": 0.2, - "law": 0.2, - "geography": 0.4 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.17330727183521114 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.29203849590946845 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.07110023147050917 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.020632655828217442 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.2730725867958359 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.5495915985997666 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.5323560958169468, - "category_acc": { - "high_school_european_history": 0.75, - "business_ethics": 0.7070707070707071, - "clinical_knowledge": 0.6136363636363636, - "medical_genetics": 0.5353535353535354, - "high_school_us_history": 0.7783251231527094, - "high_school_physics": 0.2733333333333333, - "high_school_world_history": 0.7711864406779662, - "virology": 0.4666666666666667, - "high_school_microeconomics": 0.5991561181434599, - "econometrics": 0.34513274336283184, - "college_computer_science": 0.42424242424242425, - "high_school_biology": 0.6537216828478964, - "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.42704626334519574, - "philosophy": 0.5645161290322581, - "professional_medicine": 0.5608856088560885, - "nutrition": 0.6032786885245902, - "global_facts": 0.25252525252525254, - "machine_learning": 0.35135135135135137, - "security_studies": 0.6434426229508197, - "public_relations": 0.5137614678899083, - "professional_psychology": 0.5433715220949263, - "prehistory": 0.5263157894736842, - "anatomy": 0.48507462686567165, - "human_sexuality": 0.6461538461538462, - "college_medicine": 0.47093023255813954, - "high_school_government_and_politics": 0.8229166666666666, - "college_chemistry": 0.3838383838383838, - "logical_fallacies": 0.6790123456790124, - "high_school_geography": 0.7106598984771574, - "elementary_mathematics": 0.33687002652519893, - "human_aging": 0.5765765765765766, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.7738970588235294, - "formal_logic": 0.312, - "high_school_statistics": 0.4186046511627907, - "international_law": 0.6916666666666667, - "high_school_mathematics": 0.25650557620817843, - "high_school_computer_science": 0.48484848484848486, - "conceptual_physics": 0.4188034188034188, - "miscellaneous": 0.7122762148337596, - "high_school_chemistry": 0.400990099009901, - "marketing": 0.7811158798283262, - "professional_law": 0.42204827136333983, - "management": 0.7254901960784313, - "college_physics": 0.3564356435643564, - "jurisprudence": 0.6822429906542056, - "world_religions": 0.6882352941176471, - "sociology": 0.69, - "us_foreign_policy": 0.7474747474747475, - "high_school_macroeconomics": 0.5449871465295629, - "computer_security": 0.6262626262626263, - "moral_scenarios": 0.23825503355704697, - "moral_disputes": 0.6289855072463768, - "electrical_engineering": 0.4513888888888889, - "astronomy": 0.5960264900662252, - "college_biology": 0.5104895104895105 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.24665676077265974 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.26650062266500624, - "category_acc": { - "computer_network": 0.16666666666666666, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.2619047619047619, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.21428571428571427, - "metrology_engineer": 0.27586206896551724, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.375, - "high_school_chemistry": 0.25, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.14285714285714285, - "college_economics": 0.2833333333333333, - "business_administration": 0.3157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.17647058823529413, - "teacher_qualification": 0.2653061224489796, - "high_school_politics": 0.5, - "high_school_geography": 0.25, - "middle_school_politics": 0.19230769230769232, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.375, - "logic": 0.3333333333333333, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.13157894736842105, - "professional_tour_guide": 0.17647058823529413, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.375, - "high_school_history": 0.16, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.23076923076923078, - "sports_science": 0.20833333333333334, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.375, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.1568627450980392, - "accountant": 0.3148148148148148, - "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.2777777777777778, - "physician": 0.2222222222222222 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.23297491039426524 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.25133828354342946, - "category_acc": { - "agronomy": 0.27218934911242604, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.2621951219512195, - "arts": 0.24375, - "astronomy": 0.19393939393939394, - "business_ethics": 0.23923444976076555, - "chinese_civil_service_exam": 0.23125, - "chinese_driving_rule": 0.32061068702290074, - "chinese_food_culture": 0.21323529411764705, - "chinese_foreign_policy": 0.308411214953271, - "chinese_history": 0.24148606811145512, - "chinese_literature": 0.25, - "chinese_teacher_qualification": 0.3016759776536313, - "clinical_knowledge": 0.20675105485232068, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.24299065420560748, - "college_engineering_hydrology": 0.25471698113207547, - "college_law": 0.25925925925925924, - "college_mathematics": 0.21904761904761905, - "college_medical_statistics": 0.22641509433962265, - "college_medicine": 0.27472527472527475, - "computer_science": 0.27450980392156865, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.21768707482993196, - "construction_project_management": 0.302158273381295, - "economics": 0.22012578616352202, - "education": 0.2147239263803681, - "electrical_engineering": 0.2558139534883721, - "elementary_chinese": 0.20238095238095238, - "elementary_commonsense": 0.26262626262626265, - "elementary_information_and_technology": 0.2773109243697479, - "elementary_mathematics": 0.21739130434782608, - "ethnology": 0.2518518518518518, - "food_science": 0.24475524475524477, - "genetics": 0.19318181818181818, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.28402366863905326, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.2457627118644068, - "high_school_mathematics": 0.21951219512195122, - "high_school_physics": 0.3181818181818182, - "high_school_politics": 0.26573426573426573, - "human_sexuality": 0.25396825396825395, - "international_law": 0.25405405405405407, - "journalism": 0.28488372093023256, - "jurisprudence": 0.24817518248175183, - "legal_and_moral_basis": 0.27102803738317754, - "logical": 0.22764227642276422, - "machine_learning": 0.23770491803278687, - "management": 0.24761904761904763, - "marketing": 0.2, - "marxist_theory": 0.25396825396825395, - "modern_chinese": 0.25, - "nutrition": 0.2620689655172414, - "philosophy": 0.3047619047619048, - "professional_accounting": 0.25142857142857145, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.2898936170212766, - "professional_psychology": 0.27155172413793105, - "public_relations": 0.22413793103448276, - "security_study": 0.25925925925925924, - "sociology": 0.2168141592920354, - "sports_science": 0.24848484848484848, - "traditional_chinese_medicine": 0.23783783783783785, - "virology": 0.26627218934911245, - "world_history": 0.2732919254658385, - "world_religions": 0.25625 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.3333333333333333 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.4159090909090909 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3264406779661017 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.2879581151832461 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.9333659970602646 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.5401495722799767, - "rouge2": 0.2925895887053187, - "rougeL": 0.4506841659736848, - "avg_rouge": 0.42780777565299344 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.40641906148664814, - "rouge2": 0.16610847977190274, - "rougeL": 0.32957541057011575, - "avg_rouge": 0.30070098394288886 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.963302752293578 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.8437200383509108 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.8535 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.8925 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.9425 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.7464788732394366 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.9169675090252708 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.8725490196078431 - } - } - } - }, - "sea_lion_3b": { - "model_size": "3B", - "model_link": "https://huggingface.co/aisingapore/sea-lion-3b", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.30903361344537816, - "language_acc": { - "English": 0.32016806722689073, - "Chinese": 0.28319327731092436, - "Spanish": 0.326890756302521, - "Vietnamese": 0.3058823529411765 - }, - "consistency_score_2": 0.4721288515406162, - "consistency_score_3": 0.2859243697478992, - "consistency_score_4": 0.20504201680672268, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.4865546218487395, - "English,Spanish": 0.573109243697479, - "English,Vietnamese": 0.4445378151260504, - "Chinese,Spanish": 0.39915966386554624, - "Chinese,Vietnamese": 0.5159663865546219, - "Spanish,Vietnamese": 0.4134453781512605 - }, - "3_combine": { - "English,Chinese,Spanish": 0.2907563025210084, - "English,Chinese,Vietnamese": 0.31176470588235294, - "English,Spanish,Vietnamese": 0.2823529411764706, - "Chinese,Spanish,Vietnamese": 0.25882352941176473 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.20504201680672268 - } - }, - "AC3_2": 0.3735552884952077, - "AC3_3": 0.29703018914439727, - "AC3_4": 0.24651966216920101 - }, - "prompt_2": { - "overall_acc": 0.3073529411764706, - "language_acc": { - "English": 0.3176470588235294, - "Chinese": 0.2991596638655462, - "Spanish": 0.3352941176470588, - "Vietnamese": 0.2773109243697479 - }, - "consistency_score_2": 0.5711484593837536, - "consistency_score_3": 0.4027310924369748, - "consistency_score_4": 0.31092436974789917, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.5546218487394958, - "English,Spanish": 0.6134453781512605, - "English,Vietnamese": 0.561344537815126, - "Chinese,Spanish": 0.5831932773109244, - "Chinese,Vietnamese": 0.5605042016806723, - "Spanish,Vietnamese": 0.553781512605042 - }, - "3_combine": { - "English,Chinese,Spanish": 0.41680672268907565, - "English,Chinese,Vietnamese": 0.38487394957983195, - "English,Spanish,Vietnamese": 0.4126050420168067, - "Chinese,Spanish,Vietnamese": 0.39663865546218485 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.31092436974789917 - } - }, - "AC3_2": 0.39964457360700645, - "AC3_3": 0.3486364426936797, - "AC3_4": 0.3091283403789357 - }, - "prompt_3": { - "overall_acc": 0.29537815126050415, - "language_acc": { - "English": 0.2991596638655462, - "Chinese": 0.27563025210084036, - "Spanish": 0.3302521008403361, - "Vietnamese": 0.27647058823529413 - }, - "consistency_score_2": 0.4196078431372549, - "consistency_score_3": 0.22647058823529415, - "consistency_score_4": 0.1411764705882353, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.41008403361344536, - "English,Spanish": 0.45210084033613446, - "English,Vietnamese": 0.37563025210084033, - "Chinese,Spanish": 0.39915966386554624, - "Chinese,Vietnamese": 0.4470588235294118, - "Spanish,Vietnamese": 0.4336134453781513 - }, - "3_combine": { - "English,Chinese,Spanish": 0.23361344537815126, - "English,Chinese,Vietnamese": 0.21512605042016808, - "English,Spanish,Vietnamese": 0.2218487394957983, - "Chinese,Spanish,Vietnamese": 0.23529411764705882 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.1411764705882353 - } - }, - "AC3_2": 0.34670046662201703, - "AC3_3": 0.2563749170674769, - "AC3_4": 0.19104342406312846 - }, - "prompt_4": { - "overall_acc": 0.30483193277310927, - "language_acc": { - "English": 0.3277310924369748, - "Chinese": 0.26974789915966385, - "Spanish": 0.3319327731092437, - "Vietnamese": 0.28991596638655465 - }, - "consistency_score_2": 0.6518207282913165, - "consistency_score_3": 0.5149159663865547, - "consistency_score_4": 0.44285714285714284, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.6672268907563025, - "English,Spanish": 0.6521008403361345, - "English,Vietnamese": 0.626890756302521, - "Chinese,Spanish": 0.6058823529411764, - "Chinese,Vietnamese": 0.7487394957983193, - "Spanish,Vietnamese": 0.6100840336134454 - }, - "3_combine": { - "English,Chinese,Spanish": 0.49831932773109244, - "English,Chinese,Vietnamese": 0.5571428571428572, - "English,Spanish,Vietnamese": 0.4831932773109244, - "Chinese,Spanish,Vietnamese": 0.5210084033613446 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.44285714285714284 - } - }, - "AC3_2": 0.4153979401149999, - "AC3_3": 0.3829539042212956, - "AC3_4": 0.36110464411338655 - }, - "prompt_5": { - "overall_acc": 0.30420168067226894, - "language_acc": { - "English": 0.32184873949579834, - "Chinese": 0.2773109243697479, - "Spanish": 0.326890756302521, - "Vietnamese": 0.2907563025210084 - }, - "consistency_score_2": 0.5946778711484594, - "consistency_score_3": 0.4359243697478991, - "consistency_score_4": 0.34705882352941175, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.6, - "English,Spanish": 0.5789915966386555, - "English,Vietnamese": 0.5865546218487395, - "Chinese,Spanish": 0.5168067226890757, - "Chinese,Vietnamese": 0.7588235294117647, - "Spanish,Vietnamese": 0.526890756302521 - }, - "3_combine": { - "English,Chinese,Spanish": 0.39663865546218485, - "English,Chinese,Vietnamese": 0.5126050420168067, - "English,Spanish,Vietnamese": 0.3957983193277311, - "Chinese,Spanish,Vietnamese": 0.43865546218487395 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.34705882352941175 - } - }, - "AC3_2": 0.4025055581149141, - "AC3_3": 0.35834146313350707, - "AC3_4": 0.32422011380220894 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.23904761904761904, - "language_acc": { - "Spanish": 0.26, - "English": 0.25333333333333335, - "Chinese": 0.23333333333333334, - "Vietnamese": 0.20666666666666667, - "Filipino": 0.23333333333333334, - "Indonesian": 0.24666666666666667, - "Malay": 0.24 - }, - "consistency_score_2": 0.40507936507936515, - "consistency_score_3": 0.20838095238095233, - "consistency_score_4": 0.12038095238095234, - "consistency_score_5": 0.07428571428571429, - "consistency_score_6": 0.04857142857142858, - "consistency_score_7": 0.03333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.43333333333333335, - "Spanish,Chinese": 0.14666666666666667, - "Spanish,Vietnamese": 0.4, - "Spanish,Filipino": 0.30666666666666664, - "Spanish,Indonesian": 0.3333333333333333, - "Spanish,Malay": 0.3333333333333333, - "English,Chinese": 0.2866666666666667, - "English,Vietnamese": 0.44, - "English,Filipino": 0.42, - "English,Indonesian": 0.38666666666666666, - "English,Malay": 0.4266666666666667, - "Chinese,Vietnamese": 0.43333333333333335, - "Chinese,Filipino": 0.30666666666666664, - "Chinese,Indonesian": 0.44, - "Chinese,Malay": 0.41333333333333333, - "Vietnamese,Filipino": 0.41333333333333333, - "Vietnamese,Indonesian": 0.48, - "Vietnamese,Malay": 0.47333333333333333, - "Filipino,Indonesian": 0.54, - "Filipino,Malay": 0.49333333333333335, - "Indonesian,Malay": 0.6 - }, - "3_combine": { - "Spanish,English,Chinese": 0.07333333333333333, - "Spanish,English,Vietnamese": 0.20666666666666667, - "Spanish,English,Filipino": 0.19333333333333333, - "Spanish,English,Indonesian": 0.2, - "Spanish,English,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese": 0.1, - "Spanish,Chinese,Filipino": 0.07333333333333333, - "Spanish,Chinese,Indonesian": 0.08, - "Spanish,Chinese,Malay": 0.08, - "Spanish,Vietnamese,Filipino": 0.15333333333333332, - "Spanish,Vietnamese,Indonesian": 0.2, - "Spanish,Vietnamese,Malay": 0.2, - "Spanish,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Filipino,Malay": 0.21333333333333335, - "Spanish,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese": 0.2, - "English,Chinese,Filipino": 0.13333333333333333, - "English,Chinese,Indonesian": 0.16, - "English,Chinese,Malay": 0.18666666666666668, - "English,Vietnamese,Filipino": 0.22666666666666666, - "English,Vietnamese,Indonesian": 0.24666666666666667, - "English,Vietnamese,Malay": 0.28, - "English,Filipino,Indonesian": 0.26, - "English,Filipino,Malay": 0.2733333333333333, - "English,Indonesian,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Filipino": 0.18666666666666668, - "Chinese,Vietnamese,Indonesian": 0.24666666666666667, - "Chinese,Vietnamese,Malay": 0.25333333333333335, - "Chinese,Filipino,Indonesian": 0.22666666666666666, - "Chinese,Filipino,Malay": 0.2, - "Chinese,Indonesian,Malay": 0.29333333333333333, - "Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Vietnamese,Filipino,Malay": 0.26, - "Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Filipino,Indonesian,Malay": 0.36666666666666664 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.05333333333333334, - "Spanish,English,Chinese,Filipino": 0.05333333333333334, - "Spanish,English,Chinese,Indonesian": 0.05333333333333334, - "Spanish,English,Chinese,Malay": 0.06, - "Spanish,English,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Vietnamese,Malay": 0.14, - "Spanish,English,Filipino,Indonesian": 0.14, - "Spanish,English,Filipino,Malay": 0.14, - "Spanish,English,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino": 0.06, - "Spanish,Chinese,Vietnamese,Indonesian": 0.07333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.06666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.06666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.06666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.12, - "Spanish,Vietnamese,Filipino,Malay": 0.12, - "Spanish,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.15333333333333332, - "English,Chinese,Vietnamese,Filipino": 0.10666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.12, - "English,Chinese,Vietnamese,Malay": 0.14666666666666667, - "English,Chinese,Filipino,Indonesian": 0.09333333333333334, - "English,Chinese,Filipino,Malay": 0.1, - "English,Chinese,Indonesian,Malay": 0.12666666666666668, - "English,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Vietnamese,Filipino,Malay": 0.16666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "English,Filipino,Indonesian,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.04, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.04666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.04666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.04666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.04666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.08666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,English,Filipino,Indonesian,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.06, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.05333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.06, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.06, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "English,Chinese,Vietnamese,Filipino,Malay": 0.08, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.09333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.08, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.04, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.03333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.04, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.04, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.03333333333333333 - } - }, - "AC3_2": 0.300665117651461, - "AC3_3": 0.22266334200055007, - "AC3_4": 0.16012516713944075, - "AC3_5": 0.11334780717181475, - "AC3_6": 0.08073793753105848, - "AC3_7": 0.05850815848667824 - }, - "prompt_2": { - "overall_acc": 0.26952380952380955, - "language_acc": { - "Spanish": 0.24, - "English": 0.2866666666666667, - "Chinese": 0.22666666666666666, - "Vietnamese": 0.28, - "Filipino": 0.29333333333333333, - "Indonesian": 0.24666666666666667, - "Malay": 0.31333333333333335 - }, - "consistency_score_2": 0.5873015873015874, - "consistency_score_3": 0.4260952380952381, - "consistency_score_4": 0.32952380952380944, - "consistency_score_5": 0.2625396825396825, - "consistency_score_6": 0.21238095238095236, - "consistency_score_7": 0.17333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.64, - "Spanish,Chinese": 0.38666666666666666, - "Spanish,Vietnamese": 0.48, - "Spanish,Filipino": 0.6333333333333333, - "Spanish,Indonesian": 0.58, - "Spanish,Malay": 0.6, - "English,Chinese": 0.4666666666666667, - "English,Vietnamese": 0.64, - "English,Filipino": 0.7866666666666666, - "English,Indonesian": 0.6733333333333333, - "English,Malay": 0.6733333333333333, - "Chinese,Vietnamese": 0.46, - "Chinese,Filipino": 0.43333333333333335, - "Chinese,Indonesian": 0.4866666666666667, - "Chinese,Malay": 0.4533333333333333, - "Vietnamese,Filipino": 0.64, - "Vietnamese,Indonesian": 0.6, - "Vietnamese,Malay": 0.6133333333333333, - "Filipino,Indonesian": 0.6933333333333334, - "Filipino,Malay": 0.7066666666666667, - "Indonesian,Malay": 0.6866666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.32666666666666666, - "Spanish,English,Vietnamese": 0.41333333333333333, - "Spanish,English,Filipino": 0.5466666666666666, - "Spanish,English,Indonesian": 0.4866666666666667, - "Spanish,English,Malay": 0.49333333333333335, - "Spanish,Chinese,Vietnamese": 0.26, - "Spanish,Chinese,Filipino": 0.29333333333333333, - "Spanish,Chinese,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Malay": 0.2866666666666667, - "Spanish,Vietnamese,Filipino": 0.4, - "Spanish,Vietnamese,Indonesian": 0.3933333333333333, - "Spanish,Vietnamese,Malay": 0.38666666666666666, - "Spanish,Filipino,Indonesian": 0.5, - "Spanish,Filipino,Malay": 0.49333333333333335, - "Spanish,Indonesian,Malay": 0.47333333333333333, - "English,Chinese,Vietnamese": 0.3466666666666667, - "English,Chinese,Filipino": 0.38, - "English,Chinese,Indonesian": 0.37333333333333335, - "English,Chinese,Malay": 0.3466666666666667, - "English,Vietnamese,Filipino": 0.5533333333333333, - "English,Vietnamese,Indonesian": 0.49333333333333335, - "English,Vietnamese,Malay": 0.4866666666666667, - "English,Filipino,Indonesian": 0.6, - "English,Filipino,Malay": 0.6, - "English,Indonesian,Malay": 0.5466666666666666, - "Chinese,Vietnamese,Filipino": 0.32666666666666666, - "Chinese,Vietnamese,Indonesian": 0.34, - "Chinese,Vietnamese,Malay": 0.32666666666666666, - "Chinese,Filipino,Indonesian": 0.36666666666666664, - "Chinese,Filipino,Malay": 0.35333333333333333, - "Chinese,Indonesian,Malay": 0.37333333333333335, - "Vietnamese,Filipino,Indonesian": 0.5, - "Vietnamese,Filipino,Malay": 0.5066666666666667, - "Vietnamese,Indonesian,Malay": 0.48, - "Filipino,Indonesian,Malay": 0.5733333333333334 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.24666666666666667, - "Spanish,English,Chinese,Filipino": 0.2866666666666667, - "Spanish,English,Chinese,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Malay": 0.26, - "Spanish,English,Vietnamese,Filipino": 0.37333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.36, - "Spanish,English,Vietnamese,Malay": 0.34, - "Spanish,English,Filipino,Indonesian": 0.4533333333333333, - "Spanish,English,Filipino,Malay": 0.44, - "Spanish,English,Indonesian,Malay": 0.42, - "Spanish,Chinese,Vietnamese,Filipino": 0.22, - "Spanish,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.21333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.24666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.24, - "Spanish,Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.4266666666666667, - "English,Chinese,Vietnamese,Filipino": 0.29333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.3, - "English,Chinese,Vietnamese,Malay": 0.2733333333333333, - "English,Chinese,Filipino,Indonesian": 0.3333333333333333, - "English,Chinese,Filipino,Malay": 0.31333333333333335, - "English,Chinese,Indonesian,Malay": 0.31333333333333335, - "English,Vietnamese,Filipino,Indonesian": 0.4533333333333333, - "English,Vietnamese,Filipino,Malay": 0.44, - "English,Vietnamese,Indonesian,Malay": 0.41333333333333333, - "English,Filipino,Indonesian,Malay": 0.5066666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.3, - "Chinese,Vietnamese,Filipino,Malay": 0.28, - "Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.31333333333333335, - "Vietnamese,Filipino,Indonesian,Malay": 0.42 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.22, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.22, - "Spanish,English,Chinese,Vietnamese,Malay": 0.2, - "Spanish,English,Chinese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Chinese,Filipino,Malay": 0.24, - "Spanish,English,Chinese,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.3333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.31333333333333335, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Spanish,English,Filipino,Indonesian,Malay": 0.3933333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.22, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.3, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.38, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.22, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - } - }, - "AC3_2": 0.36948428867051053, - "AC3_3": 0.3301888071292921, - "AC3_4": 0.29651903999895335, - "AC3_5": 0.26598590743949607, - "AC3_6": 0.23756446447168927, - "AC3_7": 0.21098207880540548 - }, - "prompt_3": { - "overall_acc": 0.24476190476190476, - "language_acc": { - "Spanish": 0.21333333333333335, - "English": 0.26, - "Chinese": 0.20666666666666667, - "Vietnamese": 0.26, - "Filipino": 0.26, - "Indonesian": 0.24666666666666667, - "Malay": 0.26666666666666666 - }, - "consistency_score_2": 0.5212698412698412, - "consistency_score_3": 0.34838095238095246, - "consistency_score_4": 0.2552380952380952, - "consistency_score_5": 0.19650793650793652, - "consistency_score_6": 0.15619047619047619, - "consistency_score_7": 0.12666666666666668, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5466666666666666, - "Spanish,Chinese": 0.30666666666666664, - "Spanish,Vietnamese": 0.5, - "Spanish,Filipino": 0.52, - "Spanish,Indonesian": 0.5333333333333333, - "Spanish,Malay": 0.5266666666666666, - "English,Chinese": 0.34, - "English,Vietnamese": 0.49333333333333335, - "English,Filipino": 0.6466666666666666, - "English,Indonesian": 0.5866666666666667, - "English,Malay": 0.6, - "Chinese,Vietnamese": 0.38, - "Chinese,Filipino": 0.30666666666666664, - "Chinese,Indonesian": 0.43333333333333335, - "Chinese,Malay": 0.44666666666666666, - "Vietnamese,Filipino": 0.5333333333333333, - "Vietnamese,Indonesian": 0.58, - "Vietnamese,Malay": 0.6066666666666667, - "Filipino,Indonesian": 0.66, - "Filipino,Malay": 0.6666666666666666, - "Indonesian,Malay": 0.7333333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.22, - "Spanish,English,Vietnamese": 0.34, - "Spanish,English,Filipino": 0.4266666666666667, - "Spanish,English,Indonesian": 0.4066666666666667, - "Spanish,English,Malay": 0.4, - "Spanish,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,Chinese,Filipino": 0.17333333333333334, - "Spanish,Chinese,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino": 0.3466666666666667, - "Spanish,Vietnamese,Indonesian": 0.36666666666666664, - "Spanish,Vietnamese,Malay": 0.38, - "Spanish,Filipino,Indonesian": 0.42, - "Spanish,Filipino,Malay": 0.4266666666666667, - "Spanish,Indonesian,Malay": 0.4533333333333333, - "English,Chinese,Vietnamese": 0.24, - "English,Chinese,Filipino": 0.22666666666666666, - "English,Chinese,Indonesian": 0.26666666666666666, - "English,Chinese,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino": 0.38, - "English,Vietnamese,Indonesian": 0.38, - "English,Vietnamese,Malay": 0.4, - "English,Filipino,Indonesian": 0.4866666666666667, - "English,Filipino,Malay": 0.4866666666666667, - "English,Indonesian,Malay": 0.49333333333333335, - "Chinese,Vietnamese,Filipino": 0.22666666666666666, - "Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Chinese,Vietnamese,Malay": 0.2866666666666667, - "Chinese,Filipino,Indonesian": 0.24666666666666667, - "Chinese,Filipino,Malay": 0.26, - "Chinese,Indonesian,Malay": 0.34, - "Vietnamese,Filipino,Indonesian": 0.43333333333333335, - "Vietnamese,Filipino,Malay": 0.44666666666666666, - "Vietnamese,Indonesian,Malay": 0.4866666666666667, - "Filipino,Indonesian,Malay": 0.56 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.17333333333333334, - "Spanish,English,Chinese,Filipino": 0.15333333333333332, - "Spanish,English,Chinese,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino": 0.28, - "Spanish,English,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,English,Vietnamese,Malay": 0.29333333333333333, - "Spanish,English,Filipino,Indonesian": 0.3466666666666667, - "Spanish,English,Filipino,Malay": 0.3466666666666667, - "Spanish,English,Indonesian,Malay": 0.36, - "Spanish,Chinese,Vietnamese,Filipino": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Malay": 0.19333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Filipino,Malay": 0.16666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Vietnamese,Filipino,Malay": 0.32, - "Spanish,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.4, - "English,Chinese,Vietnamese,Filipino": 0.18666666666666668, - "English,Chinese,Vietnamese,Indonesian": 0.2, - "English,Chinese,Vietnamese,Malay": 0.20666666666666667, - "English,Chinese,Filipino,Indonesian": 0.2, - "English,Chinese,Filipino,Malay": 0.21333333333333335, - "English,Chinese,Indonesian,Malay": 0.23333333333333334, - "English,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "English,Vietnamese,Filipino,Malay": 0.3466666666666667, - "English,Vietnamese,Indonesian,Malay": 0.3466666666666667, - "English,Filipino,Indonesian,Malay": 0.4266666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.19333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.3933333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Vietnamese,Malay": 0.16, - "Spanish,English,Chinese,Filipino,Indonesian": 0.14, - "Spanish,English,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.16666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,English,Vietnamese,Filipino,Malay": 0.26, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.3, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "English,Chinese,Vietnamese,Filipino,Malay": 0.18, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.18, - "English,Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.12666666666666668, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.14, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - } - }, - "AC3_2": 0.3331115189113822, - "AC3_3": 0.28752056758770056, - "AC3_4": 0.2498902493831285, - "AC3_5": 0.21799657411983395, - "AC3_6": 0.19069336043202365, - "AC3_7": 0.1669401708952255 - }, - "prompt_4": { - "overall_acc": 0.2752380952380952, - "language_acc": { - "Spanish": 0.26666666666666666, - "English": 0.3333333333333333, - "Chinese": 0.26, - "Vietnamese": 0.28, - "Filipino": 0.26, - "Indonesian": 0.28, - "Malay": 0.24666666666666667 - }, - "consistency_score_2": 0.5174603174603174, - "consistency_score_3": 0.32647619047619053, - "consistency_score_4": 0.2211428571428572, - "consistency_score_5": 0.15492063492063496, - "consistency_score_6": 0.11047619047619049, - "consistency_score_7": 0.08, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.3933333333333333, - "Spanish,Chinese": 0.31333333333333335, - "Spanish,Vietnamese": 0.3, - "Spanish,Filipino": 0.3933333333333333, - "Spanish,Indonesian": 0.36666666666666664, - "Spanish,Malay": 0.4, - "English,Chinese": 0.49333333333333335, - "English,Vietnamese": 0.5066666666666667, - "English,Filipino": 0.6133333333333333, - "English,Indonesian": 0.5733333333333334, - "English,Malay": 0.5733333333333334, - "Chinese,Vietnamese": 0.6933333333333334, - "Chinese,Filipino": 0.4066666666666667, - "Chinese,Indonesian": 0.64, - "Chinese,Malay": 0.62, - "Vietnamese,Filipino": 0.38666666666666666, - "Vietnamese,Indonesian": 0.6133333333333333, - "Vietnamese,Malay": 0.6066666666666667, - "Filipino,Indonesian": 0.6, - "Filipino,Malay": 0.6, - "Indonesian,Malay": 0.7733333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.18666666666666668, - "Spanish,English,Vietnamese": 0.18, - "Spanish,English,Filipino": 0.2733333333333333, - "Spanish,English,Indonesian": 0.22666666666666666, - "Spanish,English,Malay": 0.24666666666666667, - "Spanish,Chinese,Vietnamese": 0.20666666666666667, - "Spanish,Chinese,Filipino": 0.15333333333333332, - "Spanish,Chinese,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino": 0.13333333333333333, - "Spanish,Vietnamese,Indonesian": 0.22, - "Spanish,Vietnamese,Malay": 0.22, - "Spanish,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Filipino,Malay": 0.26, - "Spanish,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese": 0.38666666666666666, - "English,Chinese,Filipino": 0.32, - "English,Chinese,Indonesian": 0.3933333333333333, - "English,Chinese,Malay": 0.4, - "English,Vietnamese,Filipino": 0.31333333333333335, - "English,Vietnamese,Indonesian": 0.4, - "English,Vietnamese,Malay": 0.38666666666666666, - "English,Filipino,Indonesian": 0.44, - "English,Filipino,Malay": 0.44, - "English,Indonesian,Malay": 0.47333333333333333, - "Chinese,Vietnamese,Filipino": 0.2866666666666667, - "Chinese,Vietnamese,Indonesian": 0.5133333333333333, - "Chinese,Vietnamese,Malay": 0.49333333333333335, - "Chinese,Filipino,Indonesian": 0.34, - "Chinese,Filipino,Malay": 0.34, - "Chinese,Indonesian,Malay": 0.5333333333333333, - "Vietnamese,Filipino,Indonesian": 0.32, - "Vietnamese,Filipino,Malay": 0.32, - "Vietnamese,Indonesian,Malay": 0.5066666666666667, - "Filipino,Indonesian,Malay": 0.49333333333333335 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.14666666666666667, - "Spanish,English,Chinese,Filipino": 0.12666666666666668, - "Spanish,English,Chinese,Indonesian": 0.16, - "Spanish,English,Chinese,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino": 0.10666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.15333333333333332, - "Spanish,English,Vietnamese,Malay": 0.15333333333333332, - "Spanish,English,Filipino,Indonesian": 0.16666666666666666, - "Spanish,English,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Indonesian,Malay": 0.2, - "Spanish,Chinese,Vietnamese,Filipino": 0.1, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Malay": 0.18, - "Spanish,Chinese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.14, - "Spanish,Chinese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Indonesian,Malay": 0.20666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.32666666666666666, - "English,Chinese,Vietnamese,Malay": 0.32, - "English,Chinese,Filipino,Indonesian": 0.2733333333333333, - "English,Chinese,Filipino,Malay": 0.28, - "English,Chinese,Indonesian,Malay": 0.35333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "English,Vietnamese,Filipino,Malay": 0.2733333333333333, - "English,Vietnamese,Indonesian,Malay": 0.34, - "English,Filipino,Indonesian,Malay": 0.37333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.4266666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian,Malay": 0.28 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12666666666666668, - "Spanish,English,Chinese,Filipino,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.11333333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,English,Vietnamese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.14, - "Spanish,English,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.26, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.12, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - } - }, - "AC3_2": 0.3593416861686014, - "AC3_3": 0.29867559047780495, - "AC3_4": 0.24524284612974087, - "AC3_5": 0.1982526796235656, - "AC3_6": 0.15766725451526675, - "AC3_7": 0.12396782838333345 - }, - "prompt_5": { - "overall_acc": 0.25428571428571434, - "language_acc": { - "Spanish": 0.28, - "English": 0.26, - "Chinese": 0.24, - "Vietnamese": 0.23333333333333334, - "Filipino": 0.25333333333333335, - "Indonesian": 0.26, - "Malay": 0.25333333333333335 - }, - "consistency_score_2": 0.5847619047619046, - "consistency_score_3": 0.40838095238095234, - "consistency_score_4": 0.30133333333333334, - "consistency_score_5": 0.22793650793650796, - "consistency_score_6": 0.17428571428571432, - "consistency_score_7": 0.13333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5, - "Spanish,Chinese": 0.2733333333333333, - "Spanish,Vietnamese": 0.23333333333333334, - "Spanish,Filipino": 0.48, - "Spanish,Indonesian": 0.38666666666666666, - "Spanish,Malay": 0.4066666666666667, - "English,Chinese": 0.62, - "English,Vietnamese": 0.5266666666666666, - "English,Filipino": 0.6866666666666666, - "English,Indonesian": 0.6866666666666666, - "English,Malay": 0.6666666666666666, - "Chinese,Vietnamese": 0.72, - "Chinese,Filipino": 0.5733333333333334, - "Chinese,Indonesian": 0.6866666666666666, - "Chinese,Malay": 0.66, - "Vietnamese,Filipino": 0.54, - "Vietnamese,Indonesian": 0.6866666666666666, - "Vietnamese,Malay": 0.6333333333333333, - "Filipino,Indonesian": 0.7266666666666667, - "Filipino,Malay": 0.7466666666666667, - "Indonesian,Malay": 0.84 - }, - "3_combine": { - "Spanish,English,Chinese": 0.24666666666666667, - "Spanish,English,Vietnamese": 0.18666666666666668, - "Spanish,English,Filipino": 0.36666666666666664, - "Spanish,English,Indonesian": 0.32, - "Spanish,English,Malay": 0.32, - "Spanish,Chinese,Vietnamese": 0.18666666666666668, - "Spanish,Chinese,Filipino": 0.22666666666666666, - "Spanish,Chinese,Indonesian": 0.22, - "Spanish,Chinese,Malay": 0.21333333333333335, - "Spanish,Vietnamese,Filipino": 0.18666666666666668, - "Spanish,Vietnamese,Indonesian": 0.20666666666666667, - "Spanish,Vietnamese,Malay": 0.19333333333333333, - "Spanish,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Filipino,Malay": 0.36, - "Spanish,Indonesian,Malay": 0.3333333333333333, - "English,Chinese,Vietnamese": 0.47333333333333333, - "English,Chinese,Filipino": 0.47333333333333333, - "English,Chinese,Indonesian": 0.52, - "English,Chinese,Malay": 0.5, - "English,Vietnamese,Filipino": 0.41333333333333333, - "English,Vietnamese,Indonesian": 0.4666666666666667, - "English,Vietnamese,Malay": 0.44, - "English,Filipino,Indonesian": 0.5666666666666667, - "English,Filipino,Malay": 0.5733333333333334, - "English,Indonesian,Malay": 0.6066666666666667, - "Chinese,Vietnamese,Filipino": 0.44666666666666666, - "Chinese,Vietnamese,Indonesian": 0.5666666666666667, - "Chinese,Vietnamese,Malay": 0.5333333333333333, - "Chinese,Filipino,Indonesian": 0.5066666666666667, - "Chinese,Filipino,Malay": 0.5, - "Chinese,Indonesian,Malay": 0.6, - "Vietnamese,Filipino,Indonesian": 0.4866666666666667, - "Vietnamese,Filipino,Malay": 0.47333333333333333, - "Vietnamese,Indonesian,Malay": 0.5866666666666667, - "Filipino,Indonesian,Malay": 0.66 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.17333333333333334, - "Spanish,English,Chinese,Filipino": 0.20666666666666667, - "Spanish,English,Chinese,Indonesian": 0.20666666666666667, - "Spanish,English,Chinese,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.16, - "Spanish,English,Vietnamese,Indonesian": 0.18, - "Spanish,English,Vietnamese,Malay": 0.16666666666666666, - "Spanish,English,Filipino,Indonesian": 0.28, - "Spanish,English,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino": 0.16, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Malay": 0.16666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.19333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.19333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.2, - "Spanish,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Vietnamese,Filipino,Malay": 0.18, - "Spanish,Vietnamese,Indonesian,Malay": 0.18666666666666668, - "Spanish,Filipino,Indonesian,Malay": 0.30666666666666664, - "English,Chinese,Vietnamese,Filipino": 0.37333333333333335, - "English,Chinese,Vietnamese,Indonesian": 0.4266666666666667, - "English,Chinese,Vietnamese,Malay": 0.4066666666666667, - "English,Chinese,Filipino,Indonesian": 0.4266666666666667, - "English,Chinese,Filipino,Malay": 0.4266666666666667, - "English,Chinese,Indonesian,Malay": 0.47333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "English,Vietnamese,Filipino,Malay": 0.38, - "English,Vietnamese,Indonesian,Malay": 0.41333333333333333, - "English,Filipino,Indonesian,Malay": 0.5266666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.4066666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.4066666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.4866666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.47333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.44666666666666666 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Filipino,Indonesian": 0.18, - "Spanish,English,Chinese,Filipino,Malay": 0.18, - "Spanish,English,Chinese,Indonesian,Malay": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,English,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,English,Filipino,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.3466666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.38, - "English,Chinese,Filipino,Indonesian,Malay": 0.4066666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.36, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.38 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.14, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.14, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.32666666666666666 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.13333333333333333 - } - }, - "AC3_2": 0.35444138150470017, - "AC3_3": 0.31341682744828947, - "AC3_4": 0.2758176207936907, - "AC3_5": 0.2403912347850971, - "AC3_6": 0.2068190475707899, - "AC3_7": 0.1749385748934434 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.26136363636363635, - "language_acc": { - "English": 0.26704545454545453, - "Filipino": 0.23863636363636365, - "Spanish": 0.2727272727272727, - "Malay": 0.26136363636363635, - "Chinese": 0.30113636363636365, - "Indonesian": 0.2215909090909091, - "Vietnamese": 0.26704545454545453 - }, - "consistency_score_2": 0.4375, - "consistency_score_3": 0.2412337662337663, - "consistency_score_4": 0.15129870129870132, - "consistency_score_5": 0.1030844155844156, - "consistency_score_6": 0.07467532467532467, - "consistency_score_7": 0.056818181818181816, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.32386363636363635, - "English,Spanish": 0.48295454545454547, - "English,Malay": 0.42045454545454547, - "English,Chinese": 0.44886363636363635, - "English,Indonesian": 0.5, - "English,Vietnamese": 0.5681818181818182, - "Filipino,Spanish": 0.18181818181818182, - "Filipino,Malay": 0.6420454545454546, - "Filipino,Chinese": 0.2556818181818182, - "Filipino,Indonesian": 0.6590909090909091, - "Filipino,Vietnamese": 0.3977272727272727, - "Spanish,Malay": 0.3409090909090909, - "Spanish,Chinese": 0.3465909090909091, - "Spanish,Indonesian": 0.3409090909090909, - "Spanish,Vietnamese": 0.4772727272727273, - "Malay,Chinese": 0.3522727272727273, - "Malay,Indonesian": 0.6534090909090909, - "Malay,Vietnamese": 0.5113636363636364, - "Chinese,Indonesian": 0.3465909090909091, - "Chinese,Vietnamese": 0.4375, - "Indonesian,Vietnamese": 0.5 - }, - "3_combine": { - "English,Filipino,Spanish": 0.11363636363636363, - "English,Filipino,Malay": 0.2727272727272727, - "English,Filipino,Chinese": 0.16477272727272727, - "English,Filipino,Indonesian": 0.2897727272727273, - "English,Filipino,Vietnamese": 0.23863636363636365, - "English,Spanish,Malay": 0.2159090909090909, - "English,Spanish,Chinese": 0.2215909090909091, - "English,Spanish,Indonesian": 0.24431818181818182, - "English,Spanish,Vietnamese": 0.32386363636363635, - "English,Malay,Chinese": 0.2159090909090909, - "English,Malay,Indonesian": 0.3522727272727273, - "English,Malay,Vietnamese": 0.32954545454545453, - "English,Chinese,Indonesian": 0.2556818181818182, - "English,Chinese,Vietnamese": 0.29545454545454547, - "English,Indonesian,Vietnamese": 0.3465909090909091, - "Filipino,Spanish,Malay": 0.1590909090909091, - "Filipino,Spanish,Chinese": 0.08522727272727272, - "Filipino,Spanish,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Vietnamese": 0.125, - "Filipino,Malay,Chinese": 0.21022727272727273, - "Filipino,Malay,Indonesian": 0.5113636363636364, - "Filipino,Malay,Vietnamese": 0.3409090909090909, - "Filipino,Chinese,Indonesian": 0.19318181818181818, - "Filipino,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Indonesian,Vietnamese": 0.32954545454545453, - "Spanish,Malay,Chinese": 0.1534090909090909, - "Spanish,Malay,Indonesian": 0.23295454545454544, - "Spanish,Malay,Vietnamese": 0.23295454545454544, - "Spanish,Chinese,Indonesian": 0.14772727272727273, - "Spanish,Chinese,Vietnamese": 0.22727272727272727, - "Spanish,Indonesian,Vietnamese": 0.23863636363636365, - "Malay,Chinese,Indonesian": 0.23295454545454544, - "Malay,Chinese,Vietnamese": 0.2159090909090909, - "Malay,Indonesian,Vietnamese": 0.38636363636363635, - "Chinese,Indonesian,Vietnamese": 0.2159090909090909 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.10795454545454546, - "English,Filipino,Spanish,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Indonesian": 0.10795454545454546, - "English,Filipino,Spanish,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese": 0.14772727272727273, - "English,Filipino,Malay,Indonesian": 0.2556818181818182, - "English,Filipino,Malay,Vietnamese": 0.2215909090909091, - "English,Filipino,Chinese,Indonesian": 0.14204545454545456, - "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay,Chinese": 0.125, - "English,Spanish,Malay,Indonesian": 0.17045454545454544, - "English,Spanish,Malay,Vietnamese": 0.17045454545454544, - "English,Spanish,Chinese,Indonesian": 0.11931818181818182, - "English,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Malay,Chinese,Indonesian": 0.18181818181818182, - "English,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Malay,Chinese": 0.07954545454545454, - "Filipino,Spanish,Malay,Indonesian": 0.14772727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "Filipino,Spanish,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "Filipino,Malay,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "Filipino,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.10795454545454546, - "Spanish,Malay,Chinese,Vietnamese": 0.125, - "Spanish,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.0625, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.07386363636363637, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.056818181818181816, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 - } - }, - "AC3_2": 0.3272357723108996, - "AC3_3": 0.25089558369450116, - "AC3_4": 0.1916529575388513, - "AC3_5": 0.14785381652148785, - "AC3_6": 0.11616161612704826, - "AC3_7": 0.0933441558148191 - }, - "prompt_2": { - "overall_acc": 0.2353896103896104, - "language_acc": { - "English": 0.23863636363636365, - "Filipino": 0.22727272727272727, - "Spanish": 0.24431818181818182, - "Malay": 0.22727272727272727, - "Chinese": 0.26136363636363635, - "Indonesian": 0.22727272727272727, - "Vietnamese": 0.2215909090909091 - }, - "consistency_score_2": 0.6209415584415584, - "consistency_score_3": 0.4594155844155844, - "consistency_score_4": 0.35649350649350653, - "consistency_score_5": 0.28138528138528135, - "consistency_score_6": 0.22321428571428573, - "consistency_score_7": 0.17613636363636365, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.5511363636363636, - "English,Spanish": 0.5738636363636364, - "English,Malay": 0.5511363636363636, - "English,Chinese": 0.5056818181818182, - "English,Indonesian": 0.5511363636363636, - "English,Vietnamese": 0.5511363636363636, - "Filipino,Spanish": 0.6022727272727273, - "Filipino,Malay": 0.8579545454545454, - "Filipino,Chinese": 0.5454545454545454, - "Filipino,Indonesian": 0.8522727272727273, - "Filipino,Vietnamese": 0.7443181818181818, - "Spanish,Malay": 0.5852272727272727, - "Spanish,Chinese": 0.48863636363636365, - "Spanish,Indonesian": 0.5738636363636364, - "Spanish,Vietnamese": 0.5909090909090909, - "Malay,Chinese": 0.5284090909090909, - "Malay,Indonesian": 0.8579545454545454, - "Malay,Vietnamese": 0.7272727272727273, - "Chinese,Indonesian": 0.5340909090909091, - "Chinese,Vietnamese": 0.5284090909090909, - "Indonesian,Vietnamese": 0.7386363636363636 - }, - "3_combine": { - "English,Filipino,Spanish": 0.39204545454545453, - "English,Filipino,Malay": 0.48863636363636365, - "English,Filipino,Chinese": 0.35795454545454547, - "English,Filipino,Indonesian": 0.4943181818181818, - "English,Filipino,Vietnamese": 0.45454545454545453, - "English,Spanish,Malay": 0.38636363636363635, - "English,Spanish,Chinese": 0.3181818181818182, - "English,Spanish,Indonesian": 0.3806818181818182, - "English,Spanish,Vietnamese": 0.38636363636363635, - "English,Malay,Chinese": 0.35795454545454547, - "English,Malay,Indonesian": 0.48863636363636365, - "English,Malay,Vietnamese": 0.44886363636363635, - "English,Chinese,Indonesian": 0.3522727272727273, - "English,Chinese,Vietnamese": 0.3409090909090909, - "English,Indonesian,Vietnamese": 0.44886363636363635, - "Filipino,Spanish,Malay": 0.5284090909090909, - "Filipino,Spanish,Chinese": 0.3522727272727273, - "Filipino,Spanish,Indonesian": 0.5170454545454546, - "Filipino,Spanish,Vietnamese": 0.4943181818181818, - "Filipino,Malay,Chinese": 0.48863636363636365, - "Filipino,Malay,Indonesian": 0.7897727272727273, - "Filipino,Malay,Vietnamese": 0.6761363636363636, - "Filipino,Chinese,Indonesian": 0.48295454545454547, - "Filipino,Chinese,Vietnamese": 0.4318181818181818, - "Filipino,Indonesian,Vietnamese": 0.6818181818181818, - "Spanish,Malay,Chinese": 0.3465909090909091, - "Spanish,Malay,Indonesian": 0.5227272727272727, - "Spanish,Malay,Vietnamese": 0.4772727272727273, - "Spanish,Chinese,Indonesian": 0.3409090909090909, - "Spanish,Chinese,Vietnamese": 0.3522727272727273, - "Spanish,Indonesian,Vietnamese": 0.48295454545454547, - "Malay,Chinese,Indonesian": 0.48863636363636365, - "Malay,Chinese,Vietnamese": 0.42613636363636365, - "Malay,Indonesian,Vietnamese": 0.6761363636363636, - "Chinese,Indonesian,Vietnamese": 0.42613636363636365 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.3465909090909091, - "English,Filipino,Spanish,Chinese": 0.2556818181818182, - "English,Filipino,Spanish,Indonesian": 0.3465909090909091, - "English,Filipino,Spanish,Vietnamese": 0.32386363636363635, - "English,Filipino,Malay,Chinese": 0.32954545454545453, - "English,Filipino,Malay,Indonesian": 0.45454545454545453, - "English,Filipino,Malay,Vietnamese": 0.4147727272727273, - "English,Filipino,Chinese,Indonesian": 0.32954545454545453, - "English,Filipino,Chinese,Vietnamese": 0.2840909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.4147727272727273, - "English,Spanish,Malay,Chinese": 0.24431818181818182, - "English,Spanish,Malay,Indonesian": 0.3465909090909091, - "English,Spanish,Malay,Vietnamese": 0.32386363636363635, - "English,Spanish,Chinese,Indonesian": 0.24431818181818182, - "English,Spanish,Chinese,Vietnamese": 0.23863636363636365, - "English,Spanish,Indonesian,Vietnamese": 0.3181818181818182, - "English,Malay,Chinese,Indonesian": 0.32954545454545453, - "English,Malay,Chinese,Vietnamese": 0.2784090909090909, - "English,Malay,Indonesian,Vietnamese": 0.4034090909090909, - "English,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "Filipino,Spanish,Malay,Chinese": 0.32386363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.48295454545454547, - "Filipino,Spanish,Malay,Vietnamese": 0.4431818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.3181818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.2897727272727273, - "Filipino,Spanish,Indonesian,Vietnamese": 0.4431818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.4602272727272727, - "Filipino,Malay,Chinese,Vietnamese": 0.4090909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.6363636363636364, - "Filipino,Chinese,Indonesian,Vietnamese": 0.4034090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.32954545454545453, - "Spanish,Malay,Chinese,Vietnamese": 0.2897727272727273, - "Spanish,Malay,Indonesian,Vietnamese": 0.4431818181818182, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.4034090909090909 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.23295454545454544, - "English,Filipino,Spanish,Malay,Indonesian": 0.3181818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.29545454545454547, - "English,Filipino,Spanish,Chinese,Indonesian": 0.23295454545454544, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.2897727272727273, - "English,Filipino,Malay,Chinese,Indonesian": 0.3125, - "English,Filipino,Malay,Chinese,Vietnamese": 0.26704545454545453, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.3806818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "English,Spanish,Malay,Chinese,Indonesian": 0.23295454545454544, - "English,Spanish,Malay,Chinese,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2897727272727273, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.3068181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.4147727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.38636363636363635, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2784090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.2215909090909091, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1875, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.26704545454545453, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.25, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365 - } - }, - "AC3_2": 0.34137071455359913, - "AC3_3": 0.31128626042370156, - "AC3_4": 0.28355215816141, - "AC3_5": 0.2563405180710189, - "AC3_6": 0.22914032864788608, - "AC3_7": 0.20149722067902656 - }, - "prompt_3": { - "overall_acc": 0.24025974025974026, - "language_acc": { - "English": 0.2159090909090909, - "Filipino": 0.23863636363636365, - "Spanish": 0.2840909090909091, - "Malay": 0.23295454545454544, - "Chinese": 0.25, - "Indonesian": 0.23295454545454544, - "Vietnamese": 0.22727272727272727 - }, - "consistency_score_2": 0.5687229437229437, - "consistency_score_3": 0.39058441558441565, - "consistency_score_4": 0.2907467532467533, - "consistency_score_5": 0.22754329004329005, - "consistency_score_6": 0.18425324675324672, - "consistency_score_7": 0.1534090909090909, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.48295454545454547, - "English,Spanish": 0.5852272727272727, - "English,Malay": 0.4772727272727273, - "English,Chinese": 0.48295454545454547, - "English,Indonesian": 0.5056818181818182, - "English,Vietnamese": 0.5681818181818182, - "Filipino,Spanish": 0.4659090909090909, - "Filipino,Malay": 0.8806818181818182, - "Filipino,Chinese": 0.4090909090909091, - "Filipino,Indonesian": 0.8522727272727273, - "Filipino,Vietnamese": 0.6875, - "Spanish,Malay": 0.4943181818181818, - "Spanish,Chinese": 0.38636363636363635, - "Spanish,Indonesian": 0.48295454545454547, - "Spanish,Vietnamese": 0.5681818181818182, - "Malay,Chinese": 0.4318181818181818, - "Malay,Indonesian": 0.875, - "Malay,Vietnamese": 0.7045454545454546, - "Chinese,Indonesian": 0.4715909090909091, - "Chinese,Vietnamese": 0.4318181818181818, - "Indonesian,Vietnamese": 0.6988636363636364 - }, - "3_combine": { - "English,Filipino,Spanish": 0.29545454545454547, - "English,Filipino,Malay": 0.4318181818181818, - "English,Filipino,Chinese": 0.2784090909090909, - "English,Filipino,Indonesian": 0.4375, - "English,Filipino,Vietnamese": 0.3977272727272727, - "English,Spanish,Malay": 0.3125, - "English,Spanish,Chinese": 0.2840909090909091, - "English,Spanish,Indonesian": 0.3181818181818182, - "English,Spanish,Vietnamese": 0.3806818181818182, - "English,Malay,Chinese": 0.2784090909090909, - "English,Malay,Indonesian": 0.4375, - "English,Malay,Vietnamese": 0.3977272727272727, - "English,Chinese,Indonesian": 0.3125, - "English,Chinese,Vietnamese": 0.3125, - "English,Indonesian,Vietnamese": 0.4090909090909091, - "Filipino,Spanish,Malay": 0.4318181818181818, - "Filipino,Spanish,Chinese": 0.22727272727272727, - "Filipino,Spanish,Indonesian": 0.4147727272727273, - "Filipino,Spanish,Vietnamese": 0.38636363636363635, - "Filipino,Malay,Chinese": 0.38636363636363635, - "Filipino,Malay,Indonesian": 0.8068181818181818, - "Filipino,Malay,Vietnamese": 0.6420454545454546, - "Filipino,Chinese,Indonesian": 0.38636363636363635, - "Filipino,Chinese,Vietnamese": 0.3352272727272727, - "Filipino,Indonesian,Vietnamese": 0.625, - "Spanish,Malay,Chinese": 0.23863636363636365, - "Spanish,Malay,Indonesian": 0.4375, - "Spanish,Malay,Vietnamese": 0.4147727272727273, - "Spanish,Chinese,Indonesian": 0.2556818181818182, - "Spanish,Chinese,Vietnamese": 0.25, - "Spanish,Indonesian,Vietnamese": 0.3977272727272727, - "Malay,Chinese,Indonesian": 0.4147727272727273, - "Malay,Chinese,Vietnamese": 0.3409090909090909, - "Malay,Indonesian,Vietnamese": 0.6420454545454546, - "Chinese,Indonesian,Vietnamese": 0.3522727272727273 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.2727272727272727, - "English,Filipino,Spanish,Chinese": 0.18181818181818182, - "English,Filipino,Spanish,Indonesian": 0.2727272727272727, - "English,Filipino,Spanish,Vietnamese": 0.25, - "English,Filipino,Malay,Chinese": 0.26136363636363635, - "English,Filipino,Malay,Indonesian": 0.4090909090909091, - "English,Filipino,Malay,Vietnamese": 0.35795454545454547, - "English,Filipino,Chinese,Indonesian": 0.2727272727272727, - "English,Filipino,Chinese,Vietnamese": 0.23295454545454544, - "English,Filipino,Indonesian,Vietnamese": 0.36363636363636365, - "English,Spanish,Malay,Chinese": 0.19318181818181818, - "English,Spanish,Malay,Indonesian": 0.2840909090909091, - "English,Spanish,Malay,Vietnamese": 0.26136363636363635, - "English,Spanish,Chinese,Indonesian": 0.19886363636363635, - "English,Spanish,Chinese,Vietnamese": 0.20454545454545456, - "English,Spanish,Indonesian,Vietnamese": 0.26704545454545453, - "English,Malay,Chinese,Indonesian": 0.2784090909090909, - "English,Malay,Chinese,Vietnamese": 0.23863636363636365, - "English,Malay,Indonesian,Vietnamese": 0.3693181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Malay,Chinese": 0.22727272727272727, - "Filipino,Spanish,Malay,Indonesian": 0.4034090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.36363636363636365, - "Filipino,Spanish,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.3522727272727273, - "Filipino,Malay,Chinese,Indonesian": 0.375, - "Filipino,Malay,Chinese,Vietnamese": 0.32386363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.5965909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.32386363636363635, - "Spanish,Malay,Chinese,Indonesian": 0.23863636363636365, - "Spanish,Malay,Chinese,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.375, - "Spanish,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.32954545454545453 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.18181818181818182, - "English,Filipino,Spanish,Malay,Indonesian": 0.26136363636363635, - "English,Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, - "English,Filipino,Spanish,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.23295454545454544, - "English,Filipino,Malay,Chinese,Indonesian": 0.26136363636363635, - "English,Filipino,Malay,Chinese,Vietnamese": 0.2215909090909091, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.3409090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "English,Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "English,Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.3409090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.3125, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.18181818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 - } - }, - "AC3_2": 0.3378100160291282, - "AC3_3": 0.2975115465637167, - "AC3_4": 0.2631031456583058, - "AC3_5": 0.23372867728773686, - "AC3_6": 0.20856199245167728, - "AC3_7": 0.18725398308270544 - }, - "prompt_4": { - "overall_acc": 0.2573051948051948, - "language_acc": { - "English": 0.23863636363636365, - "Filipino": 0.25, - "Spanish": 0.25, - "Malay": 0.2784090909090909, - "Chinese": 0.2840909090909091, - "Indonesian": 0.2215909090909091, - "Vietnamese": 0.2784090909090909 - }, - "consistency_score_2": 0.5446428571428573, - "consistency_score_3": 0.3344155844155844, - "consistency_score_4": 0.2173701298701299, - "consistency_score_5": 0.14502164502164505, - "consistency_score_6": 0.09496753246753246, - "consistency_score_7": 0.056818181818181816, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.2840909090909091, - "English,Spanish": 0.6875, - "English,Malay": 0.48863636363636365, - "English,Chinese": 0.6875, - "English,Indonesian": 0.5170454545454546, - "English,Vietnamese": 0.7386363636363636, - "Filipino,Spanish": 0.35795454545454547, - "Filipino,Malay": 0.7215909090909091, - "Filipino,Chinese": 0.19318181818181818, - "Filipino,Indonesian": 0.6534090909090909, - "Filipino,Vietnamese": 0.2897727272727273, - "Spanish,Malay": 0.5568181818181818, - "Spanish,Chinese": 0.6647727272727273, - "Spanish,Indonesian": 0.5738636363636364, - "Spanish,Vietnamese": 0.6931818181818182, - "Malay,Chinese": 0.44886363636363635, - "Malay,Indonesian": 0.7045454545454546, - "Malay,Vietnamese": 0.5056818181818182, - "Chinese,Indonesian": 0.4602272727272727, - "Chinese,Vietnamese": 0.6818181818181818, - "Indonesian,Vietnamese": 0.5284090909090909 - }, - "3_combine": { - "English,Filipino,Spanish": 0.1875, - "English,Filipino,Malay": 0.2556818181818182, - "English,Filipino,Chinese": 0.11931818181818182, - "English,Filipino,Indonesian": 0.23863636363636365, - "English,Filipino,Vietnamese": 0.1875, - "English,Spanish,Malay": 0.38636363636363635, - "English,Spanish,Chinese": 0.5284090909090909, - "English,Spanish,Indonesian": 0.3977272727272727, - "English,Spanish,Vietnamese": 0.5681818181818182, - "English,Malay,Chinese": 0.3409090909090909, - "English,Malay,Indonesian": 0.3693181818181818, - "English,Malay,Vietnamese": 0.39204545454545453, - "English,Chinese,Indonesian": 0.35795454545454547, - "English,Chinese,Vietnamese": 0.5625, - "English,Indonesian,Vietnamese": 0.4090909090909091, - "Filipino,Spanish,Malay": 0.32386363636363635, - "Filipino,Spanish,Chinese": 0.13636363636363635, - "Filipino,Spanish,Indonesian": 0.30113636363636365, - "Filipino,Spanish,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese": 0.1875, - "Filipino,Malay,Indonesian": 0.5397727272727273, - "Filipino,Malay,Vietnamese": 0.26136363636363635, - "Filipino,Chinese,Indonesian": 0.17045454545454544, - "Filipino,Chinese,Vietnamese": 0.125, - "Filipino,Indonesian,Vietnamese": 0.23863636363636365, - "Spanish,Malay,Chinese": 0.35795454545454547, - "Spanish,Malay,Indonesian": 0.42613636363636365, - "Spanish,Malay,Vietnamese": 0.3977272727272727, - "Spanish,Chinese,Indonesian": 0.375, - "Spanish,Chinese,Vietnamese": 0.5340909090909091, - "Spanish,Indonesian,Vietnamese": 0.4090909090909091, - "Malay,Chinese,Indonesian": 0.32386363636363635, - "Malay,Chinese,Vietnamese": 0.35795454545454547, - "Malay,Indonesian,Vietnamese": 0.375, - "Chinese,Indonesian,Vietnamese": 0.3693181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.18181818181818182, - "English,Filipino,Spanish,Chinese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian": 0.1590909090909091, - "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese": 0.11363636363636363, - "English,Filipino,Malay,Indonesian": 0.21022727272727273, - "English,Filipino,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Indonesian": 0.10227272727272728, - "English,Filipino,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese": 0.29545454545454547, - "English,Spanish,Malay,Indonesian": 0.29545454545454547, - "English,Spanish,Malay,Vietnamese": 0.3352272727272727, - "English,Spanish,Chinese,Indonesian": 0.3068181818181818, - "English,Spanish,Chinese,Vietnamese": 0.44886363636363635, - "English,Spanish,Indonesian,Vietnamese": 0.3409090909090909, - "English,Malay,Chinese,Indonesian": 0.25, - "English,Malay,Chinese,Vietnamese": 0.29545454545454547, - "English,Malay,Indonesian,Vietnamese": 0.2897727272727273, - "English,Chinese,Indonesian,Vietnamese": 0.3068181818181818, - "Filipino,Spanish,Malay,Chinese": 0.13068181818181818, - "Filipino,Spanish,Malay,Indonesian": 0.2784090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Indonesian": 0.11931818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Indonesian,Vietnamese": 0.1534090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Malay,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese,Indonesian": 0.2556818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.3125, - "Spanish,Malay,Indonesian,Vietnamese": 0.29545454545454547, - "Spanish,Chinese,Indonesian,Vietnamese": 0.32386363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.25 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Chinese,Indonesian": 0.09659090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.14204545454545456, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian": 0.2159090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.26704545454545453, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.25, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 - } - }, - "AC3_2": 0.3494975418881714, - "AC3_3": 0.2908360500461584, - "AC3_4": 0.23565776728561788, - "AC3_5": 0.1854950697798318, - "AC3_6": 0.1387313722749398, - "AC3_7": 0.09308198258723403 - }, - "prompt_5": { - "overall_acc": 0.2573051948051948, - "language_acc": { - "English": 0.26136363636363635, - "Filipino": 0.24431818181818182, - "Spanish": 0.23863636363636365, - "Malay": 0.26136363636363635, - "Chinese": 0.30113636363636365, - "Indonesian": 0.23863636363636365, - "Vietnamese": 0.2556818181818182 - }, - "consistency_score_2": 0.5422077922077922, - "consistency_score_3": 0.3517857142857143, - "consistency_score_4": 0.24594155844155838, - "consistency_score_5": 0.17748917748917747, - "consistency_score_6": 0.12824675324675325, - "consistency_score_7": 0.09090909090909091, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3409090909090909, - "English,Spanish": 0.6193181818181818, - "English,Malay": 0.5568181818181818, - "English,Chinese": 0.5738636363636364, - "English,Indonesian": 0.6818181818181818, - "English,Vietnamese": 0.6818181818181818, - "Filipino,Spanish": 0.4147727272727273, - "Filipino,Malay": 0.6818181818181818, - "Filipino,Chinese": 0.29545454545454547, - "Filipino,Indonesian": 0.5056818181818182, - "Filipino,Vietnamese": 0.26136363636363635, - "Spanish,Malay": 0.5738636363636364, - "Spanish,Chinese": 0.5227272727272727, - "Spanish,Indonesian": 0.6420454545454546, - "Spanish,Vietnamese": 0.5681818181818182, - "Malay,Chinese": 0.48863636363636365, - "Malay,Indonesian": 0.7045454545454546, - "Malay,Vietnamese": 0.48863636363636365, - "Chinese,Indonesian": 0.5511363636363636, - "Chinese,Vietnamese": 0.6193181818181818, - "Indonesian,Vietnamese": 0.6136363636363636 - }, - "3_combine": { - "English,Filipino,Spanish": 0.23863636363636365, - "English,Filipino,Malay": 0.3068181818181818, - "English,Filipino,Chinese": 0.1875, - "English,Filipino,Indonesian": 0.30113636363636365, - "English,Filipino,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay": 0.4147727272727273, - "English,Spanish,Chinese": 0.3977272727272727, - "English,Spanish,Indonesian": 0.4943181818181818, - "English,Spanish,Vietnamese": 0.44886363636363635, - "English,Malay,Chinese": 0.35795454545454547, - "English,Malay,Indonesian": 0.5, - "English,Malay,Vietnamese": 0.4034090909090909, - "English,Chinese,Indonesian": 0.4318181818181818, - "English,Chinese,Vietnamese": 0.4659090909090909, - "English,Indonesian,Vietnamese": 0.5056818181818182, - "Filipino,Spanish,Malay": 0.36363636363636365, - "Filipino,Spanish,Chinese": 0.19318181818181818, - "Filipino,Spanish,Indonesian": 0.32954545454545453, - "Filipino,Spanish,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese": 0.2727272727272727, - "Filipino,Malay,Indonesian": 0.44886363636363635, - "Filipino,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Chinese,Vietnamese": 0.17045454545454544, - "Filipino,Indonesian,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Chinese": 0.32386363636363635, - "Spanish,Malay,Indonesian": 0.4943181818181818, - "Spanish,Malay,Vietnamese": 0.35795454545454547, - "Spanish,Chinese,Indonesian": 0.39204545454545453, - "Spanish,Chinese,Vietnamese": 0.39204545454545453, - "Spanish,Indonesian,Vietnamese": 0.4318181818181818, - "Malay,Chinese,Indonesian": 0.4034090909090909, - "Malay,Chinese,Vietnamese": 0.3522727272727273, - "Malay,Indonesian,Vietnamese": 0.42613636363636365, - "Chinese,Indonesian,Vietnamese": 0.4375 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.22727272727272727, - "English,Filipino,Spanish,Chinese": 0.14204545454545456, - "English,Filipino,Spanish,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Chinese": 0.17045454545454544, - "English,Filipino,Malay,Indonesian": 0.2784090909090909, - "English,Filipino,Malay,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian": 0.1590909090909091, - "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Chinese": 0.26704545454545453, - "English,Spanish,Malay,Indonesian": 0.3806818181818182, - "English,Spanish,Malay,Vietnamese": 0.3068181818181818, - "English,Spanish,Chinese,Indonesian": 0.32954545454545453, - "English,Spanish,Chinese,Vietnamese": 0.32386363636363635, - "English,Spanish,Indonesian,Vietnamese": 0.3693181818181818, - "English,Malay,Chinese,Indonesian": 0.3181818181818182, - "English,Malay,Chinese,Vietnamese": 0.30113636363636365, - "English,Malay,Indonesian,Vietnamese": 0.36363636363636365, - "English,Chinese,Indonesian,Vietnamese": 0.35795454545454547, - "Filipino,Spanish,Malay,Chinese": 0.1875, - "Filipino,Spanish,Malay,Indonesian": 0.3125, - "Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.21022727272727273, - "Filipino,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.29545454545454547, - "Spanish,Malay,Chinese,Vietnamese": 0.26704545454545453, - "Spanish,Malay,Indonesian,Vietnamese": 0.32386363636363635, - "Spanish,Chinese,Indonesian,Vietnamese": 0.32386363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.3181818181818182 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.13636363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Chinese,Indonesian": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian": 0.24431818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.22727272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.17613636363636365, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.125, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13068181818181818 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091 - } - }, - "AC3_2": 0.348994660118522, - "AC3_3": 0.2972176743343892, - "AC3_4": 0.2514950774531675, - "AC3_5": 0.21007119819960998, - "AC3_6": 0.1711756663944268, - "AC3_7": 0.1343504979482789 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.2815533980582524 - }, - "prompt_2": { - "accuracy": 0.2621359223300971 - }, - "prompt_3": { - "accuracy": 0.2621359223300971 - }, - "prompt_4": { - "accuracy": 0.2815533980582524 - }, - "prompt_5": { - "accuracy": 0.2815533980582524 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.2761904761904762 - }, - "prompt_2": { - "accuracy": 0.2761904761904762 - }, - "prompt_3": { - "accuracy": 0.29523809523809524 - }, - "prompt_4": { - "accuracy": 0.3142857142857143 - }, - "prompt_5": { - "accuracy": 0.3047619047619048 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.2523364485981308 - }, - "prompt_2": { - "accuracy": 0.2616822429906542 - }, - "prompt_3": { - "accuracy": 0.27102803738317754 - }, - "prompt_4": { - "accuracy": 0.308411214953271 - }, - "prompt_5": { - "accuracy": 0.2897196261682243 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.22, - "category_acc": { - "brand": 0.4, - "demographics": 0.4, - "biology": 0.2, - "history": 0.13333333333333333, - "literature": 0.2, - "politics": 0.1, - "culture": 0.2, - "film": 0.2, - "law": 0.3, - "geography": 0.2 - } - }, - "prompt_2": { - "accuracy": 0.21, - "category_acc": { - "brand": 0.2, - "demographics": 0.4, - "biology": 0.3, - "history": 0.13333333333333333, - "literature": 0.3, - "politics": 0.1, - "culture": 0.2, - "film": 0.2, - "law": 0.1, - "geography": 0.3 - } - }, - "prompt_3": { - "accuracy": 0.19, - "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.3, - "history": 0.2, - "literature": 0.2, - "politics": 0.1, - "culture": 0.1, - "film": 0.4, - "law": 0.1, - "geography": 0.2 - } - }, - "prompt_4": { - "accuracy": 0.26, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.3, - "history": 0.0, - "literature": 0.2, - "politics": 0.2, - "culture": 0.3, - "film": 0.5, - "law": 0.3, - "geography": 0.3 - } - }, - "prompt_5": { - "accuracy": 0.21, - "category_acc": { - "brand": 0.2, - "demographics": 0.4, - "biology": 0.3, - "history": 0.06666666666666667, - "literature": 0.3, - "politics": 0.3, - "culture": 0.0, - "film": 0.3, - "law": 0.2, - "geography": 0.2 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.05835057079330971 - }, - "prompt_2": { - "bleu_score": 0.05980902598047604 - }, - "prompt_3": { - "bleu_score": 0.06053490647231654 - }, - "prompt_4": { - "bleu_score": 0.052308155674348086 - }, - "prompt_5": { - "bleu_score": 0.036522851395988654 - } - }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.07950591420838479 - }, - "prompt_2": { - "bleu_score": 0.06665376321146874 - }, - "prompt_3": { - "bleu_score": 0.06318469460417708 - }, - "prompt_4": { - "bleu_score": 0.18945443364172493 - }, - "prompt_5": { - "bleu_score": 0.056370058895665015 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.07623845852476732 - }, - "prompt_2": { - "bleu_score": 0.05130460278781172 - }, - "prompt_3": { - "bleu_score": 0.050898246969329174 - }, - "prompt_4": { - "bleu_score": 0.16951938917668638 - }, - "prompt_5": { - "bleu_score": 0.08361543676756919 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.051534539560521044 - }, - "prompt_2": { - "bleu_score": 0.04475365078220176 - }, - "prompt_3": { - "bleu_score": 0.04628263226140173 - }, - "prompt_4": { - "bleu_score": 0.13196407800738935 - }, - "prompt_5": { - "bleu_score": 0.07729294754051765 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.0836584742661158 - }, - "prompt_2": { - "bleu_score": 0.06676038882614933 - }, - "prompt_3": { - "bleu_score": 0.06497730246829395 - }, - "prompt_4": { - "bleu_score": 0.1618826586165918 - }, - "prompt_5": { - "bleu_score": 0.0426498895453044 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.27304550758459745 - }, - "prompt_2": { - "accuracy": 0.2707117852975496 - }, - "prompt_3": { - "accuracy": 0.2823803967327888 - }, - "prompt_4": { - "accuracy": 0.2602100350058343 - }, - "prompt_5": { - "accuracy": 0.2637106184364061 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.2704326063639614, - "category_acc": { - "high_school_european_history": 0.21341463414634146, - "business_ethics": 0.24242424242424243, - "clinical_knowledge": 0.2727272727272727, - "medical_genetics": 0.3333333333333333, - "high_school_us_history": 0.2561576354679803, - "high_school_physics": 0.3, - "high_school_world_history": 0.2838983050847458, - "virology": 0.2727272727272727, - "high_school_microeconomics": 0.28270042194092826, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.24242424242424243, - "high_school_biology": 0.2686084142394822, - "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.21708185053380782, - "philosophy": 0.24838709677419354, - "professional_medicine": 0.33210332103321033, - "nutrition": 0.3180327868852459, - "global_facts": 0.30303030303030304, - "machine_learning": 0.26126126126126126, - "security_studies": 0.26639344262295084, - "public_relations": 0.27522935779816515, - "professional_psychology": 0.2520458265139116, - "prehistory": 0.30030959752321984, - "anatomy": 0.26119402985074625, - "human_sexuality": 0.33076923076923076, - "college_medicine": 0.28488372093023256, - "high_school_government_and_politics": 0.3489583333333333, - "college_chemistry": 0.1919191919191919, - "logical_fallacies": 0.2777777777777778, - "high_school_geography": 0.3604060913705584, - "elementary_mathematics": 0.2493368700265252, - "human_aging": 0.26576576576576577, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.27941176470588236, - "formal_logic": 0.288, - "high_school_statistics": 0.3023255813953488, - "international_law": 0.2916666666666667, - "high_school_mathematics": 0.2527881040892193, - "high_school_computer_science": 0.30303030303030304, - "conceptual_physics": 0.2606837606837607, - "miscellaneous": 0.2544757033248082, - "high_school_chemistry": 0.24752475247524752, - "marketing": 0.26609442060085836, - "professional_law": 0.26810176125244617, - "management": 0.2647058823529412, - "college_physics": 0.19801980198019803, - "jurisprudence": 0.2336448598130841, - "world_religions": 0.2411764705882353, - "sociology": 0.255, - "us_foreign_policy": 0.2828282828282828, - "high_school_macroeconomics": 0.30848329048843187, - "computer_security": 0.29292929292929293, - "moral_scenarios": 0.2483221476510067, - "moral_disputes": 0.22318840579710145, - "electrical_engineering": 0.3125, - "astronomy": 0.2980132450331126, - "college_biology": 0.3146853146853147 - } - }, - "prompt_2": { - "accuracy": 0.2705756167322131, - "category_acc": { - "high_school_european_history": 0.2865853658536585, - "business_ethics": 0.24242424242424243, - "clinical_knowledge": 0.2803030303030303, - "medical_genetics": 0.26262626262626265, - "high_school_us_history": 0.22660098522167488, - "high_school_physics": 0.26, - "high_school_world_history": 0.2627118644067797, - "virology": 0.22424242424242424, - "high_school_microeconomics": 0.32489451476793246, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.21212121212121213, - "high_school_biology": 0.27184466019417475, - "abstract_algebra": 0.20202020202020202, - "professional_accounting": 0.21352313167259787, - "philosophy": 0.27419354838709675, - "professional_medicine": 0.3173431734317343, - "nutrition": 0.34098360655737703, - "global_facts": 0.21212121212121213, - "machine_learning": 0.36036036036036034, - "security_studies": 0.3729508196721312, - "public_relations": 0.22018348623853212, - "professional_psychology": 0.25040916530278234, - "prehistory": 0.24458204334365324, - "anatomy": 0.20149253731343283, - "human_sexuality": 0.2846153846153846, - "college_medicine": 0.26744186046511625, - "high_school_government_and_politics": 0.3385416666666667, - "college_chemistry": 0.2828282828282828, - "logical_fallacies": 0.2654320987654321, - "high_school_geography": 0.350253807106599, - "elementary_mathematics": 0.259946949602122, - "human_aging": 0.16216216216216217, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.3161764705882353, - "formal_logic": 0.288, - "high_school_statistics": 0.33488372093023255, - "international_law": 0.18333333333333332, - "high_school_mathematics": 0.26394052044609667, - "high_school_computer_science": 0.31313131313131315, - "conceptual_physics": 0.2692307692307692, - "miscellaneous": 0.24296675191815856, - "high_school_chemistry": 0.2376237623762376, - "marketing": 0.19313304721030042, - "professional_law": 0.2720156555772994, - "management": 0.39215686274509803, - "college_physics": 0.22772277227722773, - "jurisprudence": 0.2523364485981308, - "world_religions": 0.22941176470588234, - "sociology": 0.305, - "us_foreign_policy": 0.2828282828282828, - "high_school_macroeconomics": 0.33161953727506427, - "computer_security": 0.26262626262626265, - "moral_scenarios": 0.2494407158836689, - "moral_disputes": 0.23478260869565218, - "electrical_engineering": 0.2916666666666667, - "astronomy": 0.32450331125827814, - "college_biology": 0.2937062937062937 - } - }, - "prompt_3": { - "accuracy": 0.2681444404719342, - "category_acc": { - "high_school_european_history": 0.2621951219512195, - "business_ethics": 0.2222222222222222, - "clinical_knowledge": 0.3106060606060606, - "medical_genetics": 0.2727272727272727, - "high_school_us_history": 0.26108374384236455, - "high_school_physics": 0.3333333333333333, - "high_school_world_history": 0.24152542372881355, - "virology": 0.24848484848484848, - "high_school_microeconomics": 0.33755274261603374, - "econometrics": 0.25663716814159293, - "college_computer_science": 0.24242424242424243, - "high_school_biology": 0.2783171521035599, - "abstract_algebra": 0.23232323232323232, - "professional_accounting": 0.21352313167259787, - "philosophy": 0.25806451612903225, - "professional_medicine": 0.3948339483394834, - "nutrition": 0.30491803278688523, - "global_facts": 0.16161616161616163, - "machine_learning": 0.1981981981981982, - "security_studies": 0.3729508196721312, - "public_relations": 0.1743119266055046, - "professional_psychology": 0.2144026186579378, - "prehistory": 0.23839009287925697, - "anatomy": 0.208955223880597, - "human_sexuality": 0.3076923076923077, - "college_medicine": 0.313953488372093, - "high_school_government_and_politics": 0.3645833333333333, - "college_chemistry": 0.36363636363636365, - "logical_fallacies": 0.24691358024691357, - "high_school_geography": 0.34517766497461927, - "elementary_mathematics": 0.23342175066312998, - "human_aging": 0.1891891891891892, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.29411764705882354, - "formal_logic": 0.336, - "high_school_statistics": 0.4372093023255814, - "international_law": 0.21666666666666667, - "high_school_mathematics": 0.20074349442379183, - "high_school_computer_science": 0.26262626262626265, - "conceptual_physics": 0.2564102564102564, - "miscellaneous": 0.2237851662404092, - "high_school_chemistry": 0.23267326732673269, - "marketing": 0.21888412017167383, - "professional_law": 0.26875407697325504, - "management": 0.39215686274509803, - "college_physics": 0.27722772277227725, - "jurisprudence": 0.19626168224299065, - "world_religions": 0.2, - "sociology": 0.255, - "us_foreign_policy": 0.23232323232323232, - "high_school_macroeconomics": 0.3496143958868895, - "computer_security": 0.24242424242424243, - "moral_scenarios": 0.25838926174496646, - "moral_disputes": 0.24057971014492754, - "electrical_engineering": 0.3055555555555556, - "astronomy": 0.26490066225165565, - "college_biology": 0.26573426573426573 - } - }, - "prompt_4": { - "accuracy": 0.27286378262424027, - "category_acc": { - "high_school_european_history": 0.21951219512195122, - "business_ethics": 0.2727272727272727, - "clinical_knowledge": 0.2878787878787879, - "medical_genetics": 0.32323232323232326, - "high_school_us_history": 0.29064039408866993, - "high_school_physics": 0.28, - "high_school_world_history": 0.23728813559322035, - "virology": 0.30303030303030304, - "high_school_microeconomics": 0.29535864978902954, - "econometrics": 0.3185840707964602, - "college_computer_science": 0.20202020202020202, - "high_school_biology": 0.30097087378640774, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.20284697508896798, - "philosophy": 0.27741935483870966, - "professional_medicine": 0.28044280442804426, - "nutrition": 0.31475409836065577, - "global_facts": 0.30303030303030304, - "machine_learning": 0.24324324324324326, - "security_studies": 0.3442622950819672, - "public_relations": 0.28440366972477066, - "professional_psychology": 0.25040916530278234, - "prehistory": 0.30030959752321984, - "anatomy": 0.2462686567164179, - "human_sexuality": 0.34615384615384615, - "college_medicine": 0.2558139534883721, - "high_school_government_and_politics": 0.3802083333333333, - "college_chemistry": 0.1919191919191919, - "logical_fallacies": 0.20987654320987653, - "high_school_geography": 0.3147208121827411, - "elementary_mathematics": 0.259946949602122, - "human_aging": 0.2747747747747748, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.29963235294117646, - "formal_logic": 0.272, - "high_school_statistics": 0.2744186046511628, - "international_law": 0.25833333333333336, - "high_school_mathematics": 0.24907063197026022, - "high_school_computer_science": 0.30303030303030304, - "conceptual_physics": 0.2777777777777778, - "miscellaneous": 0.2442455242966752, - "high_school_chemistry": 0.26732673267326734, - "marketing": 0.2532188841201717, - "professional_law": 0.27266797129810827, - "management": 0.2647058823529412, - "college_physics": 0.1485148514851485, - "jurisprudence": 0.32710280373831774, - "world_religions": 0.22941176470588234, - "sociology": 0.33, - "us_foreign_policy": 0.26262626262626265, - "high_school_macroeconomics": 0.31876606683804626, - "computer_security": 0.32323232323232326, - "moral_scenarios": 0.24049217002237136, - "moral_disputes": 0.26956521739130435, - "electrical_engineering": 0.2708333333333333, - "astronomy": 0.26490066225165565, - "college_biology": 0.25874125874125875 - } - }, - "prompt_5": { - "accuracy": 0.2760815159099035, - "category_acc": { - "high_school_european_history": 0.2073170731707317, - "business_ethics": 0.2828282828282828, - "clinical_knowledge": 0.2878787878787879, - "medical_genetics": 0.26262626262626265, - "high_school_us_history": 0.270935960591133, - "high_school_physics": 0.26, - "high_school_world_history": 0.2669491525423729, - "virology": 0.296969696969697, - "high_school_microeconomics": 0.29957805907172996, - "econometrics": 0.2831858407079646, - "college_computer_science": 0.20202020202020202, - "high_school_biology": 0.32362459546925565, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.23843416370106763, - "philosophy": 0.2903225806451613, - "professional_medicine": 0.2878228782287823, - "nutrition": 0.3442622950819672, - "global_facts": 0.31313131313131315, - "machine_learning": 0.2072072072072072, - "security_studies": 0.319672131147541, - "public_relations": 0.29357798165137616, - "professional_psychology": 0.25859247135842883, - "prehistory": 0.30340557275541796, - "anatomy": 0.208955223880597, - "human_sexuality": 0.3230769230769231, - "college_medicine": 0.2441860465116279, - "high_school_government_and_politics": 0.3645833333333333, - "college_chemistry": 0.21212121212121213, - "logical_fallacies": 0.24691358024691357, - "high_school_geography": 0.3248730964467005, - "elementary_mathematics": 0.2917771883289125, - "human_aging": 0.18018018018018017, - "college_mathematics": 0.26262626262626265, - "high_school_psychology": 0.2977941176470588, - "formal_logic": 0.272, - "high_school_statistics": 0.27906976744186046, - "international_law": 0.2833333333333333, - "high_school_mathematics": 0.2862453531598513, - "high_school_computer_science": 0.31313131313131315, - "conceptual_physics": 0.2863247863247863, - "miscellaneous": 0.2595907928388747, - "high_school_chemistry": 0.2871287128712871, - "marketing": 0.23605150214592274, - "professional_law": 0.27984344422700586, - "management": 0.29411764705882354, - "college_physics": 0.18811881188118812, - "jurisprudence": 0.2803738317757009, - "world_religions": 0.2529411764705882, - "sociology": 0.305, - "us_foreign_policy": 0.2727272727272727, - "high_school_macroeconomics": 0.3059125964010283, - "computer_security": 0.31313131313131315, - "moral_scenarios": 0.2494407158836689, - "moral_disputes": 0.2318840579710145, - "electrical_engineering": 0.2916666666666667, - "astronomy": 0.2847682119205298, - "college_biology": 0.2867132867132867 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.23922734026745915 - }, - "prompt_2": { - "accuracy": 0.24071322436849926 - }, - "prompt_3": { - "accuracy": 0.24294205052005943 - }, - "prompt_4": { - "accuracy": 0.2303120356612184 - }, - "prompt_5": { - "accuracy": 0.24071322436849926 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.24283935242839352, - "category_acc": { - "computer_network": 0.041666666666666664, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.21428571428571427, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.25, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.15384615384615385, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.12, - "veterinary_medicine": 0.25, - "college_economics": 0.35, - "business_administration": 0.39473684210526316, - "marxism": 0.20833333333333334, - "mao_zedong_thought": 0.2413793103448276, - "education_science": 0.2647058823529412, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.25, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.11764705882352941, - "modern_chinese_history": 0.21428571428571427, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.14814814814814814, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.25, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.10714285714285714, - "high_school_chinese": 0.125, - "high_school_history": 0.4, - "middle_school_history": 0.1111111111111111, - "civil_servant": 0.21153846153846154, - "sports_science": 0.08333333333333333, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.16666666666666666, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.16666666666666666, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.2222222222222222, - "tax_accountant": 0.18518518518518517, - "physician": 0.3148148148148148 - } - }, - "prompt_2": { - "accuracy": 0.25965130759651306, - "category_acc": { - "computer_network": 0.08333333333333333, - "operating_system": 0.375, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.23809523809523808, - "college_physics": 0.08333333333333333, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.20833333333333334, - "high_school_chemistry": 0.25, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.23076923076923078, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.2833333333333333, - "business_administration": 0.18421052631578946, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.20689655172413793, - "education_science": 0.35294117647058826, - "teacher_qualification": 0.30612244897959184, - "high_school_politics": 0.25, - "high_school_geography": 0.375, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.2222222222222222, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.10714285714285714, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.36, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.25, - "sports_science": 0.25, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.5, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.16666666666666666, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.2222222222222222, - "tax_accountant": 0.25925925925925924, - "physician": 0.3148148148148148 - } - }, - "prompt_3": { - "accuracy": 0.2658779576587796, - "category_acc": { - "computer_network": 0.041666666666666664, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.2619047619047619, - "college_physics": 0.08333333333333333, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.13043478260869565, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.11538461538461539, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.25, - "business_administration": 0.21052631578947367, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.20689655172413793, - "education_science": 0.5, - "teacher_qualification": 0.24489795918367346, - "high_school_politics": 0.25, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.3076923076923077, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.14285714285714285, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.25925925925925924, - "law": 0.20689655172413793, - "chinese_language_and_literature": 0.25, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.07142857142857142, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.4, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.23076923076923078, - "sports_science": 0.2916666666666667, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.14814814814814814, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.18518518518518517, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.25925925925925924, - "physician": 0.2222222222222222 - } - }, - "prompt_4": { - "accuracy": 0.24968866749688667, - "category_acc": { - "computer_network": 0.125, - "operating_system": 0.25, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.19047619047619047, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.047619047619047616, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.25, - "high_school_chemistry": 0.4166666666666667, - "high_school_biology": 0.16666666666666666, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.12, - "veterinary_medicine": 0.25, - "college_economics": 0.26666666666666666, - "business_administration": 0.3157894736842105, - "marxism": 0.25, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.35294117647058826, - "teacher_qualification": 0.20408163265306123, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.375, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.20833333333333334, - "logic": 0.25925925925925924, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.14705882352941177, - "legal_professional": 0.10714285714285714, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.28, - "middle_school_history": 0.2222222222222222, - "civil_servant": 0.17307692307692307, - "sports_science": 0.20833333333333334, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.25, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.16666666666666666, - "fire_engineer": 0.4166666666666667, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.24074074074074073, - "physician": 0.24074074074074073 - } - }, - "prompt_5": { - "accuracy": 0.24782067247820672, - "category_acc": { - "computer_network": 0.041666666666666664, - "operating_system": 0.375, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.19047619047619047, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.13793103448275862, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.40476190476190477, - "metrology_engineer": 0.41379310344827586, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.16666666666666666, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.11538461538461539, - "middle_school_physics": 0.16666666666666666, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.26666666666666666, - "business_administration": 0.42105263157894735, - "marxism": 0.25, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.3235294117647059, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.25, - "high_school_geography": 0.25, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.11764705882352941, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.3333333333333333, - "logic": 0.14814814814814814, - "law": 0.1724137931034483, - "chinese_language_and_literature": 0.17857142857142858, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.07142857142857142, - "high_school_chinese": 0.125, - "high_school_history": 0.36, - "middle_school_history": 0.14814814814814814, - "civil_servant": 0.23076923076923078, - "sports_science": 0.08333333333333333, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.16666666666666666, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.35294117647058826, - "accountant": 0.12962962962962962, - "fire_engineer": 0.3611111111111111, - "environmental_impact_assessment_engineer": 0.25, - "tax_accountant": 0.2037037037037037, - "physician": 0.2777777777777778 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.2831541218637993 - }, - "prompt_2": { - "accuracy": 0.2831541218637993 - }, - "prompt_3": { - "accuracy": 0.2974910394265233 - }, - "prompt_4": { - "accuracy": 0.27598566308243727 - }, - "prompt_5": { - "accuracy": 0.3010752688172043 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.2524607149024348, - "category_acc": { - "agronomy": 0.25443786982248523, - "anatomy": 0.24324324324324326, - "ancient_chinese": 0.25, - "arts": 0.25, - "astronomy": 0.24848484848484848, - "business_ethics": 0.23444976076555024, - "chinese_civil_service_exam": 0.25, - "chinese_driving_rule": 0.2748091603053435, - "chinese_food_culture": 0.25735294117647056, - "chinese_foreign_policy": 0.2803738317757009, - "chinese_history": 0.25696594427244585, - "chinese_literature": 0.2647058823529412, - "chinese_teacher_qualification": 0.21787709497206703, - "clinical_knowledge": 0.270042194092827, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.29906542056074764, - "college_engineering_hydrology": 0.2169811320754717, - "college_law": 0.17592592592592593, - "college_mathematics": 0.2, - "college_medical_statistics": 0.25471698113207547, - "college_medicine": 0.23076923076923078, - "computer_science": 0.25, - "computer_security": 0.19883040935672514, - "conceptual_physics": 0.2653061224489796, - "construction_project_management": 0.2517985611510791, - "economics": 0.2578616352201258, - "education": 0.2392638036809816, - "electrical_engineering": 0.29651162790697677, - "elementary_chinese": 0.26587301587301587, - "elementary_commonsense": 0.2777777777777778, - "elementary_information_and_technology": 0.25630252100840334, - "elementary_mathematics": 0.2826086956521739, - "ethnology": 0.2518518518518518, - "food_science": 0.24475524475524477, - "genetics": 0.22727272727272727, - "global_facts": 0.2483221476510067, - "high_school_biology": 0.2603550295857988, - "high_school_chemistry": 0.25, - "high_school_geography": 0.288135593220339, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.18181818181818182, - "high_school_politics": 0.26573426573426573, - "human_sexuality": 0.24603174603174602, - "international_law": 0.24864864864864866, - "journalism": 0.25, - "jurisprudence": 0.25547445255474455, - "legal_and_moral_basis": 0.2616822429906542, - "logical": 0.24390243902439024, - "machine_learning": 0.22131147540983606, - "management": 0.2571428571428571, - "marketing": 0.25555555555555554, - "marxist_theory": 0.25925925925925924, - "modern_chinese": 0.1896551724137931, - "nutrition": 0.2896551724137931, - "philosophy": 0.22857142857142856, - "professional_accounting": 0.2571428571428571, - "professional_law": 0.25118483412322273, - "professional_medicine": 0.2712765957446808, - "professional_psychology": 0.2974137931034483, - "public_relations": 0.28160919540229884, - "security_study": 0.25925925925925924, - "sociology": 0.2345132743362832, - "sports_science": 0.20606060606060606, - "traditional_chinese_medicine": 0.21081081081081082, - "virology": 0.25443786982248523, - "world_history": 0.2546583850931677, - "world_religions": 0.2875 - } - }, - "prompt_2": { - "accuracy": 0.2593679848040062, - "category_acc": { - "agronomy": 0.2485207100591716, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.25609756097560976, - "arts": 0.275, - "astronomy": 0.26666666666666666, - "business_ethics": 0.2727272727272727, - "chinese_civil_service_exam": 0.2625, - "chinese_driving_rule": 0.2900763358778626, - "chinese_food_culture": 0.21323529411764705, - "chinese_foreign_policy": 0.29906542056074764, - "chinese_history": 0.25696594427244585, - "chinese_literature": 0.27941176470588236, - "chinese_teacher_qualification": 0.24581005586592178, - "clinical_knowledge": 0.26582278481012656, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.29906542056074764, - "college_engineering_hydrology": 0.2641509433962264, - "college_law": 0.2037037037037037, - "college_mathematics": 0.24761904761904763, - "college_medical_statistics": 0.25471698113207547, - "college_medicine": 0.2490842490842491, - "computer_science": 0.25980392156862747, - "computer_security": 0.19298245614035087, - "conceptual_physics": 0.24489795918367346, - "construction_project_management": 0.2517985611510791, - "economics": 0.20125786163522014, - "education": 0.26380368098159507, - "electrical_engineering": 0.2558139534883721, - "elementary_chinese": 0.2698412698412698, - "elementary_commonsense": 0.22727272727272727, - "elementary_information_and_technology": 0.28991596638655465, - "elementary_mathematics": 0.28695652173913044, - "ethnology": 0.25925925925925924, - "food_science": 0.27972027972027974, - "genetics": 0.2897727272727273, - "global_facts": 0.2483221476510067, - "high_school_biology": 0.2603550295857988, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.2288135593220339, - "high_school_mathematics": 0.25609756097560976, - "high_school_physics": 0.23636363636363636, - "high_school_politics": 0.23776223776223776, - "human_sexuality": 0.2619047619047619, - "international_law": 0.2594594594594595, - "journalism": 0.27325581395348836, - "jurisprudence": 0.26277372262773724, - "legal_and_moral_basis": 0.29906542056074764, - "logical": 0.24390243902439024, - "machine_learning": 0.26229508196721313, - "management": 0.2523809523809524, - "marketing": 0.2777777777777778, - "marxist_theory": 0.25396825396825395, - "modern_chinese": 0.22413793103448276, - "nutrition": 0.2827586206896552, - "philosophy": 0.24761904761904763, - "professional_accounting": 0.24571428571428572, - "professional_law": 0.2843601895734597, - "professional_medicine": 0.26595744680851063, - "professional_psychology": 0.27155172413793105, - "public_relations": 0.27011494252873564, - "security_study": 0.28888888888888886, - "sociology": 0.25663716814159293, - "sports_science": 0.2606060606060606, - "traditional_chinese_medicine": 0.23243243243243245, - "virology": 0.2781065088757396, - "world_history": 0.22981366459627328, - "world_religions": 0.2375 - } - }, - "prompt_3": { - "accuracy": 0.2569504403384562, - "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.25, - "arts": 0.25625, - "astronomy": 0.2727272727272727, - "business_ethics": 0.28708133971291866, - "chinese_civil_service_exam": 0.23125, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.22794117647058823, - "chinese_foreign_policy": 0.2616822429906542, - "chinese_history": 0.25077399380804954, - "chinese_literature": 0.2696078431372549, - "chinese_teacher_qualification": 0.2681564245810056, - "clinical_knowledge": 0.25316455696202533, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.3177570093457944, - "college_engineering_hydrology": 0.24528301886792453, - "college_law": 0.18518518518518517, - "college_mathematics": 0.21904761904761905, - "college_medical_statistics": 0.2358490566037736, - "college_medicine": 0.2600732600732601, - "computer_science": 0.24509803921568626, - "computer_security": 0.19298245614035087, - "conceptual_physics": 0.2585034013605442, - "construction_project_management": 0.2302158273381295, - "economics": 0.2389937106918239, - "education": 0.2392638036809816, - "electrical_engineering": 0.2616279069767442, - "elementary_chinese": 0.2698412698412698, - "elementary_commonsense": 0.23737373737373738, - "elementary_information_and_technology": 0.2689075630252101, - "elementary_mathematics": 0.28695652173913044, - "ethnology": 0.23703703703703705, - "food_science": 0.27972027972027974, - "genetics": 0.2727272727272727, - "global_facts": 0.2550335570469799, - "high_school_biology": 0.23076923076923078, - "high_school_chemistry": 0.25, - "high_school_geography": 0.23728813559322035, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.21818181818181817, - "high_school_politics": 0.25874125874125875, - "human_sexuality": 0.2698412698412698, - "international_law": 0.2810810810810811, - "journalism": 0.27906976744186046, - "jurisprudence": 0.2773722627737226, - "legal_and_moral_basis": 0.3037383177570093, - "logical": 0.23577235772357724, - "machine_learning": 0.26229508196721313, - "management": 0.24285714285714285, - "marketing": 0.25555555555555554, - "marxist_theory": 0.24867724867724866, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.2620689655172414, - "philosophy": 0.23809523809523808, - "professional_accounting": 0.2571428571428571, - "professional_law": 0.23222748815165878, - "professional_medicine": 0.26861702127659576, - "professional_psychology": 0.28448275862068967, - "public_relations": 0.28160919540229884, - "security_study": 0.2814814814814815, - "sociology": 0.26548672566371684, - "sports_science": 0.24848484848484848, - "traditional_chinese_medicine": 0.23783783783783785, - "virology": 0.28402366863905326, - "world_history": 0.2422360248447205, - "world_religions": 0.25 - } - }, - "prompt_4": { - "accuracy": 0.2568640994646866, - "category_acc": { - "agronomy": 0.26627218934911245, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.2865853658536585, - "arts": 0.28125, - "astronomy": 0.30303030303030304, - "business_ethics": 0.27751196172248804, - "chinese_civil_service_exam": 0.275, - "chinese_driving_rule": 0.22137404580152673, - "chinese_food_culture": 0.3382352941176471, - "chinese_foreign_policy": 0.2523364485981308, - "chinese_history": 0.22910216718266255, - "chinese_literature": 0.28431372549019607, - "chinese_teacher_qualification": 0.21787709497206703, - "clinical_knowledge": 0.22362869198312235, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.308411214953271, - "college_engineering_hydrology": 0.16981132075471697, - "college_law": 0.26851851851851855, - "college_mathematics": 0.20952380952380953, - "college_medical_statistics": 0.25471698113207547, - "college_medicine": 0.2454212454212454, - "computer_science": 0.24019607843137256, - "computer_security": 0.2046783625730994, - "conceptual_physics": 0.2789115646258503, - "construction_project_management": 0.19424460431654678, - "economics": 0.2641509433962264, - "education": 0.25766871165644173, - "electrical_engineering": 0.26744186046511625, - "elementary_chinese": 0.26587301587301587, - "elementary_commonsense": 0.26262626262626265, - "elementary_information_and_technology": 0.24369747899159663, - "elementary_mathematics": 0.24347826086956523, - "ethnology": 0.2518518518518518, - "food_science": 0.2727272727272727, - "genetics": 0.2556818181818182, - "global_facts": 0.30201342281879195, - "high_school_biology": 0.2781065088757396, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.2796610169491525, - "high_school_mathematics": 0.21341463414634146, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.25874125874125875, - "human_sexuality": 0.23809523809523808, - "international_law": 0.23783783783783785, - "journalism": 0.2441860465116279, - "jurisprudence": 0.31873479318734793, - "legal_and_moral_basis": 0.2757009345794392, - "logical": 0.2682926829268293, - "machine_learning": 0.29508196721311475, - "management": 0.22857142857142856, - "marketing": 0.25, - "marxist_theory": 0.23809523809523808, - "modern_chinese": 0.21551724137931033, - "nutrition": 0.27586206896551724, - "philosophy": 0.23809523809523808, - "professional_accounting": 0.30857142857142855, - "professional_law": 0.22274881516587677, - "professional_medicine": 0.2579787234042553, - "professional_psychology": 0.2629310344827586, - "public_relations": 0.26436781609195403, - "security_study": 0.2962962962962963, - "sociology": 0.2610619469026549, - "sports_science": 0.2606060606060606, - "traditional_chinese_medicine": 0.22162162162162163, - "virology": 0.23076923076923078, - "world_history": 0.22981366459627328, - "world_religions": 0.2375 - } - }, - "prompt_5": { - "accuracy": 0.2547919184942152, - "category_acc": { - "agronomy": 0.24260355029585798, - "anatomy": 0.22972972972972974, - "ancient_chinese": 0.22560975609756098, - "arts": 0.275, - "astronomy": 0.2787878787878788, - "business_ethics": 0.24880382775119617, - "chinese_civil_service_exam": 0.25, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.27205882352941174, - "chinese_foreign_policy": 0.24299065420560748, - "chinese_history": 0.25386996904024767, - "chinese_literature": 0.25980392156862747, - "chinese_teacher_qualification": 0.24581005586592178, - "clinical_knowledge": 0.25738396624472576, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.2897196261682243, - "college_engineering_hydrology": 0.24528301886792453, - "college_law": 0.25, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.25471698113207547, - "college_medicine": 0.23809523809523808, - "computer_science": 0.2647058823529412, - "computer_security": 0.1695906432748538, - "conceptual_physics": 0.2653061224489796, - "construction_project_management": 0.2949640287769784, - "economics": 0.27672955974842767, - "education": 0.24539877300613497, - "electrical_engineering": 0.27325581395348836, - "elementary_chinese": 0.25793650793650796, - "elementary_commonsense": 0.2878787878787879, - "elementary_information_and_technology": 0.2689075630252101, - "elementary_mathematics": 0.2608695652173913, - "ethnology": 0.2518518518518518, - "food_science": 0.27972027972027974, - "genetics": 0.26704545454545453, - "global_facts": 0.24161073825503357, - "high_school_biology": 0.24260355029585798, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.2711864406779661, - "high_school_mathematics": 0.25, - "high_school_physics": 0.19090909090909092, - "high_school_politics": 0.2517482517482518, - "human_sexuality": 0.23809523809523808, - "international_law": 0.25405405405405407, - "journalism": 0.2441860465116279, - "jurisprudence": 0.25060827250608275, - "legal_and_moral_basis": 0.2523364485981308, - "logical": 0.24390243902439024, - "machine_learning": 0.27049180327868855, - "management": 0.2571428571428571, - "marketing": 0.2611111111111111, - "marxist_theory": 0.24338624338624337, - "modern_chinese": 0.20689655172413793, - "nutrition": 0.27586206896551724, - "philosophy": 0.22857142857142856, - "professional_accounting": 0.22857142857142856, - "professional_law": 0.27014218009478674, - "professional_medicine": 0.2712765957446808, - "professional_psychology": 0.28448275862068967, - "public_relations": 0.27586206896551724, - "security_study": 0.2962962962962963, - "sociology": 0.26991150442477874, - "sports_science": 0.23636363636363636, - "traditional_chinese_medicine": 0.21621621621621623, - "virology": 0.20710059171597633, - "world_history": 0.2484472049689441, - "world_religions": 0.3 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.2727272727272727 - }, - "prompt_2": { - "accuracy": 0.18181818181818182 - }, - "prompt_3": { - "accuracy": 0.18181818181818182 - }, - "prompt_4": { - "accuracy": 0.18181818181818182 - }, - "prompt_5": { - "accuracy": 0.18181818181818182 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.17272727272727273 - }, - "prompt_2": { - "accuracy": 0.12045454545454545 - }, - "prompt_3": { - "accuracy": 0.09772727272727273 - }, - "prompt_4": { - "accuracy": 0.17272727272727273 - }, - "prompt_5": { - "accuracy": 0.15227272727272728 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3071186440677966 - }, - "prompt_2": { - "accuracy": 0.3247457627118644 - }, - "prompt_3": { - "accuracy": 0.33559322033898303 - }, - "prompt_4": { - "accuracy": 0.3284745762711864 - }, - "prompt_5": { - "accuracy": 0.3284745762711864 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.28908002991772624 - }, - "prompt_2": { - "accuracy": 0.27898279730740466 - }, - "prompt_3": { - "accuracy": 0.2819745699326851 - }, - "prompt_4": { - "accuracy": 0.275243081525804 - }, - "prompt_5": { - "accuracy": 0.28047868362004486 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.33365997060264574 - }, - "prompt_2": { - "accuracy": 0.3365997060264576 - }, - "prompt_3": { - "accuracy": 0.32435080842724157 - }, - "prompt_4": { - "accuracy": 0.3351298383145517 - }, - "prompt_5": { - "accuracy": 0.3317001469867712 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.19718219383278152, - "rouge2": 0.05726080971801016, - "rougeL": 0.14954223042756, - "avg_rouge": 0.13466174465945058 - }, - "prompt_2": { - "rouge1": 0.16311654317927568, - "rouge2": 0.044944272379417234, - "rougeL": 0.12787691198711254, - "avg_rouge": 0.11197924251526847 - }, - "prompt_3": { - "rouge1": 0.15898599542357766, - "rouge2": 0.04389743244521572, - "rougeL": 0.1259991700442868, - "avg_rouge": 0.10962753263769338 - }, - "prompt_4": { - "rouge1": 0.18880557926975894, - "rouge2": 0.058084214916900336, - "rougeL": 0.14659548297327807, - "avg_rouge": 0.13116175905331243 - }, - "prompt_5": { - "rouge1": 0.1699611627331736, - "rouge2": 0.048706259544251194, - "rougeL": 0.13285660837921365, - "avg_rouge": 0.11717467688554616 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.20876959056891067, - "rouge2": 0.05769057381669965, - "rougeL": 0.15253206578152728, - "avg_rouge": 0.1396640767223792 - }, - "prompt_2": { - "rouge1": 0.2124161722360626, - "rouge2": 0.059033258971595566, - "rougeL": 0.15608210854722365, - "avg_rouge": 0.14251051325162728 - }, - "prompt_3": { - "rouge1": 0.21269342379488743, - "rouge2": 0.058715619562911235, - "rougeL": 0.15585288034137604, - "avg_rouge": 0.14242064123305823 - }, - "prompt_4": { - "rouge1": 0.2002333225185019, - "rouge2": 0.05538090325056703, - "rougeL": 0.14595224317552624, - "avg_rouge": 0.1338554896481984 - }, - "prompt_5": { - "rouge1": 0.204107243418519, - "rouge2": 0.055058033454769194, - "rougeL": 0.15185063159264167, - "avg_rouge": 0.13700530282197662 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.5 - }, - "prompt_2": { - "accuracy": 0.481651376146789 - }, - "prompt_3": { - "accuracy": 0.49311926605504586 - }, - "prompt_4": { - "accuracy": 0.5034403669724771 - }, - "prompt_5": { - "accuracy": 0.5355504587155964 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.5043144774688398 - }, - "prompt_2": { - "accuracy": 0.513902205177373 - }, - "prompt_3": { - "accuracy": 0.48705656759348037 - }, - "prompt_4": { - "accuracy": 0.5043144774688398 - }, - "prompt_5": { - "accuracy": 0.5043144774688398 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.499 - }, - "prompt_2": { - "accuracy": 0.5195 - }, - "prompt_3": { - "accuracy": 0.538 - }, - "prompt_4": { - "accuracy": 0.4965 - }, - "prompt_5": { - "accuracy": 0.5025 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.319 - }, - "prompt_2": { - "accuracy": 0.321 - }, - "prompt_3": { - "accuracy": 0.316 - }, - "prompt_4": { - "accuracy": 0.3275 - }, - "prompt_5": { - "accuracy": 0.318 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.4925 - }, - "prompt_2": { - "accuracy": 0.497 - }, - "prompt_3": { - "accuracy": 0.4935 - }, - "prompt_4": { - "accuracy": 0.491 - }, - "prompt_5": { - "accuracy": 0.505 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.5211267605633803 - }, - "prompt_2": { - "accuracy": 0.5492957746478874 - }, - "prompt_3": { - "accuracy": 0.5211267605633803 - }, - "prompt_4": { - "accuracy": 0.5211267605633803 - }, - "prompt_5": { - "accuracy": 0.5211267605633803 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.48736462093862815 - }, - "prompt_2": { - "accuracy": 0.49097472924187724 - }, - "prompt_3": { - "accuracy": 0.49458483754512633 - }, - "prompt_4": { - "accuracy": 0.48736462093862815 - }, - "prompt_5": { - "accuracy": 0.48375451263537905 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.46568627450980393 - }, - "prompt_2": { - "accuracy": 0.4387254901960784 - }, - "prompt_3": { - "accuracy": 0.4583333333333333 - }, - "prompt_4": { - "accuracy": 0.35294117647058826 - }, - "prompt_5": { - "accuracy": 0.46568627450980393 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": -1 - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.24000000000000002, - "language_acc": { - "Spanish": 0.21333333333333335, - "English": 0.31333333333333335, - "Chinese": 0.18666666666666668, - "Vietnamese": 0.24666666666666667, - "Filipino": 0.22666666666666666, - "Indonesian": 0.25333333333333335, - "Malay": 0.24 - }, - "consistency_score_2": 0.35873015873015873, - "consistency_score_3": 0.1430476190476191, - "consistency_score_4": 0.05942857142857142, - "consistency_score_5": 0.02444444444444445, - "consistency_score_6": 0.008571428571428572, - "consistency_score_7": 0.0, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.32, - "Spanish,Chinese": 0.36666666666666664, - "Spanish,Vietnamese": 0.3466666666666667, - "Spanish,Filipino": 0.4533333333333333, - "Spanish,Indonesian": 0.32666666666666666, - "Spanish,Malay": 0.42, - "English,Chinese": 0.32, - "English,Vietnamese": 0.41333333333333333, - "English,Filipino": 0.32666666666666666, - "English,Indonesian": 0.4, - "English,Malay": 0.37333333333333335, - "Chinese,Vietnamese": 0.32, - "Chinese,Filipino": 0.28, - "Chinese,Indonesian": 0.32666666666666666, - "Chinese,Malay": 0.37333333333333335, - "Vietnamese,Filipino": 0.37333333333333335, - "Vietnamese,Indonesian": 0.38666666666666666, - "Vietnamese,Malay": 0.37333333333333335, - "Filipino,Indonesian": 0.36666666666666664, - "Filipino,Malay": 0.26666666666666666, - "Indonesian,Malay": 0.4 - }, - "3_combine": { - "Spanish,English,Chinese": 0.10666666666666667, - "Spanish,English,Vietnamese": 0.14, - "Spanish,English,Filipino": 0.15333333333333332, - "Spanish,English,Indonesian": 0.12, - "Spanish,English,Malay": 0.14, - "Spanish,Chinese,Vietnamese": 0.13333333333333333, - "Spanish,Chinese,Filipino": 0.16, - "Spanish,Chinese,Indonesian": 0.12666666666666668, - "Spanish,Chinese,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Filipino": 0.17333333333333334, - "Spanish,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,Vietnamese,Malay": 0.17333333333333334, - "Spanish,Filipino,Indonesian": 0.18, - "Spanish,Filipino,Malay": 0.16666666666666666, - "Spanish,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese": 0.12666666666666668, - "English,Chinese,Filipino": 0.08666666666666667, - "English,Chinese,Indonesian": 0.14666666666666667, - "English,Chinese,Malay": 0.15333333333333332, - "English,Vietnamese,Filipino": 0.16, - "English,Vietnamese,Indonesian": 0.19333333333333333, - "English,Vietnamese,Malay": 0.16666666666666666, - "English,Filipino,Indonesian": 0.15333333333333332, - "English,Filipino,Malay": 0.1, - "English,Indonesian,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Filipino": 0.10666666666666667, - "Chinese,Vietnamese,Indonesian": 0.11333333333333333, - "Chinese,Vietnamese,Malay": 0.13333333333333333, - "Chinese,Filipino,Indonesian": 0.11333333333333333, - "Chinese,Filipino,Malay": 0.09333333333333334, - "Chinese,Indonesian,Malay": 0.16, - "Vietnamese,Filipino,Indonesian": 0.16666666666666666, - "Vietnamese,Filipino,Malay": 0.13333333333333333, - "Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Filipino,Indonesian,Malay": 0.12666666666666668 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.04, - "Spanish,English,Chinese,Filipino": 0.05333333333333334, - "Spanish,English,Chinese,Indonesian": 0.03333333333333333, - "Spanish,English,Chinese,Malay": 0.06, - "Spanish,English,Vietnamese,Filipino": 0.07333333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.05333333333333334, - "Spanish,English,Vietnamese,Malay": 0.07333333333333333, - "Spanish,English,Filipino,Indonesian": 0.08, - "Spanish,English,Filipino,Malay": 0.07333333333333333, - "Spanish,English,Indonesian,Malay": 0.05333333333333334, - "Spanish,Chinese,Vietnamese,Filipino": 0.06, - "Spanish,Chinese,Vietnamese,Indonesian": 0.03333333333333333, - "Spanish,Chinese,Vietnamese,Malay": 0.06, - "Spanish,Chinese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.06, - "Spanish,Chinese,Indonesian,Malay": 0.06666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.07333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.04, - "English,Chinese,Vietnamese,Indonesian": 0.06, - "English,Chinese,Vietnamese,Malay": 0.07333333333333333, - "English,Chinese,Filipino,Indonesian": 0.03333333333333333, - "English,Chinese,Filipino,Malay": 0.03333333333333333, - "English,Chinese,Indonesian,Malay": 0.08666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.08, - "English,Vietnamese,Filipino,Malay": 0.06666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.07333333333333333, - "English,Filipino,Indonesian,Malay": 0.05333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.02666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.04, - "Chinese,Vietnamese,Indonesian,Malay": 0.05333333333333334, - "Chinese,Filipino,Indonesian,Malay": 0.03333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.06666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.02, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.006666666666666667, - "Spanish,English,Chinese,Vietnamese,Malay": 0.02666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.02, - "Spanish,English,Chinese,Filipino,Malay": 0.02666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.02, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.05333333333333334, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.03333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.04, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.013333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.02666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.02, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.02, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.04666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.02, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.03333333333333333, - "English,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.03333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.0, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.013333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.006666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.02666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.006666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.0 - } - }, - "AC3_2": 0.2875927889233342, - "AC3_3": 0.1792541023898019, - "AC3_4": 0.09526717554070274, - "AC3_5": 0.0443697478823812, - "AC3_6": 0.016551724131272295, - "AC3_7": 0.0 - } - }, - "cross_logiqa": { - "prompt_1": -1 - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.1553398058252427 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.3047619047619048 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.19626168224299065 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.3, - "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.3, - "history": 0.26666666666666666, - "literature": 0.4, - "politics": 0.4, - "culture": 0.2, - "film": 0.3, - "law": 0.3, - "geography": 0.2 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.07289746026162794 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.06776690345582867 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.05547745398185548 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.06266165026510719 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.07129216151447251 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.25204200700116686 - } - }, - "mmlu_full": { - "prompt_1": -1 - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.2310549777117385 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.25529265255292655, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.2857142857142857, - "college_physics": 0.20833333333333334, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.375, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.5416666666666666, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.20833333333333334, - "middle_school_chemistry": 0.16, - "veterinary_medicine": 0.25, - "college_economics": 0.18333333333333332, - "business_administration": 0.2894736842105263, - "marxism": 0.125, - "mao_zedong_thought": 0.20689655172413793, - "education_science": 0.3235294117647059, - "teacher_qualification": 0.2857142857142857, - "high_school_politics": 0.25, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.0, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.5, - "logic": 0.2962962962962963, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.17647058823529413, - "legal_professional": 0.17857142857142858, - "high_school_chinese": 0.25, - "high_school_history": 0.48, - "middle_school_history": 0.25925925925925924, - "civil_servant": 0.21153846153846154, - "sports_science": 0.25, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.25, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.2549019607843137, - "accountant": 0.24074074074074073, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.2222222222222222, - "physician": 0.25925925925925924 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.2867383512544803 - } - }, - "cmmlu_full": { - "prompt_1": -1 - }, - "zbench": { - "prompt_1": { - "accuracy": 0.21212121212121213 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.16136363636363638 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.32508474576271185 - } - }, - "c3": { - "prompt_1": -1 - }, - "dream": { - "prompt_1": -1 - }, - "samsum": { - "prompt_1": -1 - }, - "dialogsum": { - "prompt_1": -1 - }, - "sst2": { - "prompt_1": { - "accuracy": 0.4896788990825688 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.3537871524448706 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.5515 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.3395 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.504 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.5070422535211268 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.5126353790613718 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.46078431372549017 - } - } - } - }, - "sea_lion_7b": { - "model_size": "7B", - "model_link": "https://huggingface.co/aisingapore/sea-lion-7b", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.27941176470588236, - "language_acc": { - "English": 0.2571428571428571, - "Chinese": 0.2848739495798319, - "Spanish": 0.2966386554621849, - "Vietnamese": 0.27899159663865547 - }, - "consistency_score_2": 0.5539215686274509, - "consistency_score_3": 0.3733193277310924, - "consistency_score_4": 0.2823529411764706, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.5764705882352941, - "English,Spanish": 0.6, - "English,Vietnamese": 0.5915966386554622, - "Chinese,Spanish": 0.5008403361344538, - "Chinese,Vietnamese": 0.48067226890756304, - "Spanish,Vietnamese": 0.5739495798319327 - }, - "3_combine": { - "English,Chinese,Spanish": 0.3689075630252101, - "English,Chinese,Vietnamese": 0.37899159663865545, - "English,Spanish,Vietnamese": 0.4117647058823529, - "Chinese,Spanish,Vietnamese": 0.33361344537815124 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.2823529411764706 - } - }, - "AC3_2": 0.37145328715265746, - "AC3_3": 0.31961036743243365, - "AC3_4": 0.2808746534763334 - }, - "prompt_2": { - "overall_acc": 0.26197478991596634, - "language_acc": { - "English": 0.2504201680672269, - "Chinese": 0.2672268907563025, - "Spanish": 0.2605042016806723, - "Vietnamese": 0.26974789915966385 - }, - "consistency_score_2": 0.6403361344537815, - "consistency_score_3": 0.4901260504201681, - "consistency_score_4": 0.39915966386554624, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.6487394957983194, - "English,Spanish": 0.6563025210084034, - "English,Vietnamese": 0.6403361344537815, - "Chinese,Spanish": 0.6302521008403361, - "Chinese,Vietnamese": 0.6277310924369748, - "Spanish,Vietnamese": 0.6386554621848739 - }, - "3_combine": { - "English,Chinese,Spanish": 0.49495798319327733, - "English,Chinese,Vietnamese": 0.4899159663865546, - "English,Spanish,Vietnamese": 0.49327731092436977, - "Chinese,Spanish,Vietnamese": 0.4823529411764706 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.39915966386554624 - } - }, - "AC3_2": 0.3718273152853879, - "AC3_3": 0.34144535463278525, - "AC3_4": 0.31633435066097776 - }, - "prompt_3": { - "overall_acc": 0.27415966386554624, - "language_acc": { - "English": 0.26218487394957984, - "Chinese": 0.280672268907563, - "Spanish": 0.27899159663865547, - "Vietnamese": 0.2747899159663866 - }, - "consistency_score_2": 0.5630252100840336, - "consistency_score_3": 0.3831932773109244, - "consistency_score_4": 0.27563025210084036, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.5, - "English,Spanish": 0.6084033613445378, - "English,Vietnamese": 0.6470588235294118, - "Chinese,Spanish": 0.48739495798319327, - "Chinese,Vietnamese": 0.5327731092436975, - "Spanish,Vietnamese": 0.6025210084033613 - }, - "3_combine": { - "English,Chinese,Spanish": 0.34369747899159664, - "English,Chinese,Vietnamese": 0.3722689075630252, - "English,Spanish,Vietnamese": 0.4638655462184874, - "Chinese,Spanish,Vietnamese": 0.35294117647058826 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.27563025210084036 - } - }, - "AC3_2": 0.3687567874890338, - "AC3_3": 0.31963389377736234, - "AC3_4": 0.2748929911548889 - }, - "prompt_4": { - "overall_acc": 0.276890756302521, - "language_acc": { - "English": 0.2722689075630252, - "Chinese": 0.28319327731092436, - "Spanish": 0.2773109243697479, - "Vietnamese": 0.2747899159663866 - }, - "consistency_score_2": 0.5925770308123249, - "consistency_score_3": 0.39873949579831935, - "consistency_score_4": 0.27983193277310925, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.4394957983193277, - "English,Spanish": 0.7050420168067227, - "English,Vietnamese": 0.7378151260504202, - "Chinese,Spanish": 0.5378151260504201, - "Chinese,Vietnamese": 0.45546218487394957, - "Spanish,Vietnamese": 0.6798319327731093 - }, - "3_combine": { - "English,Chinese,Spanish": 0.35042016806722687, - "English,Chinese,Vietnamese": 0.3277310924369748, - "English,Spanish,Vietnamese": 0.5689075630252101, - "Chinese,Spanish,Vietnamese": 0.34789915966386553 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.27983193277310925 - } - }, - "AC3_2": 0.3774242235119929, - "AC3_3": 0.32682752200522386, - "AC3_4": 0.2783535753424225 - }, - "prompt_5": { - "overall_acc": 0.2810924369747899, - "language_acc": { - "English": 0.26638655462184874, - "Chinese": 0.2915966386554622, - "Spanish": 0.292436974789916, - "Vietnamese": 0.2739495798319328 - }, - "consistency_score_2": 0.5507002801120449, - "consistency_score_3": 0.3586134453781513, - "consistency_score_4": 0.2504201680672269, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.40336134453781514, - "English,Spanish": 0.6436974789915967, - "English,Vietnamese": 0.6630252100840336, - "Chinese,Spanish": 0.507563025210084, - "Chinese,Vietnamese": 0.4495798319327731, - "Spanish,Vietnamese": 0.6369747899159663 - }, - "3_combine": { - "English,Chinese,Spanish": 0.30840336134453783, - "English,Chinese,Vietnamese": 0.2957983193277311, - "English,Spanish,Vietnamese": 0.49327731092436977, - "Chinese,Spanish,Vietnamese": 0.3369747899159664 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.2504201680672269 - } - }, - "AC3_2": 0.37220254657416996, - "AC3_3": 0.31515585539636076, - "AC3_4": 0.26487129234064044 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.23714285714285713, - "language_acc": { - "Spanish": 0.2733333333333333, - "English": 0.24666666666666667, - "Chinese": 0.22, - "Vietnamese": 0.18666666666666668, - "Filipino": 0.3, - "Indonesian": 0.21333333333333335, - "Malay": 0.22 - }, - "consistency_score_2": 0.5838095238095237, - "consistency_score_3": 0.4161904761904761, - "consistency_score_4": 0.3217142857142858, - "consistency_score_5": 0.2606349206349206, - "consistency_score_6": 0.22, - "consistency_score_7": 0.19333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6666666666666666, - "Spanish,Chinese": 0.48, - "Spanish,Vietnamese": 0.68, - "Spanish,Filipino": 0.5466666666666666, - "Spanish,Indonesian": 0.64, - "Spanish,Malay": 0.6266666666666667, - "English,Chinese": 0.4666666666666667, - "English,Vietnamese": 0.6533333333333333, - "English,Filipino": 0.4266666666666667, - "English,Indonesian": 0.6133333333333333, - "English,Malay": 0.6, - "Chinese,Vietnamese": 0.5266666666666666, - "Chinese,Filipino": 0.52, - "Chinese,Indonesian": 0.58, - "Chinese,Malay": 0.5133333333333333, - "Vietnamese,Filipino": 0.5066666666666667, - "Vietnamese,Indonesian": 0.66, - "Vietnamese,Malay": 0.6066666666666667, - "Filipino,Indonesian": 0.5866666666666667, - "Filipino,Malay": 0.5533333333333333, - "Indonesian,Malay": 0.8066666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.37333333333333335, - "Spanish,English,Vietnamese": 0.5266666666666666, - "Spanish,English,Filipino": 0.36, - "Spanish,English,Indonesian": 0.49333333333333335, - "Spanish,English,Malay": 0.4866666666666667, - "Spanish,Chinese,Vietnamese": 0.4, - "Spanish,Chinese,Filipino": 0.32666666666666666, - "Spanish,Chinese,Indonesian": 0.38666666666666666, - "Spanish,Chinese,Malay": 0.3466666666666667, - "Spanish,Vietnamese,Filipino": 0.4066666666666667, - "Spanish,Vietnamese,Indonesian": 0.52, - "Spanish,Vietnamese,Malay": 0.4866666666666667, - "Spanish,Filipino,Indonesian": 0.42, - "Spanish,Filipino,Malay": 0.4066666666666667, - "Spanish,Indonesian,Malay": 0.5533333333333333, - "English,Chinese,Vietnamese": 0.37333333333333335, - "English,Chinese,Filipino": 0.29333333333333333, - "English,Chinese,Indonesian": 0.38, - "English,Chinese,Malay": 0.32666666666666666, - "English,Vietnamese,Filipino": 0.3333333333333333, - "English,Vietnamese,Indonesian": 0.4866666666666667, - "English,Vietnamese,Malay": 0.44666666666666666, - "English,Filipino,Indonesian": 0.36666666666666664, - "English,Filipino,Malay": 0.3333333333333333, - "English,Indonesian,Malay": 0.52, - "Chinese,Vietnamese,Filipino": 0.3466666666666667, - "Chinese,Vietnamese,Indonesian": 0.43333333333333335, - "Chinese,Vietnamese,Malay": 0.36666666666666664, - "Chinese,Filipino,Indonesian": 0.3933333333333333, - "Chinese,Filipino,Malay": 0.34, - "Chinese,Indonesian,Malay": 0.47333333333333333, - "Vietnamese,Filipino,Indonesian": 0.44, - "Vietnamese,Filipino,Malay": 0.38666666666666666, - "Vietnamese,Indonesian,Malay": 0.5466666666666666, - "Filipino,Indonesian,Malay": 0.4866666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.3333333333333333, - "Spanish,English,Chinese,Filipino": 0.25333333333333335, - "Spanish,English,Chinese,Indonesian": 0.32, - "Spanish,English,Chinese,Malay": 0.2866666666666667, - "Spanish,English,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,English,Vietnamese,Indonesian": 0.42, - "Spanish,English,Vietnamese,Malay": 0.4, - "Spanish,English,Filipino,Indonesian": 0.31333333333333335, - "Spanish,English,Filipino,Malay": 0.30666666666666664, - "Spanish,English,Indonesian,Malay": 0.43333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.28, - "Spanish,Chinese,Vietnamese,Indonesian": 0.34, - "Spanish,Chinese,Vietnamese,Malay": 0.3, - "Spanish,Chinese,Filipino,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Filipino,Malay": 0.24666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.36, - "Spanish,Vietnamese,Filipino,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.44, - "Spanish,Filipino,Indonesian,Malay": 0.38, - "English,Chinese,Vietnamese,Filipino": 0.25333333333333335, - "English,Chinese,Vietnamese,Indonesian": 0.32666666666666666, - "English,Chinese,Vietnamese,Malay": 0.2733333333333333, - "English,Chinese,Filipino,Indonesian": 0.26666666666666666, - "English,Chinese,Filipino,Malay": 0.22666666666666666, - "English,Chinese,Indonesian,Malay": 0.32, - "English,Vietnamese,Filipino,Indonesian": 0.3, - "English,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.4, - "English,Filipino,Indonesian,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian,Malay": 0.37333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.29333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.26, - "Spanish,English,Chinese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,English,Chinese,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.28, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.26, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.36, - "Spanish,English,Filipino,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.24, - "English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333 - } - }, - "AC3_2": 0.3372820682386638, - "AC3_3": 0.3021324447684158, - "AC3_4": 0.2730295062321851, - "AC3_5": 0.24833454805506758, - "AC3_6": 0.2282499999500703, - "AC3_7": 0.21300884950803997 - }, - "prompt_2": { - "overall_acc": 0.2361904761904762, - "language_acc": { - "Spanish": 0.24666666666666667, - "English": 0.23333333333333334, - "Chinese": 0.20666666666666667, - "Vietnamese": 0.2733333333333333, - "Filipino": 0.24, - "Indonesian": 0.21333333333333335, - "Malay": 0.24 - }, - "consistency_score_2": 0.45936507936507925, - "consistency_score_3": 0.2659047619047619, - "consistency_score_4": 0.17809523809523808, - "consistency_score_5": 0.13142857142857142, - "consistency_score_6": 0.10190476190476191, - "consistency_score_7": 0.08, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.5, - "Spanish,Chinese": 0.4266666666666667, - "Spanish,Vietnamese": 0.5333333333333333, - "Spanish,Filipino": 0.46, - "Spanish,Indonesian": 0.52, - "Spanish,Malay": 0.44666666666666666, - "English,Chinese": 0.4066666666666667, - "English,Vietnamese": 0.44666666666666666, - "English,Filipino": 0.47333333333333333, - "English,Indonesian": 0.5, - "English,Malay": 0.48, - "Chinese,Vietnamese": 0.41333333333333333, - "Chinese,Filipino": 0.38666666666666666, - "Chinese,Indonesian": 0.43333333333333335, - "Chinese,Malay": 0.35333333333333333, - "Vietnamese,Filipino": 0.44, - "Vietnamese,Indonesian": 0.4066666666666667, - "Vietnamese,Malay": 0.44, - "Filipino,Indonesian": 0.47333333333333333, - "Filipino,Malay": 0.52, - "Indonesian,Malay": 0.5866666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.20666666666666667, - "Spanish,English,Vietnamese": 0.32, - "Spanish,English,Filipino": 0.28, - "Spanish,English,Indonesian": 0.31333333333333335, - "Spanish,English,Malay": 0.2733333333333333, - "Spanish,Chinese,Vietnamese": 0.30666666666666664, - "Spanish,Chinese,Filipino": 0.22666666666666666, - "Spanish,Chinese,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Malay": 0.19333333333333333, - "Spanish,Vietnamese,Filipino": 0.3, - "Spanish,Vietnamese,Indonesian": 0.30666666666666664, - "Spanish,Vietnamese,Malay": 0.28, - "Spanish,Filipino,Indonesian": 0.30666666666666664, - "Spanish,Filipino,Malay": 0.29333333333333333, - "Spanish,Indonesian,Malay": 0.3333333333333333, - "English,Chinese,Vietnamese": 0.24, - "English,Chinese,Filipino": 0.20666666666666667, - "English,Chinese,Indonesian": 0.24666666666666667, - "English,Chinese,Malay": 0.20666666666666667, - "English,Vietnamese,Filipino": 0.26, - "English,Vietnamese,Indonesian": 0.26, - "English,Vietnamese,Malay": 0.24666666666666667, - "English,Filipino,Indonesian": 0.29333333333333333, - "English,Filipino,Malay": 0.31333333333333335, - "English,Indonesian,Malay": 0.3466666666666667, - "Chinese,Vietnamese,Filipino": 0.21333333333333335, - "Chinese,Vietnamese,Indonesian": 0.22, - "Chinese,Vietnamese,Malay": 0.19333333333333333, - "Chinese,Filipino,Indonesian": 0.23333333333333334, - "Chinese,Filipino,Malay": 0.20666666666666667, - "Chinese,Indonesian,Malay": 0.26, - "Vietnamese,Filipino,Indonesian": 0.26, - "Vietnamese,Filipino,Malay": 0.2866666666666667, - "Vietnamese,Indonesian,Malay": 0.28, - "Filipino,Indonesian,Malay": 0.34 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.18666666666666668, - "Spanish,English,Chinese,Filipino": 0.14, - "Spanish,English,Chinese,Indonesian": 0.15333333333333332, - "Spanish,English,Chinese,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.21333333333333335, - "Spanish,English,Vietnamese,Indonesian": 0.20666666666666667, - "Spanish,English,Vietnamese,Malay": 0.18666666666666668, - "Spanish,English,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Filipino,Malay": 0.18666666666666668, - "Spanish,English,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino": 0.16666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Malay": 0.15333333333333332, - "Spanish,Chinese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.14666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.16666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Vietnamese,Filipino,Malay": 0.2, - "Spanish,Vietnamese,Indonesian,Malay": 0.22666666666666666, - "Spanish,Filipino,Indonesian,Malay": 0.22666666666666666, - "English,Chinese,Vietnamese,Filipino": 0.14, - "English,Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "English,Chinese,Vietnamese,Malay": 0.14666666666666667, - "English,Chinese,Filipino,Indonesian": 0.15333333333333332, - "English,Chinese,Filipino,Malay": 0.14666666666666667, - "English,Chinese,Indonesian,Malay": 0.18, - "English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "English,Vietnamese,Filipino,Malay": 0.18666666666666668, - "English,Vietnamese,Indonesian,Malay": 0.18, - "English,Filipino,Indonesian,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Chinese,Vietnamese,Filipino,Malay": 0.14, - "Chinese,Vietnamese,Indonesian,Malay": 0.16, - "Chinese,Filipino,Indonesian,Malay": 0.16, - "Vietnamese,Filipino,Indonesian,Malay": 0.2 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.12, - "Spanish,English,Chinese,Filipino,Indonesian": 0.12, - "Spanish,English,Chinese,Filipino,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.12, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Vietnamese,Filipino,Malay": 0.14666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.15333333333333332, - "Spanish,English,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.13333333333333333, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.11333333333333333, - "English,Chinese,Vietnamese,Filipino,Malay": 0.11333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "English,Chinese,Filipino,Indonesian,Malay": 0.12, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.14, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.1, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.10666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.1, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.09333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.08 - } - }, - "AC3_2": 0.3119740931067851, - "AC3_3": 0.25016836471641546, - "AC3_4": 0.20306951281359956, - "AC3_5": 0.1688823093545046, - "AC3_6": 0.14237961095721705, - "AC3_7": 0.11951807225135724 - }, - "prompt_3": { - "overall_acc": 0.23619047619047623, - "language_acc": { - "Spanish": 0.26666666666666666, - "English": 0.24666666666666667, - "Chinese": 0.22666666666666666, - "Vietnamese": 0.2866666666666667, - "Filipino": 0.20666666666666667, - "Indonesian": 0.23333333333333334, - "Malay": 0.18666666666666668 - }, - "consistency_score_2": 0.3952380952380953, - "consistency_score_3": 0.18895238095238098, - "consistency_score_4": 0.10133333333333334, - "consistency_score_5": 0.05936507936507937, - "consistency_score_6": 0.03809523809523809, - "consistency_score_7": 0.02666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.44, - "Spanish,Chinese": 0.38, - "Spanish,Vietnamese": 0.3933333333333333, - "Spanish,Filipino": 0.34, - "Spanish,Indonesian": 0.4266666666666667, - "Spanish,Malay": 0.4266666666666667, - "English,Chinese": 0.41333333333333333, - "English,Vietnamese": 0.38, - "English,Filipino": 0.2866666666666667, - "English,Indonesian": 0.46, - "English,Malay": 0.32, - "Chinese,Vietnamese": 0.4, - "Chinese,Filipino": 0.35333333333333333, - "Chinese,Indonesian": 0.4266666666666667, - "Chinese,Malay": 0.3466666666666667, - "Vietnamese,Filipino": 0.3466666666666667, - "Vietnamese,Indonesian": 0.4, - "Vietnamese,Malay": 0.4066666666666667, - "Filipino,Indonesian": 0.3466666666666667, - "Filipino,Malay": 0.4266666666666667, - "Indonesian,Malay": 0.58 - }, - "3_combine": { - "Spanish,English,Chinese": 0.20666666666666667, - "Spanish,English,Vietnamese": 0.20666666666666667, - "Spanish,English,Filipino": 0.13333333333333333, - "Spanish,English,Indonesian": 0.23333333333333334, - "Spanish,English,Malay": 0.18, - "Spanish,Chinese,Vietnamese": 0.18666666666666668, - "Spanish,Chinese,Filipino": 0.16, - "Spanish,Chinese,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Malay": 0.18666666666666668, - "Spanish,Vietnamese,Filipino": 0.15333333333333332, - "Spanish,Vietnamese,Indonesian": 0.18666666666666668, - "Spanish,Vietnamese,Malay": 0.20666666666666667, - "Spanish,Filipino,Indonesian": 0.15333333333333332, - "Spanish,Filipino,Malay": 0.2, - "Spanish,Indonesian,Malay": 0.28, - "English,Chinese,Vietnamese": 0.19333333333333333, - "English,Chinese,Filipino": 0.14666666666666667, - "English,Chinese,Indonesian": 0.23333333333333334, - "English,Chinese,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino": 0.12, - "English,Vietnamese,Indonesian": 0.21333333333333335, - "English,Vietnamese,Malay": 0.15333333333333332, - "English,Filipino,Indonesian": 0.14666666666666667, - "English,Filipino,Malay": 0.12, - "English,Indonesian,Malay": 0.24, - "Chinese,Vietnamese,Filipino": 0.16, - "Chinese,Vietnamese,Indonesian": 0.20666666666666667, - "Chinese,Vietnamese,Malay": 0.18666666666666668, - "Chinese,Filipino,Indonesian": 0.18666666666666668, - "Chinese,Filipino,Malay": 0.18666666666666668, - "Chinese,Indonesian,Malay": 0.24, - "Vietnamese,Filipino,Indonesian": 0.15333333333333332, - "Vietnamese,Filipino,Malay": 0.18, - "Vietnamese,Indonesian,Malay": 0.2733333333333333, - "Filipino,Indonesian,Malay": 0.24 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.1, - "Spanish,English,Chinese,Filipino": 0.08666666666666667, - "Spanish,English,Chinese,Indonesian": 0.14, - "Spanish,English,Chinese,Malay": 0.10666666666666667, - "Spanish,English,Vietnamese,Filipino": 0.06666666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.11333333333333333, - "Spanish,English,Vietnamese,Malay": 0.10666666666666667, - "Spanish,English,Filipino,Indonesian": 0.08, - "Spanish,English,Filipino,Malay": 0.08, - "Spanish,English,Indonesian,Malay": 0.14, - "Spanish,Chinese,Vietnamese,Filipino": 0.08, - "Spanish,Chinese,Vietnamese,Indonesian": 0.10666666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.11333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.09333333333333334, - "Spanish,Chinese,Filipino,Malay": 0.10666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.14666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.1, - "Spanish,Vietnamese,Indonesian,Malay": 0.14666666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.12, - "English,Chinese,Vietnamese,Filipino": 0.07333333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.12666666666666668, - "English,Chinese,Vietnamese,Malay": 0.08, - "English,Chinese,Filipino,Indonesian": 0.1, - "English,Chinese,Filipino,Malay": 0.06666666666666667, - "English,Chinese,Indonesian,Malay": 0.12, - "English,Vietnamese,Filipino,Indonesian": 0.07333333333333333, - "English,Vietnamese,Filipino,Malay": 0.04666666666666667, - "English,Vietnamese,Indonesian,Malay": 0.12, - "English,Filipino,Indonesian,Malay": 0.09333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian": 0.09333333333333334, - "Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.04, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.07333333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,English,Chinese,Filipino,Malay": 0.06, - "Spanish,English,Chinese,Indonesian,Malay": 0.09333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.04, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "Spanish,English,Filipino,Indonesian,Malay": 0.06, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.04, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.05333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.08666666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.08, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.06, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.05333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.02666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.06666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.06, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.04, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.02666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.02666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.06, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.05333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.03333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.04, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.02666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.02666666666666667 - } - }, - "AC3_2": 0.2956834015189269, - "AC3_3": 0.20994708989770727, - "AC3_4": 0.14182091794142818, - "AC3_5": 0.09488210318512379, - "AC3_6": 0.06560846558454585, - "AC3_7": 0.0479227052957782 - }, - "prompt_4": { - "overall_acc": 0.2542857142857143, - "language_acc": { - "Spanish": 0.25333333333333335, - "English": 0.2866666666666667, - "Chinese": 0.24, - "Vietnamese": 0.22666666666666666, - "Filipino": 0.29333333333333333, - "Indonesian": 0.24, - "Malay": 0.24 - }, - "consistency_score_2": 0.6663492063492064, - "consistency_score_3": 0.5135238095238095, - "consistency_score_4": 0.4236190476190477, - "consistency_score_5": 0.364126984126984, - "consistency_score_6": 0.320952380952381, - "consistency_score_7": 0.2866666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.7333333333333333, - "Spanish,Chinese": 0.48, - "Spanish,Vietnamese": 0.7266666666666667, - "Spanish,Filipino": 0.5666666666666667, - "Spanish,Indonesian": 0.6266666666666667, - "Spanish,Malay": 0.6733333333333333, - "English,Chinese": 0.5733333333333334, - "English,Vietnamese": 0.7266666666666667, - "English,Filipino": 0.6, - "English,Indonesian": 0.7, - "English,Malay": 0.7, - "Chinese,Vietnamese": 0.62, - "Chinese,Filipino": 0.6733333333333333, - "Chinese,Indonesian": 0.6866666666666666, - "Chinese,Malay": 0.64, - "Vietnamese,Filipino": 0.58, - "Vietnamese,Indonesian": 0.76, - "Vietnamese,Malay": 0.7533333333333333, - "Filipino,Indonesian": 0.6666666666666666, - "Filipino,Malay": 0.6666666666666666, - "Indonesian,Malay": 0.84 - }, - "3_combine": { - "Spanish,English,Chinese": 0.4066666666666667, - "Spanish,English,Vietnamese": 0.6, - "Spanish,English,Filipino": 0.46, - "Spanish,English,Indonesian": 0.54, - "Spanish,English,Malay": 0.56, - "Spanish,Chinese,Vietnamese": 0.4266666666666667, - "Spanish,Chinese,Filipino": 0.4066666666666667, - "Spanish,Chinese,Indonesian": 0.42, - "Spanish,Chinese,Malay": 0.41333333333333333, - "Spanish,Vietnamese,Filipino": 0.44666666666666666, - "Spanish,Vietnamese,Indonesian": 0.56, - "Spanish,Vietnamese,Malay": 0.5866666666666667, - "Spanish,Filipino,Indonesian": 0.44, - "Spanish,Filipino,Malay": 0.4666666666666667, - "Spanish,Indonesian,Malay": 0.5733333333333334, - "English,Chinese,Vietnamese": 0.47333333333333333, - "English,Chinese,Filipino": 0.4533333333333333, - "English,Chinese,Indonesian": 0.4866666666666667, - "English,Chinese,Malay": 0.4666666666666667, - "English,Vietnamese,Filipino": 0.4666666666666667, - "English,Vietnamese,Indonesian": 0.5933333333333334, - "English,Vietnamese,Malay": 0.5933333333333334, - "English,Filipino,Indonesian": 0.5, - "English,Filipino,Malay": 0.5, - "English,Indonesian,Malay": 0.62, - "Chinese,Vietnamese,Filipino": 0.48, - "Chinese,Vietnamese,Indonesian": 0.54, - "Chinese,Vietnamese,Malay": 0.5133333333333333, - "Chinese,Filipino,Indonesian": 0.54, - "Chinese,Filipino,Malay": 0.5333333333333333, - "Chinese,Indonesian,Malay": 0.5866666666666667, - "Vietnamese,Filipino,Indonesian": 0.52, - "Vietnamese,Filipino,Malay": 0.5133333333333333, - "Vietnamese,Indonesian,Malay": 0.68, - "Filipino,Indonesian,Malay": 0.6066666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.38, - "Spanish,English,Chinese,Filipino": 0.34, - "Spanish,English,Chinese,Indonesian": 0.38, - "Spanish,English,Chinese,Malay": 0.36, - "Spanish,English,Vietnamese,Filipino": 0.3933333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.49333333333333335, - "Spanish,English,Vietnamese,Malay": 0.5066666666666667, - "Spanish,English,Filipino,Indonesian": 0.3933333333333333, - "Spanish,English,Filipino,Malay": 0.4, - "Spanish,English,Indonesian,Malay": 0.49333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.36, - "Spanish,Chinese,Vietnamese,Indonesian": 0.4, - "Spanish,Chinese,Vietnamese,Malay": 0.38666666666666666, - "Spanish,Chinese,Filipino,Indonesian": 0.35333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.35333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.38666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian": 0.4066666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.4066666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.52, - "Spanish,Filipino,Indonesian,Malay": 0.41333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.3933333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.44, - "English,Chinese,Vietnamese,Malay": 0.41333333333333333, - "English,Chinese,Filipino,Indonesian": 0.4, - "English,Chinese,Filipino,Malay": 0.4, - "English,Chinese,Indonesian,Malay": 0.44, - "English,Vietnamese,Filipino,Indonesian": 0.4266666666666667, - "English,Vietnamese,Filipino,Malay": 0.42, - "English,Vietnamese,Indonesian,Malay": 0.5466666666666666, - "English,Filipino,Indonesian,Malay": 0.46, - "Chinese,Vietnamese,Filipino,Indonesian": 0.44, - "Chinese,Vietnamese,Filipino,Malay": 0.4266666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.49333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.5066666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.49333333333333335 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.32, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.36666666666666664, - "Spanish,English,Chinese,Vietnamese,Malay": 0.3466666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,English,Chinese,Filipino,Malay": 0.30666666666666664, - "Spanish,English,Chinese,Indonesian,Malay": 0.3466666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.36666666666666664, - "Spanish,English,Vietnamese,Filipino,Malay": 0.36, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.46, - "Spanish,English,Filipino,Indonesian,Malay": 0.36666666666666664, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.34, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.3333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.38666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.36666666666666664, - "English,Chinese,Vietnamese,Filipino,Malay": 0.35333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.38, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.4066666666666667, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.42 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.29333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.34, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.3466666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.3466666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667 - } - }, - "AC3_2": 0.3681004925708541, - "AC3_3": 0.3401410496774987, - "AC3_4": 0.31780355632930885, - "AC3_5": 0.2994514519902984, - "AC3_6": 0.28375591291188257, - "AC3_7": 0.26950704220370025 - }, - "prompt_5": { - "overall_acc": 0.23238095238095238, - "language_acc": { - "Spanish": 0.23333333333333334, - "English": 0.22, - "Chinese": 0.24666666666666667, - "Vietnamese": 0.22666666666666666, - "Filipino": 0.24666666666666667, - "Indonesian": 0.24, - "Malay": 0.21333333333333335 - }, - "consistency_score_2": 0.5247619047619048, - "consistency_score_3": 0.34400000000000003, - "consistency_score_4": 0.24476190476190476, - "consistency_score_5": 0.1822222222222222, - "consistency_score_6": 0.14095238095238094, - "consistency_score_7": 0.11333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.74, - "Spanish,Chinese": 0.4866666666666667, - "Spanish,Vietnamese": 0.6266666666666667, - "Spanish,Filipino": 0.4, - "Spanish,Indonesian": 0.5466666666666666, - "Spanish,Malay": 0.5533333333333333, - "English,Chinese": 0.49333333333333335, - "English,Vietnamese": 0.6333333333333333, - "English,Filipino": 0.36, - "English,Indonesian": 0.5266666666666666, - "English,Malay": 0.56, - "Chinese,Vietnamese": 0.49333333333333335, - "Chinese,Filipino": 0.4266666666666667, - "Chinese,Indonesian": 0.4866666666666667, - "Chinese,Malay": 0.4533333333333333, - "Vietnamese,Filipino": 0.4266666666666667, - "Vietnamese,Indonesian": 0.6466666666666666, - "Vietnamese,Malay": 0.5733333333333334, - "Filipino,Indonesian": 0.48, - "Filipino,Malay": 0.42, - "Indonesian,Malay": 0.6866666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.38, - "Spanish,English,Vietnamese": 0.5133333333333333, - "Spanish,English,Filipino": 0.28, - "Spanish,English,Indonesian": 0.4266666666666667, - "Spanish,English,Malay": 0.46, - "Spanish,Chinese,Vietnamese": 0.36666666666666664, - "Spanish,Chinese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Indonesian": 0.3333333333333333, - "Spanish,Chinese,Malay": 0.32666666666666666, - "Spanish,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,Vietnamese,Indonesian": 0.43333333333333335, - "Spanish,Vietnamese,Malay": 0.4066666666666667, - "Spanish,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Filipino,Malay": 0.26, - "Spanish,Indonesian,Malay": 0.4266666666666667, - "English,Chinese,Vietnamese": 0.35333333333333333, - "English,Chinese,Filipino": 0.22666666666666666, - "English,Chinese,Indonesian": 0.31333333333333335, - "English,Chinese,Malay": 0.3, - "English,Vietnamese,Filipino": 0.2733333333333333, - "English,Vietnamese,Indonesian": 0.4266666666666667, - "English,Vietnamese,Malay": 0.41333333333333333, - "English,Filipino,Indonesian": 0.26666666666666666, - "English,Filipino,Malay": 0.25333333333333335, - "English,Indonesian,Malay": 0.42, - "Chinese,Vietnamese,Filipino": 0.26, - "Chinese,Vietnamese,Indonesian": 0.36666666666666664, - "Chinese,Vietnamese,Malay": 0.32666666666666666, - "Chinese,Filipino,Indonesian": 0.2866666666666667, - "Chinese,Filipino,Malay": 0.26, - "Chinese,Indonesian,Malay": 0.36666666666666664, - "Vietnamese,Filipino,Indonesian": 0.34, - "Vietnamese,Filipino,Malay": 0.2866666666666667, - "Vietnamese,Indonesian,Malay": 0.4866666666666667, - "Filipino,Indonesian,Malay": 0.35333333333333333 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.29333333333333333, - "Spanish,English,Chinese,Filipino": 0.19333333333333333, - "Spanish,English,Chinese,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Malay": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino": 0.22, - "Spanish,English,Vietnamese,Indonesian": 0.35333333333333333, - "Spanish,English,Vietnamese,Malay": 0.34, - "Spanish,English,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Filipino,Malay": 0.2, - "Spanish,English,Indonesian,Malay": 0.34, - "Spanish,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,Chinese,Vietnamese,Indonesian": 0.28, - "Spanish,Chinese,Vietnamese,Malay": 0.26, - "Spanish,Chinese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,Chinese,Filipino,Malay": 0.19333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.26, - "Spanish,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Spanish,Vietnamese,Filipino,Malay": 0.2, - "Spanish,Vietnamese,Indonesian,Malay": 0.34, - "Spanish,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino": 0.17333333333333334, - "English,Chinese,Vietnamese,Indonesian": 0.26, - "English,Chinese,Vietnamese,Malay": 0.24, - "English,Chinese,Filipino,Indonesian": 0.17333333333333334, - "English,Chinese,Filipino,Malay": 0.16666666666666666, - "English,Chinese,Indonesian,Malay": 0.24, - "English,Vietnamese,Filipino,Indonesian": 0.22, - "English,Vietnamese,Filipino,Malay": 0.2, - "English,Vietnamese,Indonesian,Malay": 0.3466666666666667, - "English,Filipino,Indonesian,Malay": 0.22, - "Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Chinese,Vietnamese,Indonesian,Malay": 0.28, - "Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.14666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.21333333333333335, - "Spanish,English,Chinese,Filipino,Indonesian": 0.16, - "Spanish,English,Chinese,Filipino,Malay": 0.15333333333333332, - "Spanish,English,Chinese,Indonesian,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.28, - "Spanish,English,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.18, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.19333333333333333, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.13333333333333333, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.2, - "English,Chinese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.13333333333333333, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.12, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.17333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.13333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.12666666666666668 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.11333333333333333 - } - }, - "AC3_2": 0.3221179993584744, - "AC3_3": 0.27738268336231797, - "AC3_4": 0.2384107974027473, - "AC3_5": 0.2042674833606348, - "AC3_6": 0.1754713313426975, - "AC3_7": 0.1523599632249831 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.27516233766233766, - "language_acc": { - "English": 0.23295454545454544, - "Filipino": 0.26704545454545453, - "Spanish": 0.29545454545454547, - "Malay": 0.26136363636363635, - "Chinese": 0.2897727272727273, - "Indonesian": 0.29545454545454547, - "Vietnamese": 0.2840909090909091 - }, - "consistency_score_2": 0.40882034632034625, - "consistency_score_3": 0.21103896103896103, - "consistency_score_4": 0.12256493506493507, - "consistency_score_5": 0.07602813852813853, - "consistency_score_6": 0.047889610389610385, - "consistency_score_7": 0.028409090909090908, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.14204545454545456, - "English,Spanish": 0.4034090909090909, - "English,Malay": 0.32954545454545453, - "English,Chinese": 0.5227272727272727, - "English,Indonesian": 0.48863636363636365, - "English,Vietnamese": 0.5795454545454546, - "Filipino,Spanish": 0.4318181818181818, - "Filipino,Malay": 0.3977272727272727, - "Filipino,Chinese": 0.2784090909090909, - "Filipino,Indonesian": 0.3181818181818182, - "Filipino,Vietnamese": 0.3068181818181818, - "Spanish,Malay": 0.4318181818181818, - "Spanish,Chinese": 0.3977272727272727, - "Spanish,Indonesian": 0.4090909090909091, - "Spanish,Vietnamese": 0.4034090909090909, - "Malay,Chinese": 0.3409090909090909, - "Malay,Indonesian": 0.4715909090909091, - "Malay,Vietnamese": 0.42613636363636365, - "Chinese,Indonesian": 0.4318181818181818, - "Chinese,Vietnamese": 0.4943181818181818, - "Indonesian,Vietnamese": 0.5795454545454546 - }, - "3_combine": { - "English,Filipino,Spanish": 0.11931818181818182, - "English,Filipino,Malay": 0.08522727272727272, - "English,Filipino,Chinese": 0.08522727272727272, - "English,Filipino,Indonesian": 0.10227272727272728, - "English,Filipino,Vietnamese": 0.09090909090909091, - "English,Spanish,Malay": 0.17045454545454544, - "English,Spanish,Chinese": 0.23863636363636365, - "English,Spanish,Indonesian": 0.23863636363636365, - "English,Spanish,Vietnamese": 0.24431818181818182, - "English,Malay,Chinese": 0.21022727272727273, - "English,Malay,Indonesian": 0.24431818181818182, - "English,Malay,Vietnamese": 0.23863636363636365, - "English,Chinese,Indonesian": 0.3068181818181818, - "English,Chinese,Vietnamese": 0.36363636363636365, - "English,Indonesian,Vietnamese": 0.375, - "Filipino,Spanish,Malay": 0.23295454545454544, - "Filipino,Spanish,Chinese": 0.17045454545454544, - "Filipino,Spanish,Indonesian": 0.19886363636363635, - "Filipino,Spanish,Vietnamese": 0.1875, - "Filipino,Malay,Chinese": 0.14204545454545456, - "Filipino,Malay,Indonesian": 0.19318181818181818, - "Filipino,Malay,Vietnamese": 0.16477272727272727, - "Filipino,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Indonesian,Vietnamese": 0.20454545454545456, - "Spanish,Malay,Chinese": 0.19318181818181818, - "Spanish,Malay,Indonesian": 0.23863636363636365, - "Spanish,Malay,Vietnamese": 0.2215909090909091, - "Spanish,Chinese,Indonesian": 0.2215909090909091, - "Spanish,Chinese,Vietnamese": 0.22727272727272727, - "Spanish,Indonesian,Vietnamese": 0.2727272727272727, - "Malay,Chinese,Indonesian": 0.23863636363636365, - "Malay,Chinese,Vietnamese": 0.23295454545454544, - "Malay,Indonesian,Vietnamese": 0.30113636363636365, - "Chinese,Indonesian,Vietnamese": 0.3181818181818182 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.06818181818181818, - "English,Filipino,Spanish,Chinese": 0.07386363636363637, - "English,Filipino,Spanish,Indonesian": 0.09090909090909091, - "English,Filipino,Spanish,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Chinese": 0.05113636363636364, - "English,Filipino,Malay,Indonesian": 0.06818181818181818, - "English,Filipino,Malay,Vietnamese": 0.05113636363636364, - "English,Filipino,Chinese,Indonesian": 0.07954545454545454, - "English,Filipino,Chinese,Vietnamese": 0.06818181818181818, - "English,Filipino,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese": 0.10795454545454546, - "English,Spanish,Malay,Indonesian": 0.13636363636363635, - "English,Spanish,Malay,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Indonesian": 0.17045454545454544, - "English,Spanish,Chinese,Vietnamese": 0.17045454545454544, - "English,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Malay,Chinese,Indonesian": 0.17045454545454544, - "English,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Malay,Indonesian,Vietnamese": 0.1875, - "English,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Malay,Chinese": 0.10227272727272728, - "Filipino,Spanish,Malay,Indonesian": 0.125, - "Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Chinese,Indonesian": 0.10795454545454546, - "Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "Filipino,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "Filipino,Malay,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.08522727272727272, - "Filipino,Malay,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "Spanish,Malay,Chinese,Vietnamese": 0.125, - "Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Malay,Chinese,Indonesian,Vietnamese": 0.1875 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Indonesian": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Chinese,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.0625, - "English,Spanish,Malay,Chinese,Indonesian": 0.10227272727272728, - "English,Spanish,Malay,Chinese,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.10795454545454546, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.06818181818181818, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.056818181818181816 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 - } - }, - "AC3_2": 0.3289321931542765, - "AC3_3": 0.23887214616859187, - "AC3_4": 0.1695898488839439, - "AC3_5": 0.1191380845317414, - "AC3_6": 0.08158079355302786, - "AC3_7": 0.05150097227277732 - }, - "prompt_2": { - "overall_acc": 0.2702922077922078, - "language_acc": { - "English": 0.3068181818181818, - "Filipino": 0.2840909090909091, - "Spanish": 0.26136363636363635, - "Malay": 0.23863636363636365, - "Chinese": 0.32386363636363635, - "Indonesian": 0.23863636363636365, - "Vietnamese": 0.23863636363636365 - }, - "consistency_score_2": 0.35470779220779225, - "consistency_score_3": 0.14237012987012987, - "consistency_score_4": 0.06185064935064936, - "consistency_score_5": 0.030303030303030297, - "consistency_score_6": 0.017045454545454548, - "consistency_score_7": 0.011363636363636364, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.17613636363636365, - "English,Spanish": 0.3977272727272727, - "English,Malay": 0.19318181818181818, - "English,Chinese": 0.5227272727272727, - "English,Indonesian": 0.4034090909090909, - "English,Vietnamese": 0.4318181818181818, - "Filipino,Spanish": 0.48295454545454547, - "Filipino,Malay": 0.45454545454545453, - "Filipino,Chinese": 0.13636363636363635, - "Filipino,Indonesian": 0.38636363636363635, - "Filipino,Vietnamese": 0.2727272727272727, - "Spanish,Malay": 0.44886363636363635, - "Spanish,Chinese": 0.23863636363636365, - "Spanish,Indonesian": 0.5, - "Spanish,Vietnamese": 0.35795454545454547, - "Malay,Chinese": 0.14772727272727273, - "Malay,Indonesian": 0.42045454545454547, - "Malay,Vietnamese": 0.35795454545454547, - "Chinese,Indonesian": 0.2727272727272727, - "Chinese,Vietnamese": 0.3977272727272727, - "Indonesian,Vietnamese": 0.44886363636363635 - }, - "3_combine": { - "English,Filipino,Spanish": 0.11363636363636363, - "English,Filipino,Malay": 0.08522727272727272, - "English,Filipino,Chinese": 0.05113636363636364, - "English,Filipino,Indonesian": 0.09090909090909091, - "English,Filipino,Vietnamese": 0.06818181818181818, - "English,Spanish,Malay": 0.125, - "English,Spanish,Chinese": 0.16477272727272727, - "English,Spanish,Indonesian": 0.23863636363636365, - "English,Spanish,Vietnamese": 0.17045454545454544, - "English,Malay,Chinese": 0.056818181818181816, - "English,Malay,Indonesian": 0.11931818181818182, - "English,Malay,Vietnamese": 0.10227272727272728, - "English,Chinese,Indonesian": 0.20454545454545456, - "English,Chinese,Vietnamese": 0.2784090909090909, - "English,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Malay": 0.26704545454545453, - "Filipino,Spanish,Chinese": 0.0625, - "Filipino,Spanish,Indonesian": 0.26136363636363635, - "Filipino,Spanish,Vietnamese": 0.125, - "Filipino,Malay,Chinese": 0.0625, - "Filipino,Malay,Indonesian": 0.2159090909090909, - "Filipino,Malay,Vietnamese": 0.14772727272727273, - "Filipino,Chinese,Indonesian": 0.06818181818181818, - "Filipino,Chinese,Vietnamese": 0.07954545454545454, - "Filipino,Indonesian,Vietnamese": 0.1534090909090909, - "Spanish,Malay,Chinese": 0.0625, - "Spanish,Malay,Indonesian": 0.23863636363636365, - "Spanish,Malay,Vietnamese": 0.17045454545454544, - "Spanish,Chinese,Indonesian": 0.11931818181818182, - "Spanish,Chinese,Vietnamese": 0.125, - "Spanish,Indonesian,Vietnamese": 0.22727272727272727, - "Malay,Chinese,Indonesian": 0.07954545454545454, - "Malay,Chinese,Vietnamese": 0.07954545454545454, - "Malay,Indonesian,Vietnamese": 0.19318181818181818, - "Chinese,Indonesian,Vietnamese": 0.14772727272727273 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.0625, - "English,Filipino,Spanish,Chinese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian": 0.06818181818181818, - "English,Filipino,Spanish,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Chinese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian": 0.05113636363636364, - "English,Filipino,Malay,Vietnamese": 0.045454545454545456, - "English,Filipino,Chinese,Indonesian": 0.03409090909090909, - "English,Filipino,Chinese,Vietnamese": 0.03977272727272727, - "English,Filipino,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Malay,Chinese": 0.03409090909090909, - "English,Spanish,Malay,Indonesian": 0.09090909090909091, - "English,Spanish,Malay,Vietnamese": 0.0625, - "English,Spanish,Chinese,Indonesian": 0.10227272727272728, - "English,Spanish,Chinese,Vietnamese": 0.09090909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "English,Malay,Chinese,Indonesian": 0.03977272727272727, - "English,Malay,Chinese,Vietnamese": 0.045454545454545456, - "English,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Chinese,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay,Chinese": 0.03409090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.14204545454545456, - "Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Chinese,Indonesian": 0.03977272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.07954545454545454, - "Filipino,Malay,Chinese,Indonesian": 0.03977272727272727, - "Filipino,Malay,Chinese,Vietnamese": 0.03977272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "Filipino,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Spanish,Malay,Chinese,Indonesian": 0.03977272727272727, - "Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian": 0.03977272727272727, - "English,Filipino,Spanish,Malay,Vietnamese": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.03409090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.017045454545454544, - "English,Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.03409090909090909, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "English,Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.056818181818181816, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.07954545454545454, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 - } - }, - "AC3_2": 0.30679920723713244, - "AC3_3": 0.18650374994203234, - "AC3_4": 0.10066601284244969, - "AC3_5": 0.05449635870865228, - "AC3_6": 0.032068567015033564, - "AC3_7": 0.02181032223485529 - }, - "prompt_3": { - "overall_acc": 0.25974025974025977, - "language_acc": { - "English": 0.24431818181818182, - "Filipino": 0.2727272727272727, - "Spanish": 0.22727272727272727, - "Malay": 0.23863636363636365, - "Chinese": 0.30113636363636365, - "Indonesian": 0.2840909090909091, - "Vietnamese": 0.25 - }, - "consistency_score_2": 0.3446969696969697, - "consistency_score_3": 0.1439935064935065, - "consistency_score_4": 0.06688311688311688, - "consistency_score_5": 0.03300865800865801, - "consistency_score_6": 0.015422077922077922, - "consistency_score_7": 0.005681818181818182, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.19318181818181818, - "English,Spanish": 0.36363636363636365, - "English,Malay": 0.14204545454545456, - "English,Chinese": 0.5454545454545454, - "English,Indonesian": 0.3465909090909091, - "English,Vietnamese": 0.5568181818181818, - "Filipino,Spanish": 0.3352272727272727, - "Filipino,Malay": 0.3522727272727273, - "Filipino,Chinese": 0.19886363636363635, - "Filipino,Indonesian": 0.3409090909090909, - "Filipino,Vietnamese": 0.23295454545454544, - "Spanish,Malay": 0.3806818181818182, - "Spanish,Chinese": 0.3352272727272727, - "Spanish,Indonesian": 0.4659090909090909, - "Spanish,Vietnamese": 0.375, - "Malay,Chinese": 0.14772727272727273, - "Malay,Indonesian": 0.4375, - "Malay,Vietnamese": 0.23295454545454544, - "Chinese,Indonesian": 0.3068181818181818, - "Chinese,Vietnamese": 0.5113636363636364, - "Indonesian,Vietnamese": 0.4375 - }, - "3_combine": { - "English,Filipino,Spanish": 0.07954545454545454, - "English,Filipino,Malay": 0.05113636363636364, - "English,Filipino,Chinese": 0.08522727272727272, - "English,Filipino,Indonesian": 0.09659090909090909, - "English,Filipino,Vietnamese": 0.11931818181818182, - "English,Spanish,Malay": 0.07954545454545454, - "English,Spanish,Chinese": 0.2215909090909091, - "English,Spanish,Indonesian": 0.1875, - "English,Spanish,Vietnamese": 0.24431818181818182, - "English,Malay,Chinese": 0.056818181818181816, - "English,Malay,Indonesian": 0.09659090909090909, - "English,Malay,Vietnamese": 0.09090909090909091, - "English,Chinese,Indonesian": 0.2159090909090909, - "English,Chinese,Vietnamese": 0.3465909090909091, - "English,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay": 0.14772727272727273, - "Filipino,Spanish,Chinese": 0.09090909090909091, - "Filipino,Spanish,Indonesian": 0.17045454545454544, - "Filipino,Spanish,Vietnamese": 0.10227272727272728, - "Filipino,Malay,Chinese": 0.05113636363636364, - "Filipino,Malay,Indonesian": 0.18181818181818182, - "Filipino,Malay,Vietnamese": 0.10227272727272728, - "Filipino,Chinese,Indonesian": 0.08522727272727272, - "Filipino,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Indonesian,Vietnamese": 0.14204545454545456, - "Spanish,Malay,Chinese": 0.10227272727272728, - "Spanish,Malay,Indonesian": 0.2215909090909091, - "Spanish,Malay,Vietnamese": 0.125, - "Spanish,Chinese,Indonesian": 0.17045454545454544, - "Spanish,Chinese,Vietnamese": 0.21022727272727273, - "Spanish,Indonesian,Vietnamese": 0.2215909090909091, - "Malay,Chinese,Indonesian": 0.09090909090909091, - "Malay,Chinese,Vietnamese": 0.09090909090909091, - "Malay,Indonesian,Vietnamese": 0.17045454545454544, - "Chinese,Indonesian,Vietnamese": 0.23295454545454544 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.022727272727272728, - "English,Filipino,Spanish,Chinese": 0.03977272727272727, - "English,Filipino,Spanish,Indonesian": 0.05113636363636364, - "English,Filipino,Spanish,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese": 0.017045454545454544, - "English,Filipino,Malay,Indonesian": 0.028409090909090908, - "English,Filipino,Malay,Vietnamese": 0.03409090909090909, - "English,Filipino,Chinese,Indonesian": 0.05113636363636364, - "English,Filipino,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Indonesian,Vietnamese": 0.07954545454545454, - "English,Spanish,Malay,Chinese": 0.045454545454545456, - "English,Spanish,Malay,Indonesian": 0.0625, - "English,Spanish,Malay,Vietnamese": 0.0625, - "English,Spanish,Chinese,Indonesian": 0.125, - "English,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.14772727272727273, - "English,Malay,Chinese,Indonesian": 0.045454545454545456, - "English,Malay,Chinese,Vietnamese": 0.05113636363636364, - "English,Malay,Indonesian,Vietnamese": 0.07386363636363637, - "English,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Malay,Chinese": 0.03409090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.08522727272727272, - "Filipino,Spanish,Malay,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Chinese,Indonesian": 0.056818181818181816, - "Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, - "Filipino,Spanish,Indonesian,Vietnamese": 0.07386363636363637, - "Filipino,Malay,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Filipino,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "Filipino,Chinese,Indonesian,Vietnamese": 0.0625, - "Spanish,Malay,Chinese,Indonesian": 0.0625, - "Spanish,Malay,Chinese,Vietnamese": 0.0625, - "Spanish,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "Spanish,Chinese,Indonesian,Vietnamese": 0.125, - "Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.022727272727272728, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Chinese,Indonesian": 0.03409090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.03977272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.05113636363636364, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.10227272727272728, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.045454545454545456 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.005681818181818182, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 - } - }, - "AC3_2": 0.296248067013737, - "AC3_3": 0.18527511888423456, - "AC3_4": 0.10637473854893, - "AC3_5": 0.05857358905265673, - "AC3_6": 0.02911542733493466, - "AC3_7": 0.01112037808866546 - }, - "prompt_4": { - "overall_acc": 0.27029220779220775, - "language_acc": { - "English": 0.2556818181818182, - "Filipino": 0.24431818181818182, - "Spanish": 0.2727272727272727, - "Malay": 0.2784090909090909, - "Chinese": 0.2556818181818182, - "Indonesian": 0.3181818181818182, - "Vietnamese": 0.26704545454545453 - }, - "consistency_score_2": 0.45265151515151525, - "consistency_score_3": 0.26120129870129877, - "consistency_score_4": 0.16477272727272727, - "consistency_score_5": 0.10984848484848483, - "consistency_score_6": 0.07873376623376624, - "consistency_score_7": 0.0625, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.2727272727272727, - "English,Spanish": 0.6193181818181818, - "English,Malay": 0.4431818181818182, - "English,Chinese": 0.5056818181818182, - "English,Indonesian": 0.6136363636363636, - "English,Vietnamese": 0.6193181818181818, - "Filipino,Spanish": 0.32954545454545453, - "Filipino,Malay": 0.36363636363636365, - "Filipino,Chinese": 0.2897727272727273, - "Filipino,Indonesian": 0.26136363636363635, - "Filipino,Vietnamese": 0.29545454545454547, - "Spanish,Malay": 0.38636363636363635, - "Spanish,Chinese": 0.5170454545454546, - "Spanish,Indonesian": 0.4943181818181818, - "Spanish,Vietnamese": 0.5284090909090909, - "Malay,Chinese": 0.3465909090909091, - "Malay,Indonesian": 0.5170454545454546, - "Malay,Vietnamese": 0.5284090909090909, - "Chinese,Indonesian": 0.48863636363636365, - "Chinese,Vietnamese": 0.4318181818181818, - "Indonesian,Vietnamese": 0.6534090909090909 - }, - "3_combine": { - "English,Filipino,Spanish": 0.19886363636363635, - "English,Filipino,Malay": 0.1590909090909091, - "English,Filipino,Chinese": 0.1590909090909091, - "English,Filipino,Indonesian": 0.17045454545454544, - "English,Filipino,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay": 0.2784090909090909, - "English,Spanish,Chinese": 0.36363636363636365, - "English,Spanish,Indonesian": 0.3977272727272727, - "English,Spanish,Vietnamese": 0.4147727272727273, - "English,Malay,Chinese": 0.22727272727272727, - "English,Malay,Indonesian": 0.3522727272727273, - "English,Malay,Vietnamese": 0.3409090909090909, - "English,Chinese,Indonesian": 0.3522727272727273, - "English,Chinese,Vietnamese": 0.3352272727272727, - "English,Indonesian,Vietnamese": 0.4715909090909091, - "Filipino,Spanish,Malay": 0.16477272727272727, - "Filipino,Spanish,Chinese": 0.1875, - "Filipino,Spanish,Indonesian": 0.16477272727272727, - "Filipino,Spanish,Vietnamese": 0.19886363636363635, - "Filipino,Malay,Chinese": 0.1534090909090909, - "Filipino,Malay,Indonesian": 0.16477272727272727, - "Filipino,Malay,Vietnamese": 0.1875, - "Filipino,Chinese,Indonesian": 0.16477272727272727, - "Filipino,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Chinese": 0.19886363636363635, - "Spanish,Malay,Indonesian": 0.2784090909090909, - "Spanish,Malay,Vietnamese": 0.29545454545454547, - "Spanish,Chinese,Indonesian": 0.3068181818181818, - "Spanish,Chinese,Vietnamese": 0.30113636363636365, - "Spanish,Indonesian,Vietnamese": 0.3806818181818182, - "Malay,Chinese,Indonesian": 0.26136363636363635, - "Malay,Chinese,Vietnamese": 0.26136363636363635, - "Malay,Indonesian,Vietnamese": 0.4034090909090909, - "Chinese,Indonesian,Vietnamese": 0.32954545454545453 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.10227272727272728, - "English,Filipino,Spanish,Chinese": 0.125, - "English,Filipino,Spanish,Indonesian": 0.13068181818181818, - "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Chinese": 0.09090909090909091, - "English,Filipino,Malay,Indonesian": 0.10795454545454546, - "English,Filipino,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Indonesian": 0.125, - "English,Filipino,Chinese,Vietnamese": 0.10795454545454546, - "English,Filipino,Indonesian,Vietnamese": 0.13068181818181818, - "English,Spanish,Malay,Chinese": 0.14772727272727273, - "English,Spanish,Malay,Indonesian": 0.2159090909090909, - "English,Spanish,Malay,Vietnamese": 0.2215909090909091, - "English,Spanish,Chinese,Indonesian": 0.26136363636363635, - "English,Spanish,Chinese,Vietnamese": 0.24431818181818182, - "English,Spanish,Indonesian,Vietnamese": 0.3125, - "English,Malay,Chinese,Indonesian": 0.20454545454545456, - "English,Malay,Chinese,Vietnamese": 0.19886363636363635, - "English,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Malay,Chinese": 0.09659090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.10795454545454546, - "Filipino,Spanish,Malay,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Indonesian": 0.11931818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Malay,Chinese,Indonesian": 0.11363636363636363, - "Filipino,Malay,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Malay,Indonesian,Vietnamese": 0.125, - "Filipino,Chinese,Indonesian,Vietnamese": 0.11931818181818182, - "Spanish,Malay,Chinese,Indonesian": 0.17045454545454544, - "Spanish,Malay,Chinese,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "Spanish,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.0625, - "English,Filipino,Spanish,Malay,Indonesian": 0.07386363636363637, - "English,Filipino,Spanish,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Indonesian": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.10795454545454546, - "English,Filipino,Malay,Chinese,Indonesian": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Vietnamese": 0.07386363636363637, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay,Chinese,Indonesian": 0.13636363636363635, - "English,Spanish,Malay,Chinese,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.07954545454545454, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.09090909090909091, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.09090909090909091, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.0625, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.0625, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.08522727272727272, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.0625 - } - }, - "AC3_2": 0.3384722032738129, - "AC3_3": 0.26566900565414564, - "AC3_4": 0.2047362618253971, - "AC3_5": 0.1562115819712006, - "AC3_6": 0.12194578673642927, - "AC3_7": 0.1015243902133956 - }, - "prompt_5": { - "overall_acc": 0.2719155844155844, - "language_acc": { - "English": 0.2784090909090909, - "Filipino": 0.26136363636363635, - "Spanish": 0.30113636363636365, - "Malay": 0.26704545454545453, - "Chinese": 0.2727272727272727, - "Indonesian": 0.2556818181818182, - "Vietnamese": 0.26704545454545453 - }, - "consistency_score_2": 0.4112554112554114, - "consistency_score_3": 0.21266233766233766, - "consistency_score_4": 0.1137987012987013, - "consistency_score_5": 0.05762987012987013, - "consistency_score_6": 0.024350649350649352, - "consistency_score_7": 0.005681818181818182, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.07954545454545454, - "English,Spanish": 0.5909090909090909, - "English,Malay": 0.5056818181818182, - "English,Chinese": 0.44886363636363635, - "English,Indonesian": 0.6079545454545454, - "English,Vietnamese": 0.6761363636363636, - "Filipino,Spanish": 0.20454545454545456, - "Filipino,Malay": 0.3125, - "Filipino,Chinese": 0.17045454545454544, - "Filipino,Indonesian": 0.13068181818181818, - "Filipino,Vietnamese": 0.1590909090909091, - "Spanish,Malay": 0.5340909090909091, - "Spanish,Chinese": 0.4090909090909091, - "Spanish,Indonesian": 0.48863636363636365, - "Spanish,Vietnamese": 0.5795454545454546, - "Malay,Chinese": 0.3068181818181818, - "Malay,Indonesian": 0.5170454545454546, - "Malay,Vietnamese": 0.5738636363636364, - "Chinese,Indonesian": 0.38636363636363635, - "Chinese,Vietnamese": 0.3806818181818182, - "Indonesian,Vietnamese": 0.5738636363636364 - }, - "3_combine": { - "English,Filipino,Spanish": 0.045454545454545456, - "English,Filipino,Malay": 0.056818181818181816, - "English,Filipino,Chinese": 0.045454545454545456, - "English,Filipino,Indonesian": 0.022727272727272728, - "English,Filipino,Vietnamese": 0.03409090909090909, - "English,Spanish,Malay": 0.3693181818181818, - "English,Spanish,Chinese": 0.2784090909090909, - "English,Spanish,Indonesian": 0.4034090909090909, - "English,Spanish,Vietnamese": 0.45454545454545453, - "English,Malay,Chinese": 0.22727272727272727, - "English,Malay,Indonesian": 0.375, - "English,Malay,Vietnamese": 0.4034090909090909, - "English,Chinese,Indonesian": 0.29545454545454547, - "English,Chinese,Vietnamese": 0.30113636363636365, - "English,Indonesian,Vietnamese": 0.48295454545454547, - "Filipino,Spanish,Malay": 0.13068181818181818, - "Filipino,Spanish,Chinese": 0.08522727272727272, - "Filipino,Spanish,Indonesian": 0.045454545454545456, - "Filipino,Spanish,Vietnamese": 0.09090909090909091, - "Filipino,Malay,Chinese": 0.06818181818181818, - "Filipino,Malay,Indonesian": 0.08522727272727272, - "Filipino,Malay,Vietnamese": 0.125, - "Filipino,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Chinese,Vietnamese": 0.05113636363636364, - "Filipino,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Malay,Chinese": 0.2159090909090909, - "Spanish,Malay,Indonesian": 0.32954545454545453, - "Spanish,Malay,Vietnamese": 0.39204545454545453, - "Spanish,Chinese,Indonesian": 0.23295454545454544, - "Spanish,Chinese,Vietnamese": 0.26704545454545453, - "Spanish,Indonesian,Vietnamese": 0.3806818181818182, - "Malay,Chinese,Indonesian": 0.2159090909090909, - "Malay,Chinese,Vietnamese": 0.22727272727272727, - "Malay,Indonesian,Vietnamese": 0.3806818181818182, - "Chinese,Indonesian,Vietnamese": 0.25 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.045454545454545456, - "English,Filipino,Spanish,Chinese": 0.028409090909090908, - "English,Filipino,Spanish,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Vietnamese": 0.028409090909090908, - "English,Filipino,Malay,Chinese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian": 0.022727272727272728, - "English,Filipino,Malay,Vietnamese": 0.03409090909090909, - "English,Filipino,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese": 0.16477272727272727, - "English,Spanish,Malay,Indonesian": 0.26704545454545453, - "English,Spanish,Malay,Vietnamese": 0.30113636363636365, - "English,Spanish,Chinese,Indonesian": 0.19886363636363635, - "English,Spanish,Chinese,Vietnamese": 0.2159090909090909, - "English,Spanish,Indonesian,Vietnamese": 0.3352272727272727, - "English,Malay,Chinese,Indonesian": 0.18181818181818182, - "English,Malay,Chinese,Vietnamese": 0.18181818181818182, - "English,Malay,Indonesian,Vietnamese": 0.3125, - "English,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Malay,Chinese": 0.056818181818181816, - "Filipino,Spanish,Malay,Indonesian": 0.03977272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, - "Filipino,Spanish,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Malay,Chinese,Indonesian": 0.022727272727272728, - "Filipino,Malay,Chinese,Vietnamese": 0.045454545454545456, - "Filipino,Malay,Indonesian,Vietnamese": 0.045454545454545456, - "Filipino,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Spanish,Malay,Chinese,Indonesian": 0.14204545454545456, - "Spanish,Malay,Chinese,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Indonesian,Vietnamese": 0.2727272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Vietnamese": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.017045454545454544, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.11931818181818182, - "English,Spanish,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.11363636363636363 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.011363636363636364, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.09659090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.005681818181818182 - } - }, - "AC3_2": 0.3273755946544074, - "AC3_3": 0.23866627504918347, - "AC3_4": 0.1604485055773624, - "AC3_5": 0.09510348023471978, - "AC3_6": 0.04469845221761155, - "AC3_7": 0.011131047311248062 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.32038834951456313 - }, - "prompt_2": { - "accuracy": 0.27184466019417475 - }, - "prompt_3": { - "accuracy": 0.32038834951456313 - }, - "prompt_4": { - "accuracy": 0.2621359223300971 - }, - "prompt_5": { - "accuracy": 0.3300970873786408 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.2761904761904762 - }, - "prompt_2": { - "accuracy": 0.23809523809523808 - }, - "prompt_3": { - "accuracy": 0.3142857142857143 - }, - "prompt_4": { - "accuracy": 0.20952380952380953 - }, - "prompt_5": { - "accuracy": 0.2857142857142857 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.308411214953271 - }, - "prompt_2": { - "accuracy": 0.205607476635514 - }, - "prompt_3": { - "accuracy": 0.24299065420560748 - }, - "prompt_4": { - "accuracy": 0.22429906542056074 - }, - "prompt_5": { - "accuracy": 0.2523364485981308 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.18, - "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.1, - "history": 0.3333333333333333, - "literature": 0.0, - "politics": 0.1, - "culture": 0.2, - "film": 0.3, - "law": 0.2, - "geography": 0.2 - } - }, - "prompt_2": { - "accuracy": 0.26, - "category_acc": { - "brand": 0.5, - "demographics": 0.4, - "biology": 0.2, - "history": 0.2, - "literature": 0.2, - "politics": 0.2, - "culture": 0.2, - "film": 0.2, - "law": 0.4, - "geography": 0.2 - } - }, - "prompt_3": { - "accuracy": 0.3, - "category_acc": { - "brand": 0.4, - "demographics": 0.4, - "biology": 0.0, - "history": 0.2, - "literature": 0.5, - "politics": 0.1, - "culture": 0.4, - "film": 0.4, - "law": 0.5, - "geography": 0.2 - } - }, - "prompt_4": { - "accuracy": 0.27, - "category_acc": { - "brand": 0.1, - "demographics": 0.4, - "biology": 0.3, - "history": 0.26666666666666666, - "literature": 0.1, - "politics": 0.3, - "culture": 0.3, - "film": 0.3, - "law": 0.4, - "geography": 0.3 - } - }, - "prompt_5": { - "accuracy": 0.15, - "category_acc": { - "brand": 0.1, - "demographics": 0.2, - "biology": 0.1, - "history": 0.2, - "literature": 0.0, - "politics": 0.1, - "culture": 0.2, - "film": 0.2, - "law": 0.2, - "geography": 0.2 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.07248273525186222 - }, - "prompt_2": { - "bleu_score": 0.06518999890974515 - }, - "prompt_3": { - "bleu_score": 0.06221944486834087 - }, - "prompt_4": { - "bleu_score": 0.055074155599171476 - }, - "prompt_5": { - "bleu_score": 0.0456359134538313 - } - }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.07654704485252221 - }, - "prompt_2": { - "bleu_score": 0.06714108616110813 - }, - "prompt_3": { - "bleu_score": 0.0672314203431971 - }, - "prompt_4": { - "bleu_score": 0.07875571138457625 - }, - "prompt_5": { - "bleu_score": 0.04793686970343631 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.06467921781185289 - }, - "prompt_2": { - "bleu_score": 0.0531609454153498 - }, - "prompt_3": { - "bleu_score": 0.05175011904155169 - }, - "prompt_4": { - "bleu_score": 0.06347339251738969 - }, - "prompt_5": { - "bleu_score": 0.05269302139167072 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.039812088006341055 - }, - "prompt_2": { - "bleu_score": 0.05586728992062597 - }, - "prompt_3": { - "bleu_score": 0.05484280305808622 - }, - "prompt_4": { - "bleu_score": 0.05296472862517713 - }, - "prompt_5": { - "bleu_score": 0.06779275329860589 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.07773644340360227 - }, - "prompt_2": { - "bleu_score": 0.06724480070646238 - }, - "prompt_3": { - "bleu_score": 0.06610559722558694 - }, - "prompt_4": { - "bleu_score": 0.07872297335141586 - }, - "prompt_5": { - "bleu_score": 0.05034456802331814 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.27304550758459745 - }, - "prompt_2": { - "accuracy": 0.2613768961493582 - }, - "prompt_3": { - "accuracy": 0.27537922987164526 - }, - "prompt_4": { - "accuracy": 0.26254375729288215 - }, - "prompt_5": { - "accuracy": 0.26487747957992996 - } - }, - "mmlu_full": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.274888558692422 - }, - "prompt_2": { - "accuracy": 0.26448736998514116 - }, - "prompt_3": { - "accuracy": 0.26151560178306094 - }, - "prompt_4": { - "accuracy": 0.24962852897473997 - }, - "prompt_5": { - "accuracy": 0.24888558692421991 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.2671232876712329, - "category_acc": { - "computer_network": 0.125, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.21428571428571427, - "college_physics": 0.375, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.08333333333333333, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.08333333333333333, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.16666666666666666, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.23333333333333334, - "business_administration": 0.21052631578947367, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.20689655172413793, - "education_science": 0.14705882352941177, - "teacher_qualification": 0.1836734693877551, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.375, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.20833333333333334, - "logic": 0.37037037037037035, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.21428571428571427, - "art_studies": 0.18421052631578946, - "professional_tour_guide": 0.23529411764705882, - "legal_professional": 0.25, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.12, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.28846153846153844, - "sports_science": 0.20833333333333334, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.5833333333333334, - "clinical_medicine": 0.2222222222222222, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.25925925925925924, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.2222222222222222, - "physician": 0.25925925925925924 - } - }, - "prompt_2": { - "accuracy": 0.261519302615193, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.5833333333333334, - "computer_architecture": 0.19230769230769232, - "college_programming": 0.14285714285714285, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.11904761904761904, - "metrology_engineer": 0.27586206896551724, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.125, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.21666666666666667, - "business_administration": 0.23684210526315788, - "marxism": 0.25, - "mao_zedong_thought": 0.13793103448275862, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.16326530612244897, - "high_school_politics": 0.4583333333333333, - "high_school_geography": 0.2916666666666667, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.20833333333333334, - "logic": 0.2962962962962963, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.17857142857142858, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.2, - "middle_school_history": 0.2222222222222222, - "civil_servant": 0.25, - "sports_science": 0.3333333333333333, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.6666666666666666, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.21568627450980393, - "accountant": 0.4074074074074074, - "fire_engineer": 0.2222222222222222, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.25925925925925924, - "physician": 0.2777777777777778 - } - }, - "prompt_3": { - "accuracy": 0.2546699875466999, - "category_acc": { - "computer_network": 0.375, - "operating_system": 0.2916666666666667, - "computer_architecture": 0.11538461538461539, - "college_programming": 0.11904761904761904, - "college_physics": 0.375, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.09523809523809523, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.08333333333333333, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.3333333333333333, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.16666666666666666, - "business_administration": 0.2631578947368421, - "marxism": 0.20833333333333334, - "mao_zedong_thought": 0.20689655172413793, - "education_science": 0.20588235294117646, - "teacher_qualification": 0.16326530612244897, - "high_school_politics": 0.5, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.35714285714285715, - "ideological_and_moral_cultivation": 0.125, - "logic": 0.25925925925925924, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.25, - "art_studies": 0.2631578947368421, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.2857142857142857, - "high_school_chinese": 0.375, - "high_school_history": 0.24, - "middle_school_history": 0.18518518518518517, - "civil_servant": 0.28846153846153844, - "sports_science": 0.3333333333333333, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.375, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.17647058823529413, - "accountant": 0.4074074074074074, - "fire_engineer": 0.16666666666666666, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.2222222222222222, - "physician": 0.35185185185185186 - } - }, - "prompt_4": { - "accuracy": 0.26214196762141967, - "category_acc": { - "computer_network": 0.16666666666666666, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.19047619047619047, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.20689655172413793, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.125, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.38461538461538464, - "middle_school_physics": 0.2916666666666667, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.14285714285714285, - "college_economics": 0.23333333333333334, - "business_administration": 0.15789473684210525, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.13793103448275862, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.24489795918367346, - "high_school_politics": 0.125, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.23076923076923078, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.20833333333333334, - "logic": 0.37037037037037035, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.15789473684210525, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.21428571428571427, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.04, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.28846153846153844, - "sports_science": 0.25, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.21568627450980393, - "accountant": 0.2777777777777778, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.2962962962962963, - "physician": 0.2962962962962963 - } - }, - "prompt_5": { - "accuracy": 0.26214196762141967, - "category_acc": { - "computer_network": 0.20833333333333334, - "operating_system": 0.4166666666666667, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.19047619047619047, - "college_physics": 0.375, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.5217391304347826, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.125, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.125, - "middle_school_biology": 0.38461538461538464, - "middle_school_physics": 0.25, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.10714285714285714, - "college_economics": 0.2, - "business_administration": 0.10526315789473684, - "marxism": 0.3333333333333333, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.23529411764705882, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.16666666666666666, - "logic": 0.37037037037037035, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.21052631578947367, - "professional_tour_guide": 0.2647058823529412, - "legal_professional": 0.25, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.04, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.2692307692307692, - "sports_science": 0.25, - "plant_protection": 0.14814814814814814, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.3333333333333333, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.2777777777777778, - "tax_accountant": 0.25925925925925924, - "physician": 0.2777777777777778 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.22939068100358423 - }, - "prompt_2": { - "accuracy": 0.22580645161290322 - }, - "prompt_3": { - "accuracy": 0.25448028673835127 - }, - "prompt_4": { - "accuracy": 0.23297491039426524 - }, - "prompt_5": { - "accuracy": 0.2078853046594982 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.255309963736833, - "category_acc": { - "agronomy": 0.2603550295857988, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.2621951219512195, - "arts": 0.2625, - "astronomy": 0.24242424242424243, - "business_ethics": 0.23923444976076555, - "chinese_civil_service_exam": 0.2375, - "chinese_driving_rule": 0.26717557251908397, - "chinese_food_culture": 0.23529411764705882, - "chinese_foreign_policy": 0.22429906542056074, - "chinese_history": 0.25077399380804954, - "chinese_literature": 0.23529411764705882, - "chinese_teacher_qualification": 0.2681564245810056, - "clinical_knowledge": 0.25316455696202533, - "college_actuarial_science": 0.18867924528301888, - "college_education": 0.205607476635514, - "college_engineering_hydrology": 0.22641509433962265, - "college_law": 0.2962962962962963, - "college_mathematics": 0.1523809523809524, - "college_medical_statistics": 0.27358490566037735, - "college_medicine": 0.25274725274725274, - "computer_science": 0.2696078431372549, - "computer_security": 0.25146198830409355, - "conceptual_physics": 0.25170068027210885, - "construction_project_management": 0.2158273381294964, - "economics": 0.2893081761006289, - "education": 0.25766871165644173, - "electrical_engineering": 0.27906976744186046, - "elementary_chinese": 0.2261904761904762, - "elementary_commonsense": 0.29797979797979796, - "elementary_information_and_technology": 0.25210084033613445, - "elementary_mathematics": 0.27391304347826084, - "ethnology": 0.2518518518518518, - "food_science": 0.3146853146853147, - "genetics": 0.22727272727272727, - "global_facts": 0.26174496644295303, - "high_school_biology": 0.22485207100591717, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.2033898305084746, - "high_school_mathematics": 0.25609756097560976, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.16083916083916083, - "human_sexuality": 0.2698412698412698, - "international_law": 0.2864864864864865, - "journalism": 0.26744186046511625, - "jurisprudence": 0.26763990267639903, - "legal_and_moral_basis": 0.2803738317757009, - "logical": 0.21951219512195122, - "machine_learning": 0.28688524590163933, - "management": 0.2857142857142857, - "marketing": 0.3, - "marxist_theory": 0.2751322751322751, - "modern_chinese": 0.20689655172413793, - "nutrition": 0.2827586206896552, - "philosophy": 0.26666666666666666, - "professional_accounting": 0.22285714285714286, - "professional_law": 0.2559241706161137, - "professional_medicine": 0.26063829787234044, - "professional_psychology": 0.1896551724137931, - "public_relations": 0.2471264367816092, - "security_study": 0.2962962962962963, - "sociology": 0.2831858407079646, - "sports_science": 0.26666666666666666, - "traditional_chinese_medicine": 0.2594594594594595, - "virology": 0.27218934911242604, - "world_history": 0.2795031055900621, - "world_religions": 0.25 - } - }, - "prompt_2": { - "accuracy": 0.25582800897945085, - "category_acc": { - "agronomy": 0.25443786982248523, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.22560975609756098, - "arts": 0.3, - "astronomy": 0.23030303030303031, - "business_ethics": 0.27751196172248804, - "chinese_civil_service_exam": 0.24375, - "chinese_driving_rule": 0.25190839694656486, - "chinese_food_culture": 0.22794117647058823, - "chinese_foreign_policy": 0.2616822429906542, - "chinese_history": 0.2786377708978328, - "chinese_literature": 0.24019607843137256, - "chinese_teacher_qualification": 0.2905027932960894, - "clinical_knowledge": 0.25738396624472576, - "college_actuarial_science": 0.18867924528301888, - "college_education": 0.19626168224299065, - "college_engineering_hydrology": 0.1509433962264151, - "college_law": 0.3148148148148148, - "college_mathematics": 0.20952380952380953, - "college_medical_statistics": 0.25471698113207547, - "college_medicine": 0.2673992673992674, - "computer_science": 0.25, - "computer_security": 0.30994152046783624, - "conceptual_physics": 0.2789115646258503, - "construction_project_management": 0.2302158273381295, - "economics": 0.32075471698113206, - "education": 0.2147239263803681, - "electrical_engineering": 0.2558139534883721, - "elementary_chinese": 0.23015873015873015, - "elementary_commonsense": 0.2676767676767677, - "elementary_information_and_technology": 0.28991596638655465, - "elementary_mathematics": 0.26521739130434785, - "ethnology": 0.2962962962962963, - "food_science": 0.2727272727272727, - "genetics": 0.2215909090909091, - "global_facts": 0.2751677852348993, - "high_school_biology": 0.28402366863905326, - "high_school_chemistry": 0.2803030303030303, - "high_school_geography": 0.19491525423728814, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.24545454545454545, - "high_school_politics": 0.24475524475524477, - "human_sexuality": 0.29365079365079366, - "international_law": 0.23243243243243245, - "journalism": 0.2616279069767442, - "jurisprudence": 0.26763990267639903, - "legal_and_moral_basis": 0.27102803738317754, - "logical": 0.1951219512195122, - "machine_learning": 0.2786885245901639, - "management": 0.2857142857142857, - "marketing": 0.25555555555555554, - "marxist_theory": 0.26455026455026454, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.25517241379310346, - "philosophy": 0.26666666666666666, - "professional_accounting": 0.24, - "professional_law": 0.24170616113744076, - "professional_medicine": 0.2127659574468085, - "professional_psychology": 0.25, - "public_relations": 0.26436781609195403, - "security_study": 0.2962962962962963, - "sociology": 0.26548672566371684, - "sports_science": 0.24848484848484848, - "traditional_chinese_medicine": 0.23783783783783785, - "virology": 0.24260355029585798, - "world_history": 0.2484472049689441, - "world_religions": 0.24375 - } - }, - "prompt_3": { - "accuracy": 0.25047487480573305, - "category_acc": { - "agronomy": 0.2485207100591716, - "anatomy": 0.22972972972972974, - "ancient_chinese": 0.25609756097560976, - "arts": 0.2875, - "astronomy": 0.2606060606060606, - "business_ethics": 0.2727272727272727, - "chinese_civil_service_exam": 0.2125, - "chinese_driving_rule": 0.29770992366412213, - "chinese_food_culture": 0.27205882352941174, - "chinese_foreign_policy": 0.3177570093457944, - "chinese_history": 0.2755417956656347, - "chinese_literature": 0.2549019607843137, - "chinese_teacher_qualification": 0.25139664804469275, - "clinical_knowledge": 0.25738396624472576, - "college_actuarial_science": 0.18867924528301888, - "college_education": 0.21495327102803738, - "college_engineering_hydrology": 0.2169811320754717, - "college_law": 0.3055555555555556, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.24528301886792453, - "college_medicine": 0.26373626373626374, - "computer_science": 0.2549019607843137, - "computer_security": 0.26900584795321636, - "conceptual_physics": 0.24489795918367346, - "construction_project_management": 0.22302158273381295, - "economics": 0.27044025157232704, - "education": 0.2147239263803681, - "electrical_engineering": 0.21511627906976744, - "elementary_chinese": 0.25, - "elementary_commonsense": 0.24242424242424243, - "elementary_information_and_technology": 0.27310924369747897, - "elementary_mathematics": 0.20869565217391303, - "ethnology": 0.21481481481481482, - "food_science": 0.27972027972027974, - "genetics": 0.23863636363636365, - "global_facts": 0.22818791946308725, - "high_school_biology": 0.23668639053254437, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.1864406779661017, - "high_school_mathematics": 0.25, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.24475524475524477, - "human_sexuality": 0.21428571428571427, - "international_law": 0.2648648648648649, - "journalism": 0.27325581395348836, - "jurisprudence": 0.25547445255474455, - "legal_and_moral_basis": 0.2616822429906542, - "logical": 0.21951219512195122, - "machine_learning": 0.2540983606557377, - "management": 0.2523809523809524, - "marketing": 0.25555555555555554, - "marxist_theory": 0.20105820105820105, - "modern_chinese": 0.2672413793103448, - "nutrition": 0.27586206896551724, - "philosophy": 0.2571428571428571, - "professional_accounting": 0.22285714285714286, - "professional_law": 0.2985781990521327, - "professional_medicine": 0.21010638297872342, - "professional_psychology": 0.28879310344827586, - "public_relations": 0.28160919540229884, - "security_study": 0.28888888888888886, - "sociology": 0.252212389380531, - "sports_science": 0.2606060606060606, - "traditional_chinese_medicine": 0.2810810810810811, - "virology": 0.20710059171597633, - "world_history": 0.2546583850931677, - "world_religions": 0.21875 - } - }, - "prompt_4": { - "accuracy": 0.25375582800897944, - "category_acc": { - "agronomy": 0.28994082840236685, - "anatomy": 0.25, - "ancient_chinese": 0.17682926829268292, - "arts": 0.225, - "astronomy": 0.26666666666666666, - "business_ethics": 0.2727272727272727, - "chinese_civil_service_exam": 0.25625, - "chinese_driving_rule": 0.25190839694656486, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.24299065420560748, - "chinese_history": 0.23839009287925697, - "chinese_literature": 0.2647058823529412, - "chinese_teacher_qualification": 0.2681564245810056, - "clinical_knowledge": 0.23628691983122363, - "college_actuarial_science": 0.16981132075471697, - "college_education": 0.16822429906542055, - "college_engineering_hydrology": 0.2358490566037736, - "college_law": 0.32407407407407407, - "college_mathematics": 0.1523809523809524, - "college_medical_statistics": 0.24528301886792453, - "college_medicine": 0.2490842490842491, - "computer_science": 0.27450980392156865, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.2585034013605442, - "construction_project_management": 0.2589928057553957, - "economics": 0.2893081761006289, - "education": 0.2331288343558282, - "electrical_engineering": 0.2441860465116279, - "elementary_chinese": 0.1865079365079365, - "elementary_commonsense": 0.2727272727272727, - "elementary_information_and_technology": 0.2647058823529412, - "elementary_mathematics": 0.2782608695652174, - "ethnology": 0.2740740740740741, - "food_science": 0.3006993006993007, - "genetics": 0.21022727272727273, - "global_facts": 0.20134228187919462, - "high_school_biology": 0.27218934911242604, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.211864406779661, - "high_school_mathematics": 0.31097560975609756, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.3076923076923077, - "human_sexuality": 0.25396825396825395, - "international_law": 0.25405405405405407, - "journalism": 0.27906976744186046, - "jurisprudence": 0.25304136253041365, - "legal_and_moral_basis": 0.2803738317757009, - "logical": 0.22764227642276422, - "machine_learning": 0.319672131147541, - "management": 0.23809523809523808, - "marketing": 0.2722222222222222, - "marxist_theory": 0.30158730158730157, - "modern_chinese": 0.23275862068965517, - "nutrition": 0.2689655172413793, - "philosophy": 0.23809523809523808, - "professional_accounting": 0.2857142857142857, - "professional_law": 0.26540284360189575, - "professional_medicine": 0.24202127659574468, - "professional_psychology": 0.21551724137931033, - "public_relations": 0.2471264367816092, - "security_study": 0.25925925925925924, - "sociology": 0.2610619469026549, - "sports_science": 0.2727272727272727, - "traditional_chinese_medicine": 0.2702702702702703, - "virology": 0.26627218934911245, - "world_history": 0.2546583850931677, - "world_religions": 0.28125 - } - }, - "prompt_5": { - "accuracy": 0.24650319461232947, - "category_acc": { - "agronomy": 0.2781065088757396, - "anatomy": 0.2905405405405405, - "ancient_chinese": 0.18292682926829268, - "arts": 0.24375, - "astronomy": 0.23636363636363636, - "business_ethics": 0.2583732057416268, - "chinese_civil_service_exam": 0.2625, - "chinese_driving_rule": 0.24427480916030533, - "chinese_food_culture": 0.2426470588235294, - "chinese_foreign_policy": 0.27102803738317754, - "chinese_history": 0.2260061919504644, - "chinese_literature": 0.25980392156862747, - "chinese_teacher_qualification": 0.27932960893854747, - "clinical_knowledge": 0.25738396624472576, - "college_actuarial_science": 0.20754716981132076, - "college_education": 0.1588785046728972, - "college_engineering_hydrology": 0.2641509433962264, - "college_law": 0.3148148148148148, - "college_mathematics": 0.17142857142857143, - "college_medical_statistics": 0.2169811320754717, - "college_medicine": 0.21611721611721613, - "computer_science": 0.2647058823529412, - "computer_security": 0.24561403508771928, - "conceptual_physics": 0.2789115646258503, - "construction_project_management": 0.2589928057553957, - "economics": 0.2641509433962264, - "education": 0.24539877300613497, - "electrical_engineering": 0.23255813953488372, - "elementary_chinese": 0.16666666666666666, - "elementary_commonsense": 0.2474747474747475, - "elementary_information_and_technology": 0.25210084033613445, - "elementary_mathematics": 0.25217391304347825, - "ethnology": 0.2518518518518518, - "food_science": 0.26573426573426573, - "genetics": 0.23295454545454544, - "global_facts": 0.2080536912751678, - "high_school_biology": 0.24260355029585798, - "high_school_chemistry": 0.20454545454545456, - "high_school_geography": 0.1864406779661017, - "high_school_mathematics": 0.25609756097560976, - "high_school_physics": 0.2636363636363636, - "high_school_politics": 0.25874125874125875, - "human_sexuality": 0.24603174603174602, - "international_law": 0.24324324324324326, - "journalism": 0.20930232558139536, - "jurisprudence": 0.27007299270072993, - "legal_and_moral_basis": 0.2570093457943925, - "logical": 0.2601626016260163, - "machine_learning": 0.32786885245901637, - "management": 0.24285714285714285, - "marketing": 0.22777777777777777, - "marxist_theory": 0.25925925925925924, - "modern_chinese": 0.22413793103448276, - "nutrition": 0.2827586206896552, - "philosophy": 0.26666666666666666, - "professional_accounting": 0.28, - "professional_law": 0.26540284360189575, - "professional_medicine": 0.2393617021276596, - "professional_psychology": 0.22844827586206898, - "public_relations": 0.23563218390804597, - "security_study": 0.2518518518518518, - "sociology": 0.26548672566371684, - "sports_science": 0.22424242424242424, - "traditional_chinese_medicine": 0.2648648648648649, - "virology": 0.25443786982248523, - "world_history": 0.2670807453416149, - "world_religions": 0.2625 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.30303030303030304 - }, - "prompt_2": { - "accuracy": 0.18181818181818182 - }, - "prompt_3": { - "accuracy": 0.12121212121212122 - }, - "prompt_4": { - "accuracy": 0.3333333333333333 - }, - "prompt_5": { - "accuracy": 0.2727272727272727 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.15454545454545454 - }, - "prompt_2": { - "accuracy": 0.12272727272727273 - }, - "prompt_3": { - "accuracy": 0.1159090909090909 - }, - "prompt_4": { - "accuracy": 0.15454545454545454 - }, - "prompt_5": { - "accuracy": 0.15454545454545454 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3430508474576271 - }, - "prompt_2": { - "accuracy": 0.331864406779661 - }, - "prompt_3": { - "accuracy": 0.31966101694915255 - }, - "prompt_4": { - "accuracy": 0.3389830508474576 - }, - "prompt_5": { - "accuracy": 0.3467796610169491 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.2782348541510845 - }, - "prompt_2": { - "accuracy": 0.29768137621540763 - }, - "prompt_3": { - "accuracy": 0.2868362004487659 - }, - "prompt_4": { - "accuracy": 0.2782348541510845 - }, - "prompt_5": { - "accuracy": 0.28085265519820496 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.3591376776090152 - }, - "prompt_2": { - "accuracy": 0.34492895639392457 - }, - "prompt_3": { - "accuracy": 0.3468887800097991 - }, - "prompt_4": { - "accuracy": 0.3606075453209211 - }, - "prompt_5": { - "accuracy": 0.3601175894169525 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.20760577902863422, - "rouge2": 0.0658024090777032, - "rougeL": 0.1605171045109756, - "avg_rouge": 0.14464176420577102 - }, - "prompt_2": { - "rouge1": 0.18946379530663285, - "rouge2": 0.058845585634864514, - "rougeL": 0.1452518406974667, - "avg_rouge": 0.13118707387965467 - }, - "prompt_3": { - "rouge1": 0.15287962023593263, - "rouge2": 0.04484744853843895, - "rougeL": 0.12055190400654058, - "avg_rouge": 0.10609299092697072 - }, - "prompt_4": { - "rouge1": 0.22317838929899148, - "rouge2": 0.0711825069166511, - "rougeL": 0.16810792316962872, - "avg_rouge": 0.15415627312842375 - }, - "prompt_5": { - "rouge1": 0.20066733159105124, - "rouge2": 0.06221148338713719, - "rougeL": 0.15262007088214283, - "avg_rouge": 0.13849962862011042 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.2004050094435069, - "rouge2": 0.053388311961985825, - "rougeL": 0.14809591166837136, - "avg_rouge": 0.133963077691288 - }, - "prompt_2": { - "rouge1": 0.211359090123378, - "rouge2": 0.057218610120282926, - "rougeL": 0.15541247251061172, - "avg_rouge": 0.14133005758475756 - }, - "prompt_3": { - "rouge1": 0.21114615944932894, - "rouge2": 0.057709177177547824, - "rougeL": 0.15532401258778533, - "avg_rouge": 0.14139311640488736 - }, - "prompt_4": { - "rouge1": 0.19738912071603557, - "rouge2": 0.05363349429999815, - "rougeL": 0.1452309897263599, - "avg_rouge": 0.13208453491413122 - }, - "prompt_5": { - "rouge1": 0.1622919299663265, - "rouge2": 0.0374896647190016, - "rougeL": 0.12809729132561196, - "avg_rouge": 0.10929296200364669 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.45642201834862384 - }, - "prompt_2": { - "accuracy": 0.49770642201834864 - }, - "prompt_3": { - "accuracy": 0.5263761467889908 - }, - "prompt_4": { - "accuracy": 0.5561926605504587 - }, - "prompt_5": { - "accuracy": 0.5802752293577982 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.5052732502396932 - }, - "prompt_2": { - "accuracy": 0.5177372962607862 - }, - "prompt_3": { - "accuracy": 0.5330776605944392 - }, - "prompt_4": { - "accuracy": 0.6442953020134228 - }, - "prompt_5": { - "accuracy": 0.6912751677852349 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.4955 - }, - "prompt_2": { - "accuracy": 0.5145 - }, - "prompt_3": { - "accuracy": 0.518 - }, - "prompt_4": { - "accuracy": 0.4965 - }, - "prompt_5": { - "accuracy": 0.4715 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.3645 - }, - "prompt_2": { - "accuracy": 0.325 - }, - "prompt_3": { - "accuracy": 0.354 - }, - "prompt_4": { - "accuracy": 0.364 - }, - "prompt_5": { - "accuracy": 0.3645 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.4925 - }, - "prompt_2": { - "accuracy": 0.5105 - }, - "prompt_3": { - "accuracy": 0.5055 - }, - "prompt_4": { - "accuracy": 0.484 - }, - "prompt_5": { - "accuracy": 0.488 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.4507042253521127 - }, - "prompt_2": { - "accuracy": 0.38028169014084506 - }, - "prompt_3": { - "accuracy": 0.4225352112676056 - }, - "prompt_4": { - "accuracy": 0.4225352112676056 - }, - "prompt_5": { - "accuracy": 0.4788732394366197 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.5054151624548736 - }, - "prompt_2": { - "accuracy": 0.4981949458483754 - }, - "prompt_3": { - "accuracy": 0.5234657039711191 - }, - "prompt_4": { - "accuracy": 0.5018050541516246 - }, - "prompt_5": { - "accuracy": 0.5126353790613718 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.5367647058823529 - }, - "prompt_2": { - "accuracy": 0.5490196078431373 - }, - "prompt_3": { - "accuracy": 0.5392156862745098 - }, - "prompt_4": { - "accuracy": 0.3897058823529412 - }, - "prompt_5": { - "accuracy": 0.5343137254901961 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": -1 - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.27809523809523806, - "language_acc": { - "Spanish": 0.31333333333333335, - "English": 0.26666666666666666, - "Chinese": 0.24, - "Vietnamese": 0.28, - "Filipino": 0.22666666666666666, - "Indonesian": 0.3333333333333333, - "Malay": 0.2866666666666667 - }, - "consistency_score_2": 0.3806349206349206, - "consistency_score_3": 0.17904761904761907, - "consistency_score_4": 0.09257142857142858, - "consistency_score_5": 0.048888888888888885, - "consistency_score_6": 0.025714285714285717, - "consistency_score_7": 0.013333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.4066666666666667, - "Spanish,Chinese": 0.4266666666666667, - "Spanish,Vietnamese": 0.36666666666666664, - "Spanish,Filipino": 0.35333333333333333, - "Spanish,Indonesian": 0.34, - "Spanish,Malay": 0.36, - "English,Chinese": 0.4266666666666667, - "English,Vietnamese": 0.41333333333333333, - "English,Filipino": 0.3466666666666667, - "English,Indonesian": 0.41333333333333333, - "English,Malay": 0.37333333333333335, - "Chinese,Vietnamese": 0.43333333333333335, - "Chinese,Filipino": 0.36666666666666664, - "Chinese,Indonesian": 0.3333333333333333, - "Chinese,Malay": 0.3466666666666667, - "Vietnamese,Filipino": 0.29333333333333333, - "Vietnamese,Indonesian": 0.3333333333333333, - "Vietnamese,Malay": 0.4, - "Filipino,Indonesian": 0.38666666666666666, - "Filipino,Malay": 0.4066666666666667, - "Indonesian,Malay": 0.4666666666666667 - }, - "3_combine": { - "Spanish,English,Chinese": 0.24666666666666667, - "Spanish,English,Vietnamese": 0.18666666666666668, - "Spanish,English,Filipino": 0.16666666666666666, - "Spanish,English,Indonesian": 0.2, - "Spanish,English,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese": 0.21333333333333335, - "Spanish,Chinese,Filipino": 0.18, - "Spanish,Chinese,Indonesian": 0.15333333333333332, - "Spanish,Chinese,Malay": 0.17333333333333334, - "Spanish,Vietnamese,Filipino": 0.14, - "Spanish,Vietnamese,Indonesian": 0.12666666666666668, - "Spanish,Vietnamese,Malay": 0.16666666666666666, - "Spanish,Filipino,Indonesian": 0.14666666666666667, - "Spanish,Filipino,Malay": 0.17333333333333334, - "Spanish,Indonesian,Malay": 0.18, - "English,Chinese,Vietnamese": 0.23333333333333334, - "English,Chinese,Filipino": 0.16, - "English,Chinese,Indonesian": 0.18, - "English,Chinese,Malay": 0.16666666666666666, - "English,Vietnamese,Filipino": 0.16, - "English,Vietnamese,Indonesian": 0.18666666666666668, - "English,Vietnamese,Malay": 0.22, - "English,Filipino,Indonesian": 0.15333333333333332, - "English,Filipino,Malay": 0.16, - "English,Indonesian,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Filipino": 0.15333333333333332, - "Chinese,Vietnamese,Indonesian": 0.16666666666666666, - "Chinese,Vietnamese,Malay": 0.19333333333333333, - "Chinese,Filipino,Indonesian": 0.14666666666666667, - "Chinese,Filipino,Malay": 0.18, - "Chinese,Indonesian,Malay": 0.18666666666666668, - "Vietnamese,Filipino,Indonesian": 0.14, - "Vietnamese,Filipino,Malay": 0.17333333333333334, - "Vietnamese,Indonesian,Malay": 0.22, - "Filipino,Indonesian,Malay": 0.23333333333333334 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.14, - "Spanish,English,Chinese,Filipino": 0.1, - "Spanish,English,Chinese,Indonesian": 0.10666666666666667, - "Spanish,English,Chinese,Malay": 0.1, - "Spanish,English,Vietnamese,Filipino": 0.08, - "Spanish,English,Vietnamese,Indonesian": 0.08, - "Spanish,English,Vietnamese,Malay": 0.1, - "Spanish,English,Filipino,Indonesian": 0.06666666666666667, - "Spanish,English,Filipino,Malay": 0.09333333333333334, - "Spanish,English,Indonesian,Malay": 0.1, - "Spanish,Chinese,Vietnamese,Filipino": 0.08666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.08, - "Spanish,Chinese,Vietnamese,Malay": 0.08666666666666667, - "Spanish,Chinese,Filipino,Indonesian": 0.06666666666666667, - "Spanish,Chinese,Filipino,Malay": 0.10666666666666667, - "Spanish,Chinese,Indonesian,Malay": 0.08666666666666667, - "Spanish,Vietnamese,Filipino,Indonesian": 0.06, - "Spanish,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Spanish,Vietnamese,Indonesian,Malay": 0.1, - "Spanish,Filipino,Indonesian,Malay": 0.1, - "English,Chinese,Vietnamese,Filipino": 0.08666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.1, - "English,Chinese,Vietnamese,Malay": 0.11333333333333333, - "English,Chinese,Filipino,Indonesian": 0.06666666666666667, - "English,Chinese,Filipino,Malay": 0.06666666666666667, - "English,Chinese,Indonesian,Malay": 0.08666666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.08, - "English,Vietnamese,Filipino,Malay": 0.09333333333333334, - "English,Vietnamese,Indonesian,Malay": 0.12666666666666668, - "English,Filipino,Indonesian,Malay": 0.1, - "Chinese,Vietnamese,Filipino,Indonesian": 0.08, - "Chinese,Vietnamese,Filipino,Malay": 0.08666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.11333333333333333, - "Chinese,Filipino,Indonesian,Malay": 0.10666666666666667, - "Vietnamese,Filipino,Indonesian,Malay": 0.10666666666666667 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.05333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.06, - "Spanish,English,Chinese,Vietnamese,Malay": 0.06, - "Spanish,English,Chinese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.05333333333333334, - "Spanish,English,Chinese,Indonesian,Malay": 0.04666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.02666666666666667, - "Spanish,English,Vietnamese,Filipino,Malay": 0.04666666666666667, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.06, - "Spanish,English,Filipino,Indonesian,Malay": 0.04666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.03333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.04666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.06, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.05333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.04666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.04666666666666667, - "English,Chinese,Vietnamese,Filipino,Malay": 0.04, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.06666666666666667, - "English,Chinese,Filipino,Indonesian,Malay": 0.03333333333333333, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.05333333333333334, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.06 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.02, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.02666666666666667, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.04, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.02, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.02, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.02666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.02666666666666667 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.013333333333333334 - } - }, - "AC3_2": 0.32138427992826263, - "AC3_3": 0.21784126979361706, - "AC3_4": 0.13890471292684814, - "AC3_5": 0.08315857602634794, - "AC3_6": 0.04707568292225678, - "AC3_7": 0.025446623084950236 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.26136363636363635, - "language_acc": { - "English": 0.2727272727272727, - "Filipino": 0.2897727272727273, - "Spanish": 0.21022727272727273, - "Malay": 0.2727272727272727, - "Chinese": 0.25, - "Indonesian": 0.2727272727272727, - "Vietnamese": 0.26136363636363635 - }, - "consistency_score_2": 0.2968073593073593, - "consistency_score_3": 0.1038961038961039, - "consistency_score_4": 0.04577922077922078, - "consistency_score_5": 0.025432900432900425, - "consistency_score_6": 0.016233766233766232, - "consistency_score_7": 0.011363636363636364, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.32954545454545453, - "English,Spanish": 0.32954545454545453, - "English,Malay": 0.30113636363636365, - "English,Chinese": 0.3125, - "English,Indonesian": 0.35795454545454547, - "English,Vietnamese": 0.2897727272727273, - "Filipino,Spanish": 0.3125, - "Filipino,Malay": 0.2897727272727273, - "Filipino,Chinese": 0.2784090909090909, - "Filipino,Indonesian": 0.32386363636363635, - "Filipino,Vietnamese": 0.2784090909090909, - "Spanish,Malay": 0.2215909090909091, - "Spanish,Chinese": 0.23863636363636365, - "Spanish,Indonesian": 0.3068181818181818, - "Spanish,Vietnamese": 0.3125, - "Malay,Chinese": 0.3125, - "Malay,Indonesian": 0.2897727272727273, - "Malay,Vietnamese": 0.2897727272727273, - "Chinese,Indonesian": 0.2556818181818182, - "Chinese,Vietnamese": 0.2727272727272727, - "Indonesian,Vietnamese": 0.32954545454545453 - }, - "3_combine": { - "English,Filipino,Spanish": 0.14772727272727273, - "English,Filipino,Malay": 0.10795454545454546, - "English,Filipino,Chinese": 0.11931818181818182, - "English,Filipino,Indonesian": 0.1534090909090909, - "English,Filipino,Vietnamese": 0.09659090909090909, - "English,Spanish,Malay": 0.10227272727272728, - "English,Spanish,Chinese": 0.07954545454545454, - "English,Spanish,Indonesian": 0.14204545454545456, - "English,Spanish,Vietnamese": 0.11363636363636363, - "English,Malay,Chinese": 0.13068181818181818, - "English,Malay,Indonesian": 0.11363636363636363, - "English,Malay,Vietnamese": 0.10795454545454546, - "English,Chinese,Indonesian": 0.125, - "English,Chinese,Vietnamese": 0.07386363636363637, - "English,Indonesian,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Malay": 0.08522727272727272, - "Filipino,Spanish,Chinese": 0.08522727272727272, - "Filipino,Spanish,Indonesian": 0.11363636363636363, - "Filipino,Spanish,Vietnamese": 0.10795454545454546, - "Filipino,Malay,Chinese": 0.09090909090909091, - "Filipino,Malay,Indonesian": 0.10227272727272728, - "Filipino,Malay,Vietnamese": 0.09090909090909091, - "Filipino,Chinese,Indonesian": 0.09659090909090909, - "Filipino,Chinese,Vietnamese": 0.08522727272727272, - "Filipino,Indonesian,Vietnamese": 0.10795454545454546, - "Spanish,Malay,Chinese": 0.07954545454545454, - "Spanish,Malay,Indonesian": 0.07954545454545454, - "Spanish,Malay,Vietnamese": 0.08522727272727272, - "Spanish,Chinese,Indonesian": 0.08522727272727272, - "Spanish,Chinese,Vietnamese": 0.08522727272727272, - "Spanish,Indonesian,Vietnamese": 0.14204545454545456, - "Malay,Chinese,Indonesian": 0.09659090909090909, - "Malay,Chinese,Vietnamese": 0.09090909090909091, - "Malay,Indonesian,Vietnamese": 0.10795454545454546, - "Chinese,Indonesian,Vietnamese": 0.07386363636363637 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.05113636363636364, - "English,Filipino,Spanish,Chinese": 0.03409090909090909, - "English,Filipino,Spanish,Indonesian": 0.07954545454545454, - "English,Filipino,Spanish,Vietnamese": 0.05113636363636364, - "English,Filipino,Malay,Chinese": 0.05113636363636364, - "English,Filipino,Malay,Indonesian": 0.056818181818181816, - "English,Filipino,Malay,Vietnamese": 0.045454545454545456, - "English,Filipino,Chinese,Indonesian": 0.07386363636363637, - "English,Filipino,Chinese,Vietnamese": 0.03409090909090909, - "English,Filipino,Indonesian,Vietnamese": 0.045454545454545456, - "English,Spanish,Malay,Chinese": 0.05113636363636364, - "English,Spanish,Malay,Indonesian": 0.045454545454545456, - "English,Spanish,Malay,Vietnamese": 0.03977272727272727, - "English,Spanish,Chinese,Indonesian": 0.03977272727272727, - "English,Spanish,Chinese,Vietnamese": 0.028409090909090908, - "English,Spanish,Indonesian,Vietnamese": 0.06818181818181818, - "English,Malay,Chinese,Indonesian": 0.056818181818181816, - "English,Malay,Chinese,Vietnamese": 0.056818181818181816, - "English,Malay,Indonesian,Vietnamese": 0.06818181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Malay,Chinese": 0.03409090909090909, - "Filipino,Spanish,Malay,Indonesian": 0.03977272727272727, - "Filipino,Spanish,Malay,Vietnamese": 0.045454545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.045454545454545456, - "Filipino,Spanish,Chinese,Vietnamese": 0.03409090909090909, - "Filipino,Spanish,Indonesian,Vietnamese": 0.05113636363636364, - "Filipino,Malay,Chinese,Indonesian": 0.045454545454545456, - "Filipino,Malay,Chinese,Vietnamese": 0.03977272727272727, - "Filipino,Malay,Indonesian,Vietnamese": 0.03977272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.03409090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.03409090909090909, - "Spanish,Malay,Chinese,Vietnamese": 0.028409090909090908, - "Spanish,Malay,Indonesian,Vietnamese": 0.045454545454545456, - "Spanish,Chinese,Indonesian,Vietnamese": 0.03977272727272727, - "Malay,Chinese,Indonesian,Vietnamese": 0.03409090909090909 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.028409090909090908, - "English,Filipino,Spanish,Malay,Indonesian": 0.03409090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.022727272727272728, - "English,Filipino,Spanish,Chinese,Indonesian": 0.028409090909090908, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.03409090909090909, - "English,Filipino,Malay,Chinese,Indonesian": 0.03977272727272727, - "English,Filipino,Malay,Chinese,Vietnamese": 0.028409090909090908, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.028409090909090908, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.028409090909090908, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.028409090909090908, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.022727272727272728, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.022727272727272728, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.017045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.017045454545454544 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.011363636363636364 - } - }, - "AC3_2": 0.2779601638774038, - "AC3_3": 0.1486868686461615, - "AC3_4": 0.07791178163905774, - "AC3_5": 0.04635506001814226, - "AC3_6": 0.030568846347308103, - "AC3_7": 0.02178030302231692 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.1650485436893204 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.18095238095238095 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.22429906542056074 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.29, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.3, - "history": 0.2, - "literature": 0.3, - "politics": 0.6, - "culture": 0.2, - "film": 0.4, - "law": 0.2, - "geography": 0.3 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.07214564890700625 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.07376238182687374 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.05899427376546308 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.06828582619085334 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.07848503285774223 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.2707117852975496 - } - }, - "mmlu_full": { - "prompt_1": -1 - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.2436849925705795 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.23349937733499376, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.16666666666666666, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.19047619047619047, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.13793103448275862, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.13793103448275862, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.25, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.11538461538461539, - "middle_school_physics": 0.3333333333333333, - "middle_school_chemistry": 0.24, - "veterinary_medicine": 0.21428571428571427, - "college_economics": 0.36666666666666664, - "business_administration": 0.21052631578947367, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.1724137931034483, - "education_science": 0.17647058823529413, - "teacher_qualification": 0.22448979591836735, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.16666666666666666, - "middle_school_politics": 0.15384615384615385, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.17857142857142858, - "ideological_and_moral_cultivation": 0.2916666666666667, - "logic": 0.2222222222222222, - "law": 0.13793103448275862, - "chinese_language_and_literature": 0.14285714285714285, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.25, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.4, - "middle_school_history": 0.14814814814814814, - "civil_servant": 0.3269230769230769, - "sports_science": 0.25, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.25, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.23529411764705882, - "accountant": 0.3148148148148148, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.08333333333333333, - "tax_accountant": 0.18518518518518517, - "physician": 0.24074074074074073 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.24014336917562723 - } - }, - "cmmlu_full": { - "prompt_1": -1 - }, - "zbench": { - "prompt_1": { - "accuracy": 0.18181818181818182 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.17954545454545454 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3335593220338983 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.27786088257292446 - } - }, - "dream": { - "prompt_1": -1 - }, - "samsum": { - "prompt_1": { - "rouge1": 0.1600682306240041, - "rouge2": 0.052701017888527886, - "rougeL": 0.12407158233553614, - "avg_rouge": 0.11228027694935605 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.1527826432413544, - "rouge2": 0.04077807353494755, - "rougeL": 0.1199013337511358, - "avg_rouge": 0.10448735017581258 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.4988532110091743 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.34707574304889743 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.548 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.32 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.5025 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.5492957746478874 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.45126353790613716 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.43137254901960786 - } - } - } - }, - "mistral_7b_instruct_v0_2": { - "model_size": "7B", - "model_link": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2", - "zero_shot": { - "cross_xquad": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "cross_mmlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "cross_logiqa": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.6116504854368932 - }, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "cn_eval": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "us_eval": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "ph_eval": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "sing2eng": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_vie2eng": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_zho2eng": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_zsm2eng": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "mmlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "mmlu_full": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "c_eval": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "c_eval_full": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "cmmlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "cmmlu_full": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "zbench": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "ind_emotion": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "ocnli": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "c3": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "dream": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "samsum": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "dialogsum": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "sst2": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "cola": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "qqp": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "mnli": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "qnli": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "wnli": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "rte": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "mrpc": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": -1 - }, - "cross_mmlu": { - "prompt_1": -1 - }, - "cross_logiqa": { - "prompt_1": -1 - }, - "sg_eval": { - "prompt_1": -1 - }, - "cn_eval": { - "prompt_1": -1 - }, - "us_eval": { - "prompt_1": -1 - }, - "ph_eval": { - "prompt_1": -1 - }, - "sing2eng": { - "prompt_1": -1 - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": -1 - }, - "flores_vie2eng": { - "prompt_1": -1 - }, - "flores_zho2eng": { - "prompt_1": -1 - }, - "flores_zsm2eng": { - "prompt_1": -1 - }, - "mmlu": { - "prompt_1": -1 - }, - "mmlu_full": { - "prompt_1": -1 - }, - "c_eval": { - "prompt_1": -1 - }, - "c_eval_full": { - "prompt_1": -1 - }, - "cmmlu": { - "prompt_1": -1 - }, - "cmmlu_full": { - "prompt_1": -1 - }, - "zbench": { - "prompt_1": -1 - }, - "ind_emotion": { - "prompt_1": -1 - }, - "ocnli": { - "prompt_1": -1 - }, - "c3": { - "prompt_1": -1 - }, - "dream": { - "prompt_1": -1 - }, - "samsum": { - "prompt_1": -1 - }, - "dialogsum": { - "prompt_1": -1 - }, - "sst2": { - "prompt_1": -1 - }, - "cola": { - "prompt_1": -1 - }, - "qqp": { - "prompt_1": -1 - }, - "mnli": { - "prompt_1": -1 - }, - "qnli": { - "prompt_1": -1 - }, - "wnli": { - "prompt_1": -1 - }, - "rte": { - "prompt_1": -1 - }, - "mrpc": { - "prompt_1": -1 - } - } - }, - "sea_mistral_7b": { - "model_size": "7B", - "model_link": "https://seaeval.github.io/", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.7852941176470587, - "language_acc": { - "Spanish": 0.8243697478991596, - "English": 0.8403361344537815, - "Chinese": 0.7781512605042017, - "Vietnamese": 0.6983193277310924 - }, - "consistency_score_2": 0.7357142857142858, - "consistency_score_3": 0.6243697478991597, - "consistency_score_4": 0.5504201680672269, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8058823529411765, - "Spanish,Chinese": 0.7773109243697479, - "Spanish,Vietnamese": 0.7260504201680672, - "English,Chinese": 0.7554621848739496, - "English,Vietnamese": 0.6705882352941176, - "Chinese,Vietnamese": 0.6789915966386555 - }, - "3_combine": { - "Spanish,English,Chinese": 0.6857142857142857, - "Spanish,English,Vietnamese": 0.6235294117647059, - "Spanish,Chinese,Vietnamese": 0.6100840336134454, - "English,Chinese,Vietnamese": 0.5781512605042017 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.5504201680672269 - } - }, - "AC3_2": 0.7596961325467382, - "AC3_3": 0.6956465327813696, - "AC3_4": 0.6472068537531485 - }, - "prompt_2": { - "overall_acc": 0.8048319327731093, - "language_acc": { - "Spanish": 0.8361344537815126, - "English": 0.8823529411764706, - "Chinese": 0.7798319327731092, - "Vietnamese": 0.7210084033613445 - }, - "consistency_score_2": 0.7694677871148459, - "consistency_score_3": 0.6701680672268907, - "consistency_score_4": 0.6092436974789915, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8638655462184874, - "Spanish,Chinese": 0.7798319327731092, - "Spanish,Vietnamese": 0.7436974789915967, - "English,Chinese": 0.7932773109243697, - "English,Vietnamese": 0.7235294117647059, - "Chinese,Vietnamese": 0.7126050420168067 - }, - "3_combine": { - "Spanish,English,Chinese": 0.7294117647058823, - "Spanish,English,Vietnamese": 0.6789915966386555, - "Spanish,Chinese,Vietnamese": 0.6378151260504201, - "English,Chinese,Vietnamese": 0.634453781512605 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.6092436974789915 - } - }, - "AC3_2": 0.7867526601796346, - "AC3_3": 0.7313527603965692, - "AC3_4": 0.6935113964869889 - }, - "prompt_3": { - "overall_acc": 0.7747899159663865, - "language_acc": { - "Spanish": 0.8201680672268907, - "English": 0.8663865546218488, - "Chinese": 0.7260504201680672, - "Vietnamese": 0.6865546218487395 - }, - "consistency_score_2": 0.7303921568627452, - "consistency_score_3": 0.6172268907563025, - "consistency_score_4": 0.5428571428571428, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.853781512605042, - "Spanish,Chinese": 0.7218487394957983, - "Spanish,Vietnamese": 0.726890756302521, - "English,Chinese": 0.7336134453781512, - "English,Vietnamese": 0.7, - "Chinese,Vietnamese": 0.646218487394958 - }, - "3_combine": { - "Spanish,English,Chinese": 0.66890756302521, - "Spanish,English,Vietnamese": 0.6588235294117647, - "Spanish,Chinese,Vietnamese": 0.5747899159663865, - "English,Chinese,Vietnamese": 0.5663865546218487 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.5428571428571428 - } - }, - "AC3_2": 0.7519362448118107, - "AC3_3": 0.687091087517572, - "AC3_4": 0.6384110786687504 - }, - "prompt_4": { - "overall_acc": 0.8027310924369748, - "language_acc": { - "Spanish": 0.8302521008403362, - "English": 0.8756302521008403, - "Chinese": 0.7873949579831933, - "Vietnamese": 0.7176470588235294 - }, - "consistency_score_2": 0.761484593837535, - "consistency_score_3": 0.6596638655462185, - "consistency_score_4": 0.592436974789916, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8378151260504202, - "Spanish,Chinese": 0.7966386554621848, - "Spanish,Vietnamese": 0.7352941176470589, - "English,Chinese": 0.7789915966386555, - "English,Vietnamese": 0.711764705882353, - "Chinese,Vietnamese": 0.7084033613445379 - }, - "3_combine": { - "Spanish,English,Chinese": 0.719327731092437, - "Spanish,English,Vietnamese": 0.6588235294117647, - "Spanish,Chinese,Vietnamese": 0.6394957983193277, - "English,Chinese,Vietnamese": 0.6210084033613446 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.592436974789916 - } - }, - "AC3_2": 0.7815640326455322, - "AC3_3": 0.7241992903547596, - "AC3_4": 0.6817351846137679 - }, - "prompt_5": { - "overall_acc": 0.7939075630252102, - "language_acc": { - "Spanish": 0.8151260504201681, - "English": 0.8815126050420168, - "Chinese": 0.7756302521008404, - "Vietnamese": 0.7033613445378152 - }, - "consistency_score_2": 0.7609243697478991, - "consistency_score_3": 0.6609243697478991, - "consistency_score_4": 0.5957983193277311, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8554621848739495, - "Spanish,Chinese": 0.7815126050420168, - "Spanish,Vietnamese": 0.7327731092436974, - "English,Chinese": 0.7823529411764706, - "English,Vietnamese": 0.719327731092437, - "Chinese,Vietnamese": 0.6941176470588235 - }, - "3_combine": { - "Spanish,English,Chinese": 0.7226890756302521, - "Spanish,English,Vietnamese": 0.6714285714285714, - "Spanish,Chinese,Vietnamese": 0.6260504201680672, - "English,Chinese,Vietnamese": 0.6235294117647059 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.5957983193277311 - } - }, - "AC3_2": 0.7770661243324872, - "AC3_3": 0.7213381063120959, - "AC3_4": 0.6807322293507125 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.45428571428571424, - "language_acc": { - "Filipino": 0.38, - "Vietnamese": 0.3333333333333333, - "Chinese": 0.44, - "Spanish": 0.5533333333333333, - "Malay": 0.4066666666666667, - "Indonesian": 0.48, - "English": 0.5866666666666667 - }, - "consistency_score_2": 0.47555555555555556, - "consistency_score_3": 0.2900952380952382, - "consistency_score_4": 0.1996190476190476, - "consistency_score_5": 0.14857142857142855, - "consistency_score_6": 0.11809523809523811, - "consistency_score_7": 0.1, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.4533333333333333, - "Filipino,Chinese": 0.44, - "Filipino,Spanish": 0.48, - "Filipino,Malay": 0.4666666666666667, - "Filipino,Indonesian": 0.52, - "Filipino,English": 0.4533333333333333, - "Vietnamese,Chinese": 0.36666666666666664, - "Vietnamese,Spanish": 0.44, - "Vietnamese,Malay": 0.43333333333333335, - "Vietnamese,Indonesian": 0.44666666666666666, - "Vietnamese,English": 0.3333333333333333, - "Chinese,Spanish": 0.5266666666666666, - "Chinese,Malay": 0.47333333333333333, - "Chinese,Indonesian": 0.5533333333333333, - "Chinese,English": 0.44, - "Spanish,Malay": 0.48, - "Spanish,Indonesian": 0.5533333333333333, - "Spanish,English": 0.5666666666666667, - "Malay,Indonesian": 0.6333333333333333, - "Malay,English": 0.42, - "Indonesian,English": 0.5066666666666667 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.21333333333333335, - "Filipino,Vietnamese,Spanish": 0.25333333333333335, - "Filipino,Vietnamese,Malay": 0.26666666666666666, - "Filipino,Vietnamese,Indonesian": 0.2866666666666667, - "Filipino,Vietnamese,English": 0.20666666666666667, - "Filipino,Chinese,Spanish": 0.2866666666666667, - "Filipino,Chinese,Malay": 0.29333333333333333, - "Filipino,Chinese,Indonesian": 0.30666666666666664, - "Filipino,Chinese,English": 0.24, - "Filipino,Spanish,Malay": 0.30666666666666664, - "Filipino,Spanish,Indonesian": 0.34, - "Filipino,Spanish,English": 0.31333333333333335, - "Filipino,Malay,Indonesian": 0.35333333333333333, - "Filipino,Malay,English": 0.24666666666666667, - "Filipino,Indonesian,English": 0.30666666666666664, - "Vietnamese,Chinese,Spanish": 0.26666666666666666, - "Vietnamese,Chinese,Malay": 0.25333333333333335, - "Vietnamese,Chinese,Indonesian": 0.28, - "Vietnamese,Chinese,English": 0.18666666666666668, - "Vietnamese,Spanish,Malay": 0.2866666666666667, - "Vietnamese,Spanish,Indonesian": 0.3, - "Vietnamese,Spanish,English": 0.24666666666666667, - "Vietnamese,Malay,Indonesian": 0.34, - "Vietnamese,Malay,English": 0.20666666666666667, - "Vietnamese,Indonesian,English": 0.25333333333333335, - "Chinese,Spanish,Malay": 0.3, - "Chinese,Spanish,Indonesian": 0.37333333333333335, - "Chinese,Spanish,English": 0.32, - "Chinese,Malay,Indonesian": 0.3933333333333333, - "Chinese,Malay,English": 0.23333333333333334, - "Chinese,Indonesian,English": 0.3, - "Spanish,Malay,Indonesian": 0.38, - "Spanish,Malay,English": 0.3, - "Spanish,Indonesian,English": 0.36666666666666664, - "Malay,Indonesian,English": 0.3466666666666667 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,Malay": 0.18, - "Filipino,Vietnamese,Chinese,Indonesian": 0.17333333333333334, - "Filipino,Vietnamese,Chinese,English": 0.13333333333333333, - "Filipino,Vietnamese,Spanish,Malay": 0.2, - "Filipino,Vietnamese,Spanish,Indonesian": 0.2, - "Filipino,Vietnamese,Spanish,English": 0.15333333333333332, - "Filipino,Vietnamese,Malay,Indonesian": 0.21333333333333335, - "Filipino,Vietnamese,Malay,English": 0.15333333333333332, - "Filipino,Vietnamese,Indonesian,English": 0.18666666666666668, - "Filipino,Chinese,Spanish,Malay": 0.2, - "Filipino,Chinese,Spanish,Indonesian": 0.22, - "Filipino,Chinese,Spanish,English": 0.18, - "Filipino,Chinese,Malay,Indonesian": 0.26, - "Filipino,Chinese,Malay,English": 0.16666666666666666, - "Filipino,Chinese,Indonesian,English": 0.19333333333333333, - "Filipino,Spanish,Malay,Indonesian": 0.24, - "Filipino,Spanish,Malay,English": 0.2, - "Filipino,Spanish,Indonesian,English": 0.24, - "Filipino,Malay,Indonesian,English": 0.21333333333333335, - "Vietnamese,Chinese,Spanish,Malay": 0.20666666666666667, - "Vietnamese,Chinese,Spanish,Indonesian": 0.22, - "Vietnamese,Chinese,Spanish,English": 0.15333333333333332, - "Vietnamese,Chinese,Malay,Indonesian": 0.24, - "Vietnamese,Chinese,Malay,English": 0.14666666666666667, - "Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, - "Vietnamese,Spanish,Malay,Indonesian": 0.24, - "Vietnamese,Spanish,Malay,English": 0.17333333333333334, - "Vietnamese,Spanish,Indonesian,English": 0.2, - "Vietnamese,Malay,Indonesian,English": 0.2, - "Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, - "Chinese,Spanish,Malay,English": 0.18666666666666668, - "Chinese,Spanish,Indonesian,English": 0.24, - "Chinese,Malay,Indonesian,English": 0.21333333333333335, - "Spanish,Malay,Indonesian,English": 0.25333333333333335 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.14666666666666667, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.14, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.10666666666666667, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,Malay,English": 0.12, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.12666666666666668, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.16, - "Filipino,Vietnamese,Spanish,Malay,English": 0.12666666666666668, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14666666666666667, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.14666666666666667, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.18, - "Filipino,Chinese,Spanish,Malay,English": 0.12666666666666668, - "Filipino,Chinese,Spanish,Indonesian,English": 0.15333333333333332, - "Filipino,Chinese,Malay,Indonesian,English": 0.16, - "Filipino,Spanish,Malay,Indonesian,English": 0.16666666666666666, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, - "Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.14666666666666667, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.16666666666666666, - "Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.1, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.10666666666666667, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.1 - } - }, - "AC3_2": 0.4646773642381791, - "AC3_3": 0.35408246814953753, - "AC3_4": 0.2773617410420164, - "AC3_5": 0.2239133378098451, - "AC3_6": 0.18745899687716133, - "AC3_7": 0.16391752574362314 - }, - "prompt_2": { - "overall_acc": 0.48000000000000004, - "language_acc": { - "Filipino": 0.36, - "Vietnamese": 0.41333333333333333, - "Chinese": 0.4666666666666667, - "Spanish": 0.56, - "Malay": 0.42, - "Indonesian": 0.4866666666666667, - "English": 0.6533333333333333 - }, - "consistency_score_2": 0.534920634920635, - "consistency_score_3": 0.35790476190476195, - "consistency_score_4": 0.26628571428571424, - "consistency_score_5": 0.2111111111111111, - "consistency_score_6": 0.1742857142857143, - "consistency_score_7": 0.14666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.4066666666666667, - "Filipino,Chinese": 0.48, - "Filipino,Spanish": 0.5, - "Filipino,Malay": 0.48, - "Filipino,Indonesian": 0.5, - "Filipino,English": 0.46, - "Vietnamese,Chinese": 0.44, - "Vietnamese,Spanish": 0.5533333333333333, - "Vietnamese,Malay": 0.54, - "Vietnamese,Indonesian": 0.58, - "Vietnamese,English": 0.43333333333333335, - "Chinese,Spanish": 0.56, - "Chinese,Malay": 0.5066666666666667, - "Chinese,Indonesian": 0.58, - "Chinese,English": 0.52, - "Spanish,Malay": 0.5666666666666667, - "Spanish,Indonesian": 0.64, - "Spanish,English": 0.6733333333333333, - "Malay,Indonesian": 0.66, - "Malay,English": 0.5066666666666667, - "Indonesian,English": 0.6466666666666666 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.24, - "Filipino,Vietnamese,Spanish": 0.2866666666666667, - "Filipino,Vietnamese,Malay": 0.29333333333333333, - "Filipino,Vietnamese,Indonesian": 0.30666666666666664, - "Filipino,Vietnamese,English": 0.22666666666666666, - "Filipino,Chinese,Spanish": 0.3333333333333333, - "Filipino,Chinese,Malay": 0.2866666666666667, - "Filipino,Chinese,Indonesian": 0.32666666666666666, - "Filipino,Chinese,English": 0.3, - "Filipino,Spanish,Malay": 0.3333333333333333, - "Filipino,Spanish,Indonesian": 0.36666666666666664, - "Filipino,Spanish,English": 0.36666666666666664, - "Filipino,Malay,Indonesian": 0.36666666666666664, - "Filipino,Malay,English": 0.29333333333333333, - "Filipino,Indonesian,English": 0.36, - "Vietnamese,Chinese,Spanish": 0.35333333333333333, - "Vietnamese,Chinese,Malay": 0.32, - "Vietnamese,Chinese,Indonesian": 0.36, - "Vietnamese,Chinese,English": 0.29333333333333333, - "Vietnamese,Spanish,Malay": 0.38, - "Vietnamese,Spanish,Indonesian": 0.42, - "Vietnamese,Spanish,English": 0.38, - "Vietnamese,Malay,Indonesian": 0.43333333333333335, - "Vietnamese,Malay,English": 0.32666666666666666, - "Vietnamese,Indonesian,English": 0.38, - "Chinese,Spanish,Malay": 0.36666666666666664, - "Chinese,Spanish,Indonesian": 0.4266666666666667, - "Chinese,Spanish,English": 0.41333333333333333, - "Chinese,Malay,Indonesian": 0.41333333333333333, - "Chinese,Malay,English": 0.32666666666666666, - "Chinese,Indonesian,English": 0.42, - "Spanish,Malay,Indonesian": 0.46, - "Spanish,Malay,English": 0.41333333333333333, - "Spanish,Indonesian,English": 0.5133333333333333, - "Malay,Indonesian,English": 0.44 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.22, - "Filipino,Vietnamese,Chinese,Malay": 0.2, - "Filipino,Vietnamese,Chinese,Indonesian": 0.21333333333333335, - "Filipino,Vietnamese,Chinese,English": 0.17333333333333334, - "Filipino,Vietnamese,Spanish,Malay": 0.23333333333333334, - "Filipino,Vietnamese,Spanish,Indonesian": 0.26, - "Filipino,Vietnamese,Spanish,English": 0.21333333333333335, - "Filipino,Vietnamese,Malay,Indonesian": 0.25333333333333335, - "Filipino,Vietnamese,Malay,English": 0.18666666666666668, - "Filipino,Vietnamese,Indonesian,English": 0.22, - "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, - "Filipino,Chinese,Spanish,Indonesian": 0.26666666666666666, - "Filipino,Chinese,Spanish,English": 0.25333333333333335, - "Filipino,Chinese,Malay,Indonesian": 0.25333333333333335, - "Filipino,Chinese,Malay,English": 0.21333333333333335, - "Filipino,Chinese,Indonesian,English": 0.26, - "Filipino,Spanish,Malay,Indonesian": 0.28, - "Filipino,Spanish,Malay,English": 0.26, - "Filipino,Spanish,Indonesian,English": 0.30666666666666664, - "Filipino,Malay,Indonesian,English": 0.26666666666666666, - "Vietnamese,Chinese,Spanish,Malay": 0.2733333333333333, - "Vietnamese,Chinese,Spanish,Indonesian": 0.3, - "Vietnamese,Chinese,Spanish,English": 0.2733333333333333, - "Vietnamese,Chinese,Malay,Indonesian": 0.30666666666666664, - "Vietnamese,Chinese,Malay,English": 0.24, - "Vietnamese,Chinese,Indonesian,English": 0.26666666666666666, - "Vietnamese,Spanish,Malay,Indonesian": 0.3333333333333333, - "Vietnamese,Spanish,Malay,English": 0.2866666666666667, - "Vietnamese,Spanish,Indonesian,English": 0.32666666666666666, - "Vietnamese,Malay,Indonesian,English": 0.30666666666666664, - "Chinese,Spanish,Malay,Indonesian": 0.32666666666666666, - "Chinese,Spanish,Malay,English": 0.28, - "Chinese,Spanish,Indonesian,English": 0.36, - "Chinese,Malay,Indonesian,English": 0.31333333333333335, - "Spanish,Malay,Indonesian,English": 0.36 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.18666666666666668, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.2, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, - "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.22, - "Filipino,Vietnamese,Spanish,Malay,English": 0.18, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.20666666666666667, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.18666666666666668, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, - "Filipino,Chinese,Spanish,Malay,English": 0.18666666666666668, - "Filipino,Chinese,Spanish,Indonesian,English": 0.22666666666666666, - "Filipino,Chinese,Malay,Indonesian,English": 0.20666666666666667, - "Filipino,Spanish,Malay,Indonesian,English": 0.23333333333333334, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26, - "Vietnamese,Chinese,Spanish,Malay,English": 0.22, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.24666666666666667, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.24, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.26666666666666666, - "Chinese,Spanish,Malay,Indonesian,English": 0.2733333333333333 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14666666666666667, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.16, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.15333333333333332, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.18, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.18, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 - } - }, - "AC3_2": 0.5059743509041659, - "AC3_3": 0.41005683104903856, - "AC3_4": 0.34254211327722445, - "AC3_5": 0.293247588382006, - "AC3_6": 0.25572052397838335, - "AC3_7": 0.22468085102797647 - }, - "prompt_3": { - "overall_acc": 0.4561904761904762, - "language_acc": { - "Filipino": 0.38666666666666666, - "Vietnamese": 0.37333333333333335, - "Chinese": 0.46, - "Spanish": 0.5333333333333333, - "Malay": 0.3466666666666667, - "Indonesian": 0.4866666666666667, - "English": 0.6066666666666667 - }, - "consistency_score_2": 0.5101587301587303, - "consistency_score_3": 0.3306666666666666, - "consistency_score_4": 0.2436190476190476, - "consistency_score_5": 0.19301587301587303, - "consistency_score_6": 0.15904761904761905, - "consistency_score_7": 0.13333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.4066666666666667, - "Filipino,Chinese": 0.4866666666666667, - "Filipino,Spanish": 0.4866666666666667, - "Filipino,Malay": 0.4666666666666667, - "Filipino,Indonesian": 0.5333333333333333, - "Filipino,English": 0.4866666666666667, - "Vietnamese,Chinese": 0.46, - "Vietnamese,Spanish": 0.5, - "Vietnamese,Malay": 0.4666666666666667, - "Vietnamese,Indonesian": 0.5133333333333333, - "Vietnamese,English": 0.41333333333333333, - "Chinese,Spanish": 0.5133333333333333, - "Chinese,Malay": 0.48, - "Chinese,Indonesian": 0.5866666666666667, - "Chinese,English": 0.49333333333333335, - "Spanish,Malay": 0.5066666666666667, - "Spanish,Indonesian": 0.5866666666666667, - "Spanish,English": 0.6133333333333333, - "Malay,Indonesian": 0.64, - "Malay,English": 0.48, - "Indonesian,English": 0.5933333333333334 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.24666666666666667, - "Filipino,Vietnamese,Spanish": 0.26666666666666666, - "Filipino,Vietnamese,Malay": 0.25333333333333335, - "Filipino,Vietnamese,Indonesian": 0.29333333333333333, - "Filipino,Vietnamese,English": 0.23333333333333334, - "Filipino,Chinese,Spanish": 0.3333333333333333, - "Filipino,Chinese,Malay": 0.3, - "Filipino,Chinese,Indonesian": 0.36666666666666664, - "Filipino,Chinese,English": 0.3333333333333333, - "Filipino,Spanish,Malay": 0.32, - "Filipino,Spanish,Indonesian": 0.36666666666666664, - "Filipino,Spanish,English": 0.36, - "Filipino,Malay,Indonesian": 0.36, - "Filipino,Malay,English": 0.29333333333333333, - "Filipino,Indonesian,English": 0.37333333333333335, - "Vietnamese,Chinese,Spanish": 0.3333333333333333, - "Vietnamese,Chinese,Malay": 0.2733333333333333, - "Vietnamese,Chinese,Indonesian": 0.3333333333333333, - "Vietnamese,Chinese,English": 0.26, - "Vietnamese,Spanish,Malay": 0.31333333333333335, - "Vietnamese,Spanish,Indonesian": 0.3466666666666667, - "Vietnamese,Spanish,English": 0.3333333333333333, - "Vietnamese,Malay,Indonesian": 0.36, - "Vietnamese,Malay,English": 0.26, - "Vietnamese,Indonesian,English": 0.31333333333333335, - "Chinese,Spanish,Malay": 0.3333333333333333, - "Chinese,Spanish,Indonesian": 0.38666666666666666, - "Chinese,Spanish,English": 0.36, - "Chinese,Malay,Indonesian": 0.3933333333333333, - "Chinese,Malay,English": 0.2866666666666667, - "Chinese,Indonesian,English": 0.38, - "Spanish,Malay,Indonesian": 0.42, - "Spanish,Malay,English": 0.36, - "Spanish,Indonesian,English": 0.4266666666666667, - "Malay,Indonesian,English": 0.4 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.22, - "Filipino,Vietnamese,Chinese,Malay": 0.17333333333333334, - "Filipino,Vietnamese,Chinese,Indonesian": 0.20666666666666667, - "Filipino,Vietnamese,Chinese,English": 0.18666666666666668, - "Filipino,Vietnamese,Spanish,Malay": 0.20666666666666667, - "Filipino,Vietnamese,Spanish,Indonesian": 0.23333333333333334, - "Filipino,Vietnamese,Spanish,English": 0.20666666666666667, - "Filipino,Vietnamese,Malay,Indonesian": 0.20666666666666667, - "Filipino,Vietnamese,Malay,English": 0.18, - "Filipino,Vietnamese,Indonesian,English": 0.21333333333333335, - "Filipino,Chinese,Spanish,Malay": 0.24666666666666667, - "Filipino,Chinese,Spanish,Indonesian": 0.2733333333333333, - "Filipino,Chinese,Spanish,English": 0.26666666666666666, - "Filipino,Chinese,Malay,Indonesian": 0.26666666666666666, - "Filipino,Chinese,Malay,English": 0.22666666666666666, - "Filipino,Chinese,Indonesian,English": 0.2866666666666667, - "Filipino,Spanish,Malay,Indonesian": 0.28, - "Filipino,Spanish,Malay,English": 0.25333333333333335, - "Filipino,Spanish,Indonesian,English": 0.2866666666666667, - "Filipino,Malay,Indonesian,English": 0.26666666666666666, - "Vietnamese,Chinese,Spanish,Malay": 0.22666666666666666, - "Vietnamese,Chinese,Spanish,Indonesian": 0.2733333333333333, - "Vietnamese,Chinese,Spanish,English": 0.24666666666666667, - "Vietnamese,Chinese,Malay,Indonesian": 0.25333333333333335, - "Vietnamese,Chinese,Malay,English": 0.18666666666666668, - "Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, - "Vietnamese,Spanish,Malay,Indonesian": 0.2733333333333333, - "Vietnamese,Spanish,Malay,English": 0.23333333333333334, - "Vietnamese,Spanish,Indonesian,English": 0.26, - "Vietnamese,Malay,Indonesian,English": 0.23333333333333334, - "Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, - "Chinese,Spanish,Malay,English": 0.24666666666666667, - "Chinese,Spanish,Indonesian,English": 0.30666666666666664, - "Chinese,Malay,Indonesian,English": 0.26666666666666666, - "Spanish,Malay,Indonesian,English": 0.31333333333333335 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.19333333333333333, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.18, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16, - "Filipino,Vietnamese,Chinese,Malay,English": 0.14666666666666667, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.17333333333333334, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.18666666666666668, - "Filipino,Vietnamese,Spanish,Malay,English": 0.16666666666666666, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.16666666666666666, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.22, - "Filipino,Chinese,Spanish,Malay,English": 0.2, - "Filipino,Chinese,Spanish,Indonesian,English": 0.23333333333333334, - "Filipino,Chinese,Malay,Indonesian,English": 0.21333333333333335, - "Filipino,Spanish,Malay,Indonesian,English": 0.22666666666666666, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, - "Vietnamese,Chinese,Spanish,Malay,English": 0.18, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.22, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.18, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, - "Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.18666666666666668, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 - } - }, - "AC3_2": 0.48166760523142743, - "AC3_3": 0.3834164447184797, - "AC3_4": 0.3176198252391093, - "AC3_5": 0.2712604493702553, - "AC3_6": 0.23586318734187836, - "AC3_7": 0.20635433491383864 - }, - "prompt_4": { - "overall_acc": 0.4780952380952381, - "language_acc": { - "Filipino": 0.4066666666666667, - "Vietnamese": 0.41333333333333333, - "Chinese": 0.46, - "Spanish": 0.56, - "Malay": 0.43333333333333335, - "Indonesian": 0.5133333333333333, - "English": 0.56 - }, - "consistency_score_2": 0.505079365079365, - "consistency_score_3": 0.324952380952381, - "consistency_score_4": 0.23009523809523807, - "consistency_score_5": 0.17174603174603176, - "consistency_score_6": 0.13333333333333333, - "consistency_score_7": 0.10666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.4, - "Filipino,Chinese": 0.4533333333333333, - "Filipino,Spanish": 0.44666666666666666, - "Filipino,Malay": 0.4666666666666667, - "Filipino,Indonesian": 0.4533333333333333, - "Filipino,English": 0.44666666666666666, - "Vietnamese,Chinese": 0.4666666666666667, - "Vietnamese,Spanish": 0.47333333333333333, - "Vietnamese,Malay": 0.5, - "Vietnamese,Indonesian": 0.52, - "Vietnamese,English": 0.43333333333333335, - "Chinese,Spanish": 0.5733333333333334, - "Chinese,Malay": 0.5066666666666667, - "Chinese,Indonesian": 0.5266666666666666, - "Chinese,English": 0.5266666666666666, - "Spanish,Malay": 0.5333333333333333, - "Spanish,Indonesian": 0.5866666666666667, - "Spanish,English": 0.62, - "Malay,Indonesian": 0.6133333333333333, - "Malay,English": 0.4866666666666667, - "Indonesian,English": 0.5733333333333334 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.24666666666666667, - "Filipino,Vietnamese,Spanish": 0.24, - "Filipino,Vietnamese,Malay": 0.25333333333333335, - "Filipino,Vietnamese,Indonesian": 0.26, - "Filipino,Vietnamese,English": 0.23333333333333334, - "Filipino,Chinese,Spanish": 0.32, - "Filipino,Chinese,Malay": 0.29333333333333333, - "Filipino,Chinese,Indonesian": 0.30666666666666664, - "Filipino,Chinese,English": 0.28, - "Filipino,Spanish,Malay": 0.31333333333333335, - "Filipino,Spanish,Indonesian": 0.31333333333333335, - "Filipino,Spanish,English": 0.30666666666666664, - "Filipino,Malay,Indonesian": 0.34, - "Filipino,Malay,English": 0.28, - "Filipino,Indonesian,English": 0.30666666666666664, - "Vietnamese,Chinese,Spanish": 0.32666666666666666, - "Vietnamese,Chinese,Malay": 0.3, - "Vietnamese,Chinese,Indonesian": 0.3333333333333333, - "Vietnamese,Chinese,English": 0.2866666666666667, - "Vietnamese,Spanish,Malay": 0.32, - "Vietnamese,Spanish,Indonesian": 0.36, - "Vietnamese,Spanish,English": 0.31333333333333335, - "Vietnamese,Malay,Indonesian": 0.37333333333333335, - "Vietnamese,Malay,English": 0.2866666666666667, - "Vietnamese,Indonesian,English": 0.32666666666666666, - "Chinese,Spanish,Malay": 0.35333333333333333, - "Chinese,Spanish,Indonesian": 0.4, - "Chinese,Spanish,English": 0.4, - "Chinese,Malay,Indonesian": 0.37333333333333335, - "Chinese,Malay,English": 0.32666666666666666, - "Chinese,Indonesian,English": 0.38, - "Spanish,Malay,Indonesian": 0.4266666666666667, - "Spanish,Malay,English": 0.36666666666666664, - "Spanish,Indonesian,English": 0.43333333333333335, - "Malay,Indonesian,English": 0.3933333333333333 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.19333333333333333, - "Filipino,Vietnamese,Chinese,Malay": 0.20666666666666667, - "Filipino,Vietnamese,Chinese,Indonesian": 0.18666666666666668, - "Filipino,Vietnamese,Chinese,English": 0.16666666666666666, - "Filipino,Vietnamese,Spanish,Malay": 0.2, - "Filipino,Vietnamese,Spanish,Indonesian": 0.2, - "Filipino,Vietnamese,Spanish,English": 0.17333333333333334, - "Filipino,Vietnamese,Malay,Indonesian": 0.22, - "Filipino,Vietnamese,Malay,English": 0.18, - "Filipino,Vietnamese,Indonesian,English": 0.18666666666666668, - "Filipino,Chinese,Spanish,Malay": 0.22666666666666666, - "Filipino,Chinese,Spanish,Indonesian": 0.23333333333333334, - "Filipino,Chinese,Spanish,English": 0.22, - "Filipino,Chinese,Malay,Indonesian": 0.24666666666666667, - "Filipino,Chinese,Malay,English": 0.19333333333333333, - "Filipino,Chinese,Indonesian,English": 0.21333333333333335, - "Filipino,Spanish,Malay,Indonesian": 0.26666666666666666, - "Filipino,Spanish,Malay,English": 0.21333333333333335, - "Filipino,Spanish,Indonesian,English": 0.22, - "Filipino,Malay,Indonesian,English": 0.24, - "Vietnamese,Chinese,Spanish,Malay": 0.22666666666666666, - "Vietnamese,Chinese,Spanish,Indonesian": 0.26666666666666666, - "Vietnamese,Chinese,Spanish,English": 0.24, - "Vietnamese,Chinese,Malay,Indonesian": 0.26, - "Vietnamese,Chinese,Malay,English": 0.21333333333333335, - "Vietnamese,Chinese,Indonesian,English": 0.23333333333333334, - "Vietnamese,Spanish,Malay,Indonesian": 0.2733333333333333, - "Vietnamese,Spanish,Malay,English": 0.22, - "Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, - "Vietnamese,Malay,Indonesian,English": 0.24666666666666667, - "Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, - "Chinese,Spanish,Malay,English": 0.26, - "Chinese,Spanish,Indonesian,English": 0.30666666666666664, - "Chinese,Malay,Indonesian,English": 0.26666666666666666, - "Spanish,Malay,Indonesian,English": 0.31333333333333335 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.14, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.18, - "Filipino,Vietnamese,Chinese,Malay,English": 0.14666666666666667, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.13333333333333333, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.18666666666666668, - "Filipino,Vietnamese,Spanish,Malay,English": 0.14666666666666667, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.14, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.16, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, - "Filipino,Chinese,Spanish,Malay,English": 0.15333333333333332, - "Filipino,Chinese,Spanish,Indonesian,English": 0.16666666666666666, - "Filipino,Chinese,Malay,Indonesian,English": 0.16666666666666666, - "Filipino,Spanish,Malay,Indonesian,English": 0.18666666666666668, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, - "Vietnamese,Chinese,Spanish,Malay,English": 0.17333333333333334, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.2, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.18666666666666668, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.19333333333333333, - "Chinese,Spanish,Malay,Indonesian,English": 0.22 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.11333333333333333, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.13333333333333333, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.10666666666666667 - } - }, - "AC3_2": 0.4912169995041376, - "AC3_3": 0.38692147822054523, - "AC3_4": 0.31067189216648533, - "AC3_5": 0.25271081951120844, - "AC3_6": 0.20851505707908502, - "AC3_7": 0.17441910963358198 - }, - "prompt_5": { - "overall_acc": 0.4771428571428572, - "language_acc": { - "Filipino": 0.42, - "Vietnamese": 0.37333333333333335, - "Chinese": 0.44666666666666666, - "Spanish": 0.5933333333333334, - "Malay": 0.41333333333333333, - "Indonesian": 0.5133333333333333, - "English": 0.58 - }, - "consistency_score_2": 0.5314285714285714, - "consistency_score_3": 0.35771428571428576, - "consistency_score_4": 0.2607619047619047, - "consistency_score_5": 0.19777777777777783, - "consistency_score_6": 0.15523809523809523, - "consistency_score_7": 0.12666666666666668, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.4666666666666667, - "Filipino,Chinese": 0.5666666666666667, - "Filipino,Spanish": 0.5, - "Filipino,Malay": 0.5, - "Filipino,Indonesian": 0.56, - "Filipino,English": 0.5266666666666666, - "Vietnamese,Chinese": 0.44, - "Vietnamese,Spanish": 0.54, - "Vietnamese,Malay": 0.48, - "Vietnamese,Indonesian": 0.5133333333333333, - "Vietnamese,English": 0.4533333333333333, - "Chinese,Spanish": 0.5333333333333333, - "Chinese,Malay": 0.5266666666666666, - "Chinese,Indonesian": 0.5733333333333334, - "Chinese,English": 0.5066666666666667, - "Spanish,Malay": 0.49333333333333335, - "Spanish,Indonesian": 0.5866666666666667, - "Spanish,English": 0.6533333333333333, - "Malay,Indonesian": 0.64, - "Malay,English": 0.5133333333333333, - "Indonesian,English": 0.5866666666666667 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.29333333333333333, - "Filipino,Vietnamese,Spanish": 0.3, - "Filipino,Vietnamese,Malay": 0.30666666666666664, - "Filipino,Vietnamese,Indonesian": 0.3333333333333333, - "Filipino,Vietnamese,English": 0.26666666666666666, - "Filipino,Chinese,Spanish": 0.36, - "Filipino,Chinese,Malay": 0.36, - "Filipino,Chinese,Indonesian": 0.4, - "Filipino,Chinese,English": 0.37333333333333335, - "Filipino,Spanish,Malay": 0.32, - "Filipino,Spanish,Indonesian": 0.37333333333333335, - "Filipino,Spanish,English": 0.38666666666666666, - "Filipino,Malay,Indonesian": 0.38666666666666666, - "Filipino,Malay,English": 0.3333333333333333, - "Filipino,Indonesian,English": 0.38666666666666666, - "Vietnamese,Chinese,Spanish": 0.3466666666666667, - "Vietnamese,Chinese,Malay": 0.32, - "Vietnamese,Chinese,Indonesian": 0.3466666666666667, - "Vietnamese,Chinese,English": 0.29333333333333333, - "Vietnamese,Spanish,Malay": 0.32666666666666666, - "Vietnamese,Spanish,Indonesian": 0.37333333333333335, - "Vietnamese,Spanish,English": 0.36666666666666664, - "Vietnamese,Malay,Indonesian": 0.3933333333333333, - "Vietnamese,Malay,English": 0.29333333333333333, - "Vietnamese,Indonesian,English": 0.3466666666666667, - "Chinese,Spanish,Malay": 0.3466666666666667, - "Chinese,Spanish,Indonesian": 0.4, - "Chinese,Spanish,English": 0.3933333333333333, - "Chinese,Malay,Indonesian": 0.42, - "Chinese,Malay,English": 0.34, - "Chinese,Indonesian,English": 0.3933333333333333, - "Spanish,Malay,Indonesian": 0.4, - "Spanish,Malay,English": 0.38, - "Spanish,Indonesian,English": 0.46, - "Malay,Indonesian,English": 0.4 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.23333333333333334, - "Filipino,Vietnamese,Chinese,Malay": 0.23333333333333334, - "Filipino,Vietnamese,Chinese,Indonesian": 0.24, - "Filipino,Vietnamese,Chinese,English": 0.20666666666666667, - "Filipino,Vietnamese,Spanish,Malay": 0.22, - "Filipino,Vietnamese,Spanish,Indonesian": 0.24666666666666667, - "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, - "Filipino,Vietnamese,Malay,Indonesian": 0.25333333333333335, - "Filipino,Vietnamese,Malay,English": 0.18666666666666668, - "Filipino,Vietnamese,Indonesian,English": 0.22, - "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, - "Filipino,Chinese,Spanish,Indonesian": 0.28, - "Filipino,Chinese,Spanish,English": 0.28, - "Filipino,Chinese,Malay,Indonesian": 0.3, - "Filipino,Chinese,Malay,English": 0.24666666666666667, - "Filipino,Chinese,Indonesian,English": 0.3, - "Filipino,Spanish,Malay,Indonesian": 0.26, - "Filipino,Spanish,Malay,English": 0.25333333333333335, - "Filipino,Spanish,Indonesian,English": 0.30666666666666664, - "Filipino,Malay,Indonesian,English": 0.26, - "Vietnamese,Chinese,Spanish,Malay": 0.26, - "Vietnamese,Chinese,Spanish,Indonesian": 0.2866666666666667, - "Vietnamese,Chinese,Spanish,English": 0.26666666666666666, - "Vietnamese,Chinese,Malay,Indonesian": 0.2733333333333333, - "Vietnamese,Chinese,Malay,English": 0.22, - "Vietnamese,Chinese,Indonesian,English": 0.26, - "Vietnamese,Spanish,Malay,Indonesian": 0.2866666666666667, - "Vietnamese,Spanish,Malay,English": 0.25333333333333335, - "Vietnamese,Spanish,Indonesian,English": 0.3, - "Vietnamese,Malay,Indonesian,English": 0.26666666666666666, - "Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, - "Chinese,Spanish,Malay,English": 0.26, - "Chinese,Spanish,Indonesian,English": 0.32, - "Chinese,Malay,Indonesian,English": 0.28, - "Spanish,Malay,Indonesian,English": 0.30666666666666664 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.18, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.19333333333333333, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.18666666666666668, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2, - "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.18, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, - "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.2, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.16666666666666666, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2, - "Filipino,Chinese,Spanish,Malay,English": 0.18, - "Filipino,Chinese,Spanish,Indonesian,English": 0.23333333333333334, - "Filipino,Chinese,Malay,Indonesian,English": 0.20666666666666667, - "Filipino,Spanish,Malay,Indonesian,English": 0.2, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, - "Vietnamese,Chinese,Spanish,Malay,English": 0.2, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.23333333333333334, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.2, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.22666666666666666, - "Chinese,Spanish,Malay,Indonesian,English": 0.22 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.16, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12666666666666668 - } - }, - "AC3_2": 0.5028247672508329, - "AC3_3": 0.408886281362972, - "AC3_4": 0.3372269340939852, - "AC3_5": 0.27964252112507654, - "AC3_6": 0.2342598966927321, - "AC3_7": 0.20018927441479514 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.37581168831168826, - "language_acc": { - "Indonesian": 0.38636363636363635, - "English": 0.4943181818181818, - "Filipino": 0.30113636363636365, - "Spanish": 0.4431818181818182, - "Chinese": 0.3409090909090909, - "Malay": 0.3465909090909091, - "Vietnamese": 0.3181818181818182 - }, - "consistency_score_2": 0.4650974025974026, - "consistency_score_3": 0.27922077922077926, - "consistency_score_4": 0.18620129870129873, - "consistency_score_5": 0.1301406926406926, - "consistency_score_6": 0.09334415584415585, - "consistency_score_7": 0.06818181818181818, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.48863636363636365, - "Indonesian,Filipino": 0.45454545454545453, - "Indonesian,Spanish": 0.5852272727272727, - "Indonesian,Chinese": 0.42045454545454547, - "Indonesian,Malay": 0.5170454545454546, - "Indonesian,Vietnamese": 0.4431818181818182, - "English,Filipino": 0.38636363636363635, - "English,Spanish": 0.6761363636363636, - "English,Chinese": 0.42613636363636365, - "English,Malay": 0.42045454545454547, - "English,Vietnamese": 0.4772727272727273, - "Filipino,Spanish": 0.5056818181818182, - "Filipino,Chinese": 0.3522727272727273, - "Filipino,Malay": 0.4772727272727273, - "Filipino,Vietnamese": 0.5113636363636364, - "Spanish,Chinese": 0.4602272727272727, - "Spanish,Malay": 0.4772727272727273, - "Spanish,Vietnamese": 0.4772727272727273, - "Chinese,Malay": 0.38636363636363635, - "Chinese,Vietnamese": 0.3977272727272727, - "Malay,Vietnamese": 0.42613636363636365 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.25, - "Indonesian,English,Spanish": 0.42613636363636365, - "Indonesian,English,Chinese": 0.26136363636363635, - "Indonesian,English,Malay": 0.2840909090909091, - "Indonesian,English,Vietnamese": 0.2784090909090909, - "Indonesian,Filipino,Spanish": 0.32954545454545453, - "Indonesian,Filipino,Chinese": 0.19886363636363635, - "Indonesian,Filipino,Malay": 0.29545454545454547, - "Indonesian,Filipino,Vietnamese": 0.2784090909090909, - "Indonesian,Spanish,Chinese": 0.3181818181818182, - "Indonesian,Spanish,Malay": 0.3352272727272727, - "Indonesian,Spanish,Vietnamese": 0.3181818181818182, - "Indonesian,Chinese,Malay": 0.25, - "Indonesian,Chinese,Vietnamese": 0.22727272727272727, - "Indonesian,Malay,Vietnamese": 0.2840909090909091, - "English,Filipino,Spanish": 0.3409090909090909, - "English,Filipino,Chinese": 0.19318181818181818, - "English,Filipino,Malay": 0.24431818181818182, - "English,Filipino,Vietnamese": 0.26136363636363635, - "English,Spanish,Chinese": 0.3465909090909091, - "English,Spanish,Malay": 0.3693181818181818, - "English,Spanish,Vietnamese": 0.38636363636363635, - "English,Chinese,Malay": 0.23295454545454544, - "English,Chinese,Vietnamese": 0.23863636363636365, - "English,Malay,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Chinese": 0.22727272727272727, - "Filipino,Spanish,Malay": 0.2840909090909091, - "Filipino,Spanish,Vietnamese": 0.3181818181818182, - "Filipino,Chinese,Malay": 0.2159090909090909, - "Filipino,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Malay,Vietnamese": 0.2840909090909091, - "Spanish,Chinese,Malay": 0.2556818181818182, - "Spanish,Chinese,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Vietnamese": 0.30113636363636365, - "Chinese,Malay,Vietnamese": 0.22727272727272727 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.2215909090909091, - "Indonesian,English,Filipino,Chinese": 0.13068181818181818, - "Indonesian,English,Filipino,Malay": 0.18181818181818182, - "Indonesian,English,Filipino,Vietnamese": 0.17613636363636365, - "Indonesian,English,Spanish,Chinese": 0.24431818181818182, - "Indonesian,English,Spanish,Malay": 0.26136363636363635, - "Indonesian,English,Spanish,Vietnamese": 0.26704545454545453, - "Indonesian,English,Chinese,Malay": 0.17045454545454544, - "Indonesian,English,Chinese,Vietnamese": 0.1534090909090909, - "Indonesian,English,Malay,Vietnamese": 0.19318181818181818, - "Indonesian,Filipino,Spanish,Chinese": 0.1534090909090909, - "Indonesian,Filipino,Spanish,Malay": 0.21022727272727273, - "Indonesian,Filipino,Spanish,Vietnamese": 0.2159090909090909, - "Indonesian,Filipino,Chinese,Malay": 0.14772727272727273, - "Indonesian,Filipino,Chinese,Vietnamese": 0.13636363636363635, - "Indonesian,Filipino,Malay,Vietnamese": 0.19886363636363635, - "Indonesian,Spanish,Chinese,Malay": 0.19886363636363635, - "Indonesian,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "Indonesian,Spanish,Malay,Vietnamese": 0.2215909090909091, - "Indonesian,Chinese,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Chinese": 0.17613636363636365, - "English,Filipino,Spanish,Malay": 0.2215909090909091, - "English,Filipino,Spanish,Vietnamese": 0.24431818181818182, - "English,Filipino,Chinese,Malay": 0.13068181818181818, - "English,Filipino,Chinese,Vietnamese": 0.125, - "English,Filipino,Malay,Vietnamese": 0.16477272727272727, - "English,Spanish,Chinese,Malay": 0.2159090909090909, - "English,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Vietnamese": 0.2556818181818182, - "English,Chinese,Malay,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Chinese,Malay": 0.14772727272727273, - "Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, - "Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, - "Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.11931818181818182, - "Indonesian,English,Filipino,Spanish,Malay": 0.1590909090909091, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.17045454545454544, - "Indonesian,English,Filipino,Chinese,Malay": 0.09659090909090909, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09090909090909091, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.13068181818181818, - "Indonesian,English,Spanish,Chinese,Malay": 0.1590909090909091, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.1875, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.11931818181818182, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Chinese,Malay,Vietnamese": 0.09659090909090909, - "English,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.08522727272727272, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.125, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 - } - }, - "AC3_2": 0.41571447370075176, - "AC3_3": 0.32039459915609536, - "AC3_4": 0.24902137863478216, - "AC3_5": 0.193332002184165, - "AC3_6": 0.14954444341395234, - "AC3_7": 0.11542296823060973 - }, - "prompt_2": { - "overall_acc": 0.40259740259740256, - "language_acc": { - "Indonesian": 0.4375, - "English": 0.5, - "Filipino": 0.3068181818181818, - "Spanish": 0.5056818181818182, - "Chinese": 0.3977272727272727, - "Malay": 0.3409090909090909, - "Vietnamese": 0.32954545454545453 - }, - "consistency_score_2": 0.5270562770562771, - "consistency_score_3": 0.3478896103896103, - "consistency_score_4": 0.25422077922077924, - "consistency_score_5": 0.19507575757575757, - "consistency_score_6": 0.1525974025974026, - "consistency_score_7": 0.11931818181818182, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6534090909090909, - "Indonesian,Filipino": 0.42613636363636365, - "Indonesian,Spanish": 0.6647727272727273, - "Indonesian,Chinese": 0.5170454545454546, - "Indonesian,Malay": 0.5852272727272727, - "Indonesian,Vietnamese": 0.5170454545454546, - "English,Filipino": 0.4034090909090909, - "English,Spanish": 0.6590909090909091, - "English,Chinese": 0.5340909090909091, - "English,Malay": 0.5568181818181818, - "English,Vietnamese": 0.4943181818181818, - "Filipino,Spanish": 0.5, - "Filipino,Chinese": 0.4318181818181818, - "Filipino,Malay": 0.6022727272727273, - "Filipino,Vietnamese": 0.5284090909090909, - "Spanish,Chinese": 0.5056818181818182, - "Spanish,Malay": 0.5397727272727273, - "Spanish,Vietnamese": 0.5, - "Chinese,Malay": 0.44886363636363635, - "Chinese,Vietnamese": 0.44886363636363635, - "Malay,Vietnamese": 0.5511363636363636 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.3125, - "Indonesian,English,Spanish": 0.5227272727272727, - "Indonesian,English,Chinese": 0.4034090909090909, - "Indonesian,English,Malay": 0.44886363636363635, - "Indonesian,English,Vietnamese": 0.3806818181818182, - "Indonesian,Filipino,Spanish": 0.35795454545454547, - "Indonesian,Filipino,Chinese": 0.2556818181818182, - "Indonesian,Filipino,Malay": 0.3465909090909091, - "Indonesian,Filipino,Vietnamese": 0.29545454545454547, - "Indonesian,Spanish,Chinese": 0.39204545454545453, - "Indonesian,Spanish,Malay": 0.4318181818181818, - "Indonesian,Spanish,Vietnamese": 0.3977272727272727, - "Indonesian,Chinese,Malay": 0.32954545454545453, - "Indonesian,Chinese,Vietnamese": 0.3068181818181818, - "Indonesian,Malay,Vietnamese": 0.3806818181818182, - "English,Filipino,Spanish": 0.3465909090909091, - "English,Filipino,Chinese": 0.26704545454545453, - "English,Filipino,Malay": 0.3352272727272727, - "English,Filipino,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese": 0.39204545454545453, - "English,Spanish,Malay": 0.42045454545454547, - "English,Spanish,Vietnamese": 0.3693181818181818, - "English,Chinese,Malay": 0.3352272727272727, - "English,Chinese,Vietnamese": 0.3068181818181818, - "English,Malay,Vietnamese": 0.3693181818181818, - "Filipino,Spanish,Chinese": 0.2840909090909091, - "Filipino,Spanish,Malay": 0.3693181818181818, - "Filipino,Spanish,Vietnamese": 0.3181818181818182, - "Filipino,Chinese,Malay": 0.3068181818181818, - "Filipino,Chinese,Vietnamese": 0.26136363636363635, - "Filipino,Malay,Vietnamese": 0.3693181818181818, - "Spanish,Chinese,Malay": 0.3352272727272727, - "Spanish,Chinese,Vietnamese": 0.30113636363636365, - "Spanish,Malay,Vietnamese": 0.35795454545454547, - "Chinese,Malay,Vietnamese": 0.2897727272727273 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.2784090909090909, - "Indonesian,English,Filipino,Chinese": 0.21022727272727273, - "Indonesian,English,Filipino,Malay": 0.2727272727272727, - "Indonesian,English,Filipino,Vietnamese": 0.2215909090909091, - "Indonesian,English,Spanish,Chinese": 0.3352272727272727, - "Indonesian,English,Spanish,Malay": 0.3693181818181818, - "Indonesian,English,Spanish,Vietnamese": 0.3181818181818182, - "Indonesian,English,Chinese,Malay": 0.2784090909090909, - "Indonesian,English,Chinese,Vietnamese": 0.2556818181818182, - "Indonesian,English,Malay,Vietnamese": 0.30113636363636365, - "Indonesian,Filipino,Spanish,Chinese": 0.23295454545454544, - "Indonesian,Filipino,Spanish,Malay": 0.2840909090909091, - "Indonesian,Filipino,Spanish,Vietnamese": 0.26136363636363635, - "Indonesian,Filipino,Chinese,Malay": 0.20454545454545456, - "Indonesian,Filipino,Chinese,Vietnamese": 0.17613636363636365, - "Indonesian,Filipino,Malay,Vietnamese": 0.25, - "Indonesian,Spanish,Chinese,Malay": 0.2840909090909091, - "Indonesian,Spanish,Chinese,Vietnamese": 0.25, - "Indonesian,Spanish,Malay,Vietnamese": 0.3068181818181818, - "Indonesian,Chinese,Malay,Vietnamese": 0.22727272727272727, - "English,Filipino,Spanish,Chinese": 0.23295454545454544, - "English,Filipino,Spanish,Malay": 0.2784090909090909, - "English,Filipino,Spanish,Vietnamese": 0.24431818181818182, - "English,Filipino,Chinese,Malay": 0.22727272727272727, - "English,Filipino,Chinese,Vietnamese": 0.1875, - "English,Filipino,Malay,Vietnamese": 0.23863636363636365, - "English,Spanish,Chinese,Malay": 0.2840909090909091, - "English,Spanish,Chinese,Vietnamese": 0.23863636363636365, - "English,Spanish,Malay,Vietnamese": 0.2897727272727273, - "English,Chinese,Malay,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Chinese,Malay": 0.22727272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.2556818181818182, - "Filipino,Chinese,Malay,Vietnamese": 0.20454545454545456, - "Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.19886363636363635, - "Indonesian,English,Filipino,Spanish,Malay": 0.23863636363636365, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.20454545454545456, - "Indonesian,English,Filipino,Chinese,Malay": 0.17613636363636365, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.14772727272727273, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.19886363636363635, - "Indonesian,English,Spanish,Chinese,Malay": 0.25, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.26136363636363635, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.19318181818181818, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.1875, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635, - "English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, - "English,Filipino,Chinese,Malay,Vietnamese": 0.16477272727272727, - "English,Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.125, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 - } - }, - "AC3_2": 0.4564957742580791, - "AC3_3": 0.37324950623027153, - "AC3_4": 0.3116497935598341, - "AC3_5": 0.262809169057637, - "AC3_6": 0.22131085284993843, - "AC3_7": 0.18408030535141162 - }, - "prompt_3": { - "overall_acc": 0.38068181818181823, - "language_acc": { - "Indonesian": 0.4375, - "English": 0.4715909090909091, - "Filipino": 0.2840909090909091, - "Spanish": 0.48863636363636365, - "Chinese": 0.3693181818181818, - "Malay": 0.3125, - "Vietnamese": 0.30113636363636365 - }, - "consistency_score_2": 0.4945887445887446, - "consistency_score_3": 0.31022727272727274, - "consistency_score_4": 0.21818181818181814, - "consistency_score_5": 0.16233766233766234, - "consistency_score_6": 0.12256493506493506, - "consistency_score_7": 0.09090909090909091, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6420454545454546, - "Indonesian,Filipino": 0.45454545454545453, - "Indonesian,Spanish": 0.6420454545454546, - "Indonesian,Chinese": 0.4659090909090909, - "Indonesian,Malay": 0.5681818181818182, - "Indonesian,Vietnamese": 0.5, - "English,Filipino": 0.3977272727272727, - "English,Spanish": 0.6534090909090909, - "English,Chinese": 0.5056818181818182, - "English,Malay": 0.5, - "English,Vietnamese": 0.4431818181818182, - "Filipino,Spanish": 0.4659090909090909, - "Filipino,Chinese": 0.39204545454545453, - "Filipino,Malay": 0.5284090909090909, - "Filipino,Vietnamese": 0.5397727272727273, - "Spanish,Chinese": 0.4659090909090909, - "Spanish,Malay": 0.48295454545454547, - "Spanish,Vietnamese": 0.48863636363636365, - "Chinese,Malay": 0.4034090909090909, - "Chinese,Vietnamese": 0.3522727272727273, - "Malay,Vietnamese": 0.4943181818181818 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.3125, - "Indonesian,English,Spanish": 0.5056818181818182, - "Indonesian,English,Chinese": 0.35795454545454547, - "Indonesian,English,Malay": 0.3977272727272727, - "Indonesian,English,Vietnamese": 0.3352272727272727, - "Indonesian,Filipino,Spanish": 0.3352272727272727, - "Indonesian,Filipino,Chinese": 0.23863636363636365, - "Indonesian,Filipino,Malay": 0.3465909090909091, - "Indonesian,Filipino,Vietnamese": 0.3181818181818182, - "Indonesian,Spanish,Chinese": 0.32954545454545453, - "Indonesian,Spanish,Malay": 0.39204545454545453, - "Indonesian,Spanish,Vietnamese": 0.35795454545454547, - "Indonesian,Chinese,Malay": 0.2840909090909091, - "Indonesian,Chinese,Vietnamese": 0.23863636363636365, - "Indonesian,Malay,Vietnamese": 0.36363636363636365, - "English,Filipino,Spanish": 0.3352272727272727, - "English,Filipino,Chinese": 0.23295454545454544, - "English,Filipino,Malay": 0.30113636363636365, - "English,Filipino,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese": 0.3693181818181818, - "English,Spanish,Malay": 0.3806818181818182, - "English,Spanish,Vietnamese": 0.3352272727272727, - "English,Chinese,Malay": 0.2897727272727273, - "English,Chinese,Vietnamese": 0.2215909090909091, - "English,Malay,Vietnamese": 0.32954545454545453, - "Filipino,Spanish,Chinese": 0.23863636363636365, - "Filipino,Spanish,Malay": 0.3181818181818182, - "Filipino,Spanish,Vietnamese": 0.3181818181818182, - "Filipino,Chinese,Malay": 0.23863636363636365, - "Filipino,Chinese,Vietnamese": 0.21022727272727273, - "Filipino,Malay,Vietnamese": 0.3352272727272727, - "Spanish,Chinese,Malay": 0.25, - "Spanish,Chinese,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Vietnamese": 0.3181818181818182, - "Chinese,Malay,Vietnamese": 0.2159090909090909 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.26704545454545453, - "Indonesian,English,Filipino,Chinese": 0.19318181818181818, - "Indonesian,English,Filipino,Malay": 0.2556818181818182, - "Indonesian,English,Filipino,Vietnamese": 0.23295454545454544, - "Indonesian,English,Spanish,Chinese": 0.2784090909090909, - "Indonesian,English,Spanish,Malay": 0.3181818181818182, - "Indonesian,English,Spanish,Vietnamese": 0.2784090909090909, - "Indonesian,English,Chinese,Malay": 0.24431818181818182, - "Indonesian,English,Chinese,Vietnamese": 0.18181818181818182, - "Indonesian,English,Malay,Vietnamese": 0.26704545454545453, - "Indonesian,Filipino,Spanish,Chinese": 0.1875, - "Indonesian,Filipino,Spanish,Malay": 0.26136363636363635, - "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, - "Indonesian,Filipino,Chinese,Malay": 0.1875, - "Indonesian,Filipino,Chinese,Vietnamese": 0.16477272727272727, - "Indonesian,Filipino,Malay,Vietnamese": 0.25, - "Indonesian,Spanish,Chinese,Malay": 0.2159090909090909, - "Indonesian,Spanish,Chinese,Vietnamese": 0.1875, - "Indonesian,Spanish,Malay,Vietnamese": 0.26704545454545453, - "Indonesian,Chinese,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Chinese": 0.1875, - "English,Filipino,Spanish,Malay": 0.2556818181818182, - "English,Filipino,Spanish,Vietnamese": 0.25, - "English,Filipino,Chinese,Malay": 0.17613636363636365, - "English,Filipino,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Malay,Vietnamese": 0.23295454545454544, - "English,Spanish,Chinese,Malay": 0.22727272727272727, - "English,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Vietnamese": 0.26136363636363635, - "English,Chinese,Malay,Vietnamese": 0.1875, - "Filipino,Spanish,Chinese,Malay": 0.1534090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, - "Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, - "Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.1590909090909091, - "Indonesian,English,Filipino,Spanish,Malay": 0.2159090909090909, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.21022727272727273, - "Indonesian,English,Filipino,Chinese,Malay": 0.1590909090909091, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.13068181818181818, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.19886363636363635, - "Indonesian,English,Spanish,Chinese,Malay": 0.19318181818181818, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.2215909090909091, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.1534090909090909, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.125, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Malay": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, - "English,Filipino,Chinese,Malay,Vietnamese": 0.125, - "English,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.125, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 - } - }, - "AC3_2": 0.4302234086904843, - "AC3_3": 0.34186229062037643, - "AC3_4": 0.2773848541886305, - "AC3_5": 0.22761244730147298, - "AC3_6": 0.185428885593652, - "AC3_7": 0.14676889372572346 - }, - "prompt_4": { - "overall_acc": 0.3993506493506494, - "language_acc": { - "Indonesian": 0.4090909090909091, - "English": 0.5170454545454546, - "Filipino": 0.29545454545454547, - "Spanish": 0.4715909090909091, - "Chinese": 0.39204545454545453, - "Malay": 0.3806818181818182, - "Vietnamese": 0.32954545454545453 - }, - "consistency_score_2": 0.46617965367965364, - "consistency_score_3": 0.27451298701298704, - "consistency_score_4": 0.17889610389610386, - "consistency_score_5": 0.1231060606060606, - "consistency_score_6": 0.0892857142857143, - "consistency_score_7": 0.06818181818181818, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6022727272727273, - "Indonesian,Filipino": 0.4147727272727273, - "Indonesian,Spanish": 0.5852272727272727, - "Indonesian,Chinese": 0.4431818181818182, - "Indonesian,Malay": 0.5227272727272727, - "Indonesian,Vietnamese": 0.4375, - "English,Filipino": 0.4147727272727273, - "English,Spanish": 0.6306818181818182, - "English,Chinese": 0.4943181818181818, - "English,Malay": 0.45454545454545453, - "English,Vietnamese": 0.4318181818181818, - "Filipino,Spanish": 0.4715909090909091, - "Filipino,Chinese": 0.3352272727272727, - "Filipino,Malay": 0.42045454545454547, - "Filipino,Vietnamese": 0.4772727272727273, - "Spanish,Chinese": 0.48863636363636365, - "Spanish,Malay": 0.5056818181818182, - "Spanish,Vietnamese": 0.4602272727272727, - "Chinese,Malay": 0.3806818181818182, - "Chinese,Vietnamese": 0.4090909090909091, - "Malay,Vietnamese": 0.4090909090909091 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.26704545454545453, - "Indonesian,English,Spanish": 0.4602272727272727, - "Indonesian,English,Chinese": 0.3465909090909091, - "Indonesian,English,Malay": 0.3522727272727273, - "Indonesian,English,Vietnamese": 0.2897727272727273, - "Indonesian,Filipino,Spanish": 0.2897727272727273, - "Indonesian,Filipino,Chinese": 0.17045454545454544, - "Indonesian,Filipino,Malay": 0.26136363636363635, - "Indonesian,Filipino,Vietnamese": 0.23295454545454544, - "Indonesian,Spanish,Chinese": 0.32954545454545453, - "Indonesian,Spanish,Malay": 0.3465909090909091, - "Indonesian,Spanish,Vietnamese": 0.29545454545454547, - "Indonesian,Chinese,Malay": 0.25, - "Indonesian,Chinese,Vietnamese": 0.22727272727272727, - "Indonesian,Malay,Vietnamese": 0.2727272727272727, - "English,Filipino,Spanish": 0.3068181818181818, - "English,Filipino,Chinese": 0.2215909090909091, - "English,Filipino,Malay": 0.24431818181818182, - "English,Filipino,Vietnamese": 0.25, - "English,Spanish,Chinese": 0.375, - "English,Spanish,Malay": 0.35795454545454547, - "English,Spanish,Vietnamese": 0.32386363636363635, - "English,Chinese,Malay": 0.2727272727272727, - "English,Chinese,Vietnamese": 0.24431818181818182, - "English,Malay,Vietnamese": 0.2556818181818182, - "Filipino,Spanish,Chinese": 0.2215909090909091, - "Filipino,Spanish,Malay": 0.2727272727272727, - "Filipino,Spanish,Vietnamese": 0.2727272727272727, - "Filipino,Chinese,Malay": 0.17613636363636365, - "Filipino,Chinese,Vietnamese": 0.1875, - "Filipino,Malay,Vietnamese": 0.24431818181818182, - "Spanish,Chinese,Malay": 0.2727272727272727, - "Spanish,Chinese,Vietnamese": 0.2556818181818182, - "Spanish,Malay,Vietnamese": 0.26704545454545453, - "Chinese,Malay,Vietnamese": 0.19318181818181818 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.21022727272727273, - "Indonesian,English,Filipino,Chinese": 0.14204545454545456, - "Indonesian,English,Filipino,Malay": 0.1875, - "Indonesian,English,Filipino,Vietnamese": 0.1590909090909091, - "Indonesian,English,Spanish,Chinese": 0.29545454545454547, - "Indonesian,English,Spanish,Malay": 0.2840909090909091, - "Indonesian,English,Spanish,Vietnamese": 0.24431818181818182, - "Indonesian,English,Chinese,Malay": 0.2215909090909091, - "Indonesian,English,Chinese,Vietnamese": 0.17613636363636365, - "Indonesian,English,Malay,Vietnamese": 0.19886363636363635, - "Indonesian,Filipino,Spanish,Chinese": 0.13636363636363635, - "Indonesian,Filipino,Spanish,Malay": 0.18181818181818182, - "Indonesian,Filipino,Spanish,Vietnamese": 0.16477272727272727, - "Indonesian,Filipino,Chinese,Malay": 0.11931818181818182, - "Indonesian,Filipino,Chinese,Vietnamese": 0.10227272727272728, - "Indonesian,Filipino,Malay,Vietnamese": 0.1875, - "Indonesian,Spanish,Chinese,Malay": 0.19886363636363635, - "Indonesian,Spanish,Chinese,Vietnamese": 0.17045454545454544, - "Indonesian,Spanish,Malay,Vietnamese": 0.19318181818181818, - "Indonesian,Chinese,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese": 0.18181818181818182, - "English,Filipino,Spanish,Malay": 0.19886363636363635, - "English,Filipino,Spanish,Vietnamese": 0.19318181818181818, - "English,Filipino,Chinese,Malay": 0.14772727272727273, - "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Vietnamese": 0.17613636363636365, - "English,Spanish,Chinese,Malay": 0.23295454545454544, - "English,Spanish,Chinese,Vietnamese": 0.20454545454545456, - "English,Spanish,Malay,Vietnamese": 0.20454545454545456, - "English,Chinese,Malay,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Malay": 0.13068181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, - "Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.125, - "Indonesian,English,Filipino,Spanish,Malay": 0.14772727272727273, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.13068181818181818, - "Indonesian,English,Filipino,Chinese,Malay": 0.10795454545454546, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.08522727272727272, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.14204545454545456, - "Indonesian,English,Spanish,Chinese,Malay": 0.1875, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.16477272727272727, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.125, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Chinese,Malay": 0.125, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, - "English,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 - } - }, - "AC3_2": 0.4301851633333358, - "AC3_3": 0.32536831989965537, - "AC3_4": 0.24709961567002184, - "AC3_5": 0.18819735413751068, - "AC3_6": 0.1459420977394731, - "AC3_7": 0.11647727270235952 - }, - "prompt_5": { - "overall_acc": 0.39123376623376627, - "language_acc": { - "Indonesian": 0.4090909090909091, - "English": 0.5227272727272727, - "Filipino": 0.2784090909090909, - "Spanish": 0.4772727272727273, - "Chinese": 0.39204545454545453, - "Malay": 0.375, - "Vietnamese": 0.2840909090909091 - }, - "consistency_score_2": 0.4764610389610388, - "consistency_score_3": 0.2923701298701298, - "consistency_score_4": 0.20129870129870137, - "consistency_score_5": 0.14989177489177488, - "consistency_score_6": 0.12012987012987013, - "consistency_score_7": 0.10227272727272728, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6079545454545454, - "Indonesian,Filipino": 0.42045454545454547, - "Indonesian,Spanish": 0.6306818181818182, - "Indonesian,Chinese": 0.42045454545454547, - "Indonesian,Malay": 0.6022727272727273, - "Indonesian,Vietnamese": 0.45454545454545453, - "English,Filipino": 0.4147727272727273, - "English,Spanish": 0.6931818181818182, - "English,Chinese": 0.4659090909090909, - "English,Malay": 0.5568181818181818, - "English,Vietnamese": 0.4034090909090909, - "Filipino,Spanish": 0.48295454545454547, - "Filipino,Chinese": 0.30113636363636365, - "Filipino,Malay": 0.45454545454545453, - "Filipino,Vietnamese": 0.4943181818181818, - "Spanish,Chinese": 0.4602272727272727, - "Spanish,Malay": 0.5170454545454546, - "Spanish,Vietnamese": 0.4772727272727273, - "Chinese,Malay": 0.3693181818181818, - "Chinese,Vietnamese": 0.36363636363636365, - "Malay,Vietnamese": 0.4147727272727273 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.2840909090909091, - "Indonesian,English,Spanish": 0.4943181818181818, - "Indonesian,English,Chinese": 0.32954545454545453, - "Indonesian,English,Malay": 0.42613636363636365, - "Indonesian,English,Vietnamese": 0.29545454545454547, - "Indonesian,Filipino,Spanish": 0.32954545454545453, - "Indonesian,Filipino,Chinese": 0.1875, - "Indonesian,Filipino,Malay": 0.32386363636363635, - "Indonesian,Filipino,Vietnamese": 0.2556818181818182, - "Indonesian,Spanish,Chinese": 0.3409090909090909, - "Indonesian,Spanish,Malay": 0.42613636363636365, - "Indonesian,Spanish,Vietnamese": 0.3465909090909091, - "Indonesian,Chinese,Malay": 0.26704545454545453, - "Indonesian,Chinese,Vietnamese": 0.2215909090909091, - "Indonesian,Malay,Vietnamese": 0.3068181818181818, - "English,Filipino,Spanish": 0.3522727272727273, - "English,Filipino,Chinese": 0.20454545454545456, - "English,Filipino,Malay": 0.2784090909090909, - "English,Filipino,Vietnamese": 0.24431818181818182, - "English,Spanish,Chinese": 0.3693181818181818, - "English,Spanish,Malay": 0.4147727272727273, - "English,Spanish,Vietnamese": 0.3352272727272727, - "English,Chinese,Malay": 0.2784090909090909, - "English,Chinese,Vietnamese": 0.2215909090909091, - "English,Malay,Vietnamese": 0.2784090909090909, - "Filipino,Spanish,Chinese": 0.2159090909090909, - "Filipino,Spanish,Malay": 0.3068181818181818, - "Filipino,Spanish,Vietnamese": 0.2897727272727273, - "Filipino,Chinese,Malay": 0.17613636363636365, - "Filipino,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Vietnamese": 0.2556818181818182, - "Spanish,Chinese,Malay": 0.26704545454545453, - "Spanish,Chinese,Vietnamese": 0.23863636363636365, - "Spanish,Malay,Vietnamese": 0.30113636363636365, - "Chinese,Malay,Vietnamese": 0.19318181818181818 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.24431818181818182, - "Indonesian,English,Filipino,Chinese": 0.1590909090909091, - "Indonesian,English,Filipino,Malay": 0.23863636363636365, - "Indonesian,English,Filipino,Vietnamese": 0.18181818181818182, - "Indonesian,English,Spanish,Chinese": 0.2897727272727273, - "Indonesian,English,Spanish,Malay": 0.3465909090909091, - "Indonesian,English,Spanish,Vietnamese": 0.2556818181818182, - "Indonesian,English,Chinese,Malay": 0.23295454545454544, - "Indonesian,English,Chinese,Vietnamese": 0.17613636363636365, - "Indonesian,English,Malay,Vietnamese": 0.22727272727272727, - "Indonesian,Filipino,Spanish,Chinese": 0.17613636363636365, - "Indonesian,Filipino,Spanish,Malay": 0.2556818181818182, - "Indonesian,Filipino,Spanish,Vietnamese": 0.21022727272727273, - "Indonesian,Filipino,Chinese,Malay": 0.14772727272727273, - "Indonesian,Filipino,Chinese,Vietnamese": 0.125, - "Indonesian,Filipino,Malay,Vietnamese": 0.19886363636363635, - "Indonesian,Spanish,Chinese,Malay": 0.23295454545454544, - "Indonesian,Spanish,Chinese,Vietnamese": 0.1875, - "Indonesian,Spanish,Malay,Vietnamese": 0.26136363636363635, - "Indonesian,Chinese,Malay,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Chinese": 0.18181818181818182, - "English,Filipino,Spanish,Malay": 0.24431818181818182, - "English,Filipino,Spanish,Vietnamese": 0.2215909090909091, - "English,Filipino,Chinese,Malay": 0.14772727272727273, - "English,Filipino,Chinese,Vietnamese": 0.13636363636363635, - "English,Filipino,Malay,Vietnamese": 0.17613636363636365, - "English,Spanish,Chinese,Malay": 0.23295454545454544, - "English,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay,Vietnamese": 0.24431818181818182, - "English,Chinese,Malay,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Chinese,Malay": 0.1590909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Malay,Vietnamese": 0.125, - "Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.14772727272727273, - "Indonesian,English,Filipino,Spanish,Malay": 0.20454545454545456, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.16477272727272727, - "Indonesian,English,Filipino,Chinese,Malay": 0.13636363636363635, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.11363636363636363, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.1534090909090909, - "Indonesian,English,Spanish,Chinese,Malay": 0.20454545454545456, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.21022727272727273, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.14204545454545456, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.17045454545454544, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Malay": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Malay,Vietnamese": 0.16477272727272727, - "English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.10795454545454546, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.10227272727272728, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 - } - }, - "AC3_2": 0.4296617787703529, - "AC3_3": 0.33465305765088565, - "AC3_4": 0.26582458632766914, - "AC3_5": 0.2167435064534523, - "AC3_6": 0.18381776949610867, - "AC3_7": 0.16215610044561096 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.5533980582524272 - }, - "prompt_2": { - "accuracy": 0.5631067961165048 - }, - "prompt_3": { - "accuracy": 0.5631067961165048 - }, - "prompt_4": { - "accuracy": 0.3106796116504854 - }, - "prompt_5": { - "accuracy": 0.47572815533980584 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.3238095238095238 - }, - "prompt_2": { - "accuracy": 0.3523809523809524 - }, - "prompt_3": { - "accuracy": 0.34285714285714286 - }, - "prompt_4": { - "accuracy": 0.2857142857142857 - }, - "prompt_5": { - "accuracy": 0.3523809523809524 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.5420560747663551 - }, - "prompt_2": { - "accuracy": 0.5981308411214953 - }, - "prompt_3": { - "accuracy": 0.6635514018691588 - }, - "prompt_4": { - "accuracy": 0.3644859813084112 - }, - "prompt_5": { - "accuracy": 0.5607476635514018 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.4, - "category_acc": { - "brand": 0.3, - "demographics": 0.0, - "biology": 0.4, - "history": 0.2, - "literature": 0.3, - "politics": 0.8, - "culture": 0.5, - "film": 0.4, - "law": 0.3, - "geography": 0.7 - } - }, - "prompt_2": { - "accuracy": 0.43, - "category_acc": { - "brand": 0.5, - "demographics": 0.0, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.7, - "culture": 0.6, - "film": 0.4, - "law": 0.4, - "geography": 0.6 - } - }, - "prompt_3": { - "accuracy": 0.39, - "category_acc": { - "brand": 0.4, - "demographics": 0.0, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.7, - "culture": 0.3, - "film": 0.3, - "law": 0.3, - "geography": 0.7 - } - }, - "prompt_4": { - "accuracy": 0.33, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.3, - "history": 0.26666666666666666, - "literature": 0.5, - "politics": 0.3, - "culture": 0.4, - "film": 0.2, - "law": 0.3, - "geography": 0.4 - } - }, - "prompt_5": { - "accuracy": 0.39, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.3, - "history": 0.4, - "literature": 0.3, - "politics": 0.6, - "culture": 0.4, - "film": 0.4, - "law": 0.3, - "geography": 0.6 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.11019567971189798 - }, - "prompt_2": { - "bleu_score": 0.06811345832897743 - }, - "prompt_3": { - "bleu_score": 0.07173070353196886 - }, - "prompt_4": { - "bleu_score": 0.16582932130400574 - }, - "prompt_5": { - "bleu_score": 0.0715846198154993 - } - }, - "indommlu": { - "prompt_1": { - "accuracy": 0.3340009346418319, - "category_acc": { - "History": 0.3253012048192771, - "Geography": 0.32040816326530613, - "Lampungic": 0.2925170068027211, - "Social science": 0.44908180300500833, - "Balinese": 0.3036093418259023, - "Makassarese": 0.3118279569892473, - "Banjarese": 0.3888888888888889, - "Chemistry": 0.21021897810218979, - "Biology": 0.3254437869822485, - "Science": 0.4055727554179567, - "Christian religion": 0.3283582089552239, - "Art": 0.2995008319467554, - "Islam religion": 0.3357041251778094, - "Hindu religion": 0.3466666666666667, - "Madurese": 0.31864406779661014, - "Sport": 0.3918918918918919, - "Indonesian language": 0.36581569115815693, - "Physics": 0.298989898989899, - "Minangkabau culture": 0.3065326633165829, - "Dayak language": 0.28440366972477066, - "Sociology": 0.3064516129032258, - "Economy": 0.29713114754098363, - "Sundanese": 0.3146067415730337, - "Javanese": 0.26310483870967744, - "Civic education": 0.4005722460658083 - } - }, - "prompt_2": { - "accuracy": 0.352560251018092, - "category_acc": { - "History": 0.3433734939759036, - "Geography": 0.3081632653061224, - "Lampungic": 0.24489795918367346, - "Social science": 0.5141903171953256, - "Balinese": 0.26963906581740976, - "Makassarese": 0.27956989247311825, - "Banjarese": 0.3472222222222222, - "Chemistry": 0.24087591240875914, - "Biology": 0.3301775147928994, - "Science": 0.46439628482972134, - "Christian religion": 0.43283582089552236, - "Art": 0.3910149750415973, - "Islam religion": 0.4395448079658606, - "Hindu religion": 0.43333333333333335, - "Madurese": 0.31864406779661014, - "Sport": 0.36486486486486486, - "Indonesian language": 0.35118306351183065, - "Physics": 0.3292929292929293, - "Minangkabau culture": 0.35678391959798994, - "Dayak language": 0.3302752293577982, - "Sociology": 0.3125, - "Economy": 0.319672131147541, - "Sundanese": 0.2912705272255834, - "Javanese": 0.2973790322580645, - "Civic education": 0.43919885550786836 - } - }, - "prompt_3": { - "accuracy": 0.3484878830362508, - "category_acc": { - "History": 0.3755020080321285, - "Geography": 0.28775510204081634, - "Lampungic": 0.23809523809523808, - "Social science": 0.5275459098497496, - "Balinese": 0.30148619957537154, - "Makassarese": 0.2956989247311828, - "Banjarese": 0.3125, - "Chemistry": 0.22335766423357664, - "Biology": 0.3207100591715976, - "Science": 0.42105263157894735, - "Christian religion": 0.48756218905472637, - "Art": 0.40266222961730447, - "Islam religion": 0.43385490753911804, - "Hindu religion": 0.31333333333333335, - "Madurese": 0.31186440677966104, - "Sport": 0.40540540540540543, - "Indonesian language": 0.34775840597758406, - "Physics": 0.33131313131313134, - "Minangkabau culture": 0.36683417085427134, - "Dayak language": 0.29357798165137616, - "Sociology": 0.3165322580645161, - "Economy": 0.305327868852459, - "Sundanese": 0.29818496110630943, - "Javanese": 0.2963709677419355, - "Civic education": 0.4177396280400572 - } - }, - "prompt_4": { - "accuracy": 0.33987582615661927, - "category_acc": { - "History": 0.3353413654618474, - "Geography": 0.27755102040816326, - "Lampungic": 0.30612244897959184, - "Social science": 0.4540901502504174, - "Balinese": 0.32908704883227174, - "Makassarese": 0.3333333333333333, - "Banjarese": 0.4305555555555556, - "Chemistry": 0.23503649635036497, - "Biology": 0.30532544378698223, - "Science": 0.3797729618163055, - "Christian religion": 0.3880597014925373, - "Art": 0.3277870216306156, - "Islam religion": 0.37268847795163584, - "Hindu religion": 0.25333333333333335, - "Madurese": 0.2983050847457627, - "Sport": 0.43243243243243246, - "Indonesian language": 0.3844956413449564, - "Physics": 0.2828282828282828, - "Minangkabau culture": 0.3165829145728643, - "Dayak language": 0.28440366972477066, - "Sociology": 0.35080645161290325, - "Economy": 0.29098360655737704, - "Sundanese": 0.31374243733794294, - "Javanese": 0.27419354838709675, - "Civic education": 0.36909871244635195 - } - }, - "prompt_5": { - "accuracy": 0.3554309366446358, - "category_acc": { - "History": 0.357429718875502, - "Geography": 0.31020408163265306, - "Lampungic": 0.2653061224489796, - "Social science": 0.4607679465776294, - "Balinese": 0.3184713375796178, - "Makassarese": 0.2956989247311828, - "Banjarese": 0.3472222222222222, - "Chemistry": 0.25255474452554744, - "Biology": 0.34437869822485206, - "Science": 0.391124871001032, - "Christian religion": 0.40298507462686567, - "Art": 0.3793677204658902, - "Islam religion": 0.38549075391180654, - "Hindu religion": 0.36, - "Madurese": 0.27796610169491526, - "Sport": 0.47297297297297297, - "Indonesian language": 0.39321295143212953, - "Physics": 0.2787878787878788, - "Minangkabau culture": 0.35678391959798994, - "Dayak language": 0.26605504587155965, - "Sociology": 0.36088709677419356, - "Economy": 0.32786885245901637, - "Sundanese": 0.33016421780466726, - "Javanese": 0.2963709677419355, - "Civic education": 0.39914163090128757 - } - } - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.10701947359874414 - }, - "prompt_2": { - "bleu_score": 0.10500695436368739 - }, - "prompt_3": { - "bleu_score": 0.11420689867404787 - }, - "prompt_4": { - "bleu_score": 0.1300280593061721 - }, - "prompt_5": { - "bleu_score": 0.08455793773419258 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.07199046785155615 - }, - "prompt_2": { - "bleu_score": 0.07723858215690355 - }, - "prompt_3": { - "bleu_score": 0.0867870336630168 - }, - "prompt_4": { - "bleu_score": 0.06310585954563377 - }, - "prompt_5": { - "bleu_score": 0.06263968316226737 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.06263723220528784 - }, - "prompt_2": { - "bleu_score": 0.05904179358838399 - }, - "prompt_3": { - "bleu_score": 0.05644144765969993 - }, - "prompt_4": { - "bleu_score": 0.0727928412763168 - }, - "prompt_5": { - "bleu_score": 0.05354019964746781 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.09400915338404318 - }, - "prompt_2": { - "bleu_score": 0.097180972527284 - }, - "prompt_3": { - "bleu_score": 0.10694124342130719 - }, - "prompt_4": { - "bleu_score": 0.11373484458656416 - }, - "prompt_5": { - "bleu_score": 0.07005924515462118 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.5134189031505251 - }, - "prompt_2": { - "accuracy": 0.5040840140023337 - }, - "prompt_3": { - "accuracy": 0.5309218203033839 - }, - "prompt_4": { - "accuracy": 0.47141190198366395 - }, - "prompt_5": { - "accuracy": 0.5425904317386231 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.5249910618519843, - "category_acc": { - "high_school_european_history": 0.6951219512195121, - "business_ethics": 0.5151515151515151, - "clinical_knowledge": 0.5871212121212122, - "medical_genetics": 0.6060606060606061, - "high_school_us_history": 0.7142857142857143, - "high_school_physics": 0.35333333333333333, - "high_school_world_history": 0.7203389830508474, - "virology": 0.43636363636363634, - "high_school_microeconomics": 0.5485232067510548, - "econometrics": 0.37168141592920356, - "college_computer_science": 0.494949494949495, - "high_school_biology": 0.627831715210356, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.39501779359430605, - "philosophy": 0.5903225806451613, - "professional_medicine": 0.6789667896678967, - "nutrition": 0.580327868852459, - "global_facts": 0.37373737373737376, - "machine_learning": 0.35135135135135137, - "security_studies": 0.6352459016393442, - "public_relations": 0.5504587155963303, - "professional_psychology": 0.5531914893617021, - "prehistory": 0.5975232198142415, - "anatomy": 0.5223880597014925, - "human_sexuality": 0.6384615384615384, - "college_medicine": 0.5174418604651163, - "high_school_government_and_politics": 0.7239583333333334, - "college_chemistry": 0.43434343434343436, - "logical_fallacies": 0.6172839506172839, - "high_school_geography": 0.6802030456852792, - "elementary_mathematics": 0.3713527851458886, - "human_aging": 0.5540540540540541, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.7058823529411765, - "formal_logic": 0.32, - "high_school_statistics": 0.4, - "international_law": 0.625, - "high_school_mathematics": 0.31970260223048325, - "high_school_computer_science": 0.5151515151515151, - "conceptual_physics": 0.4230769230769231, - "miscellaneous": 0.6483375959079284, - "high_school_chemistry": 0.41089108910891087, - "marketing": 0.7553648068669528, - "professional_law": 0.4279191128506197, - "management": 0.6274509803921569, - "college_physics": 0.39603960396039606, - "jurisprudence": 0.6355140186915887, - "world_religions": 0.711764705882353, - "sociology": 0.64, - "us_foreign_policy": 0.7474747474747475, - "high_school_macroeconomics": 0.5038560411311054, - "computer_security": 0.6464646464646465, - "moral_scenarios": 0.24272930648769575, - "moral_disputes": 0.5797101449275363, - "electrical_engineering": 0.4930555555555556, - "astronomy": 0.5496688741721855, - "college_biology": 0.5524475524475524 - } - }, - "prompt_2": { - "accuracy": 0.5272792277440115, - "category_acc": { - "high_school_european_history": 0.676829268292683, - "business_ethics": 0.494949494949495, - "clinical_knowledge": 0.571969696969697, - "medical_genetics": 0.5454545454545454, - "high_school_us_history": 0.729064039408867, - "high_school_physics": 0.3, - "high_school_world_history": 0.7203389830508474, - "virology": 0.41818181818181815, - "high_school_microeconomics": 0.5232067510548524, - "econometrics": 0.3893805309734513, - "college_computer_science": 0.48484848484848486, - "high_school_biology": 0.6407766990291263, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.4128113879003559, - "philosophy": 0.6225806451612903, - "professional_medicine": 0.6531365313653137, - "nutrition": 0.5901639344262295, - "global_facts": 0.35353535353535354, - "machine_learning": 0.3963963963963964, - "security_studies": 0.639344262295082, - "public_relations": 0.5596330275229358, - "professional_psychology": 0.5548281505728314, - "prehistory": 0.6377708978328174, - "anatomy": 0.5074626865671642, - "human_sexuality": 0.6, - "college_medicine": 0.5465116279069767, - "high_school_government_and_politics": 0.7083333333333334, - "college_chemistry": 0.36363636363636365, - "logical_fallacies": 0.654320987654321, - "high_school_geography": 0.6548223350253807, - "elementary_mathematics": 0.3421750663129973, - "human_aging": 0.545045045045045, - "college_mathematics": 0.36363636363636365, - "high_school_psychology": 0.6746323529411765, - "formal_logic": 0.296, - "high_school_statistics": 0.413953488372093, - "international_law": 0.6583333333333333, - "high_school_mathematics": 0.31970260223048325, - "high_school_computer_science": 0.5454545454545454, - "conceptual_physics": 0.39316239316239315, - "miscellaneous": 0.7148337595907929, - "high_school_chemistry": 0.42574257425742573, - "marketing": 0.759656652360515, - "professional_law": 0.41487279843444225, - "management": 0.6764705882352942, - "college_physics": 0.39603960396039606, - "jurisprudence": 0.7289719626168224, - "world_religions": 0.7529411764705882, - "sociology": 0.705, - "us_foreign_policy": 0.7373737373737373, - "high_school_macroeconomics": 0.4781491002570694, - "computer_security": 0.6262626262626263, - "moral_scenarios": 0.24161073825503357, - "moral_disputes": 0.5826086956521739, - "electrical_engineering": 0.4791666666666667, - "astronomy": 0.5496688741721855, - "college_biology": 0.6013986013986014 - } - }, - "prompt_3": { - "accuracy": 0.556381837683232, - "category_acc": { - "high_school_european_history": 0.676829268292683, - "business_ethics": 0.5252525252525253, - "clinical_knowledge": 0.6212121212121212, - "medical_genetics": 0.5757575757575758, - "high_school_us_history": 0.7241379310344828, - "high_school_physics": 0.2866666666666667, - "high_school_world_history": 0.7203389830508474, - "virology": 0.4727272727272727, - "high_school_microeconomics": 0.5611814345991561, - "econometrics": 0.415929203539823, - "college_computer_science": 0.5050505050505051, - "high_school_biology": 0.686084142394822, - "abstract_algebra": 0.3333333333333333, - "professional_accounting": 0.4234875444839858, - "philosophy": 0.6419354838709678, - "professional_medicine": 0.6568265682656826, - "nutrition": 0.6295081967213115, - "global_facts": 0.31313131313131315, - "machine_learning": 0.40540540540540543, - "security_studies": 0.6352459016393442, - "public_relations": 0.6330275229357798, - "professional_psychology": 0.5793780687397708, - "prehistory": 0.6625386996904025, - "anatomy": 0.5373134328358209, - "human_sexuality": 0.6384615384615384, - "college_medicine": 0.5930232558139535, - "high_school_government_and_politics": 0.8072916666666666, - "college_chemistry": 0.41414141414141414, - "logical_fallacies": 0.6666666666666666, - "high_school_geography": 0.7106598984771574, - "elementary_mathematics": 0.363395225464191, - "human_aging": 0.6036036036036037, - "college_mathematics": 0.36363636363636365, - "high_school_psychology": 0.7389705882352942, - "formal_logic": 0.328, - "high_school_statistics": 0.3953488372093023, - "international_law": 0.7166666666666667, - "high_school_mathematics": 0.31970260223048325, - "high_school_computer_science": 0.5454545454545454, - "conceptual_physics": 0.43162393162393164, - "miscellaneous": 0.7595907928388747, - "high_school_chemistry": 0.4405940594059406, - "marketing": 0.8068669527896996, - "professional_law": 0.4337899543378995, - "management": 0.7156862745098039, - "college_physics": 0.36633663366336633, - "jurisprudence": 0.7476635514018691, - "world_religions": 0.7823529411764706, - "sociology": 0.815, - "us_foreign_policy": 0.7676767676767676, - "high_school_macroeconomics": 0.5398457583547558, - "computer_security": 0.6565656565656566, - "moral_scenarios": 0.24049217002237136, - "moral_disputes": 0.6057971014492753, - "electrical_engineering": 0.5763888888888888, - "astronomy": 0.6026490066225165, - "college_biology": 0.6573426573426573 - } - }, - "prompt_4": { - "accuracy": 0.48430461208437614, - "category_acc": { - "high_school_european_history": 0.6646341463414634, - "business_ethics": 0.45454545454545453, - "clinical_knowledge": 0.5075757575757576, - "medical_genetics": 0.494949494949495, - "high_school_us_history": 0.7241379310344828, - "high_school_physics": 0.36666666666666664, - "high_school_world_history": 0.6991525423728814, - "virology": 0.4121212121212121, - "high_school_microeconomics": 0.459915611814346, - "econometrics": 0.36283185840707965, - "college_computer_science": 0.4444444444444444, - "high_school_biology": 0.5825242718446602, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.4306049822064057, - "philosophy": 0.5161290322580645, - "professional_medicine": 0.6642066420664207, - "nutrition": 0.521311475409836, - "global_facts": 0.2828282828282828, - "machine_learning": 0.32432432432432434, - "security_studies": 0.6147540983606558, - "public_relations": 0.46788990825688076, - "professional_psychology": 0.5237315875613748, - "prehistory": 0.5263157894736842, - "anatomy": 0.44029850746268656, - "human_sexuality": 0.6384615384615384, - "college_medicine": 0.4941860465116279, - "high_school_government_and_politics": 0.6354166666666666, - "college_chemistry": 0.41414141414141414, - "logical_fallacies": 0.49382716049382713, - "high_school_geography": 0.6142131979695431, - "elementary_mathematics": 0.32625994694960214, - "human_aging": 0.509009009009009, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.5919117647058824, - "formal_logic": 0.304, - "high_school_statistics": 0.386046511627907, - "international_law": 0.5833333333333334, - "high_school_mathematics": 0.3271375464684015, - "high_school_computer_science": 0.5555555555555556, - "conceptual_physics": 0.4188034188034188, - "miscellaneous": 0.5677749360613811, - "high_school_chemistry": 0.400990099009901, - "marketing": 0.6437768240343348, - "professional_law": 0.4187866927592955, - "management": 0.6372549019607843, - "college_physics": 0.37623762376237624, - "jurisprudence": 0.5981308411214953, - "world_religions": 0.6176470588235294, - "sociology": 0.535, - "us_foreign_policy": 0.6565656565656566, - "high_school_macroeconomics": 0.44473007712082263, - "computer_security": 0.5252525252525253, - "moral_scenarios": 0.24161073825503357, - "moral_disputes": 0.5072463768115942, - "electrical_engineering": 0.4652777777777778, - "astronomy": 0.5364238410596026, - "college_biology": 0.5874125874125874 - } - }, - "prompt_5": { - "accuracy": 0.552592062924562, - "category_acc": { - "high_school_european_history": 0.7012195121951219, - "business_ethics": 0.5252525252525253, - "clinical_knowledge": 0.5946969696969697, - "medical_genetics": 0.5959595959595959, - "high_school_us_history": 0.7093596059113301, - "high_school_physics": 0.3333333333333333, - "high_school_world_history": 0.690677966101695, - "virology": 0.46060606060606063, - "high_school_microeconomics": 0.5822784810126582, - "econometrics": 0.415929203539823, - "college_computer_science": 0.5151515151515151, - "high_school_biology": 0.6957928802588996, - "abstract_algebra": 0.26262626262626265, - "professional_accounting": 0.4306049822064057, - "philosophy": 0.6709677419354839, - "professional_medicine": 0.6568265682656826, - "nutrition": 0.6163934426229508, - "global_facts": 0.36363636363636365, - "machine_learning": 0.38738738738738737, - "security_studies": 0.6762295081967213, - "public_relations": 0.6238532110091743, - "professional_psychology": 0.5810147299509002, - "prehistory": 0.6470588235294118, - "anatomy": 0.5447761194029851, - "human_sexuality": 0.6076923076923076, - "college_medicine": 0.5755813953488372, - "high_school_government_and_politics": 0.796875, - "college_chemistry": 0.47474747474747475, - "logical_fallacies": 0.6358024691358025, - "high_school_geography": 0.7360406091370558, - "elementary_mathematics": 0.35543766578249336, - "human_aging": 0.6036036036036037, - "college_mathematics": 0.32323232323232326, - "high_school_psychology": 0.7426470588235294, - "formal_logic": 0.4, - "high_school_statistics": 0.46511627906976744, - "international_law": 0.7, - "high_school_mathematics": 0.31226765799256506, - "high_school_computer_science": 0.5454545454545454, - "conceptual_physics": 0.4444444444444444, - "miscellaneous": 0.7301790281329923, - "high_school_chemistry": 0.4504950495049505, - "marketing": 0.8111587982832618, - "professional_law": 0.42400521852576645, - "management": 0.7549019607843137, - "college_physics": 0.38613861386138615, - "jurisprudence": 0.719626168224299, - "world_religions": 0.788235294117647, - "sociology": 0.7, - "us_foreign_policy": 0.7575757575757576, - "high_school_macroeconomics": 0.519280205655527, - "computer_security": 0.696969696969697, - "moral_scenarios": 0.24272930648769575, - "moral_disputes": 0.5739130434782609, - "electrical_engineering": 0.5208333333333334, - "astronomy": 0.5827814569536424, - "college_biology": 0.6503496503496503 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.34843982169390786 - }, - "prompt_2": { - "accuracy": 0.33803863298662706 - }, - "prompt_3": { - "accuracy": 0.32540861812778604 - }, - "prompt_4": { - "accuracy": 0.36701337295690933 - }, - "prompt_5": { - "accuracy": 0.3291233283803863 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.3331257783312578, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.375, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.35714285714285715, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.4583333333333333, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.3448275862068966, - "high_school_mathematics": 0.08695652173913043, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.3333333333333333, - "business_administration": 0.2894736842105263, - "marxism": 0.25, - "mao_zedong_thought": 0.6551724137931034, - "education_science": 0.35294117647058826, - "teacher_qualification": 0.3877551020408163, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.5882352941176471, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.4583333333333333, - "logic": 0.48148148148148145, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.125, - "high_school_history": 0.48, - "middle_school_history": 0.3333333333333333, - "civil_servant": 0.38461538461538464, - "sports_science": 0.16666666666666666, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.16666666666666666, - "clinical_medicine": 0.25925925925925924, - "urban_and_rural_planner": 0.27450980392156865, - "accountant": 0.4074074074074074, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.19444444444444445, - "tax_accountant": 0.25925925925925924, - "physician": 0.35185185185185186 - } - }, - "prompt_2": { - "accuracy": 0.32938978829389787, - "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.375, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.30952380952380953, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.047619047619047616, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.27586206896551724, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.08333333333333333, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.38461538461538464, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.35, - "business_administration": 0.2631578947368421, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.5172413793103449, - "education_science": 0.35294117647058826, - "teacher_qualification": 0.3673469387755102, - "high_school_politics": 0.20833333333333334, - "high_school_geography": 0.4583333333333333, - "middle_school_politics": 0.2692307692307692, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.32142857142857145, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.37037037037037035, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.17857142857142858, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.4642857142857143, - "high_school_chinese": 0.25, - "high_school_history": 0.44, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.34615384615384615, - "sports_science": 0.25, - "plant_protection": 0.2222222222222222, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.37254901960784315, - "accountant": 0.3888888888888889, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.3888888888888889, - "tax_accountant": 0.18518518518518517, - "physician": 0.4074074074074074 - } - }, - "prompt_3": { - "accuracy": 0.33997509339975096, - "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.40476190476190477, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.3103448275862069, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.09523809523809523, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.25, - "high_school_chemistry": 0.25, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.34615384615384615, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.17857142857142858, - "college_economics": 0.31666666666666665, - "business_administration": 0.39473684210526316, - "marxism": 0.375, - "mao_zedong_thought": 0.5517241379310345, - "education_science": 0.4117647058823529, - "teacher_qualification": 0.42857142857142855, - "high_school_politics": 0.16666666666666666, - "high_school_geography": 0.375, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.4583333333333333, - "logic": 0.48148148148148145, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.14285714285714285, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.47058823529411764, - "legal_professional": 0.4642857142857143, - "high_school_chinese": 0.20833333333333334, - "high_school_history": 0.36, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.36538461538461536, - "sports_science": 0.16666666666666666, - "plant_protection": 0.37037037037037035, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.35294117647058826, - "accountant": 0.4074074074074074, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.3611111111111111, - "tax_accountant": 0.25925925925925924, - "physician": 0.3888888888888889 - } - }, - "prompt_4": { - "accuracy": 0.3318804483188045, - "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.16666666666666666, - "computer_architecture": 0.46153846153846156, - "college_programming": 0.38095238095238093, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.2413793103448276, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.25, - "high_school_chemistry": 0.20833333333333334, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.2692307692307692, - "middle_school_physics": 0.3333333333333333, - "middle_school_chemistry": 0.2, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.35, - "business_administration": 0.3157894736842105, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.5172413793103449, - "education_science": 0.38235294117647056, - "teacher_qualification": 0.42857142857142855, - "high_school_politics": 0.25, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.46153846153846156, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.5416666666666666, - "logic": 0.48148148148148145, - "law": 0.41379310344827586, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.5, - "high_school_chinese": 0.08333333333333333, - "high_school_history": 0.48, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.40384615384615385, - "sports_science": 0.125, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.18518518518518517, - "urban_and_rural_planner": 0.3333333333333333, - "accountant": 0.42592592592592593, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.3888888888888889, - "tax_accountant": 0.16666666666666666, - "physician": 0.2962962962962963 - } - }, - "prompt_5": { - "accuracy": 0.32503113325031135, - "category_acc": { - "computer_network": 0.375, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.40476190476190477, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.3448275862068966, - "advanced_mathematics": 0.4583333333333333, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.047619047619047616, - "electrical_engineer": 0.23809523809523808, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.125, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.3076923076923077, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.28, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.3, - "business_administration": 0.3157894736842105, - "marxism": 0.2916666666666667, - "mao_zedong_thought": 0.7241379310344828, - "education_science": 0.3235294117647059, - "teacher_qualification": 0.3877551020408163, - "high_school_politics": 0.2916666666666667, - "high_school_geography": 0.20833333333333334, - "middle_school_politics": 0.4230769230769231, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.48148148148148145, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.5, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.5, - "high_school_chinese": 0.08333333333333333, - "high_school_history": 0.4, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.2692307692307692, - "sports_science": 0.25, - "plant_protection": 0.37037037037037035, - "basic_medicine": 0.16666666666666666, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.29411764705882354, - "accountant": 0.3333333333333333, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.18518518518518517, - "physician": 0.2777777777777778 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.3333333333333333 - }, - "prompt_2": { - "accuracy": 0.34408602150537637 - }, - "prompt_3": { - "accuracy": 0.32974910394265233 - }, - "prompt_4": { - "accuracy": 0.35125448028673834 - }, - "prompt_5": { - "accuracy": 0.2939068100358423 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.3423415644966327, - "category_acc": { - "agronomy": 0.33727810650887574, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.2804878048780488, - "arts": 0.3125, - "astronomy": 0.28484848484848485, - "business_ethics": 0.3875598086124402, - "chinese_civil_service_exam": 0.30625, - "chinese_driving_rule": 0.4580152671755725, - "chinese_food_culture": 0.27205882352941174, - "chinese_foreign_policy": 0.45794392523364486, - "chinese_history": 0.4117647058823529, - "chinese_literature": 0.29411764705882354, - "chinese_teacher_qualification": 0.39664804469273746, - "clinical_knowledge": 0.33755274261603374, - "college_actuarial_science": 0.29245283018867924, - "college_education": 0.4205607476635514, - "college_engineering_hydrology": 0.3018867924528302, - "college_law": 0.32407407407407407, - "college_mathematics": 0.17142857142857143, - "college_medical_statistics": 0.29245283018867924, - "college_medicine": 0.2564102564102564, - "computer_science": 0.4019607843137255, - "computer_security": 0.30409356725146197, - "conceptual_physics": 0.3401360544217687, - "construction_project_management": 0.31654676258992803, - "economics": 0.4339622641509434, - "education": 0.36809815950920244, - "electrical_engineering": 0.37209302325581395, - "elementary_chinese": 0.3373015873015873, - "elementary_commonsense": 0.31313131313131315, - "elementary_information_and_technology": 0.42016806722689076, - "elementary_mathematics": 0.30869565217391304, - "ethnology": 0.32592592592592595, - "food_science": 0.3706293706293706, - "genetics": 0.3181818181818182, - "global_facts": 0.3959731543624161, - "high_school_biology": 0.30177514792899407, - "high_school_chemistry": 0.2803030303030303, - "high_school_geography": 0.3305084745762712, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.2909090909090909, - "high_school_politics": 0.3776223776223776, - "human_sexuality": 0.3333333333333333, - "international_law": 0.2810810810810811, - "journalism": 0.37209302325581395, - "jurisprudence": 0.3844282238442822, - "legal_and_moral_basis": 0.514018691588785, - "logical": 0.2845528455284553, - "machine_learning": 0.23770491803278687, - "management": 0.42857142857142855, - "marketing": 0.4166666666666667, - "marxist_theory": 0.43386243386243384, - "modern_chinese": 0.3017241379310345, - "nutrition": 0.30344827586206896, - "philosophy": 0.3047619047619048, - "professional_accounting": 0.3142857142857143, - "professional_law": 0.2890995260663507, - "professional_medicine": 0.30851063829787234, - "professional_psychology": 0.33189655172413796, - "public_relations": 0.43103448275862066, - "security_study": 0.34074074074074073, - "sociology": 0.39823008849557523, - "sports_science": 0.3515151515151515, - "traditional_chinese_medicine": 0.23783783783783785, - "virology": 0.34911242603550297, - "world_history": 0.37888198757763975, - "world_religions": 0.31875 - } - }, - "prompt_2": { - "accuracy": 0.3413918148851666, - "category_acc": { - "agronomy": 0.3136094674556213, - "anatomy": 0.24324324324324326, - "ancient_chinese": 0.27439024390243905, - "arts": 0.34375, - "astronomy": 0.2727272727272727, - "business_ethics": 0.3684210526315789, - "chinese_civil_service_exam": 0.275, - "chinese_driving_rule": 0.4732824427480916, - "chinese_food_culture": 0.3602941176470588, - "chinese_foreign_policy": 0.4485981308411215, - "chinese_history": 0.39009287925696595, - "chinese_literature": 0.30392156862745096, - "chinese_teacher_qualification": 0.4245810055865922, - "clinical_knowledge": 0.3206751054852321, - "college_actuarial_science": 0.3018867924528302, - "college_education": 0.4205607476635514, - "college_engineering_hydrology": 0.37735849056603776, - "college_law": 0.2962962962962963, - "college_mathematics": 0.26666666666666666, - "college_medical_statistics": 0.330188679245283, - "college_medicine": 0.34065934065934067, - "computer_science": 0.39215686274509803, - "computer_security": 0.34502923976608185, - "conceptual_physics": 0.3741496598639456, - "construction_project_management": 0.33093525179856115, - "economics": 0.41509433962264153, - "education": 0.36809815950920244, - "electrical_engineering": 0.3313953488372093, - "elementary_chinese": 0.26587301587301587, - "elementary_commonsense": 0.31313131313131315, - "elementary_information_and_technology": 0.35714285714285715, - "elementary_mathematics": 0.27391304347826084, - "ethnology": 0.34814814814814815, - "food_science": 0.4825174825174825, - "genetics": 0.3125, - "global_facts": 0.3691275167785235, - "high_school_biology": 0.25443786982248523, - "high_school_chemistry": 0.20454545454545456, - "high_school_geography": 0.423728813559322, - "high_school_mathematics": 0.25, - "high_school_physics": 0.3, - "high_school_politics": 0.36363636363636365, - "human_sexuality": 0.373015873015873, - "international_law": 0.3027027027027027, - "journalism": 0.3953488372093023, - "jurisprudence": 0.3819951338199513, - "legal_and_moral_basis": 0.5514018691588785, - "logical": 0.2926829268292683, - "machine_learning": 0.2540983606557377, - "management": 0.3333333333333333, - "marketing": 0.3888888888888889, - "marxist_theory": 0.3968253968253968, - "modern_chinese": 0.3103448275862069, - "nutrition": 0.3310344827586207, - "philosophy": 0.34285714285714286, - "professional_accounting": 0.3314285714285714, - "professional_law": 0.27014218009478674, - "professional_medicine": 0.2845744680851064, - "professional_psychology": 0.29310344827586204, - "public_relations": 0.39655172413793105, - "security_study": 0.34074074074074073, - "sociology": 0.3672566371681416, - "sports_science": 0.37575757575757573, - "traditional_chinese_medicine": 0.2594594594594595, - "virology": 0.34911242603550297, - "world_history": 0.42857142857142855, - "world_religions": 0.30625 - } - }, - "prompt_3": { - "accuracy": 0.3379381799343809, - "category_acc": { - "agronomy": 0.33727810650887574, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.21341463414634146, - "arts": 0.3125, - "astronomy": 0.2606060606060606, - "business_ethics": 0.35406698564593303, - "chinese_civil_service_exam": 0.3, - "chinese_driving_rule": 0.48091603053435117, - "chinese_food_culture": 0.33088235294117646, - "chinese_foreign_policy": 0.4485981308411215, - "chinese_history": 0.3591331269349845, - "chinese_literature": 0.25980392156862747, - "chinese_teacher_qualification": 0.40782122905027934, - "clinical_knowledge": 0.3333333333333333, - "college_actuarial_science": 0.3113207547169811, - "college_education": 0.40186915887850466, - "college_engineering_hydrology": 0.3490566037735849, - "college_law": 0.25, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.3113207547169811, - "college_medicine": 0.3516483516483517, - "computer_science": 0.37745098039215685, - "computer_security": 0.3508771929824561, - "conceptual_physics": 0.35374149659863946, - "construction_project_management": 0.3597122302158273, - "economics": 0.3710691823899371, - "education": 0.32515337423312884, - "electrical_engineering": 0.3546511627906977, - "elementary_chinese": 0.26587301587301587, - "elementary_commonsense": 0.29797979797979796, - "elementary_information_and_technology": 0.39915966386554624, - "elementary_mathematics": 0.29130434782608694, - "ethnology": 0.37777777777777777, - "food_science": 0.3916083916083916, - "genetics": 0.3068181818181818, - "global_facts": 0.4161073825503356, - "high_school_biology": 0.31952662721893493, - "high_school_chemistry": 0.25757575757575757, - "high_school_geography": 0.3728813559322034, - "high_school_mathematics": 0.23170731707317074, - "high_school_physics": 0.2545454545454545, - "high_school_politics": 0.36363636363636365, - "human_sexuality": 0.3888888888888889, - "international_law": 0.35135135135135137, - "journalism": 0.36046511627906974, - "jurisprudence": 0.36009732360097324, - "legal_and_moral_basis": 0.5700934579439252, - "logical": 0.2926829268292683, - "machine_learning": 0.3114754098360656, - "management": 0.3523809523809524, - "marketing": 0.36666666666666664, - "marxist_theory": 0.3862433862433862, - "modern_chinese": 0.31896551724137934, - "nutrition": 0.3310344827586207, - "philosophy": 0.2571428571428571, - "professional_accounting": 0.35428571428571426, - "professional_law": 0.26066350710900477, - "professional_medicine": 0.300531914893617, - "professional_psychology": 0.3448275862068966, - "public_relations": 0.3620689655172414, - "security_study": 0.34074074074074073, - "sociology": 0.3893805309734513, - "sports_science": 0.32727272727272727, - "traditional_chinese_medicine": 0.24324324324324326, - "virology": 0.33727810650887574, - "world_history": 0.40372670807453415, - "world_religions": 0.30625 - } - }, - "prompt_4": { - "accuracy": 0.3463995855638059, - "category_acc": { - "agronomy": 0.33136094674556216, - "anatomy": 0.23648648648648649, - "ancient_chinese": 0.27439024390243905, - "arts": 0.35, - "astronomy": 0.2787878787878788, - "business_ethics": 0.3349282296650718, - "chinese_civil_service_exam": 0.3125, - "chinese_driving_rule": 0.4351145038167939, - "chinese_food_culture": 0.34558823529411764, - "chinese_foreign_policy": 0.48598130841121495, - "chinese_history": 0.39628482972136225, - "chinese_literature": 0.2647058823529412, - "chinese_teacher_qualification": 0.4301675977653631, - "clinical_knowledge": 0.379746835443038, - "college_actuarial_science": 0.3490566037735849, - "college_education": 0.4485981308411215, - "college_engineering_hydrology": 0.33962264150943394, - "college_law": 0.25, - "college_mathematics": 0.20952380952380953, - "college_medical_statistics": 0.4056603773584906, - "college_medicine": 0.336996336996337, - "computer_science": 0.36764705882352944, - "computer_security": 0.2982456140350877, - "conceptual_physics": 0.3197278911564626, - "construction_project_management": 0.34532374100719426, - "economics": 0.42138364779874216, - "education": 0.3987730061349693, - "electrical_engineering": 0.3372093023255814, - "elementary_chinese": 0.31746031746031744, - "elementary_commonsense": 0.29797979797979796, - "elementary_information_and_technology": 0.3949579831932773, - "elementary_mathematics": 0.2826086956521739, - "ethnology": 0.35555555555555557, - "food_science": 0.43356643356643354, - "genetics": 0.2727272727272727, - "global_facts": 0.3422818791946309, - "high_school_biology": 0.31952662721893493, - "high_school_chemistry": 0.2878787878787879, - "high_school_geography": 0.3474576271186441, - "high_school_mathematics": 0.2865853658536585, - "high_school_physics": 0.3, - "high_school_politics": 0.3986013986013986, - "human_sexuality": 0.3888888888888889, - "international_law": 0.32432432432432434, - "journalism": 0.4011627906976744, - "jurisprudence": 0.3819951338199513, - "legal_and_moral_basis": 0.5747663551401869, - "logical": 0.3170731707317073, - "machine_learning": 0.21311475409836064, - "management": 0.38095238095238093, - "marketing": 0.35555555555555557, - "marxist_theory": 0.41798941798941797, - "modern_chinese": 0.3017241379310345, - "nutrition": 0.3448275862068966, - "philosophy": 0.34285714285714286, - "professional_accounting": 0.35428571428571426, - "professional_law": 0.2985781990521327, - "professional_medicine": 0.2579787234042553, - "professional_psychology": 0.33620689655172414, - "public_relations": 0.41954022988505746, - "security_study": 0.35555555555555557, - "sociology": 0.41150442477876104, - "sports_science": 0.3393939393939394, - "traditional_chinese_medicine": 0.23783783783783785, - "virology": 0.31952662721893493, - "world_history": 0.40993788819875776, - "world_religions": 0.34375 - } - }, - "prompt_5": { - "accuracy": 0.3384562251769988, - "category_acc": { - "agronomy": 0.3668639053254438, - "anatomy": 0.2635135135135135, - "ancient_chinese": 0.23170731707317074, - "arts": 0.3, - "astronomy": 0.2606060606060606, - "business_ethics": 0.40669856459330145, - "chinese_civil_service_exam": 0.3625, - "chinese_driving_rule": 0.46564885496183206, - "chinese_food_culture": 0.3014705882352941, - "chinese_foreign_policy": 0.411214953271028, - "chinese_history": 0.4055727554179567, - "chinese_literature": 0.3137254901960784, - "chinese_teacher_qualification": 0.3854748603351955, - "clinical_knowledge": 0.34177215189873417, - "college_actuarial_science": 0.32075471698113206, - "college_education": 0.4205607476635514, - "college_engineering_hydrology": 0.3113207547169811, - "college_law": 0.2777777777777778, - "college_mathematics": 0.2, - "college_medical_statistics": 0.2830188679245283, - "college_medicine": 0.315018315018315, - "computer_science": 0.39215686274509803, - "computer_security": 0.3333333333333333, - "conceptual_physics": 0.36054421768707484, - "construction_project_management": 0.28776978417266186, - "economics": 0.3836477987421384, - "education": 0.32515337423312884, - "electrical_engineering": 0.3546511627906977, - "elementary_chinese": 0.2896825396825397, - "elementary_commonsense": 0.2777777777777778, - "elementary_information_and_technology": 0.41596638655462187, - "elementary_mathematics": 0.28695652173913044, - "ethnology": 0.37777777777777777, - "food_science": 0.4405594405594406, - "genetics": 0.2727272727272727, - "global_facts": 0.3691275167785235, - "high_school_biology": 0.2603550295857988, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.3559322033898305, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.34545454545454546, - "high_school_politics": 0.36363636363636365, - "human_sexuality": 0.3888888888888889, - "international_law": 0.3081081081081081, - "journalism": 0.38953488372093026, - "jurisprudence": 0.36009732360097324, - "legal_and_moral_basis": 0.5280373831775701, - "logical": 0.24390243902439024, - "machine_learning": 0.28688524590163933, - "management": 0.38571428571428573, - "marketing": 0.4, - "marxist_theory": 0.4074074074074074, - "modern_chinese": 0.2672413793103448, - "nutrition": 0.3448275862068966, - "philosophy": 0.3047619047619048, - "professional_accounting": 0.3657142857142857, - "professional_law": 0.2843601895734597, - "professional_medicine": 0.26063829787234044, - "professional_psychology": 0.36637931034482757, - "public_relations": 0.39655172413793105, - "security_study": 0.32592592592592595, - "sociology": 0.33185840707964603, - "sports_science": 0.3575757575757576, - "traditional_chinese_medicine": 0.24864864864864866, - "virology": 0.3609467455621302, - "world_history": 0.35403726708074534, - "world_religions": 0.325 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.21212121212121213 - }, - "prompt_2": { - "accuracy": 0.30303030303030304 - }, - "prompt_3": { - "accuracy": 0.21212121212121213 - }, - "prompt_4": { - "accuracy": 0.3333333333333333 - }, - "prompt_5": { - "accuracy": 0.24242424242424243 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.16818181818181818 - }, - "prompt_2": { - "accuracy": 0.16590909090909092 - }, - "prompt_3": { - "accuracy": 0.11363636363636363 - }, - "prompt_4": { - "accuracy": 0.5431818181818182 - }, - "prompt_5": { - "accuracy": 0.5159090909090909 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3264406779661017 - }, - "prompt_2": { - "accuracy": 0.336271186440678 - }, - "prompt_3": { - "accuracy": 0.3369491525423729 - }, - "prompt_4": { - "accuracy": 0.32508474576271185 - }, - "prompt_5": { - "accuracy": 0.3288135593220339 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.5100972326103216 - }, - "prompt_2": { - "accuracy": 0.4913986537023186 - }, - "prompt_3": { - "accuracy": 0.4760658189977562 - }, - "prompt_4": { - "accuracy": 0.5220643231114436 - }, - "prompt_5": { - "accuracy": 0.5287958115183246 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.7751102400783929 - }, - "prompt_2": { - "accuracy": 0.7427731504164625 - }, - "prompt_3": { - "accuracy": 0.7898089171974523 - }, - "prompt_4": { - "accuracy": 0.7751102400783929 - }, - "prompt_5": { - "accuracy": 0.7956883880450759 - } - }, - "samsum": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "dialogsum": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "sst2": { - "prompt_1": { - "accuracy": 0.8738532110091743 - }, - "prompt_2": { - "accuracy": 0.7236238532110092 - }, - "prompt_3": { - "accuracy": 0.6915137614678899 - }, - "prompt_4": { - "accuracy": 0.7775229357798165 - }, - "prompt_5": { - "accuracy": 0.8165137614678899 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.4956855225311601 - }, - "prompt_2": { - "accuracy": 0.5062320230105465 - }, - "prompt_3": { - "accuracy": 0.5436241610738255 - }, - "prompt_4": { - "accuracy": 0.5580057526366251 - }, - "prompt_5": { - "accuracy": 0.5906040268456376 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.5395 - }, - "prompt_2": { - "accuracy": 0.625 - }, - "prompt_3": { - "accuracy": 0.5405 - }, - "prompt_4": { - "accuracy": 0.5265 - }, - "prompt_5": { - "accuracy": 0.6 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.39 - }, - "prompt_2": { - "accuracy": 0.381 - }, - "prompt_3": { - "accuracy": 0.406 - }, - "prompt_4": { - "accuracy": 0.392 - }, - "prompt_5": { - "accuracy": 0.351 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.521 - }, - "prompt_2": { - "accuracy": 0.516 - }, - "prompt_3": { - "accuracy": 0.5105 - }, - "prompt_4": { - "accuracy": 0.525 - }, - "prompt_5": { - "accuracy": 0.4985 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.4647887323943662 - }, - "prompt_2": { - "accuracy": 0.5070422535211268 - }, - "prompt_3": { - "accuracy": 0.4788732394366197 - }, - "prompt_4": { - "accuracy": 0.43661971830985913 - }, - "prompt_5": { - "accuracy": 0.5633802816901409 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.5776173285198556 - }, - "prompt_2": { - "accuracy": 0.5415162454873647 - }, - "prompt_3": { - "accuracy": 0.5631768953068592 - }, - "prompt_4": { - "accuracy": 0.5740072202166066 - }, - "prompt_5": { - "accuracy": 0.5342960288808665 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.5024509803921569 - }, - "prompt_2": { - "accuracy": 0.678921568627451 - }, - "prompt_3": { - "accuracy": 0.44607843137254904 - }, - "prompt_4": { - "accuracy": 0.39705882352941174 - }, - "prompt_5": { - "accuracy": 0.4803921568627451 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.8151260504201681, - "language_acc": { - "Spanish": 0.8403361344537815, - "English": 0.8815126050420168, - "Chinese": 0.8042016806722689, - "Vietnamese": 0.7344537815126051 - }, - "consistency_score_2": 0.7530812324929972, - "consistency_score_3": 0.6453781512605042, - "consistency_score_4": 0.5705882352941176, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8411764705882353, - "Spanish,Chinese": 0.7756302521008404, - "Spanish,Vietnamese": 0.7159663865546219, - "English,Chinese": 0.7848739495798319, - "English,Vietnamese": 0.7218487394957983, - "Chinese,Vietnamese": 0.6789915966386555 - }, - "3_combine": { - "Spanish,English,Chinese": 0.7109243697478992, - "Spanish,English,Vietnamese": 0.6529411764705882, - "Spanish,Chinese,Vietnamese": 0.6050420168067226, - "English,Chinese,Vietnamese": 0.6126050420168068 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.5705882352941176 - } - }, - "AC3_2": 0.7828762655764503, - "AC3_3": 0.7203875796085455, - "AC3_4": 0.6712802767681662 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.4123809523809524, - "language_acc": { - "Filipino": 0.3333333333333333, - "Vietnamese": 0.36666666666666664, - "Chinese": 0.36666666666666664, - "Spanish": 0.5066666666666667, - "Malay": 0.3933333333333333, - "Indonesian": 0.47333333333333333, - "English": 0.44666666666666666 - }, - "consistency_score_2": 0.4012698412698412, - "consistency_score_3": 0.19257142857142862, - "consistency_score_4": 0.10609523809523808, - "consistency_score_5": 0.06761904761904762, - "consistency_score_6": 0.049523809523809526, - "consistency_score_7": 0.04, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.4, - "Filipino,Chinese": 0.34, - "Filipino,Spanish": 0.38, - "Filipino,Malay": 0.36666666666666664, - "Filipino,Indonesian": 0.44666666666666666, - "Filipino,English": 0.44, - "Vietnamese,Chinese": 0.36, - "Vietnamese,Spanish": 0.38, - "Vietnamese,Malay": 0.37333333333333335, - "Vietnamese,Indonesian": 0.36, - "Vietnamese,English": 0.4066666666666667, - "Chinese,Spanish": 0.4266666666666667, - "Chinese,Malay": 0.46, - "Chinese,Indonesian": 0.4066666666666667, - "Chinese,English": 0.38666666666666666, - "Spanish,Malay": 0.3933333333333333, - "Spanish,Indonesian": 0.4533333333333333, - "Spanish,English": 0.42, - "Malay,Indonesian": 0.46, - "Malay,English": 0.35333333333333333, - "Indonesian,English": 0.41333333333333333 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.15333333333333332, - "Filipino,Vietnamese,Spanish": 0.18, - "Filipino,Vietnamese,Malay": 0.16666666666666666, - "Filipino,Vietnamese,Indonesian": 0.18666666666666668, - "Filipino,Vietnamese,English": 0.20666666666666667, - "Filipino,Chinese,Spanish": 0.16666666666666666, - "Filipino,Chinese,Malay": 0.21333333333333335, - "Filipino,Chinese,Indonesian": 0.2, - "Filipino,Chinese,English": 0.18, - "Filipino,Spanish,Malay": 0.16666666666666666, - "Filipino,Spanish,Indonesian": 0.20666666666666667, - "Filipino,Spanish,English": 0.19333333333333333, - "Filipino,Malay,Indonesian": 0.2, - "Filipino,Malay,English": 0.18, - "Filipino,Indonesian,English": 0.22, - "Vietnamese,Chinese,Spanish": 0.21333333333333335, - "Vietnamese,Chinese,Malay": 0.19333333333333333, - "Vietnamese,Chinese,Indonesian": 0.18666666666666668, - "Vietnamese,Chinese,English": 0.18, - "Vietnamese,Spanish,Malay": 0.18666666666666668, - "Vietnamese,Spanish,Indonesian": 0.19333333333333333, - "Vietnamese,Spanish,English": 0.19333333333333333, - "Vietnamese,Malay,Indonesian": 0.18, - "Vietnamese,Malay,English": 0.16666666666666666, - "Vietnamese,Indonesian,English": 0.17333333333333334, - "Chinese,Spanish,Malay": 0.23333333333333334, - "Chinese,Spanish,Indonesian": 0.24666666666666667, - "Chinese,Spanish,English": 0.19333333333333333, - "Chinese,Malay,Indonesian": 0.24, - "Chinese,Malay,English": 0.20666666666666667, - "Chinese,Indonesian,English": 0.18666666666666668, - "Spanish,Malay,Indonesian": 0.20666666666666667, - "Spanish,Malay,English": 0.16, - "Spanish,Indonesian,English": 0.20666666666666667, - "Malay,Indonesian,English": 0.17333333333333334 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.11333333333333333, - "Filipino,Vietnamese,Chinese,Malay": 0.08666666666666667, - "Filipino,Vietnamese,Chinese,Indonesian": 0.10666666666666667, - "Filipino,Vietnamese,Chinese,English": 0.10666666666666667, - "Filipino,Vietnamese,Spanish,Malay": 0.11333333333333333, - "Filipino,Vietnamese,Spanish,Indonesian": 0.09333333333333334, - "Filipino,Vietnamese,Spanish,English": 0.10666666666666667, - "Filipino,Vietnamese,Malay,Indonesian": 0.08666666666666667, - "Filipino,Vietnamese,Malay,English": 0.09333333333333334, - "Filipino,Vietnamese,Indonesian,English": 0.11333333333333333, - "Filipino,Chinese,Spanish,Malay": 0.10666666666666667, - "Filipino,Chinese,Spanish,Indonesian": 0.10666666666666667, - "Filipino,Chinese,Spanish,English": 0.09333333333333334, - "Filipino,Chinese,Malay,Indonesian": 0.12666666666666668, - "Filipino,Chinese,Malay,English": 0.12, - "Filipino,Chinese,Indonesian,English": 0.10666666666666667, - "Filipino,Spanish,Malay,Indonesian": 0.1, - "Filipino,Spanish,Malay,English": 0.09333333333333334, - "Filipino,Spanish,Indonesian,English": 0.11333333333333333, - "Filipino,Malay,Indonesian,English": 0.1, - "Vietnamese,Chinese,Spanish,Malay": 0.12666666666666668, - "Vietnamese,Chinese,Spanish,Indonesian": 0.14666666666666667, - "Vietnamese,Chinese,Spanish,English": 0.11333333333333333, - "Vietnamese,Chinese,Malay,Indonesian": 0.09333333333333334, - "Vietnamese,Chinese,Malay,English": 0.11333333333333333, - "Vietnamese,Chinese,Indonesian,English": 0.11333333333333333, - "Vietnamese,Spanish,Malay,Indonesian": 0.1, - "Vietnamese,Spanish,Malay,English": 0.08, - "Vietnamese,Spanish,Indonesian,English": 0.10666666666666667, - "Vietnamese,Malay,Indonesian,English": 0.08666666666666667, - "Chinese,Spanish,Malay,Indonesian": 0.14, - "Chinese,Spanish,Malay,English": 0.11333333333333333, - "Chinese,Spanish,Indonesian,English": 0.10666666666666667, - "Chinese,Malay,Indonesian,English": 0.09333333333333334, - "Spanish,Malay,Indonesian,English": 0.09333333333333334 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.07333333333333333, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.08, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.08, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.05333333333333334, - "Filipino,Vietnamese,Chinese,Malay,English": 0.07333333333333333, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.08, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.06, - "Filipino,Vietnamese,Spanish,Malay,English": 0.06666666666666667, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.06666666666666667, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.06, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.06666666666666667, - "Filipino,Chinese,Spanish,Malay,English": 0.07333333333333333, - "Filipino,Chinese,Spanish,Indonesian,English": 0.06, - "Filipino,Chinese,Malay,Indonesian,English": 0.06, - "Filipino,Spanish,Malay,Indonesian,English": 0.06, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.08, - "Vietnamese,Chinese,Spanish,Malay,English": 0.07333333333333333, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.08666666666666667, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.06, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.05333333333333334, - "Chinese,Spanish,Malay,Indonesian,English": 0.05333333333333334 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.04666666666666667, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.06, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.06, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.04666666666666667, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.04666666666666667, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.04, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.04666666666666667 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.04 - } - }, - "AC3_2": 0.4067495308197118, - "AC3_3": 0.2625422813527869, - "AC3_4": 0.1687701619631802, - "AC3_5": 0.11618669687677705, - "AC3_6": 0.0884280804914105, - "AC3_7": 0.07292631577335314 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.387987012987013, - "language_acc": { - "Indonesian": 0.4772727272727273, - "English": 0.44886363636363635, - "Filipino": 0.30113636363636365, - "Spanish": 0.4431818181818182, - "Chinese": 0.375, - "Malay": 0.32386363636363635, - "Vietnamese": 0.3465909090909091 - }, - "consistency_score_2": 0.41693722943722944, - "consistency_score_3": 0.21915584415584421, - "consistency_score_4": 0.12938311688311688, - "consistency_score_5": 0.07954545454545456, - "consistency_score_6": 0.04870129870129869, - "consistency_score_7": 0.028409090909090908, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6079545454545454, - "Indonesian,Filipino": 0.375, - "Indonesian,Spanish": 0.5056818181818182, - "Indonesian,Chinese": 0.4375, - "Indonesian,Malay": 0.5170454545454546, - "Indonesian,Vietnamese": 0.45454545454545453, - "English,Filipino": 0.3977272727272727, - "English,Spanish": 0.48295454545454547, - "English,Chinese": 0.4090909090909091, - "English,Malay": 0.45454545454545453, - "English,Vietnamese": 0.4318181818181818, - "Filipino,Spanish": 0.3352272727272727, - "Filipino,Chinese": 0.3125, - "Filipino,Malay": 0.3465909090909091, - "Filipino,Vietnamese": 0.375, - "Spanish,Chinese": 0.42613636363636365, - "Spanish,Malay": 0.42613636363636365, - "Spanish,Vietnamese": 0.35795454545454547, - "Chinese,Malay": 0.3977272727272727, - "Chinese,Vietnamese": 0.32954545454545453, - "Malay,Vietnamese": 0.375 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.25, - "Indonesian,English,Spanish": 0.35795454545454547, - "Indonesian,English,Chinese": 0.29545454545454547, - "Indonesian,English,Malay": 0.3465909090909091, - "Indonesian,English,Vietnamese": 0.3181818181818182, - "Indonesian,Filipino,Spanish": 0.2159090909090909, - "Indonesian,Filipino,Chinese": 0.17613636363636365, - "Indonesian,Filipino,Malay": 0.21022727272727273, - "Indonesian,Filipino,Vietnamese": 0.19886363636363635, - "Indonesian,Spanish,Chinese": 0.26704545454545453, - "Indonesian,Spanish,Malay": 0.29545454545454547, - "Indonesian,Spanish,Vietnamese": 0.23295454545454544, - "Indonesian,Chinese,Malay": 0.25, - "Indonesian,Chinese,Vietnamese": 0.19318181818181818, - "Indonesian,Malay,Vietnamese": 0.2784090909090909, - "English,Filipino,Spanish": 0.19886363636363635, - "English,Filipino,Chinese": 0.1590909090909091, - "English,Filipino,Malay": 0.19886363636363635, - "English,Filipino,Vietnamese": 0.19318181818181818, - "English,Spanish,Chinese": 0.26136363636363635, - "English,Spanish,Malay": 0.2727272727272727, - "English,Spanish,Vietnamese": 0.22727272727272727, - "English,Chinese,Malay": 0.23295454545454544, - "English,Chinese,Vietnamese": 0.1875, - "English,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Chinese": 0.1534090909090909, - "Filipino,Spanish,Malay": 0.16477272727272727, - "Filipino,Spanish,Vietnamese": 0.1534090909090909, - "Filipino,Chinese,Malay": 0.13068181818181818, - "Filipino,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Malay,Vietnamese": 0.16477272727272727, - "Spanish,Chinese,Malay": 0.22727272727272727, - "Spanish,Chinese,Vietnamese": 0.16477272727272727, - "Spanish,Malay,Vietnamese": 0.19318181818181818, - "Chinese,Malay,Vietnamese": 0.1590909090909091 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.1534090909090909, - "Indonesian,English,Filipino,Chinese": 0.125, - "Indonesian,English,Filipino,Malay": 0.16477272727272727, - "Indonesian,English,Filipino,Vietnamese": 0.14772727272727273, - "Indonesian,English,Spanish,Chinese": 0.20454545454545456, - "Indonesian,English,Spanish,Malay": 0.22727272727272727, - "Indonesian,English,Spanish,Vietnamese": 0.18181818181818182, - "Indonesian,English,Chinese,Malay": 0.1875, - "Indonesian,English,Chinese,Vietnamese": 0.14772727272727273, - "Indonesian,English,Malay,Vietnamese": 0.19318181818181818, - "Indonesian,Filipino,Spanish,Chinese": 0.125, - "Indonesian,Filipino,Spanish,Malay": 0.125, - "Indonesian,Filipino,Spanish,Vietnamese": 0.11931818181818182, - "Indonesian,Filipino,Chinese,Malay": 0.09090909090909091, - "Indonesian,Filipino,Chinese,Vietnamese": 0.08522727272727272, - "Indonesian,Filipino,Malay,Vietnamese": 0.13068181818181818, - "Indonesian,Spanish,Chinese,Malay": 0.1590909090909091, - "Indonesian,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "Indonesian,Spanish,Malay,Vietnamese": 0.1534090909090909, - "Indonesian,Chinese,Malay,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese": 0.10227272727272728, - "English,Filipino,Spanish,Malay": 0.10795454545454546, - "English,Filipino,Spanish,Vietnamese": 0.10795454545454546, - "English,Filipino,Chinese,Malay": 0.09090909090909091, - "English,Filipino,Chinese,Vietnamese": 0.07954545454545454, - "English,Filipino,Malay,Vietnamese": 0.11931818181818182, - "English,Spanish,Chinese,Malay": 0.16477272727272727, - "English,Spanish,Chinese,Vietnamese": 0.125, - "English,Spanish,Malay,Vietnamese": 0.14204545454545456, - "English,Chinese,Malay,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Chinese,Malay": 0.07386363636363637, - "Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, - "Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, - "Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.08522727272727272, - "Indonesian,English,Filipino,Spanish,Malay": 0.09659090909090909, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09090909090909091, - "Indonesian,English,Filipino,Chinese,Malay": 0.07954545454545454, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.06818181818181818, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.10227272727272728, - "Indonesian,English,Spanish,Chinese,Malay": 0.13636363636363635, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.11931818181818182, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.10227272727272728, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.0625, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.06818181818181818, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.05113636363636364, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Malay": 0.056818181818181816, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.056818181818181816, - "English,Filipino,Spanish,Malay,Vietnamese": 0.06818181818181818, - "English,Filipino,Chinese,Malay,Vietnamese": 0.056818181818181816, - "English,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03977272727272727 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.05113636363636364, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.05113636363636364, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.056818181818181816, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.045454545454545456, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.03409090909090909 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.028409090909090908 - } - }, - "AC3_2": 0.4019415038243417, - "AC3_3": 0.2800975761744372, - "AC3_4": 0.19405437671219364, - "AC3_5": 0.13202335855762026, - "AC3_6": 0.08653985417802944, - "AC3_7": 0.05294169766582552 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.20388349514563106 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.23809523809523808 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.42990654205607476 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.37, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.6, - "culture": 0.5, - "film": 0.3, - "law": 0.4, - "geography": 0.3 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.08710988278002887 - } - }, - "indommlu": { - "prompt_1": { - "accuracy": 0.39114760664930903, - "category_acc": { - "History": 0.4036144578313253, - "Geography": 0.3346938775510204, - "Lampungic": 0.3333333333333333, - "Social science": 0.5258764607679466, - "Balinese": 0.2760084925690021, - "Makassarese": 0.3279569892473118, - "Banjarese": 0.3055555555555556, - "Chemistry": 0.24525547445255474, - "Biology": 0.3585798816568047, - "Science": 0.46439628482972134, - "Christian religion": 0.4527363184079602, - "Art": 0.43926788685524126, - "Islam religion": 0.4665718349928876, - "Hindu religion": 0.4266666666666667, - "Madurese": 0.3050847457627119, - "Sport": 0.44594594594594594, - "Indonesian language": 0.44800747198007473, - "Physics": 0.301010101010101, - "Minangkabau culture": 0.35175879396984927, - "Dayak language": 0.22935779816513763, - "Sociology": 0.3790322580645161, - "Economy": 0.375, - "Sundanese": 0.3353500432152118, - "Javanese": 0.30544354838709675, - "Civic education": 0.4663805436337625 - } - } - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.1201169256288979 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.0971127191886138 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.07748224463156872 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.11766283521247839 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.4294049008168028 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.41358598498391136, - "category_acc": { - "high_school_european_history": 0.6097560975609756, - "business_ethics": 0.43434343434343436, - "clinical_knowledge": 0.4015151515151515, - "medical_genetics": 0.47474747474747475, - "high_school_us_history": 0.5763546798029556, - "high_school_physics": 0.3333333333333333, - "high_school_world_history": 0.5508474576271186, - "virology": 0.38181818181818183, - "high_school_microeconomics": 0.4345991561181435, - "econometrics": 0.336283185840708, - "college_computer_science": 0.3838383838383838, - "high_school_biology": 0.5048543689320388, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.3274021352313167, - "philosophy": 0.4645161290322581, - "professional_medicine": 0.6051660516605166, - "nutrition": 0.4459016393442623, - "global_facts": 0.2727272727272727, - "machine_learning": 0.40540540540540543, - "security_studies": 0.5040983606557377, - "public_relations": 0.3577981651376147, - "professional_psychology": 0.45662847790507366, - "prehistory": 0.4241486068111455, - "anatomy": 0.3208955223880597, - "human_sexuality": 0.4153846153846154, - "college_medicine": 0.38372093023255816, - "high_school_government_and_politics": 0.5677083333333334, - "college_chemistry": 0.3434343434343434, - "logical_fallacies": 0.4691358024691358, - "high_school_geography": 0.5126903553299492, - "elementary_mathematics": 0.3156498673740053, - "human_aging": 0.4774774774774775, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.4852941176470588, - "formal_logic": 0.392, - "high_school_statistics": 0.40930232558139534, - "international_law": 0.4583333333333333, - "high_school_mathematics": 0.24535315985130113, - "high_school_computer_science": 0.46464646464646464, - "conceptual_physics": 0.3547008547008547, - "miscellaneous": 0.4437340153452685, - "high_school_chemistry": 0.33663366336633666, - "marketing": 0.47639484978540775, - "professional_law": 0.3522504892367906, - "management": 0.4117647058823529, - "college_physics": 0.31683168316831684, - "jurisprudence": 0.514018691588785, - "world_religions": 0.48823529411764705, - "sociology": 0.555, - "us_foreign_policy": 0.47474747474747475, - "high_school_macroeconomics": 0.390745501285347, - "computer_security": 0.47474747474747475, - "moral_scenarios": 0.25615212527964204, - "moral_disputes": 0.4028985507246377, - "electrical_engineering": 0.3472222222222222, - "astronomy": 0.44370860927152317, - "college_biology": 0.5104895104895105 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.3395245170876672 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.32627646326276466, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.16666666666666666, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.35714285714285715, - "college_physics": 0.16666666666666666, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.09523809523809523, - "metrology_engineer": 0.1724137931034483, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.5416666666666666, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.4166666666666667, - "middle_school_chemistry": 0.4, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.38333333333333336, - "business_administration": 0.18421052631578946, - "marxism": 0.375, - "mao_zedong_thought": 0.4482758620689655, - "education_science": 0.35294117647058826, - "teacher_qualification": 0.40816326530612246, - "high_school_politics": 0.25, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.38461538461538464, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.5416666666666666, - "logic": 0.2222222222222222, - "law": 0.41379310344827586, - "chinese_language_and_literature": 0.32142857142857145, - "art_studies": 0.42105263157894735, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.35714285714285715, - "high_school_chinese": 0.16666666666666666, - "high_school_history": 0.44, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.36538461538461536, - "sports_science": 0.2916666666666667, - "plant_protection": 0.25925925925925924, - "basic_medicine": 0.20833333333333334, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.3137254901960784, - "accountant": 0.4074074074074074, - "fire_engineer": 0.25, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.2222222222222222, - "physician": 0.3333333333333333 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.37992831541218636 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.37696425487825935, - "category_acc": { - "agronomy": 0.3609467455621302, - "anatomy": 0.3108108108108108, - "ancient_chinese": 0.2804878048780488, - "arts": 0.3625, - "astronomy": 0.32727272727272727, - "business_ethics": 0.3827751196172249, - "chinese_civil_service_exam": 0.35, - "chinese_driving_rule": 0.5038167938931297, - "chinese_food_culture": 0.3161764705882353, - "chinese_foreign_policy": 0.3644859813084112, - "chinese_history": 0.4148606811145511, - "chinese_literature": 0.3431372549019608, - "chinese_teacher_qualification": 0.4301675977653631, - "clinical_knowledge": 0.3291139240506329, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.42990654205607476, - "college_engineering_hydrology": 0.3490566037735849, - "college_law": 0.35185185185185186, - "college_mathematics": 0.21904761904761905, - "college_medical_statistics": 0.3018867924528302, - "college_medicine": 0.32234432234432236, - "computer_science": 0.4215686274509804, - "computer_security": 0.47368421052631576, - "conceptual_physics": 0.36054421768707484, - "construction_project_management": 0.38848920863309355, - "economics": 0.4088050314465409, - "education": 0.48466257668711654, - "electrical_engineering": 0.4011627906976744, - "elementary_chinese": 0.3134920634920635, - "elementary_commonsense": 0.3787878787878788, - "elementary_information_and_technology": 0.4831932773109244, - "elementary_mathematics": 0.2826086956521739, - "ethnology": 0.4074074074074074, - "food_science": 0.4125874125874126, - "genetics": 0.36363636363636365, - "global_facts": 0.4161073825503356, - "high_school_biology": 0.31952662721893493, - "high_school_chemistry": 0.2196969696969697, - "high_school_geography": 0.3983050847457627, - "high_school_mathematics": 0.3048780487804878, - "high_school_physics": 0.34545454545454546, - "high_school_politics": 0.32867132867132864, - "human_sexuality": 0.42857142857142855, - "international_law": 0.3675675675675676, - "journalism": 0.45348837209302323, - "jurisprudence": 0.3746958637469586, - "legal_and_moral_basis": 0.6448598130841121, - "logical": 0.34146341463414637, - "machine_learning": 0.30327868852459017, - "management": 0.44761904761904764, - "marketing": 0.4444444444444444, - "marxist_theory": 0.4497354497354497, - "modern_chinese": 0.2413793103448276, - "nutrition": 0.3586206896551724, - "philosophy": 0.3142857142857143, - "professional_accounting": 0.38285714285714284, - "professional_law": 0.3033175355450237, - "professional_medicine": 0.300531914893617, - "professional_psychology": 0.3922413793103448, - "public_relations": 0.45977011494252873, - "security_study": 0.3925925925925926, - "sociology": 0.4690265486725664, - "sports_science": 0.4303030303030303, - "traditional_chinese_medicine": 0.2918918918918919, - "virology": 0.3905325443786982, - "world_history": 0.40993788819875776, - "world_religions": 0.4 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.36363636363636365 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.4863636363636364 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3403389830508475 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.5426327599102468 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.7961783439490446 - } - }, - "samsum": { - "prompt_1": -1 - }, - "dialogsum": { - "prompt_1": -1 - }, - "sst2": { - "prompt_1": { - "accuracy": 0.5103211009174312 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.3374880153403643 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.5915 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.4 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.5185 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.5352112676056338 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.4981949458483754 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.4338235294117647 - } - } - } - }, - "sea_mistral_highest_acc_inst_7b": { - "model_size": "7B", - "model_link": "https://seaeval.github.io/", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.8838235294117648, - "language_acc": { - "Spanish": 0.9, - "English": 0.9243697478991597, - "Chinese": 0.8705882352941177, - "Vietnamese": 0.8403361344537815 - }, - "consistency_score_2": 0.8591036414565826, - "consistency_score_3": 0.7989495798319327, - "consistency_score_4": 0.761344537815126, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.9210084033613445, - "Spanish,Chinese": 0.8596638655462185, - "Spanish,Vietnamese": 0.8436974789915966, - "English,Chinese": 0.8689075630252101, - "English,Vietnamese": 0.8420168067226891, - "Chinese,Vietnamese": 0.819327731092437 - }, - "3_combine": { - "Spanish,English,Chinese": 0.8336134453781513, - "Spanish,English,Vietnamese": 0.8117647058823529, - "Spanish,Chinese,Vietnamese": 0.7739495798319328, - "English,Chinese,Vietnamese": 0.7764705882352941 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.761344537815126 - } - }, - "AC3_2": 0.8712882846399156, - "AC3_3": 0.8392461628347567, - "AC3_4": 0.8180248934730036 - }, - "prompt_2": { - "overall_acc": 0.882983193277311, - "language_acc": { - "Spanish": 0.9058823529411765, - "English": 0.9201680672268907, - "Chinese": 0.8663865546218488, - "Vietnamese": 0.8394957983193277 - }, - "consistency_score_2": 0.8591036414565826, - "consistency_score_3": 0.7983193277310924, - "consistency_score_4": 0.7588235294117647, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.9176470588235294, - "Spanish,Chinese": 0.8638655462184874, - "Spanish,Vietnamese": 0.8403361344537815, - "English,Chinese": 0.8689075630252101, - "English,Vietnamese": 0.8403361344537815, - "Chinese,Vietnamese": 0.8235294117647058 - }, - "3_combine": { - "Spanish,English,Chinese": 0.8336134453781513, - "Spanish,English,Vietnamese": 0.807563025210084, - "Spanish,Chinese,Vietnamese": 0.7747899159663866, - "English,Chinese,Vietnamese": 0.7773109243697479 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.7588235294117647 - } - }, - "AC3_2": 0.8708797535477946, - "AC3_3": 0.8385195887177843, - "AC3_4": 0.8162086484768751 - }, - "prompt_3": { - "overall_acc": 0.8819327731092438, - "language_acc": { - "Spanish": 0.9008403361344538, - "English": 0.9168067226890756, - "Chinese": 0.8697478991596639, - "Vietnamese": 0.8403361344537815 - }, - "consistency_score_2": 0.8585434173669467, - "consistency_score_3": 0.7972689075630252, - "consistency_score_4": 0.7579831932773109, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.9092436974789916, - "Spanish,Chinese": 0.8647058823529412, - "Spanish,Vietnamese": 0.8411764705882353, - "English,Chinese": 0.8680672268907563, - "English,Vietnamese": 0.8378151260504202, - "Chinese,Vietnamese": 0.8302521008403362 - }, - "3_combine": { - "Spanish,English,Chinese": 0.8294117647058824, - "Spanish,English,Vietnamese": 0.8042016806722689, - "Spanish,Chinese,Vietnamese": 0.7781512605042017, - "English,Chinese,Vietnamese": 0.7773109243697479 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.7579831932773109 - } - }, - "AC3_2": 0.8700809364849192, - "AC3_3": 0.837466501626489, - "AC3_4": 0.8152737497269066 - }, - "prompt_4": { - "overall_acc": 0.8806722689075631, - "language_acc": { - "Spanish": 0.8983193277310925, - "English": 0.9117647058823529, - "Chinese": 0.8714285714285714, - "Vietnamese": 0.8411764705882353 - }, - "consistency_score_2": 0.857703081232493, - "consistency_score_3": 0.7953781512605042, - "consistency_score_4": 0.753781512605042, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.9100840336134454, - "Spanish,Chinese": 0.8722689075630252, - "Spanish,Vietnamese": 0.8436974789915966, - "English,Chinese": 0.8647058823529412, - "English,Vietnamese": 0.83109243697479, - "Chinese,Vietnamese": 0.8243697478991596 - }, - "3_combine": { - "Spanish,English,Chinese": 0.83109243697479, - "Spanish,English,Vietnamese": 0.8016806722689076, - "Spanish,Chinese,Vietnamese": 0.7798319327731092, - "English,Chinese,Vietnamese": 0.7689075630252101 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.753781512605042 - } - }, - "AC3_2": 0.8690359288558928, - "AC3_3": 0.8358549034557785, - "AC3_4": 0.8123013112203107 - }, - "prompt_5": { - "overall_acc": 0.8733193277310926, - "language_acc": { - "Spanish": 0.8949579831932774, - "English": 0.9, - "Chinese": 0.8689075630252101, - "Vietnamese": 0.8294117647058824 - }, - "consistency_score_2": 0.8476190476190476, - "consistency_score_3": 0.7819327731092437, - "consistency_score_4": 0.7369747899159664, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8957983193277311, - "Spanish,Chinese": 0.873109243697479, - "Spanish,Vietnamese": 0.8294117647058824, - "English,Chinese": 0.8588235294117647, - "English,Vietnamese": 0.8134453781512605, - "Chinese,Vietnamese": 0.8151260504201681 - }, - "3_combine": { - "Spanish,English,Chinese": 0.8218487394957983, - "Spanish,English,Vietnamese": 0.780672268907563, - "Spanish,Chinese,Vietnamese": 0.7705882352941177, - "English,Chinese,Vietnamese": 0.7546218487394958 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.7369747899159664 - } - }, - "AC3_2": 0.8602772852283137, - "AC3_3": 0.8251033221522406, - "AC3_4": 0.7993748731873246 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.5047619047619047, - "language_acc": { - "Filipino": 0.42, - "Vietnamese": 0.38666666666666666, - "Chinese": 0.5133333333333333, - "Spanish": 0.54, - "Malay": 0.46, - "Indonesian": 0.54, - "English": 0.6733333333333333 - }, - "consistency_score_2": 0.5987301587301587, - "consistency_score_3": 0.44209523809523815, - "consistency_score_4": 0.3537142857142857, - "consistency_score_5": 0.29714285714285715, - "consistency_score_6": 0.2590476190476191, - "consistency_score_7": 0.23333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.5, - "Filipino,Chinese": 0.5333333333333333, - "Filipino,Spanish": 0.5266666666666666, - "Filipino,Malay": 0.4866666666666667, - "Filipino,Indonesian": 0.54, - "Filipino,English": 0.5066666666666667, - "Vietnamese,Chinese": 0.5533333333333333, - "Vietnamese,Spanish": 0.5066666666666667, - "Vietnamese,Malay": 0.58, - "Vietnamese,Indonesian": 0.5466666666666666, - "Vietnamese,English": 0.4666666666666667, - "Chinese,Spanish": 0.6133333333333333, - "Chinese,Malay": 0.6333333333333333, - "Chinese,Indonesian": 0.7066666666666667, - "Chinese,English": 0.64, - "Spanish,Malay": 0.68, - "Spanish,Indonesian": 0.7466666666666667, - "Spanish,English": 0.7466666666666667, - "Malay,Indonesian": 0.7333333333333333, - "Malay,English": 0.62, - "Indonesian,English": 0.7066666666666667 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.36666666666666664, - "Filipino,Vietnamese,Spanish": 0.3333333333333333, - "Filipino,Vietnamese,Malay": 0.3466666666666667, - "Filipino,Vietnamese,Indonesian": 0.34, - "Filipino,Vietnamese,English": 0.32, - "Filipino,Chinese,Spanish": 0.38666666666666666, - "Filipino,Chinese,Malay": 0.38, - "Filipino,Chinese,Indonesian": 0.4266666666666667, - "Filipino,Chinese,English": 0.3933333333333333, - "Filipino,Spanish,Malay": 0.3933333333333333, - "Filipino,Spanish,Indonesian": 0.43333333333333335, - "Filipino,Spanish,English": 0.43333333333333335, - "Filipino,Malay,Indonesian": 0.41333333333333333, - "Filipino,Malay,English": 0.37333333333333335, - "Filipino,Indonesian,English": 0.4, - "Vietnamese,Chinese,Spanish": 0.4, - "Vietnamese,Chinese,Malay": 0.43333333333333335, - "Vietnamese,Chinese,Indonesian": 0.44666666666666666, - "Vietnamese,Chinese,English": 0.3933333333333333, - "Vietnamese,Spanish,Malay": 0.4266666666666667, - "Vietnamese,Spanish,Indonesian": 0.44666666666666666, - "Vietnamese,Spanish,English": 0.4066666666666667, - "Vietnamese,Malay,Indonesian": 0.47333333333333333, - "Vietnamese,Malay,English": 0.4, - "Vietnamese,Indonesian,English": 0.4, - "Chinese,Spanish,Malay": 0.49333333333333335, - "Chinese,Spanish,Indonesian": 0.56, - "Chinese,Spanish,English": 0.5266666666666666, - "Chinese,Malay,Indonesian": 0.5533333333333333, - "Chinese,Malay,English": 0.48, - "Chinese,Indonesian,English": 0.5533333333333333, - "Spanish,Malay,Indonesian": 0.6066666666666667, - "Spanish,Malay,English": 0.54, - "Spanish,Indonesian,English": 0.6333333333333333, - "Malay,Indonesian,English": 0.56 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.29333333333333333, - "Filipino,Vietnamese,Chinese,Malay": 0.29333333333333333, - "Filipino,Vietnamese,Chinese,Indonesian": 0.30666666666666664, - "Filipino,Vietnamese,Chinese,English": 0.29333333333333333, - "Filipino,Vietnamese,Spanish,Malay": 0.2866666666666667, - "Filipino,Vietnamese,Spanish,Indonesian": 0.3, - "Filipino,Vietnamese,Spanish,English": 0.28, - "Filipino,Vietnamese,Malay,Indonesian": 0.3, - "Filipino,Vietnamese,Malay,English": 0.2733333333333333, - "Filipino,Vietnamese,Indonesian,English": 0.2733333333333333, - "Filipino,Chinese,Spanish,Malay": 0.32, - "Filipino,Chinese,Spanish,Indonesian": 0.35333333333333333, - "Filipino,Chinese,Spanish,English": 0.34, - "Filipino,Chinese,Malay,Indonesian": 0.3466666666666667, - "Filipino,Chinese,Malay,English": 0.31333333333333335, - "Filipino,Chinese,Indonesian,English": 0.34, - "Filipino,Spanish,Malay,Indonesian": 0.37333333333333335, - "Filipino,Spanish,Malay,English": 0.3333333333333333, - "Filipino,Spanish,Indonesian,English": 0.37333333333333335, - "Filipino,Malay,Indonesian,English": 0.34, - "Vietnamese,Chinese,Spanish,Malay": 0.36, - "Vietnamese,Chinese,Spanish,Indonesian": 0.38666666666666666, - "Vietnamese,Chinese,Spanish,English": 0.3466666666666667, - "Vietnamese,Chinese,Malay,Indonesian": 0.4, - "Vietnamese,Chinese,Malay,English": 0.3466666666666667, - "Vietnamese,Chinese,Indonesian,English": 0.35333333333333333, - "Vietnamese,Spanish,Malay,Indonesian": 0.4066666666666667, - "Vietnamese,Spanish,Malay,English": 0.35333333333333333, - "Vietnamese,Spanish,Indonesian,English": 0.38666666666666666, - "Vietnamese,Malay,Indonesian,English": 0.35333333333333333, - "Chinese,Spanish,Malay,Indonesian": 0.47333333333333333, - "Chinese,Spanish,Malay,English": 0.42, - "Chinese,Spanish,Indonesian,English": 0.49333333333333335, - "Chinese,Malay,Indonesian,English": 0.44, - "Spanish,Malay,Indonesian,English": 0.5266666666666666 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.26, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.28, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.26, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2733333333333333, - "Filipino,Vietnamese,Chinese,Malay,English": 0.25333333333333335, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.26, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.28, - "Filipino,Vietnamese,Spanish,Malay,English": 0.24666666666666667, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.26666666666666666, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.24666666666666667, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.31333333333333335, - "Filipino,Chinese,Spanish,Malay,English": 0.28, - "Filipino,Chinese,Spanish,Indonesian,English": 0.31333333333333335, - "Filipino,Chinese,Malay,Indonesian,English": 0.2866666666666667, - "Filipino,Spanish,Malay,Indonesian,English": 0.32666666666666666, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.36, - "Vietnamese,Chinese,Spanish,Malay,English": 0.31333333333333335, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.34, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.32, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.3466666666666667, - "Chinese,Spanish,Malay,Indonesian,English": 0.41333333333333333 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.26, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.23333333333333334, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.25333333333333335, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.23333333333333334, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.24666666666666667, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.2733333333333333, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.31333333333333335 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334 - } - }, - "AC3_2": 0.5477450818726597, - "AC3_3": 0.4713548102307988, - "AC3_4": 0.4159497923379944, - "AC3_5": 0.37407533079827865, - "AC3_6": 0.3423821398435487, - "AC3_7": 0.3191397849029982 - }, - "prompt_2": { - "overall_acc": 0.5066666666666666, - "language_acc": { - "Filipino": 0.46, - "Vietnamese": 0.3933333333333333, - "Chinese": 0.52, - "Spanish": 0.54, - "Malay": 0.4666666666666667, - "Indonesian": 0.5333333333333333, - "English": 0.6333333333333333 - }, - "consistency_score_2": 0.5825396825396826, - "consistency_score_3": 0.42361904761904773, - "consistency_score_4": 0.33257142857142846, - "consistency_score_5": 0.2726984126984127, - "consistency_score_6": 0.23047619047619047, - "consistency_score_7": 0.2, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.46, - "Filipino,Chinese": 0.5266666666666666, - "Filipino,Spanish": 0.5266666666666666, - "Filipino,Malay": 0.5, - "Filipino,Indonesian": 0.5266666666666666, - "Filipino,English": 0.5466666666666666, - "Vietnamese,Chinese": 0.52, - "Vietnamese,Spanish": 0.5066666666666667, - "Vietnamese,Malay": 0.58, - "Vietnamese,Indonesian": 0.5533333333333333, - "Vietnamese,English": 0.48, - "Chinese,Spanish": 0.64, - "Chinese,Malay": 0.6066666666666667, - "Chinese,Indonesian": 0.6333333333333333, - "Chinese,English": 0.6133333333333333, - "Spanish,Malay": 0.66, - "Spanish,Indonesian": 0.72, - "Spanish,English": 0.68, - "Malay,Indonesian": 0.72, - "Malay,English": 0.5733333333333334, - "Indonesian,English": 0.66 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.3333333333333333, - "Filipino,Vietnamese,Spanish": 0.31333333333333335, - "Filipino,Vietnamese,Malay": 0.34, - "Filipino,Vietnamese,Indonesian": 0.34, - "Filipino,Vietnamese,English": 0.31333333333333335, - "Filipino,Chinese,Spanish": 0.3933333333333333, - "Filipino,Chinese,Malay": 0.36666666666666664, - "Filipino,Chinese,Indonesian": 0.37333333333333335, - "Filipino,Chinese,English": 0.3933333333333333, - "Filipino,Spanish,Malay": 0.4, - "Filipino,Spanish,Indonesian": 0.43333333333333335, - "Filipino,Spanish,English": 0.43333333333333335, - "Filipino,Malay,Indonesian": 0.4266666666666667, - "Filipino,Malay,English": 0.38, - "Filipino,Indonesian,English": 0.4, - "Vietnamese,Chinese,Spanish": 0.4, - "Vietnamese,Chinese,Malay": 0.4066666666666667, - "Vietnamese,Chinese,Indonesian": 0.41333333333333333, - "Vietnamese,Chinese,English": 0.37333333333333335, - "Vietnamese,Spanish,Malay": 0.4266666666666667, - "Vietnamese,Spanish,Indonesian": 0.44666666666666666, - "Vietnamese,Spanish,English": 0.38666666666666666, - "Vietnamese,Malay,Indonesian": 0.46, - "Vietnamese,Malay,English": 0.37333333333333335, - "Vietnamese,Indonesian,English": 0.4066666666666667, - "Chinese,Spanish,Malay": 0.4866666666666667, - "Chinese,Spanish,Indonesian": 0.54, - "Chinese,Spanish,English": 0.49333333333333335, - "Chinese,Malay,Indonesian": 0.5066666666666667, - "Chinese,Malay,English": 0.4266666666666667, - "Chinese,Indonesian,English": 0.49333333333333335, - "Spanish,Malay,Indonesian": 0.5733333333333334, - "Spanish,Malay,English": 0.49333333333333335, - "Spanish,Indonesian,English": 0.5666666666666667, - "Malay,Indonesian,English": 0.5133333333333333 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.26666666666666666, - "Filipino,Vietnamese,Chinese,Malay": 0.2733333333333333, - "Filipino,Vietnamese,Chinese,Indonesian": 0.2733333333333333, - "Filipino,Vietnamese,Chinese,English": 0.26, - "Filipino,Vietnamese,Spanish,Malay": 0.28, - "Filipino,Vietnamese,Spanish,Indonesian": 0.3, - "Filipino,Vietnamese,Spanish,English": 0.26, - "Filipino,Vietnamese,Malay,Indonesian": 0.30666666666666664, - "Filipino,Vietnamese,Malay,English": 0.26, - "Filipino,Vietnamese,Indonesian,English": 0.26666666666666666, - "Filipino,Chinese,Spanish,Malay": 0.32666666666666666, - "Filipino,Chinese,Spanish,Indonesian": 0.3333333333333333, - "Filipino,Chinese,Spanish,English": 0.32666666666666666, - "Filipino,Chinese,Malay,Indonesian": 0.31333333333333335, - "Filipino,Chinese,Malay,English": 0.2866666666666667, - "Filipino,Chinese,Indonesian,English": 0.30666666666666664, - "Filipino,Spanish,Malay,Indonesian": 0.36666666666666664, - "Filipino,Spanish,Malay,English": 0.3333333333333333, - "Filipino,Spanish,Indonesian,English": 0.36, - "Filipino,Malay,Indonesian,English": 0.3333333333333333, - "Vietnamese,Chinese,Spanish,Malay": 0.3466666666666667, - "Vietnamese,Chinese,Spanish,Indonesian": 0.38, - "Vietnamese,Chinese,Spanish,English": 0.31333333333333335, - "Vietnamese,Chinese,Malay,Indonesian": 0.36666666666666664, - "Vietnamese,Chinese,Malay,English": 0.3, - "Vietnamese,Chinese,Indonesian,English": 0.3333333333333333, - "Vietnamese,Spanish,Malay,Indonesian": 0.3933333333333333, - "Vietnamese,Spanish,Malay,English": 0.32666666666666666, - "Vietnamese,Spanish,Indonesian,English": 0.36, - "Vietnamese,Malay,Indonesian,English": 0.35333333333333333, - "Chinese,Spanish,Malay,Indonesian": 0.44666666666666666, - "Chinese,Spanish,Malay,English": 0.3933333333333333, - "Chinese,Spanish,Indonesian,English": 0.44, - "Chinese,Malay,Indonesian,English": 0.3933333333333333, - "Spanish,Malay,Indonesian,English": 0.46 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.24666666666666667, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.26, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.22, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.25333333333333335, - "Filipino,Vietnamese,Chinese,Malay,English": 0.21333333333333335, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.28, - "Filipino,Vietnamese,Spanish,Malay,English": 0.23333333333333334, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.24666666666666667, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, - "Filipino,Chinese,Spanish,Malay,English": 0.26666666666666666, - "Filipino,Chinese,Spanish,Indonesian,English": 0.28, - "Filipino,Chinese,Malay,Indonesian,English": 0.25333333333333335, - "Filipino,Spanish,Malay,Indonesian,English": 0.30666666666666664, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.34, - "Vietnamese,Chinese,Spanish,Malay,English": 0.2733333333333333, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.30666666666666664, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.29333333333333333, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.32, - "Chinese,Spanish,Malay,Indonesian,English": 0.36666666666666664 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.24666666666666667, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.2, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.21333333333333335, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.20666666666666667, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.23333333333333334, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.24, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2733333333333333 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2 - } - }, - "AC3_2": 0.5419605556677205, - "AC3_3": 0.4614359813863799, - "AC3_4": 0.40156150698801174, - "AC3_5": 0.354563475854031, - "AC3_6": 0.3168303186478029, - "AC3_7": 0.28679245278960486 - }, - "prompt_3": { - "overall_acc": 0.4923809523809524, - "language_acc": { - "Filipino": 0.36666666666666664, - "Vietnamese": 0.4266666666666667, - "Chinese": 0.47333333333333333, - "Spanish": 0.56, - "Malay": 0.44, - "Indonesian": 0.56, - "English": 0.62 - }, - "consistency_score_2": 0.513968253968254, - "consistency_score_3": 0.33790476190476193, - "consistency_score_4": 0.24666666666666665, - "consistency_score_5": 0.19206349206349205, - "consistency_score_6": 0.15714285714285717, - "consistency_score_7": 0.13333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.4266666666666667, - "Filipino,Chinese": 0.42, - "Filipino,Spanish": 0.4533333333333333, - "Filipino,Malay": 0.4666666666666667, - "Filipino,Indonesian": 0.46, - "Filipino,English": 0.44666666666666666, - "Vietnamese,Chinese": 0.49333333333333335, - "Vietnamese,Spanish": 0.4066666666666667, - "Vietnamese,Malay": 0.56, - "Vietnamese,Indonesian": 0.5066666666666667, - "Vietnamese,English": 0.41333333333333333, - "Chinese,Spanish": 0.5066666666666667, - "Chinese,Malay": 0.5466666666666666, - "Chinese,Indonesian": 0.5933333333333334, - "Chinese,English": 0.5466666666666666, - "Spanish,Malay": 0.56, - "Spanish,Indonesian": 0.6133333333333333, - "Spanish,English": 0.58, - "Malay,Indonesian": 0.64, - "Malay,English": 0.52, - "Indonesian,English": 0.6333333333333333 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.28, - "Filipino,Vietnamese,Spanish": 0.24666666666666667, - "Filipino,Vietnamese,Malay": 0.2866666666666667, - "Filipino,Vietnamese,Indonesian": 0.28, - "Filipino,Vietnamese,English": 0.25333333333333335, - "Filipino,Chinese,Spanish": 0.26, - "Filipino,Chinese,Malay": 0.29333333333333333, - "Filipino,Chinese,Indonesian": 0.2866666666666667, - "Filipino,Chinese,English": 0.2866666666666667, - "Filipino,Spanish,Malay": 0.31333333333333335, - "Filipino,Spanish,Indonesian": 0.3333333333333333, - "Filipino,Spanish,English": 0.31333333333333335, - "Filipino,Malay,Indonesian": 0.34, - "Filipino,Malay,English": 0.3, - "Filipino,Indonesian,English": 0.31333333333333335, - "Vietnamese,Chinese,Spanish": 0.2866666666666667, - "Vietnamese,Chinese,Malay": 0.35333333333333333, - "Vietnamese,Chinese,Indonesian": 0.36, - "Vietnamese,Chinese,English": 0.31333333333333335, - "Vietnamese,Spanish,Malay": 0.3333333333333333, - "Vietnamese,Spanish,Indonesian": 0.34, - "Vietnamese,Spanish,English": 0.28, - "Vietnamese,Malay,Indonesian": 0.4066666666666667, - "Vietnamese,Malay,English": 0.30666666666666664, - "Vietnamese,Indonesian,English": 0.3333333333333333, - "Chinese,Spanish,Malay": 0.38, - "Chinese,Spanish,Indonesian": 0.4066666666666667, - "Chinese,Spanish,English": 0.36, - "Chinese,Malay,Indonesian": 0.43333333333333335, - "Chinese,Malay,English": 0.36, - "Chinese,Indonesian,English": 0.44666666666666666, - "Spanish,Malay,Indonesian": 0.4666666666666667, - "Spanish,Malay,English": 0.38666666666666666, - "Spanish,Indonesian,English": 0.4533333333333333, - "Malay,Indonesian,English": 0.43333333333333335 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.18, - "Filipino,Vietnamese,Chinese,Malay": 0.21333333333333335, - "Filipino,Vietnamese,Chinese,Indonesian": 0.22, - "Filipino,Vietnamese,Chinese,English": 0.21333333333333335, - "Filipino,Vietnamese,Spanish,Malay": 0.20666666666666667, - "Filipino,Vietnamese,Spanish,Indonesian": 0.22, - "Filipino,Vietnamese,Spanish,English": 0.18, - "Filipino,Vietnamese,Malay,Indonesian": 0.24, - "Filipino,Vietnamese,Malay,English": 0.20666666666666667, - "Filipino,Vietnamese,Indonesian,English": 0.20666666666666667, - "Filipino,Chinese,Spanish,Malay": 0.22666666666666666, - "Filipino,Chinese,Spanish,Indonesian": 0.22, - "Filipino,Chinese,Spanish,English": 0.2, - "Filipino,Chinese,Malay,Indonesian": 0.24666666666666667, - "Filipino,Chinese,Malay,English": 0.22666666666666666, - "Filipino,Chinese,Indonesian,English": 0.23333333333333334, - "Filipino,Spanish,Malay,Indonesian": 0.28, - "Filipino,Spanish,Malay,English": 0.22, - "Filipino,Spanish,Indonesian,English": 0.24666666666666667, - "Filipino,Malay,Indonesian,English": 0.25333333333333335, - "Vietnamese,Chinese,Spanish,Malay": 0.25333333333333335, - "Vietnamese,Chinese,Spanish,Indonesian": 0.24666666666666667, - "Vietnamese,Chinese,Spanish,English": 0.21333333333333335, - "Vietnamese,Chinese,Malay,Indonesian": 0.31333333333333335, - "Vietnamese,Chinese,Malay,English": 0.24666666666666667, - "Vietnamese,Chinese,Indonesian,English": 0.2733333333333333, - "Vietnamese,Spanish,Malay,Indonesian": 0.2866666666666667, - "Vietnamese,Spanish,Malay,English": 0.22666666666666666, - "Vietnamese,Spanish,Indonesian,English": 0.24666666666666667, - "Vietnamese,Malay,Indonesian,English": 0.26666666666666666, - "Chinese,Spanish,Malay,Indonesian": 0.3333333333333333, - "Chinese,Spanish,Malay,English": 0.28, - "Chinese,Spanish,Indonesian,English": 0.32666666666666666, - "Chinese,Malay,Indonesian,English": 0.32666666666666666, - "Spanish,Malay,Indonesian,English": 0.35333333333333333 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.14666666666666667, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.2, - "Filipino,Vietnamese,Chinese,Malay,English": 0.18666666666666668, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.18666666666666668, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.19333333333333333, - "Filipino,Vietnamese,Spanish,Malay,English": 0.15333333333333332, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.18, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.20666666666666667, - "Filipino,Chinese,Spanish,Malay,English": 0.17333333333333334, - "Filipino,Chinese,Spanish,Indonesian,English": 0.18, - "Filipino,Chinese,Malay,Indonesian,English": 0.20666666666666667, - "Filipino,Spanish,Malay,Indonesian,English": 0.20666666666666667, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, - "Vietnamese,Chinese,Spanish,Malay,English": 0.18666666666666668, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.2, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.23333333333333334, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, - "Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.14, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.14, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.17333333333333334, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16666666666666666, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.18 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333 - } - }, - "AC3_2": 0.5029430673978224, - "AC3_3": 0.40077256683579116, - "AC3_4": 0.3286769759005441, - "AC3_5": 0.2763362487448546, - "AC3_6": 0.23824884788958675, - "AC3_7": 0.209842719398224 - }, - "prompt_4": { - "overall_acc": 0.5066666666666666, - "language_acc": { - "Filipino": 0.44, - "Vietnamese": 0.38666666666666666, - "Chinese": 0.5133333333333333, - "Spanish": 0.5466666666666666, - "Malay": 0.49333333333333335, - "Indonesian": 0.5333333333333333, - "English": 0.6333333333333333 - }, - "consistency_score_2": 0.5622222222222222, - "consistency_score_3": 0.3952380952380952, - "consistency_score_4": 0.3064761904761905, - "consistency_score_5": 0.253015873015873, - "consistency_score_6": 0.21809523809523812, - "consistency_score_7": 0.19333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.44666666666666666, - "Filipino,Chinese": 0.49333333333333335, - "Filipino,Spanish": 0.5, - "Filipino,Malay": 0.4533333333333333, - "Filipino,Indonesian": 0.5, - "Filipino,English": 0.47333333333333333, - "Vietnamese,Chinese": 0.5333333333333333, - "Vietnamese,Spanish": 0.5, - "Vietnamese,Malay": 0.56, - "Vietnamese,Indonesian": 0.5266666666666666, - "Vietnamese,English": 0.4533333333333333, - "Chinese,Spanish": 0.5933333333333334, - "Chinese,Malay": 0.5733333333333334, - "Chinese,Indonesian": 0.64, - "Chinese,English": 0.6, - "Spanish,Malay": 0.6466666666666666, - "Spanish,Indonesian": 0.74, - "Spanish,English": 0.66, - "Malay,Indonesian": 0.72, - "Malay,English": 0.54, - "Indonesian,English": 0.6533333333333333 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.32, - "Filipino,Vietnamese,Spanish": 0.29333333333333333, - "Filipino,Vietnamese,Malay": 0.29333333333333333, - "Filipino,Vietnamese,Indonesian": 0.3, - "Filipino,Vietnamese,English": 0.28, - "Filipino,Chinese,Spanish": 0.32666666666666666, - "Filipino,Chinese,Malay": 0.3333333333333333, - "Filipino,Chinese,Indonesian": 0.36, - "Filipino,Chinese,English": 0.34, - "Filipino,Spanish,Malay": 0.36, - "Filipino,Spanish,Indonesian": 0.41333333333333333, - "Filipino,Spanish,English": 0.37333333333333335, - "Filipino,Malay,Indonesian": 0.38666666666666666, - "Filipino,Malay,English": 0.31333333333333335, - "Filipino,Indonesian,English": 0.36666666666666664, - "Vietnamese,Chinese,Spanish": 0.37333333333333335, - "Vietnamese,Chinese,Malay": 0.3933333333333333, - "Vietnamese,Chinese,Indonesian": 0.3933333333333333, - "Vietnamese,Chinese,English": 0.35333333333333333, - "Vietnamese,Spanish,Malay": 0.4066666666666667, - "Vietnamese,Spanish,Indonesian": 0.43333333333333335, - "Vietnamese,Spanish,English": 0.36, - "Vietnamese,Malay,Indonesian": 0.44666666666666666, - "Vietnamese,Malay,English": 0.35333333333333333, - "Vietnamese,Indonesian,English": 0.37333333333333335, - "Chinese,Spanish,Malay": 0.44, - "Chinese,Spanish,Indonesian": 0.52, - "Chinese,Spanish,English": 0.4666666666666667, - "Chinese,Malay,Indonesian": 0.4866666666666667, - "Chinese,Malay,English": 0.3933333333333333, - "Chinese,Indonesian,English": 0.48, - "Spanish,Malay,Indonesian": 0.5733333333333334, - "Spanish,Malay,English": 0.4666666666666667, - "Spanish,Indonesian,English": 0.5666666666666667, - "Malay,Indonesian,English": 0.49333333333333335 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.24666666666666667, - "Filipino,Vietnamese,Chinese,Malay": 0.25333333333333335, - "Filipino,Vietnamese,Chinese,Indonesian": 0.26, - "Filipino,Vietnamese,Chinese,English": 0.24666666666666667, - "Filipino,Vietnamese,Spanish,Malay": 0.25333333333333335, - "Filipino,Vietnamese,Spanish,Indonesian": 0.2733333333333333, - "Filipino,Vietnamese,Spanish,English": 0.23333333333333334, - "Filipino,Vietnamese,Malay,Indonesian": 0.26, - "Filipino,Vietnamese,Malay,English": 0.22666666666666666, - "Filipino,Vietnamese,Indonesian,English": 0.24666666666666667, - "Filipino,Chinese,Spanish,Malay": 0.2733333333333333, - "Filipino,Chinese,Spanish,Indonesian": 0.30666666666666664, - "Filipino,Chinese,Spanish,English": 0.2733333333333333, - "Filipino,Chinese,Malay,Indonesian": 0.29333333333333333, - "Filipino,Chinese,Malay,English": 0.25333333333333335, - "Filipino,Chinese,Indonesian,English": 0.29333333333333333, - "Filipino,Spanish,Malay,Indonesian": 0.34, - "Filipino,Spanish,Malay,English": 0.2866666666666667, - "Filipino,Spanish,Indonesian,English": 0.3333333333333333, - "Filipino,Malay,Indonesian,English": 0.30666666666666664, - "Vietnamese,Chinese,Spanish,Malay": 0.32, - "Vietnamese,Chinese,Spanish,Indonesian": 0.34, - "Vietnamese,Chinese,Spanish,English": 0.29333333333333333, - "Vietnamese,Chinese,Malay,Indonesian": 0.34, - "Vietnamese,Chinese,Malay,English": 0.2866666666666667, - "Vietnamese,Chinese,Indonesian,English": 0.31333333333333335, - "Vietnamese,Spanish,Malay,Indonesian": 0.38, - "Vietnamese,Spanish,Malay,English": 0.31333333333333335, - "Vietnamese,Spanish,Indonesian,English": 0.3333333333333333, - "Vietnamese,Malay,Indonesian,English": 0.32666666666666666, - "Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, - "Chinese,Spanish,Malay,English": 0.36, - "Chinese,Spanish,Indonesian,English": 0.42, - "Chinese,Malay,Indonesian,English": 0.38, - "Spanish,Malay,Indonesian,English": 0.44666666666666666 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.22666666666666666, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.24, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.20666666666666667, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.22666666666666666, - "Filipino,Vietnamese,Chinese,Malay,English": 0.2, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.24666666666666667, - "Filipino,Vietnamese,Spanish,Malay,English": 0.21333333333333335, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.22666666666666666, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.22, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, - "Filipino,Chinese,Spanish,Malay,English": 0.23333333333333334, - "Filipino,Chinese,Spanish,Indonesian,English": 0.26, - "Filipino,Chinese,Malay,Indonesian,English": 0.25333333333333335, - "Filipino,Spanish,Malay,Indonesian,English": 0.2866666666666667, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.30666666666666664, - "Vietnamese,Chinese,Spanish,Malay,English": 0.26666666666666666, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.28, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.28, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.3, - "Chinese,Spanish,Malay,Indonesian,English": 0.3466666666666667 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.19333333333333333, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.20666666666666667, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.2, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.23333333333333334, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.26 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.19333333333333333 - } - }, - "AC3_2": 0.533000692950828, - "AC3_3": 0.44406898974308984, - "AC3_4": 0.3819286327319795, - "AC3_5": 0.33749547286266424, - "AC3_6": 0.3049321068348429, - "AC3_7": 0.279873015833034 - }, - "prompt_5": { - "overall_acc": 0.49047619047619045, - "language_acc": { - "Filipino": 0.4066666666666667, - "Vietnamese": 0.3933333333333333, - "Chinese": 0.5, - "Spanish": 0.49333333333333335, - "Malay": 0.49333333333333335, - "Indonesian": 0.5466666666666666, - "English": 0.6 - }, - "consistency_score_2": 0.5482539682539683, - "consistency_score_3": 0.38019047619047625, - "consistency_score_4": 0.291047619047619, - "consistency_score_5": 0.2358730158730159, - "consistency_score_6": 0.19904761904761906, - "consistency_score_7": 0.17333333333333334, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.48, - "Filipino,Chinese": 0.4866666666666667, - "Filipino,Spanish": 0.46, - "Filipino,Malay": 0.47333333333333333, - "Filipino,Indonesian": 0.4666666666666667, - "Filipino,English": 0.4666666666666667, - "Vietnamese,Chinese": 0.5133333333333333, - "Vietnamese,Spanish": 0.49333333333333335, - "Vietnamese,Malay": 0.5866666666666667, - "Vietnamese,Indonesian": 0.5666666666666667, - "Vietnamese,English": 0.4266666666666667, - "Chinese,Spanish": 0.5466666666666666, - "Chinese,Malay": 0.5933333333333334, - "Chinese,Indonesian": 0.66, - "Chinese,English": 0.5266666666666666, - "Spanish,Malay": 0.6866666666666666, - "Spanish,Indonesian": 0.6866666666666666, - "Spanish,English": 0.6, - "Malay,Indonesian": 0.72, - "Malay,English": 0.5133333333333333, - "Indonesian,English": 0.56 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.32666666666666666, - "Filipino,Vietnamese,Spanish": 0.2866666666666667, - "Filipino,Vietnamese,Malay": 0.34, - "Filipino,Vietnamese,Indonesian": 0.3333333333333333, - "Filipino,Vietnamese,English": 0.2866666666666667, - "Filipino,Chinese,Spanish": 0.32666666666666666, - "Filipino,Chinese,Malay": 0.35333333333333333, - "Filipino,Chinese,Indonesian": 0.36, - "Filipino,Chinese,English": 0.32666666666666666, - "Filipino,Spanish,Malay": 0.36, - "Filipino,Spanish,Indonesian": 0.36666666666666664, - "Filipino,Spanish,English": 0.31333333333333335, - "Filipino,Malay,Indonesian": 0.38, - "Filipino,Malay,English": 0.30666666666666664, - "Filipino,Indonesian,English": 0.31333333333333335, - "Vietnamese,Chinese,Spanish": 0.35333333333333333, - "Vietnamese,Chinese,Malay": 0.41333333333333333, - "Vietnamese,Chinese,Indonesian": 0.43333333333333335, - "Vietnamese,Chinese,English": 0.3333333333333333, - "Vietnamese,Spanish,Malay": 0.4266666666666667, - "Vietnamese,Spanish,Indonesian": 0.42, - "Vietnamese,Spanish,English": 0.32, - "Vietnamese,Malay,Indonesian": 0.4666666666666667, - "Vietnamese,Malay,English": 0.3333333333333333, - "Vietnamese,Indonesian,English": 0.3333333333333333, - "Chinese,Spanish,Malay": 0.4533333333333333, - "Chinese,Spanish,Indonesian": 0.48, - "Chinese,Spanish,English": 0.38666666666666666, - "Chinese,Malay,Indonesian": 0.5133333333333333, - "Chinese,Malay,English": 0.36666666666666664, - "Chinese,Indonesian,English": 0.41333333333333333, - "Spanish,Malay,Indonesian": 0.5733333333333334, - "Spanish,Malay,English": 0.4266666666666667, - "Spanish,Indonesian,English": 0.4533333333333333, - "Malay,Indonesian,English": 0.4266666666666667 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.24, - "Filipino,Vietnamese,Chinese,Malay": 0.2733333333333333, - "Filipino,Vietnamese,Chinese,Indonesian": 0.2866666666666667, - "Filipino,Vietnamese,Chinese,English": 0.24, - "Filipino,Vietnamese,Spanish,Malay": 0.26666666666666666, - "Filipino,Vietnamese,Spanish,Indonesian": 0.2733333333333333, - "Filipino,Vietnamese,Spanish,English": 0.2, - "Filipino,Vietnamese,Malay,Indonesian": 0.29333333333333333, - "Filipino,Vietnamese,Malay,English": 0.23333333333333334, - "Filipino,Vietnamese,Indonesian,English": 0.23333333333333334, - "Filipino,Chinese,Spanish,Malay": 0.2866666666666667, - "Filipino,Chinese,Spanish,Indonesian": 0.29333333333333333, - "Filipino,Chinese,Spanish,English": 0.24666666666666667, - "Filipino,Chinese,Malay,Indonesian": 0.30666666666666664, - "Filipino,Chinese,Malay,English": 0.25333333333333335, - "Filipino,Chinese,Indonesian,English": 0.26666666666666666, - "Filipino,Spanish,Malay,Indonesian": 0.3333333333333333, - "Filipino,Spanish,Malay,English": 0.25333333333333335, - "Filipino,Spanish,Indonesian,English": 0.26666666666666666, - "Filipino,Malay,Indonesian,English": 0.26, - "Vietnamese,Chinese,Spanish,Malay": 0.3333333333333333, - "Vietnamese,Chinese,Spanish,Indonesian": 0.3333333333333333, - "Vietnamese,Chinese,Spanish,English": 0.26, - "Vietnamese,Chinese,Malay,Indonesian": 0.38666666666666666, - "Vietnamese,Chinese,Malay,English": 0.2866666666666667, - "Vietnamese,Chinese,Indonesian,English": 0.29333333333333333, - "Vietnamese,Spanish,Malay,Indonesian": 0.37333333333333335, - "Vietnamese,Spanish,Malay,English": 0.2733333333333333, - "Vietnamese,Spanish,Indonesian,English": 0.28, - "Vietnamese,Malay,Indonesian,English": 0.2866666666666667, - "Chinese,Spanish,Malay,Indonesian": 0.4266666666666667, - "Chinese,Spanish,Malay,English": 0.31333333333333335, - "Chinese,Spanish,Indonesian,English": 0.3333333333333333, - "Chinese,Malay,Indonesian,English": 0.32, - "Spanish,Malay,Indonesian,English": 0.38 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.23333333333333334, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.24, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.18, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.26, - "Filipino,Vietnamese,Chinese,Malay,English": 0.20666666666666667, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.22, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.26, - "Filipino,Vietnamese,Spanish,Malay,English": 0.18666666666666668, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.20666666666666667, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, - "Filipino,Chinese,Spanish,Malay,English": 0.21333333333333335, - "Filipino,Chinese,Spanish,Indonesian,English": 0.22, - "Filipino,Chinese,Malay,Indonesian,English": 0.22, - "Filipino,Spanish,Malay,Indonesian,English": 0.24, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.32, - "Vietnamese,Chinese,Spanish,Malay,English": 0.24, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.24, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.26, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.25333333333333335, - "Chinese,Spanish,Malay,Indonesian,English": 0.2866666666666667 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.17333333333333334, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.19333333333333333, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.18666666666666668, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.2, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22666666666666666 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334 - } - }, - "AC3_2": 0.5177581790164023, - "AC3_3": 0.42834849095516725, - "AC3_4": 0.3653168993184125, - "AC3_5": 0.3185522810084245, - "AC3_6": 0.28317548009573784, - "AC3_7": 0.2561453849446744 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.4537337662337663, - "language_acc": { - "Indonesian": 0.5, - "English": 0.5397727272727273, - "Filipino": 0.3522727272727273, - "Spanish": 0.4943181818181818, - "Chinese": 0.4943181818181818, - "Malay": 0.4147727272727273, - "Vietnamese": 0.3806818181818182 - }, - "consistency_score_2": 0.5676406926406927, - "consistency_score_3": 0.3974025974025973, - "consistency_score_4": 0.30227272727272725, - "consistency_score_5": 0.240530303030303, - "consistency_score_6": 0.19724025974025974, - "consistency_score_7": 0.16477272727272727, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.7215909090909091, - "Indonesian,Filipino": 0.4943181818181818, - "Indonesian,Spanish": 0.7443181818181818, - "Indonesian,Chinese": 0.6477272727272727, - "Indonesian,Malay": 0.7045454545454546, - "Indonesian,Vietnamese": 0.5056818181818182, - "English,Filipino": 0.45454545454545453, - "English,Spanish": 0.6875, - "English,Chinese": 0.6363636363636364, - "English,Malay": 0.6079545454545454, - "English,Vietnamese": 0.48295454545454547, - "Filipino,Spanish": 0.45454545454545453, - "Filipino,Chinese": 0.4318181818181818, - "Filipino,Malay": 0.5, - "Filipino,Vietnamese": 0.5170454545454546, - "Spanish,Chinese": 0.6193181818181818, - "Spanish,Malay": 0.6079545454545454, - "Spanish,Vietnamese": 0.5113636363636364, - "Chinese,Malay": 0.5909090909090909, - "Chinese,Vietnamese": 0.48863636363636365, - "Malay,Vietnamese": 0.5113636363636364 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.38636363636363635, - "Indonesian,English,Spanish": 0.5909090909090909, - "Indonesian,English,Chinese": 0.5397727272727273, - "Indonesian,English,Malay": 0.5454545454545454, - "Indonesian,English,Vietnamese": 0.38636363636363635, - "Indonesian,Filipino,Spanish": 0.38636363636363635, - "Indonesian,Filipino,Chinese": 0.3522727272727273, - "Indonesian,Filipino,Malay": 0.4034090909090909, - "Indonesian,Filipino,Vietnamese": 0.3181818181818182, - "Indonesian,Spanish,Chinese": 0.5284090909090909, - "Indonesian,Spanish,Malay": 0.5454545454545454, - "Indonesian,Spanish,Vietnamese": 0.4090909090909091, - "Indonesian,Chinese,Malay": 0.4943181818181818, - "Indonesian,Chinese,Vietnamese": 0.35795454545454547, - "Indonesian,Malay,Vietnamese": 0.3977272727272727, - "English,Filipino,Spanish": 0.3465909090909091, - "English,Filipino,Chinese": 0.3352272727272727, - "English,Filipino,Malay": 0.3465909090909091, - "English,Filipino,Vietnamese": 0.2897727272727273, - "English,Spanish,Chinese": 0.5170454545454546, - "English,Spanish,Malay": 0.48295454545454547, - "English,Spanish,Vietnamese": 0.3806818181818182, - "English,Chinese,Malay": 0.4602272727272727, - "English,Chinese,Vietnamese": 0.36363636363636365, - "English,Malay,Vietnamese": 0.3522727272727273, - "Filipino,Spanish,Chinese": 0.3181818181818182, - "Filipino,Spanish,Malay": 0.3409090909090909, - "Filipino,Spanish,Vietnamese": 0.3068181818181818, - "Filipino,Chinese,Malay": 0.3181818181818182, - "Filipino,Chinese,Vietnamese": 0.2840909090909091, - "Filipino,Malay,Vietnamese": 0.3125, - "Spanish,Chinese,Malay": 0.4431818181818182, - "Spanish,Chinese,Vietnamese": 0.35795454545454547, - "Spanish,Malay,Vietnamese": 0.36363636363636365, - "Chinese,Malay,Vietnamese": 0.3465909090909091 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.3181818181818182, - "Indonesian,English,Filipino,Chinese": 0.3125, - "Indonesian,English,Filipino,Malay": 0.32386363636363635, - "Indonesian,English,Filipino,Vietnamese": 0.25, - "Indonesian,English,Spanish,Chinese": 0.4715909090909091, - "Indonesian,English,Spanish,Malay": 0.45454545454545453, - "Indonesian,English,Spanish,Vietnamese": 0.3352272727272727, - "Indonesian,English,Chinese,Malay": 0.42613636363636365, - "Indonesian,English,Chinese,Vietnamese": 0.3068181818181818, - "Indonesian,English,Malay,Vietnamese": 0.32386363636363635, - "Indonesian,Filipino,Spanish,Chinese": 0.29545454545454547, - "Indonesian,Filipino,Spanish,Malay": 0.3181818181818182, - "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, - "Indonesian,Filipino,Chinese,Malay": 0.30113636363636365, - "Indonesian,Filipino,Chinese,Vietnamese": 0.24431818181818182, - "Indonesian,Filipino,Malay,Vietnamese": 0.26136363636363635, - "Indonesian,Spanish,Chinese,Malay": 0.4034090909090909, - "Indonesian,Spanish,Chinese,Vietnamese": 0.3068181818181818, - "Indonesian,Spanish,Malay,Vietnamese": 0.32386363636363635, - "Indonesian,Chinese,Malay,Vietnamese": 0.30113636363636365, - "English,Filipino,Spanish,Chinese": 0.29545454545454547, - "English,Filipino,Spanish,Malay": 0.2727272727272727, - "English,Filipino,Spanish,Vietnamese": 0.23295454545454544, - "English,Filipino,Chinese,Malay": 0.2784090909090909, - "English,Filipino,Chinese,Vietnamese": 0.23863636363636365, - "English,Filipino,Malay,Vietnamese": 0.22727272727272727, - "English,Spanish,Chinese,Malay": 0.39204545454545453, - "English,Spanish,Chinese,Vietnamese": 0.3068181818181818, - "English,Spanish,Malay,Vietnamese": 0.29545454545454547, - "English,Chinese,Malay,Vietnamese": 0.2784090909090909, - "Filipino,Spanish,Chinese,Malay": 0.2556818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Chinese,Malay,Vietnamese": 0.23295454545454544, - "Spanish,Chinese,Malay,Vietnamese": 0.2784090909090909 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.2784090909090909, - "Indonesian,English,Filipino,Spanish,Malay": 0.26704545454545453, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.21022727272727273, - "Indonesian,English,Filipino,Chinese,Malay": 0.26704545454545453, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.2159090909090909, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.2159090909090909, - "Indonesian,English,Spanish,Chinese,Malay": 0.3693181818181818, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2784090909090909, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.2784090909090909, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.26136363636363635, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.25, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.2159090909090909, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182, - "English,Filipino,Spanish,Chinese,Malay": 0.23863636363636365, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, - "English,Filipino,Chinese,Malay,Vietnamese": 0.19886363636363635, - "English,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1875 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.23295454545454544, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.18181818181818182, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1875, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.23295454545454544, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727 - } - }, - "AC3_2": 0.5043355981272797, - "AC3_3": 0.42370408529840126, - "AC3_4": 0.3628311241638483, - "AC3_5": 0.31439541552900846, - "AC3_6": 0.274955873261516, - "AC3_7": 0.2417531614975395 - }, - "prompt_2": { - "overall_acc": 0.4310064935064935, - "language_acc": { - "Indonesian": 0.48863636363636365, - "English": 0.4659090909090909, - "Filipino": 0.3465909090909091, - "Spanish": 0.4943181818181818, - "Chinese": 0.48863636363636365, - "Malay": 0.4090909090909091, - "Vietnamese": 0.32386363636363635 - }, - "consistency_score_2": 0.5546536796536797, - "consistency_score_3": 0.3774350649350649, - "consistency_score_4": 0.27402597402597406, - "consistency_score_5": 0.20427489177489178, - "consistency_score_6": 0.15503246753246752, - "consistency_score_7": 0.11931818181818182, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6931818181818182, - "Indonesian,Filipino": 0.4715909090909091, - "Indonesian,Spanish": 0.7159090909090909, - "Indonesian,Chinese": 0.6420454545454546, - "Indonesian,Malay": 0.6988636363636364, - "Indonesian,Vietnamese": 0.5056818181818182, - "English,Filipino": 0.4602272727272727, - "English,Spanish": 0.6363636363636364, - "English,Chinese": 0.5852272727272727, - "English,Malay": 0.5909090909090909, - "English,Vietnamese": 0.4431818181818182, - "Filipino,Spanish": 0.4772727272727273, - "Filipino,Chinese": 0.4602272727272727, - "Filipino,Malay": 0.5056818181818182, - "Filipino,Vietnamese": 0.5284090909090909, - "Spanish,Chinese": 0.5738636363636364, - "Spanish,Malay": 0.6477272727272727, - "Spanish,Vietnamese": 0.48295454545454547, - "Chinese,Malay": 0.5852272727272727, - "Chinese,Vietnamese": 0.4375, - "Malay,Vietnamese": 0.5056818181818182 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.36363636363636365, - "Indonesian,English,Spanish": 0.5511363636363636, - "Indonesian,English,Chinese": 0.5, - "Indonesian,English,Malay": 0.5113636363636364, - "Indonesian,English,Vietnamese": 0.3465909090909091, - "Indonesian,Filipino,Spanish": 0.375, - "Indonesian,Filipino,Chinese": 0.32954545454545453, - "Indonesian,Filipino,Malay": 0.39204545454545453, - "Indonesian,Filipino,Vietnamese": 0.3125, - "Indonesian,Spanish,Chinese": 0.4943181818181818, - "Indonesian,Spanish,Malay": 0.5454545454545454, - "Indonesian,Spanish,Vietnamese": 0.39204545454545453, - "Indonesian,Chinese,Malay": 0.48863636363636365, - "Indonesian,Chinese,Vietnamese": 0.32954545454545453, - "Indonesian,Malay,Vietnamese": 0.38636363636363635, - "English,Filipino,Spanish": 0.3409090909090909, - "English,Filipino,Chinese": 0.32386363636363635, - "English,Filipino,Malay": 0.3465909090909091, - "English,Filipino,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese": 0.44886363636363635, - "English,Spanish,Malay": 0.4602272727272727, - "English,Spanish,Vietnamese": 0.32954545454545453, - "English,Chinese,Malay": 0.4147727272727273, - "English,Chinese,Vietnamese": 0.30113636363636365, - "English,Malay,Vietnamese": 0.32386363636363635, - "Filipino,Spanish,Chinese": 0.3068181818181818, - "Filipino,Spanish,Malay": 0.3693181818181818, - "Filipino,Spanish,Vietnamese": 0.3181818181818182, - "Filipino,Chinese,Malay": 0.3181818181818182, - "Filipino,Chinese,Vietnamese": 0.2897727272727273, - "Filipino,Malay,Vietnamese": 0.3181818181818182, - "Spanish,Chinese,Malay": 0.42613636363636365, - "Spanish,Chinese,Vietnamese": 0.3068181818181818, - "Spanish,Malay,Vietnamese": 0.36363636363636365, - "Chinese,Malay,Vietnamese": 0.3068181818181818 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.29545454545454547, - "Indonesian,English,Filipino,Chinese": 0.2840909090909091, - "Indonesian,English,Filipino,Malay": 0.30113636363636365, - "Indonesian,English,Filipino,Vietnamese": 0.22727272727272727, - "Indonesian,English,Spanish,Chinese": 0.4147727272727273, - "Indonesian,English,Spanish,Malay": 0.42613636363636365, - "Indonesian,English,Spanish,Vietnamese": 0.2840909090909091, - "Indonesian,English,Chinese,Malay": 0.3806818181818182, - "Indonesian,English,Chinese,Vietnamese": 0.26136363636363635, - "Indonesian,English,Malay,Vietnamese": 0.2784090909090909, - "Indonesian,Filipino,Spanish,Chinese": 0.26136363636363635, - "Indonesian,Filipino,Spanish,Malay": 0.3125, - "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, - "Indonesian,Filipino,Chinese,Malay": 0.2784090909090909, - "Indonesian,Filipino,Chinese,Vietnamese": 0.22727272727272727, - "Indonesian,Filipino,Malay,Vietnamese": 0.26704545454545453, - "Indonesian,Spanish,Chinese,Malay": 0.39204545454545453, - "Indonesian,Spanish,Chinese,Vietnamese": 0.26136363636363635, - "Indonesian,Spanish,Malay,Vietnamese": 0.3181818181818182, - "Indonesian,Chinese,Malay,Vietnamese": 0.26136363636363635, - "English,Filipino,Spanish,Chinese": 0.25, - "English,Filipino,Spanish,Malay": 0.25, - "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, - "English,Filipino,Chinese,Malay": 0.26136363636363635, - "English,Filipino,Chinese,Vietnamese": 0.2159090909090909, - "English,Filipino,Malay,Vietnamese": 0.21022727272727273, - "English,Spanish,Chinese,Malay": 0.3465909090909091, - "English,Spanish,Chinese,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay,Vietnamese": 0.24431818181818182, - "English,Chinese,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Chinese,Malay": 0.24431818181818182, - "Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Malay,Vietnamese": 0.2159090909090909, - "Spanish,Chinese,Malay,Vietnamese": 0.22727272727272727 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.23295454545454544, - "Indonesian,English,Filipino,Spanish,Malay": 0.23863636363636365, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.18181818181818182, - "Indonesian,English,Filipino,Chinese,Malay": 0.23863636363636365, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1875, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.1875, - "Indonesian,English,Spanish,Chinese,Malay": 0.32954545454545453, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2159090909090909, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.22727272727272727, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.20454545454545456, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.2215909090909091, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, - "English,Filipino,Spanish,Chinese,Malay": 0.19886363636363635, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, - "English,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 - } - }, - "AC3_2": 0.4850745603067087, - "AC3_3": 0.40244582220025865, - "AC3_4": 0.33503981622646506, - "AC3_5": 0.27718049620628593, - "AC3_6": 0.22803944666396958, - "AC3_7": 0.18689662104407584 - }, - "prompt_3": { - "overall_acc": 0.43831168831168826, - "language_acc": { - "Indonesian": 0.4659090909090909, - "English": 0.5, - "Filipino": 0.36363636363636365, - "Spanish": 0.4715909090909091, - "Chinese": 0.4943181818181818, - "Malay": 0.4318181818181818, - "Vietnamese": 0.3409090909090909 - }, - "consistency_score_2": 0.5462662337662337, - "consistency_score_3": 0.36980519480519464, - "consistency_score_4": 0.2683441558441559, - "consistency_score_5": 0.20075757575757583, - "consistency_score_6": 0.15178571428571427, - "consistency_score_7": 0.11363636363636363, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6761363636363636, - "Indonesian,Filipino": 0.4772727272727273, - "Indonesian,Spanish": 0.6988636363636364, - "Indonesian,Chinese": 0.6136363636363636, - "Indonesian,Malay": 0.6875, - "Indonesian,Vietnamese": 0.5113636363636364, - "English,Filipino": 0.4375, - "English,Spanish": 0.625, - "English,Chinese": 0.6022727272727273, - "English,Malay": 0.5738636363636364, - "English,Vietnamese": 0.42045454545454547, - "Filipino,Spanish": 0.44886363636363635, - "Filipino,Chinese": 0.4318181818181818, - "Filipino,Malay": 0.5284090909090909, - "Filipino,Vietnamese": 0.5227272727272727, - "Spanish,Chinese": 0.5852272727272727, - "Spanish,Malay": 0.6136363636363636, - "Spanish,Vietnamese": 0.4943181818181818, - "Chinese,Malay": 0.5625, - "Chinese,Vietnamese": 0.42045454545454547, - "Malay,Vietnamese": 0.5397727272727273 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.35795454545454547, - "Indonesian,English,Spanish": 0.5227272727272727, - "Indonesian,English,Chinese": 0.48295454545454547, - "Indonesian,English,Malay": 0.4943181818181818, - "Indonesian,English,Vietnamese": 0.3522727272727273, - "Indonesian,Filipino,Spanish": 0.36363636363636365, - "Indonesian,Filipino,Chinese": 0.3352272727272727, - "Indonesian,Filipino,Malay": 0.4034090909090909, - "Indonesian,Filipino,Vietnamese": 0.3125, - "Indonesian,Spanish,Chinese": 0.4659090909090909, - "Indonesian,Spanish,Malay": 0.5170454545454546, - "Indonesian,Spanish,Vietnamese": 0.3977272727272727, - "Indonesian,Chinese,Malay": 0.4659090909090909, - "Indonesian,Chinese,Vietnamese": 0.3125, - "Indonesian,Malay,Vietnamese": 0.4034090909090909, - "English,Filipino,Spanish": 0.3125, - "English,Filipino,Chinese": 0.3068181818181818, - "English,Filipino,Malay": 0.3465909090909091, - "English,Filipino,Vietnamese": 0.26136363636363635, - "English,Spanish,Chinese": 0.44886363636363635, - "English,Spanish,Malay": 0.42613636363636365, - "English,Spanish,Vietnamese": 0.32386363636363635, - "English,Chinese,Malay": 0.4147727272727273, - "English,Chinese,Vietnamese": 0.2897727272727273, - "English,Malay,Vietnamese": 0.32954545454545453, - "Filipino,Spanish,Chinese": 0.30113636363636365, - "Filipino,Spanish,Malay": 0.35795454545454547, - "Filipino,Spanish,Vietnamese": 0.2897727272727273, - "Filipino,Chinese,Malay": 0.32386363636363635, - "Filipino,Chinese,Vietnamese": 0.2840909090909091, - "Filipino,Malay,Vietnamese": 0.3465909090909091, - "Spanish,Chinese,Malay": 0.4090909090909091, - "Spanish,Chinese,Vietnamese": 0.29545454545454547, - "Spanish,Malay,Vietnamese": 0.38636363636363635, - "Chinese,Malay,Vietnamese": 0.30113636363636365 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.2727272727272727, - "Indonesian,English,Filipino,Chinese": 0.2727272727272727, - "Indonesian,English,Filipino,Malay": 0.3068181818181818, - "Indonesian,English,Filipino,Vietnamese": 0.2159090909090909, - "Indonesian,English,Spanish,Chinese": 0.39204545454545453, - "Indonesian,English,Spanish,Malay": 0.38636363636363635, - "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, - "Indonesian,English,Chinese,Malay": 0.375, - "Indonesian,English,Chinese,Vietnamese": 0.2556818181818182, - "Indonesian,English,Malay,Vietnamese": 0.2897727272727273, - "Indonesian,Filipino,Spanish,Chinese": 0.26704545454545453, - "Indonesian,Filipino,Spanish,Malay": 0.3068181818181818, - "Indonesian,Filipino,Spanish,Vietnamese": 0.23863636363636365, - "Indonesian,Filipino,Chinese,Malay": 0.2897727272727273, - "Indonesian,Filipino,Chinese,Vietnamese": 0.2159090909090909, - "Indonesian,Filipino,Malay,Vietnamese": 0.2727272727272727, - "Indonesian,Spanish,Chinese,Malay": 0.36363636363636365, - "Indonesian,Spanish,Chinese,Vietnamese": 0.24431818181818182, - "Indonesian,Spanish,Malay,Vietnamese": 0.32954545454545453, - "Indonesian,Chinese,Malay,Vietnamese": 0.2556818181818182, - "English,Filipino,Spanish,Chinese": 0.23295454545454544, - "English,Filipino,Spanish,Malay": 0.24431818181818182, - "English,Filipino,Spanish,Vietnamese": 0.19318181818181818, - "English,Filipino,Chinese,Malay": 0.26136363636363635, - "English,Filipino,Chinese,Vietnamese": 0.19886363636363635, - "English,Filipino,Malay,Vietnamese": 0.2159090909090909, - "English,Spanish,Chinese,Malay": 0.32386363636363635, - "English,Spanish,Chinese,Vietnamese": 0.23295454545454544, - "English,Spanish,Malay,Vietnamese": 0.25, - "English,Chinese,Malay,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Chinese,Malay": 0.25, - "Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Malay,Vietnamese": 0.22727272727272727, - "Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.2159090909090909, - "Indonesian,English,Filipino,Spanish,Malay": 0.22727272727272727, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.17045454545454544, - "Indonesian,English,Filipino,Chinese,Malay": 0.23863636363636365, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.17045454545454544, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.19318181818181818, - "Indonesian,English,Spanish,Chinese,Malay": 0.30113636363636365, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.23295454545454544, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.2159090909090909, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.22727272727272727, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.21022727272727273, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.21022727272727273, - "English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "English,Filipino,Chinese,Malay,Vietnamese": 0.17613636363636365, - "English,Spanish,Chinese,Malay,Vietnamese": 0.1875, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.1534090909090909, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 - } - }, - "AC3_2": 0.48637059555450246, - "AC3_3": 0.4011546909805022, - "AC3_4": 0.33288730561759317, - "AC3_5": 0.27538295738897717, - "AC3_6": 0.22548634305473386, - "AC3_7": 0.18048128338976094 - }, - "prompt_4": { - "overall_acc": 0.4375, - "language_acc": { - "Indonesian": 0.4659090909090909, - "English": 0.4943181818181818, - "Filipino": 0.3465909090909091, - "Spanish": 0.48295454545454547, - "Chinese": 0.4943181818181818, - "Malay": 0.3977272727272727, - "Vietnamese": 0.3806818181818182 - }, - "consistency_score_2": 0.5514069264069262, - "consistency_score_3": 0.37061688311688307, - "consistency_score_4": 0.2641233766233766, - "consistency_score_5": 0.1931818181818182, - "consistency_score_6": 0.14366883116883117, - "consistency_score_7": 0.10795454545454546, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6363636363636364, - "Indonesian,Filipino": 0.5, - "Indonesian,Spanish": 0.6988636363636364, - "Indonesian,Chinese": 0.6477272727272727, - "Indonesian,Malay": 0.7272727272727273, - "Indonesian,Vietnamese": 0.5738636363636364, - "English,Filipino": 0.3806818181818182, - "English,Spanish": 0.6079545454545454, - "English,Chinese": 0.5852272727272727, - "English,Malay": 0.5625, - "English,Vietnamese": 0.42045454545454547, - "Filipino,Spanish": 0.4659090909090909, - "Filipino,Chinese": 0.44886363636363635, - "Filipino,Malay": 0.4943181818181818, - "Filipino,Vietnamese": 0.5056818181818182, - "Spanish,Chinese": 0.6022727272727273, - "Spanish,Malay": 0.6079545454545454, - "Spanish,Vietnamese": 0.5170454545454546, - "Chinese,Malay": 0.5625, - "Chinese,Vietnamese": 0.4943181818181818, - "Malay,Vietnamese": 0.5397727272727273 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.32386363636363635, - "Indonesian,English,Spanish": 0.4943181818181818, - "Indonesian,English,Chinese": 0.48295454545454547, - "Indonesian,English,Malay": 0.4943181818181818, - "Indonesian,English,Vietnamese": 0.3522727272727273, - "Indonesian,Filipino,Spanish": 0.38636363636363635, - "Indonesian,Filipino,Chinese": 0.3409090909090909, - "Indonesian,Filipino,Malay": 0.39204545454545453, - "Indonesian,Filipino,Vietnamese": 0.3352272727272727, - "Indonesian,Spanish,Chinese": 0.5, - "Indonesian,Spanish,Malay": 0.5284090909090909, - "Indonesian,Spanish,Vietnamese": 0.4147727272727273, - "Indonesian,Chinese,Malay": 0.48863636363636365, - "Indonesian,Chinese,Vietnamese": 0.38636363636363635, - "Indonesian,Malay,Vietnamese": 0.4431818181818182, - "English,Filipino,Spanish": 0.2840909090909091, - "English,Filipino,Chinese": 0.2840909090909091, - "English,Filipino,Malay": 0.2784090909090909, - "English,Filipino,Vietnamese": 0.23295454545454544, - "English,Spanish,Chinese": 0.4431818181818182, - "English,Spanish,Malay": 0.42045454545454547, - "English,Spanish,Vietnamese": 0.3181818181818182, - "English,Chinese,Malay": 0.4034090909090909, - "English,Chinese,Vietnamese": 0.3068181818181818, - "English,Malay,Vietnamese": 0.3068181818181818, - "Filipino,Spanish,Chinese": 0.3068181818181818, - "Filipino,Spanish,Malay": 0.3352272727272727, - "Filipino,Spanish,Vietnamese": 0.30113636363636365, - "Filipino,Chinese,Malay": 0.2897727272727273, - "Filipino,Chinese,Vietnamese": 0.2897727272727273, - "Filipino,Malay,Vietnamese": 0.3181818181818182, - "Spanish,Chinese,Malay": 0.42613636363636365, - "Spanish,Chinese,Vietnamese": 0.3465909090909091, - "Spanish,Malay,Vietnamese": 0.375, - "Chinese,Malay,Vietnamese": 0.3409090909090909 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.25, - "Indonesian,English,Filipino,Chinese": 0.2556818181818182, - "Indonesian,English,Filipino,Malay": 0.25, - "Indonesian,English,Filipino,Vietnamese": 0.21022727272727273, - "Indonesian,English,Spanish,Chinese": 0.39204545454545453, - "Indonesian,English,Spanish,Malay": 0.39204545454545453, - "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, - "Indonesian,English,Chinese,Malay": 0.375, - "Indonesian,English,Chinese,Vietnamese": 0.2727272727272727, - "Indonesian,English,Malay,Vietnamese": 0.2840909090909091, - "Indonesian,Filipino,Spanish,Chinese": 0.2727272727272727, - "Indonesian,Filipino,Spanish,Malay": 0.30113636363636365, - "Indonesian,Filipino,Spanish,Vietnamese": 0.2556818181818182, - "Indonesian,Filipino,Chinese,Malay": 0.26704545454545453, - "Indonesian,Filipino,Chinese,Vietnamese": 0.23863636363636365, - "Indonesian,Filipino,Malay,Vietnamese": 0.26704545454545453, - "Indonesian,Spanish,Chinese,Malay": 0.38636363636363635, - "Indonesian,Spanish,Chinese,Vietnamese": 0.2897727272727273, - "Indonesian,Spanish,Malay,Vietnamese": 0.3352272727272727, - "Indonesian,Chinese,Malay,Vietnamese": 0.3068181818181818, - "English,Filipino,Spanish,Chinese": 0.2159090909090909, - "English,Filipino,Spanish,Malay": 0.21022727272727273, - "English,Filipino,Spanish,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Malay": 0.21022727272727273, - "English,Filipino,Chinese,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Vietnamese": 0.17045454545454544, - "English,Spanish,Chinese,Malay": 0.32954545454545453, - "English,Spanish,Chinese,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay,Vietnamese": 0.23863636363636365, - "English,Chinese,Malay,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Chinese,Malay": 0.22727272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Malay,Vietnamese": 0.26136363636363635 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.19886363636363635, - "Indonesian,English,Filipino,Spanish,Malay": 0.19318181818181818, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.16477272727272727, - "Indonesian,English,Filipino,Chinese,Malay": 0.19886363636363635, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.17613636363636365, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.1590909090909091, - "Indonesian,English,Spanish,Chinese,Malay": 0.3125, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2215909090909091, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.23295454545454544, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.2215909090909091, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.2159090909090909, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.1875, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.19318181818181818, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.23863636363636365, - "English,Filipino,Spanish,Chinese,Malay": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, - "English,Spanish,Chinese,Malay,Vietnamese": 0.1875, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.125, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11363636363636363 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 - } - }, - "AC3_2": 0.48789329680428845, - "AC3_3": 0.40129067893788767, - "AC3_4": 0.32939032850464955, - "AC3_5": 0.2680180179755215, - "AC3_6": 0.21630586588456857, - "AC3_7": 0.1731770833015842 - }, - "prompt_5": { - "overall_acc": 0.4391233766233766, - "language_acc": { - "Indonesian": 0.4715909090909091, - "English": 0.5113636363636364, - "Filipino": 0.3352272727272727, - "Spanish": 0.4943181818181818, - "Chinese": 0.48295454545454547, - "Malay": 0.39204545454545453, - "Vietnamese": 0.38636363636363635 - }, - "consistency_score_2": 0.5462662337662338, - "consistency_score_3": 0.3625, - "consistency_score_4": 0.25649350649350655, - "consistency_score_5": 0.18614718614718617, - "consistency_score_6": 0.13555194805194806, - "consistency_score_7": 0.09659090909090909, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6306818181818182, - "Indonesian,Filipino": 0.4659090909090909, - "Indonesian,Spanish": 0.6875, - "Indonesian,Chinese": 0.6477272727272727, - "Indonesian,Malay": 0.6931818181818182, - "Indonesian,Vietnamese": 0.5397727272727273, - "English,Filipino": 0.4090909090909091, - "English,Spanish": 0.6534090909090909, - "English,Chinese": 0.5625, - "English,Malay": 0.5738636363636364, - "English,Vietnamese": 0.4431818181818182, - "Filipino,Spanish": 0.42045454545454547, - "Filipino,Chinese": 0.4375, - "Filipino,Malay": 0.4659090909090909, - "Filipino,Vietnamese": 0.5113636363636364, - "Spanish,Chinese": 0.5738636363636364, - "Spanish,Malay": 0.6193181818181818, - "Spanish,Vietnamese": 0.5511363636363636, - "Chinese,Malay": 0.5568181818181818, - "Chinese,Vietnamese": 0.5, - "Malay,Vietnamese": 0.5284090909090909 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.3181818181818182, - "Indonesian,English,Spanish": 0.5056818181818182, - "Indonesian,English,Chinese": 0.4602272727272727, - "Indonesian,English,Malay": 0.4715909090909091, - "Indonesian,English,Vietnamese": 0.3465909090909091, - "Indonesian,Filipino,Spanish": 0.32954545454545453, - "Indonesian,Filipino,Chinese": 0.3181818181818182, - "Indonesian,Filipino,Malay": 0.375, - "Indonesian,Filipino,Vietnamese": 0.3181818181818182, - "Indonesian,Spanish,Chinese": 0.48295454545454547, - "Indonesian,Spanish,Malay": 0.5227272727272727, - "Indonesian,Spanish,Vietnamese": 0.4147727272727273, - "Indonesian,Chinese,Malay": 0.4772727272727273, - "Indonesian,Chinese,Vietnamese": 0.375, - "Indonesian,Malay,Vietnamese": 0.4090909090909091, - "English,Filipino,Spanish": 0.2897727272727273, - "English,Filipino,Chinese": 0.26704545454545453, - "English,Filipino,Malay": 0.2897727272727273, - "English,Filipino,Vietnamese": 0.23863636363636365, - "English,Spanish,Chinese": 0.4375, - "English,Spanish,Malay": 0.44886363636363635, - "English,Spanish,Vietnamese": 0.3522727272727273, - "English,Chinese,Malay": 0.375, - "English,Chinese,Vietnamese": 0.3125, - "English,Malay,Vietnamese": 0.3068181818181818, - "Filipino,Spanish,Chinese": 0.26704545454545453, - "Filipino,Spanish,Malay": 0.3125, - "Filipino,Spanish,Vietnamese": 0.29545454545454547, - "Filipino,Chinese,Malay": 0.2784090909090909, - "Filipino,Chinese,Vietnamese": 0.3068181818181818, - "Filipino,Malay,Vietnamese": 0.30113636363636365, - "Spanish,Chinese,Malay": 0.4147727272727273, - "Spanish,Chinese,Vietnamese": 0.3522727272727273, - "Spanish,Malay,Vietnamese": 0.38636363636363635, - "Chinese,Malay,Vietnamese": 0.32954545454545453 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.24431818181818182, - "Indonesian,English,Filipino,Chinese": 0.24431818181818182, - "Indonesian,English,Filipino,Malay": 0.25, - "Indonesian,English,Filipino,Vietnamese": 0.21022727272727273, - "Indonesian,English,Spanish,Chinese": 0.3806818181818182, - "Indonesian,English,Spanish,Malay": 0.3977272727272727, - "Indonesian,English,Spanish,Vietnamese": 0.2897727272727273, - "Indonesian,English,Chinese,Malay": 0.3352272727272727, - "Indonesian,English,Chinese,Vietnamese": 0.26136363636363635, - "Indonesian,English,Malay,Vietnamese": 0.2727272727272727, - "Indonesian,Filipino,Spanish,Chinese": 0.23863636363636365, - "Indonesian,Filipino,Spanish,Malay": 0.2784090909090909, - "Indonesian,Filipino,Spanish,Vietnamese": 0.23863636363636365, - "Indonesian,Filipino,Chinese,Malay": 0.25, - "Indonesian,Filipino,Chinese,Vietnamese": 0.23863636363636365, - "Indonesian,Filipino,Malay,Vietnamese": 0.24431818181818182, - "Indonesian,Spanish,Chinese,Malay": 0.38636363636363635, - "Indonesian,Spanish,Chinese,Vietnamese": 0.29545454545454547, - "Indonesian,Spanish,Malay,Vietnamese": 0.3409090909090909, - "Indonesian,Chinese,Malay,Vietnamese": 0.2897727272727273, - "English,Filipino,Spanish,Chinese": 0.21022727272727273, - "English,Filipino,Spanish,Malay": 0.2159090909090909, - "English,Filipino,Spanish,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Malay": 0.19318181818181818, - "English,Filipino,Chinese,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Vietnamese": 0.16477272727272727, - "English,Spanish,Chinese,Malay": 0.32386363636363635, - "English,Spanish,Chinese,Vietnamese": 0.25, - "English,Spanish,Malay,Vietnamese": 0.2556818181818182, - "English,Chinese,Malay,Vietnamese": 0.21022727272727273, - "Filipino,Spanish,Chinese,Malay": 0.20454545454545456, - "Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, - "Filipino,Chinese,Malay,Vietnamese": 0.21022727272727273, - "Spanish,Chinese,Malay,Vietnamese": 0.2556818181818182 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.19318181818181818, - "Indonesian,English,Filipino,Spanish,Malay": 0.19886363636363635, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.16477272727272727, - "Indonesian,English,Filipino,Chinese,Malay": 0.18181818181818182, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.17613636363636365, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.1534090909090909, - "Indonesian,English,Spanish,Chinese,Malay": 0.30113636363636365, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.2159090909090909, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.23863636363636365, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.19318181818181818, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.19886363636363635, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.18181818181818182, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.18181818181818182, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.24431818181818182, - "English,Filipino,Spanish,Chinese,Malay": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Chinese,Malay,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Malay,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.1534090909090909, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.125, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10227272727272728 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 - } - }, - "AC3_2": 0.48686990516893125, - "AC3_3": 0.397149655681522, - "AC3_4": 0.3238342753629844, - "AC3_5": 0.26145987284168354, - "AC3_6": 0.2071570731167882, - "AC3_7": 0.15835055093462852 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.5922330097087378 - }, - "prompt_2": { - "accuracy": 0.6213592233009708 - }, - "prompt_3": { - "accuracy": 0.6116504854368932 - }, - "prompt_4": { - "accuracy": 0.5728155339805825 - }, - "prompt_5": { - "accuracy": 0.6310679611650486 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.4 - }, - "prompt_2": { - "accuracy": 0.38095238095238093 - }, - "prompt_3": { - "accuracy": 0.41904761904761906 - }, - "prompt_4": { - "accuracy": 0.3904761904761905 - }, - "prompt_5": { - "accuracy": 0.44761904761904764 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.6074766355140186 - }, - "prompt_2": { - "accuracy": 0.6074766355140186 - }, - "prompt_3": { - "accuracy": 0.6448598130841121 - }, - "prompt_4": { - "accuracy": 0.6542056074766355 - }, - "prompt_5": { - "accuracy": 0.6542056074766355 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.38, - "category_acc": { - "brand": 0.4, - "demographics": 0.0, - "biology": 0.2, - "history": 0.2, - "literature": 0.2, - "politics": 0.6, - "culture": 0.8, - "film": 0.4, - "law": 0.2, - "geography": 0.7 - } - }, - "prompt_2": { - "accuracy": 0.39, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.3, - "history": 0.2, - "literature": 0.2, - "politics": 0.6, - "culture": 0.6, - "film": 0.5, - "law": 0.2, - "geography": 0.7 - } - }, - "prompt_3": { - "accuracy": 0.36, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.2, - "politics": 0.5, - "culture": 0.6, - "film": 0.2, - "law": 0.2, - "geography": 0.7 - } - }, - "prompt_4": { - "accuracy": 0.39, - "category_acc": { - "brand": 0.3, - "demographics": 0.0, - "biology": 0.2, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.6, - "culture": 0.6, - "film": 0.5, - "law": 0.3, - "geography": 0.7 - } - }, - "prompt_5": { - "accuracy": 0.38, - "category_acc": { - "brand": 0.4, - "demographics": 0.0, - "biology": 0.2, - "history": 0.26666666666666666, - "literature": 0.2, - "politics": 0.6, - "culture": 0.6, - "film": 0.4, - "law": 0.3, - "geography": 0.7 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.2574017356444316 - }, - "prompt_2": { - "bleu_score": 0.2697397453044214 - }, - "prompt_3": { - "bleu_score": 0.2710806888883188 - }, - "prompt_4": { - "bleu_score": 0.252810003719897 - }, - "prompt_5": { - "bleu_score": 0.223438870981906 - } - }, - "indommlu": { - "prompt_1": { - "accuracy": 0.49495961012083584, - "category_acc": { - "History": 0.4819277108433735, - "Geography": 0.46938775510204084, - "Lampungic": 0.3877551020408163, - "Social science": 0.7328881469115192, - "Balinese": 0.34182590233545646, - "Makassarese": 0.34946236559139787, - "Banjarese": 0.4027777777777778, - "Chemistry": 0.32554744525547447, - "Biology": 0.43431952662721895, - "Science": 0.6171310629514963, - "Christian religion": 0.5920398009950248, - "Art": 0.6039933444259568, - "Islam religion": 0.6002844950213371, - "Hindu religion": 0.5066666666666667, - "Madurese": 0.3152542372881356, - "Sport": 0.5337837837837838, - "Indonesian language": 0.5703611457036114, - "Physics": 0.4202020202020202, - "Minangkabau culture": 0.39195979899497485, - "Dayak language": 0.26605504587155965, - "Sociology": 0.4879032258064516, - "Economy": 0.4733606557377049, - "Sundanese": 0.3777009507346586, - "Javanese": 0.3538306451612903, - "Civic education": 0.5951359084406295 - } - }, - "prompt_2": { - "accuracy": 0.4969624140463315, - "category_acc": { - "History": 0.4799196787148594, - "Geography": 0.47346938775510206, - "Lampungic": 0.3469387755102041, - "Social science": 0.7445742904841403, - "Balinese": 0.3375796178343949, - "Makassarese": 0.3602150537634409, - "Banjarese": 0.3680555555555556, - "Chemistry": 0.32408759124087594, - "Biology": 0.44260355029585796, - "Science": 0.6243550051599587, - "Christian religion": 0.6119402985074627, - "Art": 0.6173044925124792, - "Islam religion": 0.6287339971550497, - "Hindu religion": 0.52, - "Madurese": 0.3152542372881356, - "Sport": 0.527027027027027, - "Indonesian language": 0.5625778331257784, - "Physics": 0.40404040404040403, - "Minangkabau culture": 0.3768844221105528, - "Dayak language": 0.27522935779816515, - "Sociology": 0.48185483870967744, - "Economy": 0.4651639344262295, - "Sundanese": 0.38547968885047534, - "Javanese": 0.3719758064516129, - "Civic education": 0.597997138769671 - } - }, - "prompt_3": { - "accuracy": 0.49302356632619, - "category_acc": { - "History": 0.4738955823293173, - "Geography": 0.463265306122449, - "Lampungic": 0.3401360544217687, - "Social science": 0.7345575959933222, - "Balinese": 0.31210191082802546, - "Makassarese": 0.3548387096774194, - "Banjarese": 0.4097222222222222, - "Chemistry": 0.31532846715328466, - "Biology": 0.44023668639053254, - "Science": 0.6181630546955624, - "Christian religion": 0.6069651741293532, - "Art": 0.610648918469218, - "Islam religion": 0.6244665718349929, - "Hindu religion": 0.5133333333333333, - "Madurese": 0.3050847457627119, - "Sport": 0.527027027027027, - "Indonesian language": 0.5619551681195517, - "Physics": 0.4080808080808081, - "Minangkabau culture": 0.40703517587939697, - "Dayak language": 0.3211009174311927, - "Sociology": 0.4798387096774194, - "Economy": 0.45901639344262296, - "Sundanese": 0.37251512532411407, - "Javanese": 0.3780241935483871, - "Civic education": 0.5851216022889842 - } - }, - "prompt_4": { - "accuracy": 0.4936244075038387, - "category_acc": { - "History": 0.4738955823293173, - "Geography": 0.45510204081632655, - "Lampungic": 0.35374149659863946, - "Social science": 0.7245409015025042, - "Balinese": 0.34182590233545646, - "Makassarese": 0.3763440860215054, - "Banjarese": 0.3888888888888889, - "Chemistry": 0.3167883211678832, - "Biology": 0.4461538461538462, - "Science": 0.6160990712074303, - "Christian religion": 0.6069651741293532, - "Art": 0.6073211314475874, - "Islam religion": 0.6116642958748222, - "Hindu religion": 0.5, - "Madurese": 0.2847457627118644, - "Sport": 0.5337837837837838, - "Indonesian language": 0.5675591531755916, - "Physics": 0.402020202020202, - "Minangkabau culture": 0.38190954773869346, - "Dayak language": 0.27522935779816515, - "Sociology": 0.49193548387096775, - "Economy": 0.4672131147540984, - "Sundanese": 0.3777009507346586, - "Javanese": 0.3629032258064516, - "Civic education": 0.5994277539341917 - } - }, - "prompt_5": { - "accuracy": 0.4945590493357367, - "category_acc": { - "History": 0.4779116465863454, - "Geography": 0.4448979591836735, - "Lampungic": 0.3469387755102041, - "Social science": 0.7145242070116862, - "Balinese": 0.3333333333333333, - "Makassarese": 0.3709677419354839, - "Banjarese": 0.4097222222222222, - "Chemistry": 0.32408759124087594, - "Biology": 0.44260355029585796, - "Science": 0.6202270381836945, - "Christian religion": 0.6268656716417911, - "Art": 0.6023294509151415, - "Islam religion": 0.6031294452347084, - "Hindu religion": 0.5, - "Madurese": 0.3152542372881356, - "Sport": 0.5540540540540541, - "Indonesian language": 0.5691158156911582, - "Physics": 0.41414141414141414, - "Minangkabau culture": 0.3969849246231156, - "Dayak language": 0.28440366972477066, - "Sociology": 0.4717741935483871, - "Economy": 0.4672131147540984, - "Sundanese": 0.3802938634399309, - "Javanese": 0.37399193548387094, - "Civic education": 0.5908440629470673 - } - } - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.2904210030825523 - }, - "prompt_2": { - "bleu_score": 0.36955498730132713 - }, - "prompt_3": { - "bleu_score": 0.36547048717499736 - }, - "prompt_4": { - "bleu_score": 0.3656155033164473 - }, - "prompt_5": { - "bleu_score": 0.3613834087838442 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.1361309104530537 - }, - "prompt_2": { - "bleu_score": 0.22571437892643548 - }, - "prompt_3": { - "bleu_score": 0.2207638338470559 - }, - "prompt_4": { - "bleu_score": 0.21907103091395946 - }, - "prompt_5": { - "bleu_score": 0.18624794146447873 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.2141645930096053 - }, - "prompt_2": { - "bleu_score": 0.22386319490546705 - }, - "prompt_3": { - "bleu_score": 0.22159896503568985 - }, - "prompt_4": { - "bleu_score": 0.22119680625556593 - }, - "prompt_5": { - "bleu_score": 0.20898856132141055 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.20246195784882284 - }, - "prompt_2": { - "bleu_score": 0.34339277822910913 - }, - "prompt_3": { - "bleu_score": 0.3414705878211933 - }, - "prompt_4": { - "bleu_score": 0.3421444160411124 - }, - "prompt_5": { - "bleu_score": 0.31648254812909576 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.544924154025671 - }, - "prompt_2": { - "accuracy": 0.5694282380396732 - }, - "prompt_3": { - "accuracy": 0.5472578763127188 - }, - "prompt_4": { - "accuracy": 0.5530921820303384 - }, - "prompt_5": { - "accuracy": 0.5390898483080513 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.5572398998927423, - "category_acc": { - "high_school_european_history": 0.7073170731707317, - "business_ethics": 0.6262626262626263, - "clinical_knowledge": 0.6363636363636364, - "medical_genetics": 0.6262626262626263, - "high_school_us_history": 0.7438423645320197, - "high_school_physics": 0.26666666666666666, - "high_school_world_history": 0.7288135593220338, - "virology": 0.4666666666666667, - "high_school_microeconomics": 0.5864978902953587, - "econometrics": 0.34513274336283184, - "college_computer_science": 0.5555555555555556, - "high_school_biology": 0.6699029126213593, - "abstract_algebra": 0.35353535353535354, - "professional_accounting": 0.4128113879003559, - "philosophy": 0.5903225806451613, - "professional_medicine": 0.6494464944649446, - "nutrition": 0.5868852459016394, - "global_facts": 0.35353535353535354, - "machine_learning": 0.3963963963963964, - "security_studies": 0.5737704918032787, - "public_relations": 0.6605504587155964, - "professional_psychology": 0.5417348608837971, - "prehistory": 0.5882352941176471, - "anatomy": 0.5298507462686567, - "human_sexuality": 0.6384615384615384, - "college_medicine": 0.6104651162790697, - "high_school_government_and_politics": 0.8020833333333334, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.7037037037037037, - "high_school_geography": 0.7258883248730964, - "elementary_mathematics": 0.3740053050397878, - "human_aging": 0.6441441441441441, - "college_mathematics": 0.3434343434343434, - "high_school_psychology": 0.7867647058823529, - "formal_logic": 0.48, - "high_school_statistics": 0.413953488372093, - "international_law": 0.7, - "high_school_mathematics": 0.3159851301115242, - "high_school_computer_science": 0.5959595959595959, - "conceptual_physics": 0.45726495726495725, - "miscellaneous": 0.7902813299232737, - "high_school_chemistry": 0.43564356435643564, - "marketing": 0.8412017167381974, - "professional_law": 0.4187866927592955, - "management": 0.7647058823529411, - "college_physics": 0.36633663366336633, - "jurisprudence": 0.6261682242990654, - "world_religions": 0.8411764705882353, - "sociology": 0.765, - "us_foreign_policy": 0.7474747474747475, - "high_school_macroeconomics": 0.5295629820051414, - "computer_security": 0.7272727272727273, - "moral_scenarios": 0.25727069351230425, - "moral_disputes": 0.6202898550724638, - "electrical_engineering": 0.5138888888888888, - "astronomy": 0.5695364238410596, - "college_biology": 0.6223776223776224 - } - }, - "prompt_2": { - "accuracy": 0.5602431176260279, - "category_acc": { - "high_school_european_history": 0.7195121951219512, - "business_ethics": 0.5555555555555556, - "clinical_knowledge": 0.6553030303030303, - "medical_genetics": 0.6262626262626263, - "high_school_us_history": 0.7536945812807881, - "high_school_physics": 0.32666666666666666, - "high_school_world_history": 0.7372881355932204, - "virology": 0.4727272727272727, - "high_school_microeconomics": 0.6033755274261603, - "econometrics": 0.3805309734513274, - "college_computer_science": 0.5454545454545454, - "high_school_biology": 0.6763754045307443, - "abstract_algebra": 0.31313131313131315, - "professional_accounting": 0.4234875444839858, - "philosophy": 0.6096774193548387, - "professional_medicine": 0.6346863468634686, - "nutrition": 0.5770491803278689, - "global_facts": 0.36363636363636365, - "machine_learning": 0.4144144144144144, - "security_studies": 0.6311475409836066, - "public_relations": 0.6697247706422018, - "professional_psychology": 0.5482815057283142, - "prehistory": 0.6130030959752322, - "anatomy": 0.5298507462686567, - "human_sexuality": 0.6384615384615384, - "college_medicine": 0.5988372093023255, - "high_school_government_and_politics": 0.796875, - "college_chemistry": 0.42424242424242425, - "logical_fallacies": 0.6975308641975309, - "high_school_geography": 0.751269035532995, - "elementary_mathematics": 0.3819628647214854, - "human_aging": 0.6306306306306306, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.7849264705882353, - "formal_logic": 0.464, - "high_school_statistics": 0.4372093023255814, - "international_law": 0.7, - "high_school_mathematics": 0.2788104089219331, - "high_school_computer_science": 0.5959595959595959, - "conceptual_physics": 0.48717948717948717, - "miscellaneous": 0.789002557544757, - "high_school_chemistry": 0.4405940594059406, - "marketing": 0.8583690987124464, - "professional_law": 0.39399869536855836, - "management": 0.7647058823529411, - "college_physics": 0.4158415841584158, - "jurisprudence": 0.6542056074766355, - "world_religions": 0.8176470588235294, - "sociology": 0.77, - "us_foreign_policy": 0.7676767676767676, - "high_school_macroeconomics": 0.5424164524421594, - "computer_security": 0.7171717171717171, - "moral_scenarios": 0.25279642058165547, - "moral_disputes": 0.6434782608695652, - "electrical_engineering": 0.5347222222222222, - "astronomy": 0.5629139072847682, - "college_biology": 0.6713286713286714 - } - }, - "prompt_3": { - "accuracy": 0.5461565963532357, - "category_acc": { - "high_school_european_history": 0.6951219512195121, - "business_ethics": 0.6161616161616161, - "clinical_knowledge": 0.6212121212121212, - "medical_genetics": 0.6262626262626263, - "high_school_us_history": 0.7586206896551724, - "high_school_physics": 0.31333333333333335, - "high_school_world_history": 0.7245762711864406, - "virology": 0.4727272727272727, - "high_school_microeconomics": 0.6118143459915611, - "econometrics": 0.36283185840707965, - "college_computer_science": 0.5353535353535354, - "high_school_biology": 0.6472491909385113, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.40213523131672596, - "philosophy": 0.603225806451613, - "professional_medicine": 0.6125461254612546, - "nutrition": 0.5540983606557377, - "global_facts": 0.35353535353535354, - "machine_learning": 0.3963963963963964, - "security_studies": 0.6065573770491803, - "public_relations": 0.6880733944954128, - "professional_psychology": 0.5270049099836334, - "prehistory": 0.5882352941176471, - "anatomy": 0.4925373134328358, - "human_sexuality": 0.6153846153846154, - "college_medicine": 0.5523255813953488, - "high_school_government_and_politics": 0.7708333333333334, - "college_chemistry": 0.42424242424242425, - "logical_fallacies": 0.6604938271604939, - "high_school_geography": 0.7055837563451777, - "elementary_mathematics": 0.3740053050397878, - "human_aging": 0.6261261261261262, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.7536764705882353, - "formal_logic": 0.48, - "high_school_statistics": 0.4046511627906977, - "international_law": 0.6916666666666667, - "high_school_mathematics": 0.30111524163568776, - "high_school_computer_science": 0.5757575757575758, - "conceptual_physics": 0.44017094017094016, - "miscellaneous": 0.7774936061381074, - "high_school_chemistry": 0.43564356435643564, - "marketing": 0.7939914163090128, - "professional_law": 0.4090019569471624, - "management": 0.7549019607843137, - "college_physics": 0.3564356435643564, - "jurisprudence": 0.6635514018691588, - "world_religions": 0.8176470588235294, - "sociology": 0.715, - "us_foreign_policy": 0.7171717171717171, - "high_school_macroeconomics": 0.519280205655527, - "computer_security": 0.6868686868686869, - "moral_scenarios": 0.25615212527964204, - "moral_disputes": 0.6144927536231884, - "electrical_engineering": 0.5347222222222222, - "astronomy": 0.5496688741721855, - "college_biology": 0.6153846153846154 - } - }, - "prompt_4": { - "accuracy": 0.5447264926707186, - "category_acc": { - "high_school_european_history": 0.7134146341463414, - "business_ethics": 0.5858585858585859, - "clinical_knowledge": 0.5909090909090909, - "medical_genetics": 0.6262626262626263, - "high_school_us_history": 0.7586206896551724, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.7161016949152542, - "virology": 0.47878787878787876, - "high_school_microeconomics": 0.6033755274261603, - "econometrics": 0.35398230088495575, - "college_computer_science": 0.5252525252525253, - "high_school_biology": 0.6601941747572816, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.4128113879003559, - "philosophy": 0.5838709677419355, - "professional_medicine": 0.6051660516605166, - "nutrition": 0.6, - "global_facts": 0.35353535353535354, - "machine_learning": 0.35135135135135137, - "security_studies": 0.5942622950819673, - "public_relations": 0.6422018348623854, - "professional_psychology": 0.5122749590834698, - "prehistory": 0.6037151702786377, - "anatomy": 0.5074626865671642, - "human_sexuality": 0.6461538461538462, - "college_medicine": 0.5523255813953488, - "high_school_government_and_politics": 0.765625, - "college_chemistry": 0.42424242424242425, - "logical_fallacies": 0.6728395061728395, - "high_school_geography": 0.7360406091370558, - "elementary_mathematics": 0.3793103448275862, - "human_aging": 0.6441441441441441, - "college_mathematics": 0.32323232323232326, - "high_school_psychology": 0.7555147058823529, - "formal_logic": 0.488, - "high_school_statistics": 0.3953488372093023, - "international_law": 0.7, - "high_school_mathematics": 0.30111524163568776, - "high_school_computer_science": 0.5959595959595959, - "conceptual_physics": 0.452991452991453, - "miscellaneous": 0.7583120204603581, - "high_school_chemistry": 0.4158415841584158, - "marketing": 0.8240343347639485, - "professional_law": 0.40769732550554466, - "management": 0.7647058823529411, - "college_physics": 0.3564356435643564, - "jurisprudence": 0.6261682242990654, - "world_religions": 0.8176470588235294, - "sociology": 0.75, - "us_foreign_policy": 0.7070707070707071, - "high_school_macroeconomics": 0.4910025706940874, - "computer_security": 0.6868686868686869, - "moral_scenarios": 0.2550335570469799, - "moral_disputes": 0.5942028985507246, - "electrical_engineering": 0.5208333333333334, - "astronomy": 0.5562913907284768, - "college_biology": 0.6293706293706294 - } - }, - "prompt_5": { - "accuracy": 0.5319985698963174, - "category_acc": { - "high_school_european_history": 0.6829268292682927, - "business_ethics": 0.5757575757575758, - "clinical_knowledge": 0.5871212121212122, - "medical_genetics": 0.5858585858585859, - "high_school_us_history": 0.7536945812807881, - "high_school_physics": 0.2733333333333333, - "high_school_world_history": 0.7076271186440678, - "virology": 0.44242424242424244, - "high_school_microeconomics": 0.5738396624472574, - "econometrics": 0.35398230088495575, - "college_computer_science": 0.5050505050505051, - "high_school_biology": 0.6472491909385113, - "abstract_algebra": 0.2727272727272727, - "professional_accounting": 0.4199288256227758, - "philosophy": 0.5580645161290323, - "professional_medicine": 0.5977859778597786, - "nutrition": 0.5540983606557377, - "global_facts": 0.36363636363636365, - "machine_learning": 0.3783783783783784, - "security_studies": 0.6024590163934426, - "public_relations": 0.5963302752293578, - "professional_psychology": 0.5090016366612111, - "prehistory": 0.5758513931888545, - "anatomy": 0.4925373134328358, - "human_sexuality": 0.6615384615384615, - "college_medicine": 0.5465116279069767, - "high_school_government_and_politics": 0.7552083333333334, - "college_chemistry": 0.36363636363636365, - "logical_fallacies": 0.6296296296296297, - "high_school_geography": 0.7309644670050761, - "elementary_mathematics": 0.3925729442970822, - "human_aging": 0.6306306306306306, - "college_mathematics": 0.29292929292929293, - "high_school_psychology": 0.7316176470588235, - "formal_logic": 0.48, - "high_school_statistics": 0.4232558139534884, - "international_law": 0.6916666666666667, - "high_school_mathematics": 0.2936802973977695, - "high_school_computer_science": 0.5454545454545454, - "conceptual_physics": 0.44871794871794873, - "miscellaneous": 0.7583120204603581, - "high_school_chemistry": 0.4306930693069307, - "marketing": 0.759656652360515, - "professional_law": 0.41226353555120676, - "management": 0.7450980392156863, - "college_physics": 0.3465346534653465, - "jurisprudence": 0.6728971962616822, - "world_religions": 0.7823529411764706, - "sociology": 0.755, - "us_foreign_policy": 0.6767676767676768, - "high_school_macroeconomics": 0.4832904884318766, - "computer_security": 0.6666666666666666, - "moral_scenarios": 0.22930648769574943, - "moral_disputes": 0.5594202898550724, - "electrical_engineering": 0.5, - "astronomy": 0.48344370860927155, - "college_biology": 0.5734265734265734 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.4227340267459138 - }, - "prompt_2": { - "accuracy": 0.4279346210995542 - }, - "prompt_3": { - "accuracy": 0.4323922734026746 - }, - "prompt_4": { - "accuracy": 0.4078751857355126 - }, - "prompt_5": { - "accuracy": 0.43313521545319467 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.4246575342465753, - "category_acc": { - "computer_network": 0.5, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.42857142857142855, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.47619047619047616, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.41379310344827586, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.625, - "middle_school_chemistry": 0.56, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.45, - "business_administration": 0.4473684210526316, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.6122448979591837, - "high_school_politics": 0.375, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.5384615384615384, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.4642857142857143, - "ideological_and_moral_cultivation": 0.75, - "logic": 0.48148148148148145, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.5, - "legal_professional": 0.5, - "high_school_chinese": 0.375, - "high_school_history": 0.56, - "middle_school_history": 0.6296296296296297, - "civil_servant": 0.36538461538461536, - "sports_science": 0.375, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.4583333333333333, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.45098039215686275, - "accountant": 0.37037037037037035, - "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.2962962962962963, - "physician": 0.3888888888888889 - } - }, - "prompt_2": { - "accuracy": 0.42528019925280197, - "category_acc": { - "computer_network": 0.5416666666666666, - "operating_system": 0.5833333333333334, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.42857142857142855, - "college_physics": 0.375, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.42857142857142855, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.5172413793103449, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.25, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.46153846153846156, - "middle_school_physics": 0.625, - "middle_school_chemistry": 0.52, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.4166666666666667, - "business_administration": 0.42105263157894735, - "marxism": 0.4583333333333333, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.6326530612244898, - "high_school_politics": 0.375, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.6538461538461539, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.6666666666666666, - "logic": 0.5185185185185185, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.5294117647058824, - "legal_professional": 0.5357142857142857, - "high_school_chinese": 0.375, - "high_school_history": 0.52, - "middle_school_history": 0.6666666666666666, - "civil_servant": 0.34615384615384615, - "sports_science": 0.25, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.5, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.43137254901960786, - "accountant": 0.37037037037037035, - "fire_engineer": 0.3611111111111111, - "environmental_impact_assessment_engineer": 0.4444444444444444, - "tax_accountant": 0.2962962962962963, - "physician": 0.42592592592592593 - } - }, - "prompt_3": { - "accuracy": 0.4339975093399751, - "category_acc": { - "computer_network": 0.5416666666666666, - "operating_system": 0.5833333333333334, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.47619047619047616, - "college_physics": 0.375, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.47619047619047616, - "electrical_engineer": 0.38095238095238093, - "metrology_engineer": 0.4482758620689655, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.25, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.46153846153846156, - "middle_school_physics": 0.6666666666666666, - "middle_school_chemistry": 0.52, - "veterinary_medicine": 0.42857142857142855, - "college_economics": 0.45, - "business_administration": 0.42105263157894735, - "marxism": 0.5, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.5, - "teacher_qualification": 0.6122448979591837, - "high_school_politics": 0.4166666666666667, - "high_school_geography": 0.375, - "middle_school_politics": 0.5, - "middle_school_geography": 0.35294117647058826, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.7916666666666666, - "logic": 0.5185185185185185, - "law": 0.41379310344827586, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.5588235294117647, - "legal_professional": 0.5, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.52, - "middle_school_history": 0.6296296296296297, - "civil_servant": 0.40384615384615385, - "sports_science": 0.25, - "plant_protection": 0.5185185185185185, - "basic_medicine": 0.5, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.4117647058823529, - "accountant": 0.3888888888888889, - "fire_engineer": 0.3611111111111111, - "environmental_impact_assessment_engineer": 0.5, - "tax_accountant": 0.25925925925925924, - "physician": 0.4444444444444444 - } - }, - "prompt_4": { - "accuracy": 0.41594022415940224, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.5833333333333334, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.38095238095238093, - "college_physics": 0.375, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.41379310344827586, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.25, - "high_school_chemistry": 0.375, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.625, - "middle_school_chemistry": 0.6, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.36666666666666664, - "business_administration": 0.42105263157894735, - "marxism": 0.375, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.5, - "teacher_qualification": 0.5102040816326531, - "high_school_politics": 0.375, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.5384615384615384, - "middle_school_geography": 0.5294117647058824, - "modern_chinese_history": 0.5, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.48148148148148145, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.42857142857142855, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.4411764705882353, - "legal_professional": 0.5, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.48, - "middle_school_history": 0.6296296296296297, - "civil_servant": 0.34615384615384615, - "sports_science": 0.375, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.5, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.4117647058823529, - "accountant": 0.37037037037037035, - "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.3611111111111111, - "tax_accountant": 0.35185185185185186, - "physician": 0.42592592592592593 - } - }, - "prompt_5": { - "accuracy": 0.4246575342465753, - "category_acc": { - "computer_network": 0.5833333333333334, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.42857142857142855, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.4482758620689655, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.38461538461538464, - "middle_school_physics": 0.625, - "middle_school_chemistry": 0.44, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.48333333333333334, - "business_administration": 0.42105263157894735, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.5, - "teacher_qualification": 0.5714285714285714, - "high_school_politics": 0.4583333333333333, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.5, - "middle_school_geography": 0.5294117647058824, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.7916666666666666, - "logic": 0.48148148148148145, - "law": 0.3103448275862069, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.39473684210526316, - "professional_tour_guide": 0.47058823529411764, - "legal_professional": 0.5357142857142857, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.52, - "middle_school_history": 0.6296296296296297, - "civil_servant": 0.36538461538461536, - "sports_science": 0.2916666666666667, - "plant_protection": 0.5555555555555556, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.43137254901960786, - "accountant": 0.35185185185185186, - "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.4444444444444444, - "tax_accountant": 0.2962962962962963, - "physician": 0.4444444444444444 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.4157706093189964 - }, - "prompt_2": { - "accuracy": 0.3978494623655914 - }, - "prompt_3": { - "accuracy": 0.4121863799283154 - }, - "prompt_4": { - "accuracy": 0.4014336917562724 - }, - "prompt_5": { - "accuracy": 0.3942652329749104 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.41504058021067175, - "category_acc": { - "agronomy": 0.34911242603550297, - "anatomy": 0.22297297297297297, - "ancient_chinese": 0.25609756097560976, - "arts": 0.43125, - "astronomy": 0.3090909090909091, - "business_ethics": 0.46411483253588515, - "chinese_civil_service_exam": 0.3375, - "chinese_driving_rule": 0.6183206106870229, - "chinese_food_culture": 0.3161764705882353, - "chinese_foreign_policy": 0.48598130841121495, - "chinese_history": 0.43653250773993807, - "chinese_literature": 0.3088235294117647, - "chinese_teacher_qualification": 0.4860335195530726, - "clinical_knowledge": 0.34177215189873417, - "college_actuarial_science": 0.2830188679245283, - "college_education": 0.4766355140186916, - "college_engineering_hydrology": 0.4528301886792453, - "college_law": 0.3888888888888889, - "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.36792452830188677, - "college_medicine": 0.38095238095238093, - "computer_science": 0.47058823529411764, - "computer_security": 0.5263157894736842, - "conceptual_physics": 0.4421768707482993, - "construction_project_management": 0.3669064748201439, - "economics": 0.4968553459119497, - "education": 0.44785276073619634, - "electrical_engineering": 0.42441860465116277, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.37373737373737376, - "elementary_information_and_technology": 0.6176470588235294, - "elementary_mathematics": 0.30434782608695654, - "ethnology": 0.4888888888888889, - "food_science": 0.45454545454545453, - "genetics": 0.3409090909090909, - "global_facts": 0.4563758389261745, - "high_school_biology": 0.34911242603550297, - "high_school_chemistry": 0.32575757575757575, - "high_school_geography": 0.4152542372881356, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.3090909090909091, - "high_school_politics": 0.4195804195804196, - "human_sexuality": 0.4365079365079365, - "international_law": 0.3783783783783784, - "journalism": 0.47674418604651164, - "jurisprudence": 0.49148418491484186, - "legal_and_moral_basis": 0.7757009345794392, - "logical": 0.37398373983739835, - "machine_learning": 0.3114754098360656, - "management": 0.5190476190476191, - "marketing": 0.5666666666666667, - "marxist_theory": 0.48148148148148145, - "modern_chinese": 0.3620689655172414, - "nutrition": 0.3793103448275862, - "philosophy": 0.47619047619047616, - "professional_accounting": 0.4342857142857143, - "professional_law": 0.3459715639810427, - "professional_medicine": 0.28191489361702127, - "professional_psychology": 0.4827586206896552, - "public_relations": 0.5574712643678161, - "security_study": 0.45925925925925926, - "sociology": 0.504424778761062, - "sports_science": 0.41818181818181815, - "traditional_chinese_medicine": 0.3027027027027027, - "virology": 0.39644970414201186, - "world_history": 0.391304347826087, - "world_religions": 0.41875 - } - }, - "prompt_2": { - "accuracy": 0.41849421516145746, - "category_acc": { - "agronomy": 0.33727810650887574, - "anatomy": 0.22297297297297297, - "ancient_chinese": 0.2804878048780488, - "arts": 0.4125, - "astronomy": 0.3090909090909091, - "business_ethics": 0.44019138755980863, - "chinese_civil_service_exam": 0.33125, - "chinese_driving_rule": 0.6335877862595419, - "chinese_food_culture": 0.3088235294117647, - "chinese_foreign_policy": 0.4766355140186916, - "chinese_history": 0.4086687306501548, - "chinese_literature": 0.3137254901960784, - "chinese_teacher_qualification": 0.4972067039106145, - "clinical_knowledge": 0.3459915611814346, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.5233644859813084, - "college_engineering_hydrology": 0.4811320754716981, - "college_law": 0.37037037037037035, - "college_mathematics": 0.2857142857142857, - "college_medical_statistics": 0.3490566037735849, - "college_medicine": 0.34798534798534797, - "computer_science": 0.4803921568627451, - "computer_security": 0.4853801169590643, - "conceptual_physics": 0.4421768707482993, - "construction_project_management": 0.38848920863309355, - "economics": 0.4779874213836478, - "education": 0.48466257668711654, - "electrical_engineering": 0.43023255813953487, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.3888888888888889, - "elementary_information_and_technology": 0.6428571428571429, - "elementary_mathematics": 0.3, - "ethnology": 0.4962962962962963, - "food_science": 0.46153846153846156, - "genetics": 0.3693181818181818, - "global_facts": 0.4429530201342282, - "high_school_biology": 0.35502958579881655, - "high_school_chemistry": 0.29545454545454547, - "high_school_geography": 0.3983050847457627, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.33636363636363636, - "high_school_politics": 0.43356643356643354, - "human_sexuality": 0.4444444444444444, - "international_law": 0.3837837837837838, - "journalism": 0.5232558139534884, - "jurisprudence": 0.49391727493917276, - "legal_and_moral_basis": 0.7757009345794392, - "logical": 0.3983739837398374, - "machine_learning": 0.36065573770491804, - "management": 0.5428571428571428, - "marketing": 0.5555555555555556, - "marxist_theory": 0.4708994708994709, - "modern_chinese": 0.35344827586206895, - "nutrition": 0.42758620689655175, - "philosophy": 0.45714285714285713, - "professional_accounting": 0.44571428571428573, - "professional_law": 0.33175355450236965, - "professional_medicine": 0.3058510638297872, - "professional_psychology": 0.4698275862068966, - "public_relations": 0.5517241379310345, - "security_study": 0.5185185185185185, - "sociology": 0.4823008849557522, - "sports_science": 0.41818181818181815, - "traditional_chinese_medicine": 0.3081081081081081, - "virology": 0.4319526627218935, - "world_history": 0.40993788819875776, - "world_religions": 0.43125 - } - }, - "prompt_3": { - "accuracy": 0.41745812467622173, - "category_acc": { - "agronomy": 0.3254437869822485, - "anatomy": 0.20270270270270271, - "ancient_chinese": 0.2621951219512195, - "arts": 0.41875, - "astronomy": 0.2545454545454545, - "business_ethics": 0.45933014354066987, - "chinese_civil_service_exam": 0.325, - "chinese_driving_rule": 0.5954198473282443, - "chinese_food_culture": 0.3161764705882353, - "chinese_foreign_policy": 0.48598130841121495, - "chinese_history": 0.3931888544891641, - "chinese_literature": 0.3137254901960784, - "chinese_teacher_qualification": 0.5027932960893855, - "clinical_knowledge": 0.34177215189873417, - "college_actuarial_science": 0.3018867924528302, - "college_education": 0.5233644859813084, - "college_engineering_hydrology": 0.49056603773584906, - "college_law": 0.37037037037037035, - "college_mathematics": 0.2857142857142857, - "college_medical_statistics": 0.3490566037735849, - "college_medicine": 0.34798534798534797, - "computer_science": 0.49019607843137253, - "computer_security": 0.49707602339181284, - "conceptual_physics": 0.41496598639455784, - "construction_project_management": 0.381294964028777, - "economics": 0.5031446540880503, - "education": 0.49693251533742333, - "electrical_engineering": 0.436046511627907, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.3888888888888889, - "elementary_information_and_technology": 0.6596638655462185, - "elementary_mathematics": 0.3, - "ethnology": 0.4740740740740741, - "food_science": 0.46153846153846156, - "genetics": 0.375, - "global_facts": 0.42953020134228187, - "high_school_biology": 0.3609467455621302, - "high_school_chemistry": 0.30303030303030304, - "high_school_geography": 0.423728813559322, - "high_school_mathematics": 0.27439024390243905, - "high_school_physics": 0.34545454545454546, - "high_school_politics": 0.4195804195804196, - "human_sexuality": 0.46825396825396826, - "international_law": 0.3837837837837838, - "journalism": 0.4883720930232558, - "jurisprudence": 0.48418491484184917, - "legal_and_moral_basis": 0.7757009345794392, - "logical": 0.3821138211382114, - "machine_learning": 0.36885245901639346, - "management": 0.5142857142857142, - "marketing": 0.5555555555555556, - "marxist_theory": 0.4656084656084656, - "modern_chinese": 0.3620689655172414, - "nutrition": 0.4206896551724138, - "philosophy": 0.4380952380952381, - "professional_accounting": 0.42857142857142855, - "professional_law": 0.3459715639810427, - "professional_medicine": 0.324468085106383, - "professional_psychology": 0.47844827586206895, - "public_relations": 0.5517241379310345, - "security_study": 0.5111111111111111, - "sociology": 0.4823008849557522, - "sports_science": 0.4, - "traditional_chinese_medicine": 0.34054054054054056, - "virology": 0.4319526627218935, - "world_history": 0.40372670807453415, - "world_religions": 0.41875 - } - }, - "prompt_4": { - "accuracy": 0.40960110516318426, - "category_acc": { - "agronomy": 0.33136094674556216, - "anatomy": 0.22297297297297297, - "ancient_chinese": 0.24390243902439024, - "arts": 0.4, - "astronomy": 0.30303030303030304, - "business_ethics": 0.45933014354066987, - "chinese_civil_service_exam": 0.3625, - "chinese_driving_rule": 0.6106870229007634, - "chinese_food_culture": 0.3014705882352941, - "chinese_foreign_policy": 0.45794392523364486, - "chinese_history": 0.4117647058823529, - "chinese_literature": 0.30392156862745096, - "chinese_teacher_qualification": 0.5139664804469274, - "clinical_knowledge": 0.350210970464135, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.514018691588785, - "college_engineering_hydrology": 0.44339622641509435, - "college_law": 0.39814814814814814, - "college_mathematics": 0.3333333333333333, - "college_medical_statistics": 0.39622641509433965, - "college_medicine": 0.31868131868131866, - "computer_science": 0.49019607843137253, - "computer_security": 0.5087719298245614, - "conceptual_physics": 0.41496598639455784, - "construction_project_management": 0.381294964028777, - "economics": 0.48427672955974843, - "education": 0.48466257668711654, - "electrical_engineering": 0.4127906976744186, - "elementary_chinese": 0.2857142857142857, - "elementary_commonsense": 0.37373737373737376, - "elementary_information_and_technology": 0.6386554621848739, - "elementary_mathematics": 0.2782608695652174, - "ethnology": 0.4888888888888889, - "food_science": 0.44755244755244755, - "genetics": 0.3465909090909091, - "global_facts": 0.4228187919463087, - "high_school_biology": 0.33727810650887574, - "high_school_chemistry": 0.3409090909090909, - "high_school_geography": 0.3898305084745763, - "high_school_mathematics": 0.21951219512195122, - "high_school_physics": 0.3181818181818182, - "high_school_politics": 0.45454545454545453, - "human_sexuality": 0.42857142857142855, - "international_law": 0.3621621621621622, - "journalism": 0.47674418604651164, - "jurisprudence": 0.44768856447688565, - "legal_and_moral_basis": 0.7523364485981309, - "logical": 0.37398373983739835, - "machine_learning": 0.3524590163934426, - "management": 0.5142857142857142, - "marketing": 0.55, - "marxist_theory": 0.48148148148148145, - "modern_chinese": 0.3879310344827586, - "nutrition": 0.43448275862068964, - "philosophy": 0.44761904761904764, - "professional_accounting": 0.41714285714285715, - "professional_law": 0.35071090047393366, - "professional_medicine": 0.2872340425531915, - "professional_psychology": 0.47844827586206895, - "public_relations": 0.5344827586206896, - "security_study": 0.45925925925925926, - "sociology": 0.49557522123893805, - "sports_science": 0.41818181818181815, - "traditional_chinese_medicine": 0.31351351351351353, - "virology": 0.40236686390532544, - "world_history": 0.37888198757763975, - "world_religions": 0.4 - } - }, - "prompt_5": { - "accuracy": 0.41875323778276635, - "category_acc": { - "agronomy": 0.33136094674556216, - "anatomy": 0.24324324324324326, - "ancient_chinese": 0.2804878048780488, - "arts": 0.4125, - "astronomy": 0.296969696969697, - "business_ethics": 0.47368421052631576, - "chinese_civil_service_exam": 0.35, - "chinese_driving_rule": 0.6259541984732825, - "chinese_food_culture": 0.3235294117647059, - "chinese_foreign_policy": 0.4953271028037383, - "chinese_history": 0.43962848297213625, - "chinese_literature": 0.30392156862745096, - "chinese_teacher_qualification": 0.5083798882681564, - "clinical_knowledge": 0.35864978902953587, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.5420560747663551, - "college_engineering_hydrology": 0.4811320754716981, - "college_law": 0.35185185185185186, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.41509433962264153, - "college_medicine": 0.3772893772893773, - "computer_science": 0.47058823529411764, - "computer_security": 0.5029239766081871, - "conceptual_physics": 0.4421768707482993, - "construction_project_management": 0.381294964028777, - "economics": 0.48427672955974843, - "education": 0.44785276073619634, - "electrical_engineering": 0.4127906976744186, - "elementary_chinese": 0.30158730158730157, - "elementary_commonsense": 0.40404040404040403, - "elementary_information_and_technology": 0.6596638655462185, - "elementary_mathematics": 0.33043478260869563, - "ethnology": 0.5037037037037037, - "food_science": 0.45454545454545453, - "genetics": 0.35795454545454547, - "global_facts": 0.44966442953020136, - "high_school_biology": 0.35502958579881655, - "high_school_chemistry": 0.25, - "high_school_geography": 0.4152542372881356, - "high_school_mathematics": 0.25, - "high_school_physics": 0.35454545454545455, - "high_school_politics": 0.4195804195804196, - "human_sexuality": 0.4444444444444444, - "international_law": 0.3837837837837838, - "journalism": 0.5058139534883721, - "jurisprudence": 0.48905109489051096, - "legal_and_moral_basis": 0.7616822429906542, - "logical": 0.3902439024390244, - "machine_learning": 0.36885245901639346, - "management": 0.5190476190476191, - "marketing": 0.5388888888888889, - "marxist_theory": 0.48148148148148145, - "modern_chinese": 0.3706896551724138, - "nutrition": 0.36551724137931035, - "philosophy": 0.47619047619047616, - "professional_accounting": 0.44, - "professional_law": 0.35071090047393366, - "professional_medicine": 0.3058510638297872, - "professional_psychology": 0.4698275862068966, - "public_relations": 0.5574712643678161, - "security_study": 0.4666666666666667, - "sociology": 0.48672566371681414, - "sports_science": 0.41818181818181815, - "traditional_chinese_medicine": 0.31351351351351353, - "virology": 0.40828402366863903, - "world_history": 0.391304347826087, - "world_religions": 0.4 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.30303030303030304 - }, - "prompt_2": { - "accuracy": 0.3939393939393939 - }, - "prompt_3": { - "accuracy": 0.30303030303030304 - }, - "prompt_4": { - "accuracy": 0.3333333333333333 - }, - "prompt_5": { - "accuracy": 0.3939393939393939 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.7090909090909091 - }, - "prompt_2": { - "accuracy": 0.6545454545454545 - }, - "prompt_3": { - "accuracy": 0.6613636363636364 - }, - "prompt_4": { - "accuracy": 0.6909090909090909 - }, - "prompt_5": { - "accuracy": 0.6818181818181818 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.4105084745762712 - }, - "prompt_2": { - "accuracy": 0.42915254237288136 - }, - "prompt_3": { - "accuracy": 0.4105084745762712 - }, - "prompt_4": { - "accuracy": 0.41966101694915253 - }, - "prompt_5": { - "accuracy": 0.408135593220339 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.7857142857142857 - }, - "prompt_2": { - "accuracy": 0.7756170531039641 - }, - "prompt_3": { - "accuracy": 0.7729992520568437 - }, - "prompt_4": { - "accuracy": 0.7789827973074046 - }, - "prompt_5": { - "accuracy": 0.7756170531039641 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.8564429201371877 - }, - "prompt_2": { - "accuracy": 0.850563449289564 - }, - "prompt_3": { - "accuracy": 0.842234198922097 - }, - "prompt_4": { - "accuracy": 0.8593826555609995 - }, - "prompt_5": { - "accuracy": 0.8535031847133758 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.33790427030878784, - "rouge2": 0.13123765327646858, - "rougeL": 0.26011814401921435, - "avg_rouge": 0.24308668920149024 - }, - "prompt_2": { - "rouge1": 0.37154166310910414, - "rouge2": 0.14640033772856853, - "rougeL": 0.28670668600872545, - "avg_rouge": 0.26821622894879943 - }, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "dialogsum": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": { - "rouge1": 0.2302453407208971, - "rouge2": 0.06366341833152594, - "rougeL": 0.1696771173569469, - "avg_rouge": 0.15452862546979 - }, - "prompt_5": { - "rouge1": 0.23135431548840976, - "rouge2": 0.06432053752605707, - "rougeL": 0.17093322020657767, - "avg_rouge": 0.15553602440701483 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.9151376146788991 - }, - "prompt_2": { - "accuracy": 0.9002293577981652 - }, - "prompt_3": { - "accuracy": 0.9139908256880734 - }, - "prompt_4": { - "accuracy": 0.9094036697247706 - }, - "prompt_5": { - "accuracy": 0.9220183486238532 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.7813998082454459 - }, - "prompt_2": { - "accuracy": 0.7785234899328859 - }, - "prompt_3": { - "accuracy": 0.7727708533077661 - }, - "prompt_4": { - "accuracy": 0.7813998082454459 - }, - "prompt_5": { - "accuracy": 0.7785234899328859 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.693 - }, - "prompt_2": { - "accuracy": 0.695 - }, - "prompt_3": { - "accuracy": 0.6865 - }, - "prompt_4": { - "accuracy": 0.6905 - }, - "prompt_5": { - "accuracy": 0.688 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.5645 - }, - "prompt_2": { - "accuracy": 0.587 - }, - "prompt_3": { - "accuracy": 0.5575 - }, - "prompt_4": { - "accuracy": 0.5735 - }, - "prompt_5": { - "accuracy": 0.5555 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.6055 - }, - "prompt_2": { - "accuracy": 0.6415 - }, - "prompt_3": { - "accuracy": 0.649 - }, - "prompt_4": { - "accuracy": 0.629 - }, - "prompt_5": { - "accuracy": 0.6405 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.5633802816901409 - }, - "prompt_2": { - "accuracy": 0.5633802816901409 - }, - "prompt_3": { - "accuracy": 0.5211267605633803 - }, - "prompt_4": { - "accuracy": 0.5492957746478874 - }, - "prompt_5": { - "accuracy": 0.5915492957746479 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.6209386281588448 - }, - "prompt_2": { - "accuracy": 0.5740072202166066 - }, - "prompt_3": { - "accuracy": 0.6534296028880866 - }, - "prompt_4": { - "accuracy": 0.7220216606498195 - }, - "prompt_5": { - "accuracy": 0.592057761732852 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.6936274509803921 - }, - "prompt_2": { - "accuracy": 0.7058823529411765 - }, - "prompt_3": { - "accuracy": 0.6544117647058824 - }, - "prompt_4": { - "accuracy": 0.7034313725490197 - }, - "prompt_5": { - "accuracy": 0.6151960784313726 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.8642857142857143, - "language_acc": { - "Spanish": 0.8865546218487395, - "English": 0.9184873949579831, - "Chinese": 0.8504201680672269, - "Vietnamese": 0.8016806722689076 - }, - "consistency_score_2": 0.8295518207282914, - "consistency_score_3": 0.7560924369747899, - "consistency_score_4": 0.704201680672269, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8991596638655462, - "Spanish,Chinese": 0.8470588235294118, - "Spanish,Vietnamese": 0.7924369747899159, - "English,Chinese": 0.8546218487394958, - "English,Vietnamese": 0.8042016806722689, - "Chinese,Vietnamese": 0.7798319327731092 - }, - "3_combine": { - "Spanish,English,Chinese": 0.8092436974789916, - "Spanish,English,Vietnamese": 0.7571428571428571, - "Spanish,Chinese,Vietnamese": 0.7243697478991596, - "English,Chinese,Vietnamese": 0.7336134453781512 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.704201680672269 - } - }, - "AC3_2": 0.8465626402202849, - "AC3_3": 0.8065770220464982, - "AC3_4": 0.7760743943479798 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.52, - "language_acc": { - "Filipino": 0.46, - "Vietnamese": 0.41333333333333333, - "Chinese": 0.5066666666666667, - "Spanish": 0.5666666666666667, - "Malay": 0.48, - "Indonesian": 0.58, - "English": 0.6333333333333333 - }, - "consistency_score_2": 0.5955555555555557, - "consistency_score_3": 0.43542857142857133, - "consistency_score_4": 0.34323809523809534, - "consistency_score_5": 0.2844444444444444, - "consistency_score_6": 0.2457142857142857, - "consistency_score_7": 0.22, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.49333333333333335, - "Filipino,Chinese": 0.52, - "Filipino,Spanish": 0.5333333333333333, - "Filipino,Malay": 0.5066666666666667, - "Filipino,Indonesian": 0.56, - "Filipino,English": 0.56, - "Vietnamese,Chinese": 0.44, - "Vietnamese,Spanish": 0.5266666666666666, - "Vietnamese,Malay": 0.5933333333333334, - "Vietnamese,Indonesian": 0.5333333333333333, - "Vietnamese,English": 0.5066666666666667, - "Chinese,Spanish": 0.6333333333333333, - "Chinese,Malay": 0.6, - "Chinese,Indonesian": 0.6533333333333333, - "Chinese,English": 0.66, - "Spanish,Malay": 0.64, - "Spanish,Indonesian": 0.72, - "Spanish,English": 0.7466666666666667, - "Malay,Indonesian": 0.6866666666666666, - "Malay,English": 0.6533333333333333, - "Indonesian,English": 0.74 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.30666666666666664, - "Filipino,Vietnamese,Spanish": 0.34, - "Filipino,Vietnamese,Malay": 0.36, - "Filipino,Vietnamese,Indonesian": 0.35333333333333333, - "Filipino,Vietnamese,English": 0.3466666666666667, - "Filipino,Chinese,Spanish": 0.4, - "Filipino,Chinese,Malay": 0.36, - "Filipino,Chinese,Indonesian": 0.41333333333333333, - "Filipino,Chinese,English": 0.4066666666666667, - "Filipino,Spanish,Malay": 0.38666666666666666, - "Filipino,Spanish,Indonesian": 0.4533333333333333, - "Filipino,Spanish,English": 0.44666666666666666, - "Filipino,Malay,Indonesian": 0.42, - "Filipino,Malay,English": 0.3933333333333333, - "Filipino,Indonesian,English": 0.46, - "Vietnamese,Chinese,Spanish": 0.36666666666666664, - "Vietnamese,Chinese,Malay": 0.37333333333333335, - "Vietnamese,Chinese,Indonesian": 0.36666666666666664, - "Vietnamese,Chinese,English": 0.36, - "Vietnamese,Spanish,Malay": 0.4266666666666667, - "Vietnamese,Spanish,Indonesian": 0.43333333333333335, - "Vietnamese,Spanish,English": 0.42, - "Vietnamese,Malay,Indonesian": 0.4533333333333333, - "Vietnamese,Malay,English": 0.43333333333333335, - "Vietnamese,Indonesian,English": 0.4266666666666667, - "Chinese,Spanish,Malay": 0.47333333333333333, - "Chinese,Spanish,Indonesian": 0.52, - "Chinese,Spanish,English": 0.5466666666666666, - "Chinese,Malay,Indonesian": 0.5, - "Chinese,Malay,English": 0.4866666666666667, - "Chinese,Indonesian,English": 0.5533333333333333, - "Spanish,Malay,Indonesian": 0.54, - "Spanish,Malay,English": 0.5333333333333333, - "Spanish,Indonesian,English": 0.62, - "Malay,Indonesian,English": 0.56 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.26, - "Filipino,Vietnamese,Chinese,Malay": 0.26666666666666666, - "Filipino,Vietnamese,Chinese,Indonesian": 0.2733333333333333, - "Filipino,Vietnamese,Chinese,English": 0.2733333333333333, - "Filipino,Vietnamese,Spanish,Malay": 0.2866666666666667, - "Filipino,Vietnamese,Spanish,Indonesian": 0.3, - "Filipino,Vietnamese,Spanish,English": 0.28, - "Filipino,Vietnamese,Malay,Indonesian": 0.30666666666666664, - "Filipino,Vietnamese,Malay,English": 0.2866666666666667, - "Filipino,Vietnamese,Indonesian,English": 0.3, - "Filipino,Chinese,Spanish,Malay": 0.30666666666666664, - "Filipino,Chinese,Spanish,Indonesian": 0.3466666666666667, - "Filipino,Chinese,Spanish,English": 0.35333333333333333, - "Filipino,Chinese,Malay,Indonesian": 0.3333333333333333, - "Filipino,Chinese,Malay,English": 0.31333333333333335, - "Filipino,Chinese,Indonesian,English": 0.37333333333333335, - "Filipino,Spanish,Malay,Indonesian": 0.36, - "Filipino,Spanish,Malay,English": 0.34, - "Filipino,Spanish,Indonesian,English": 0.4066666666666667, - "Filipino,Malay,Indonesian,English": 0.35333333333333333, - "Vietnamese,Chinese,Spanish,Malay": 0.3333333333333333, - "Vietnamese,Chinese,Spanish,Indonesian": 0.3333333333333333, - "Vietnamese,Chinese,Spanish,English": 0.32666666666666666, - "Vietnamese,Chinese,Malay,Indonesian": 0.3333333333333333, - "Vietnamese,Chinese,Malay,English": 0.32666666666666666, - "Vietnamese,Chinese,Indonesian,English": 0.3333333333333333, - "Vietnamese,Spanish,Malay,Indonesian": 0.38, - "Vietnamese,Spanish,Malay,English": 0.38, - "Vietnamese,Spanish,Indonesian,English": 0.37333333333333335, - "Vietnamese,Malay,Indonesian,English": 0.38, - "Chinese,Spanish,Malay,Indonesian": 0.41333333333333333, - "Chinese,Spanish,Malay,English": 0.41333333333333333, - "Chinese,Spanish,Indonesian,English": 0.4666666666666667, - "Chinese,Malay,Indonesian,English": 0.43333333333333335, - "Spanish,Malay,Indonesian,English": 0.4666666666666667 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.24, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.24, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.24, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.25333333333333335, - "Filipino,Vietnamese,Chinese,Malay,English": 0.24666666666666667, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.26, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.26666666666666666, - "Filipino,Vietnamese,Spanish,Malay,English": 0.25333333333333335, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.26666666666666666, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.26, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.2866666666666667, - "Filipino,Chinese,Spanish,Malay,English": 0.28, - "Filipino,Chinese,Spanish,Indonesian,English": 0.32666666666666666, - "Filipino,Chinese,Malay,Indonesian,English": 0.3, - "Filipino,Spanish,Malay,Indonesian,English": 0.32, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.30666666666666664, - "Vietnamese,Chinese,Spanish,Malay,English": 0.30666666666666664, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.30666666666666664, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.30666666666666664, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.34, - "Chinese,Spanish,Malay,Indonesian,English": 0.36666666666666664 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.22666666666666666, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.22666666666666666, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.23333333333333334, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.24, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.24, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.2866666666666667 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.22 - } - }, - "AC3_2": 0.5552191234562055, - "AC3_3": 0.47397129181642045, - "AC3_4": 0.4135216239591574, - "AC3_5": 0.3677348065841213, - "AC3_6": 0.33373134323999776, - "AC3_7": 0.3091891891474069 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.44155844155844154, - "language_acc": { - "Indonesian": 0.4715909090909091, - "English": 0.5454545454545454, - "Filipino": 0.3068181818181818, - "Spanish": 0.5056818181818182, - "Chinese": 0.4943181818181818, - "Malay": 0.44886363636363635, - "Vietnamese": 0.3181818181818182 - }, - "consistency_score_2": 0.5197510822510824, - "consistency_score_3": 0.33344155844155837, - "consistency_score_4": 0.2375, - "consistency_score_5": 0.1810064935064935, - "consistency_score_6": 0.1444805194805195, - "consistency_score_7": 0.11931818181818182, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6647727272727273, - "Indonesian,Filipino": 0.45454545454545453, - "Indonesian,Spanish": 0.6534090909090909, - "Indonesian,Chinese": 0.5568181818181818, - "Indonesian,Malay": 0.6477272727272727, - "Indonesian,Vietnamese": 0.48295454545454547, - "English,Filipino": 0.4318181818181818, - "English,Spanish": 0.6477272727272727, - "English,Chinese": 0.6079545454545454, - "English,Malay": 0.6136363636363636, - "English,Vietnamese": 0.4147727272727273, - "Filipino,Spanish": 0.4147727272727273, - "Filipino,Chinese": 0.4090909090909091, - "Filipino,Malay": 0.4318181818181818, - "Filipino,Vietnamese": 0.4943181818181818, - "Spanish,Chinese": 0.5511363636363636, - "Spanish,Malay": 0.6022727272727273, - "Spanish,Vietnamese": 0.44886363636363635, - "Chinese,Malay": 0.5284090909090909, - "Chinese,Vietnamese": 0.39204545454545453, - "Malay,Vietnamese": 0.4659090909090909 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.32954545454545453, - "Indonesian,English,Spanish": 0.5113636363636364, - "Indonesian,English,Chinese": 0.45454545454545453, - "Indonesian,English,Malay": 0.4943181818181818, - "Indonesian,English,Vietnamese": 0.32386363636363635, - "Indonesian,Filipino,Spanish": 0.3125, - "Indonesian,Filipino,Chinese": 0.2840909090909091, - "Indonesian,Filipino,Malay": 0.32386363636363635, - "Indonesian,Filipino,Vietnamese": 0.26704545454545453, - "Indonesian,Spanish,Chinese": 0.4090909090909091, - "Indonesian,Spanish,Malay": 0.48863636363636365, - "Indonesian,Spanish,Vietnamese": 0.3409090909090909, - "Indonesian,Chinese,Malay": 0.4147727272727273, - "Indonesian,Chinese,Vietnamese": 0.2727272727272727, - "Indonesian,Malay,Vietnamese": 0.32954545454545453, - "English,Filipino,Spanish": 0.30113636363636365, - "English,Filipino,Chinese": 0.3125, - "English,Filipino,Malay": 0.30113636363636365, - "English,Filipino,Vietnamese": 0.24431818181818182, - "English,Spanish,Chinese": 0.4659090909090909, - "English,Spanish,Malay": 0.4715909090909091, - "English,Spanish,Vietnamese": 0.30113636363636365, - "English,Chinese,Malay": 0.42613636363636365, - "English,Chinese,Vietnamese": 0.2727272727272727, - "English,Malay,Vietnamese": 0.29545454545454547, - "Filipino,Spanish,Chinese": 0.2556818181818182, - "Filipino,Spanish,Malay": 0.2897727272727273, - "Filipino,Spanish,Vietnamese": 0.24431818181818182, - "Filipino,Chinese,Malay": 0.26136363636363635, - "Filipino,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Vietnamese": 0.24431818181818182, - "Spanish,Chinese,Malay": 0.3977272727272727, - "Spanish,Chinese,Vietnamese": 0.25, - "Spanish,Malay,Vietnamese": 0.30113636363636365, - "Chinese,Malay,Vietnamese": 0.2556818181818182 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.26136363636363635, - "Indonesian,English,Filipino,Chinese": 0.25, - "Indonesian,English,Filipino,Malay": 0.26136363636363635, - "Indonesian,English,Filipino,Vietnamese": 0.1875, - "Indonesian,English,Spanish,Chinese": 0.36363636363636365, - "Indonesian,English,Spanish,Malay": 0.4147727272727273, - "Indonesian,English,Spanish,Vietnamese": 0.26704545454545453, - "Indonesian,English,Chinese,Malay": 0.3693181818181818, - "Indonesian,English,Chinese,Vietnamese": 0.24431818181818182, - "Indonesian,English,Malay,Vietnamese": 0.24431818181818182, - "Indonesian,Filipino,Spanish,Chinese": 0.2215909090909091, - "Indonesian,Filipino,Spanish,Malay": 0.26136363636363635, - "Indonesian,Filipino,Spanish,Vietnamese": 0.18181818181818182, - "Indonesian,Filipino,Chinese,Malay": 0.21022727272727273, - "Indonesian,Filipino,Chinese,Vietnamese": 0.17045454545454544, - "Indonesian,Filipino,Malay,Vietnamese": 0.1875, - "Indonesian,Spanish,Chinese,Malay": 0.3409090909090909, - "Indonesian,Spanish,Chinese,Vietnamese": 0.2215909090909091, - "Indonesian,Spanish,Malay,Vietnamese": 0.2556818181818182, - "Indonesian,Chinese,Malay,Vietnamese": 0.2215909090909091, - "English,Filipino,Spanish,Chinese": 0.23863636363636365, - "English,Filipino,Spanish,Malay": 0.25, - "English,Filipino,Spanish,Vietnamese": 0.17045454545454544, - "English,Filipino,Chinese,Malay": 0.2215909090909091, - "English,Filipino,Chinese,Vietnamese": 0.18181818181818182, - "English,Filipino,Malay,Vietnamese": 0.17613636363636365, - "English,Spanish,Chinese,Malay": 0.35795454545454547, - "English,Spanish,Chinese,Vietnamese": 0.23295454545454544, - "English,Spanish,Malay,Vietnamese": 0.23863636363636365, - "English,Chinese,Malay,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Chinese,Malay": 0.19886363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Malay,Vietnamese": 0.1590909090909091, - "Filipino,Chinese,Malay,Vietnamese": 0.17045454545454544, - "Spanish,Chinese,Malay,Vietnamese": 0.19886363636363635 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.20454545454545456, - "Indonesian,English,Filipino,Spanish,Malay": 0.23295454545454544, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.1534090909090909, - "Indonesian,English,Filipino,Chinese,Malay": 0.19886363636363635, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.1590909090909091, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.1534090909090909, - "Indonesian,English,Spanish,Chinese,Malay": 0.3181818181818182, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.21022727272727273, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.2159090909090909, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.20454545454545456, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.1875, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.14204545454545456, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.14204545454545456, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.1875, - "English,Filipino,Spanish,Chinese,Malay": 0.19318181818181818, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, - "English,Spanish,Chinese,Malay,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.18181818181818182, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.13636363636363635, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.13636363636363635, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.18181818181818182, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.125 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182 - } - }, - "AC3_2": 0.477474678381947, - "AC3_3": 0.3799585416187732, - "AC3_4": 0.308869232562691, - "AC3_5": 0.2567601889223365, - "AC3_6": 0.2177213367977589, - "AC3_7": 0.18787001706951698 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.6504854368932039 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.3238095238095238 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.6542056074766355 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.42, - "category_acc": { - "brand": 0.3, - "demographics": 0.4, - "biology": 0.4, - "history": 0.2, - "literature": 0.2, - "politics": 0.7, - "culture": 0.6, - "film": 0.4, - "law": 0.4, - "geography": 0.7 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.30846091455013425 - } - }, - "indommlu": { - "prompt_1": { - "accuracy": 0.4982308565324788, - "category_acc": { - "History": 0.5160642570281124, - "Geography": 0.44693877551020406, - "Lampungic": 0.3129251700680272, - "Social science": 0.7395659432387313, - "Balinese": 0.32908704883227174, - "Makassarese": 0.41397849462365593, - "Banjarese": 0.4305555555555556, - "Chemistry": 0.30802919708029197, - "Biology": 0.46982248520710057, - "Science": 0.6099071207430341, - "Christian religion": 0.6467661691542289, - "Art": 0.6173044925124792, - "Islam religion": 0.6017069701280228, - "Hindu religion": 0.5, - "Madurese": 0.288135593220339, - "Sport": 0.5540540540540541, - "Indonesian language": 0.571917808219178, - "Physics": 0.4121212121212121, - "Minangkabau culture": 0.3768844221105528, - "Dayak language": 0.25688073394495414, - "Sociology": 0.4838709677419355, - "Economy": 0.4979508196721312, - "Sundanese": 0.3863439930855661, - "Javanese": 0.34274193548387094, - "Civic education": 0.6080114449213162 - } - } - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.37647845181633705 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.22941829839366545 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.2291244503731969 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.35845911812249015 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.5834305717619603 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.5736145870575616, - "category_acc": { - "high_school_european_history": 0.7317073170731707, - "business_ethics": 0.5656565656565656, - "clinical_knowledge": 0.625, - "medical_genetics": 0.5757575757575758, - "high_school_us_history": 0.7635467980295566, - "high_school_physics": 0.36666666666666664, - "high_school_world_history": 0.7457627118644068, - "virology": 0.4727272727272727, - "high_school_microeconomics": 0.620253164556962, - "econometrics": 0.3893805309734513, - "college_computer_science": 0.5050505050505051, - "high_school_biology": 0.7152103559870551, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.4234875444839858, - "philosophy": 0.6451612903225806, - "professional_medicine": 0.6383763837638377, - "nutrition": 0.6295081967213115, - "global_facts": 0.3434343434343434, - "machine_learning": 0.3963963963963964, - "security_studies": 0.6598360655737705, - "public_relations": 0.6697247706422018, - "professional_psychology": 0.5875613747954174, - "prehistory": 0.6253869969040248, - "anatomy": 0.5, - "human_sexuality": 0.7, - "college_medicine": 0.5813953488372093, - "high_school_government_and_politics": 0.8177083333333334, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.691358024691358, - "high_school_geography": 0.7766497461928934, - "elementary_mathematics": 0.3793103448275862, - "human_aging": 0.6531531531531531, - "college_mathematics": 0.3434343434343434, - "high_school_psychology": 0.7996323529411765, - "formal_logic": 0.44, - "high_school_statistics": 0.3767441860465116, - "international_law": 0.725, - "high_school_mathematics": 0.30111524163568776, - "high_school_computer_science": 0.6161616161616161, - "conceptual_physics": 0.48717948717948717, - "miscellaneous": 0.789002557544757, - "high_school_chemistry": 0.4801980198019802, - "marketing": 0.8454935622317596, - "professional_law": 0.4272667971298108, - "management": 0.7843137254901961, - "college_physics": 0.32673267326732675, - "jurisprudence": 0.6728971962616822, - "world_religions": 0.8058823529411765, - "sociology": 0.825, - "us_foreign_policy": 0.8383838383838383, - "high_school_macroeconomics": 0.5861182519280206, - "computer_security": 0.6666666666666666, - "moral_scenarios": 0.2796420581655481, - "moral_disputes": 0.6579710144927536, - "electrical_engineering": 0.5208333333333334, - "astronomy": 0.5894039735099338, - "college_biology": 0.6643356643356644 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.4539375928677563 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.42714819427148193, - "category_acc": { - "computer_network": 0.4583333333333333, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.46153846153846156, - "college_programming": 0.5, - "college_physics": 0.25, - "college_chemistry": 0.2413793103448276, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.30434782608695654, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.2608695652173913, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.5769230769230769, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.48, - "veterinary_medicine": 0.39285714285714285, - "college_economics": 0.5, - "business_administration": 0.42105263157894735, - "marxism": 0.5, - "mao_zedong_thought": 0.6551724137931034, - "education_science": 0.5, - "teacher_qualification": 0.7142857142857143, - "high_school_politics": 0.4583333333333333, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.5384615384615384, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.75, - "logic": 0.5925925925925926, - "law": 0.41379310344827586, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.2631578947368421, - "professional_tour_guide": 0.47058823529411764, - "legal_professional": 0.4642857142857143, - "high_school_chinese": 0.25, - "high_school_history": 0.52, - "middle_school_history": 0.5925925925925926, - "civil_servant": 0.36538461538461536, - "sports_science": 0.20833333333333334, - "plant_protection": 0.48148148148148145, - "basic_medicine": 0.3333333333333333, - "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.5294117647058824, - "accountant": 0.3888888888888889, - "fire_engineer": 0.3611111111111111, - "environmental_impact_assessment_engineer": 0.4722222222222222, - "tax_accountant": 0.3148148148148148, - "physician": 0.4444444444444444 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.3942652329749104 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.4283370747711967, - "category_acc": { - "agronomy": 0.40236686390532544, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.23170731707317074, - "arts": 0.375, - "astronomy": 0.296969696969697, - "business_ethics": 0.5263157894736842, - "chinese_civil_service_exam": 0.39375, - "chinese_driving_rule": 0.6564885496183206, - "chinese_food_culture": 0.3161764705882353, - "chinese_foreign_policy": 0.4953271028037383, - "chinese_history": 0.4613003095975232, - "chinese_literature": 0.30392156862745096, - "chinese_teacher_qualification": 0.5251396648044693, - "clinical_knowledge": 0.35864978902953587, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.514018691588785, - "college_engineering_hydrology": 0.46226415094339623, - "college_law": 0.35185185185185186, - "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.39622641509433965, - "college_medicine": 0.336996336996337, - "computer_science": 0.5049019607843137, - "computer_security": 0.5497076023391813, - "conceptual_physics": 0.3945578231292517, - "construction_project_management": 0.4316546762589928, - "economics": 0.4716981132075472, - "education": 0.5030674846625767, - "electrical_engineering": 0.4069767441860465, - "elementary_chinese": 0.3055555555555556, - "elementary_commonsense": 0.42424242424242425, - "elementary_information_and_technology": 0.6974789915966386, - "elementary_mathematics": 0.3347826086956522, - "ethnology": 0.43703703703703706, - "food_science": 0.45454545454545453, - "genetics": 0.42045454545454547, - "global_facts": 0.42953020134228187, - "high_school_biology": 0.3431952662721893, - "high_school_chemistry": 0.3181818181818182, - "high_school_geography": 0.423728813559322, - "high_school_mathematics": 0.23170731707317074, - "high_school_physics": 0.35454545454545455, - "high_school_politics": 0.4125874125874126, - "human_sexuality": 0.48412698412698413, - "international_law": 0.3567567567567568, - "journalism": 0.5232558139534884, - "jurisprudence": 0.49878345498783455, - "legal_and_moral_basis": 0.7897196261682243, - "logical": 0.34959349593495936, - "machine_learning": 0.4344262295081967, - "management": 0.5142857142857142, - "marketing": 0.55, - "marxist_theory": 0.5343915343915344, - "modern_chinese": 0.3017241379310345, - "nutrition": 0.4482758620689655, - "philosophy": 0.4666666666666667, - "professional_accounting": 0.4857142857142857, - "professional_law": 0.3459715639810427, - "professional_medicine": 0.300531914893617, - "professional_psychology": 0.44396551724137934, - "public_relations": 0.47701149425287354, - "security_study": 0.5481481481481482, - "sociology": 0.4646017699115044, - "sports_science": 0.41818181818181815, - "traditional_chinese_medicine": 0.3567567567567568, - "virology": 0.4319526627218935, - "world_history": 0.453416149068323, - "world_religions": 0.44375 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.36363636363636365 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.6704545454545454 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.4471186440677966 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.7868362004487659 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.8588926996570309 - } - }, - "samsum": { - "prompt_1": -1 - }, - "dialogsum": { - "prompt_1": -1 - }, - "sst2": { - "prompt_1": { - "accuracy": 0.9369266055045872 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.7181208053691275 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.6595 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.5855 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.5955 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.5633802816901409 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.6534296028880866 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.5098039215686274 - } - } - } - }, - "regional_sea_mistral_inst_7b_latest": { - "model_size": "7B", - "model_link": "https://seaeval.github.io/", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.9039915966386555, - "language_acc": { - "English": 0.938655462184874, - "Chinese": 0.8831932773109243, - "Spanish": 0.9226890756302522, - "Vietnamese": 0.8714285714285714 - }, - "consistency_score_2": 0.8787114845938376, - "consistency_score_3": 0.8247899159663865, - "consistency_score_4": 0.7857142857142857, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.892436974789916, - "English,Spanish": 0.9319327731092437, - "English,Vietnamese": 0.8621848739495799, - "Chinese,Spanish": 0.8865546218487395, - "Chinese,Vietnamese": 0.8361344537815126, - "Spanish,Vietnamese": 0.8630252100840337 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8588235294117647, - "English,Chinese,Vietnamese": 0.8033613445378152, - "English,Spanish,Vietnamese": 0.8369747899159664, - "Chinese,Spanish,Vietnamese": 0.8 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7857142857142857 - } - }, - "AC3_2": 0.8911722947704406, - "AC3_3": 0.8625764997443353, - "AC3_4": 0.8407133087724684 - }, - "prompt_2": { - "overall_acc": 0.8987394957983194, - "language_acc": { - "English": 0.9327731092436975, - "Chinese": 0.8848739495798319, - "Spanish": 0.9159663865546218, - "Vietnamese": 0.8613445378151261 - }, - "consistency_score_2": 0.8753501400560223, - "consistency_score_3": 0.8207983193277311, - "consistency_score_4": 0.7840336134453781, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8857142857142857, - "English,Spanish": 0.9201680672268907, - "English,Vietnamese": 0.8588235294117647, - "Chinese,Spanish": 0.8781512605042017, - "Chinese,Vietnamese": 0.8470588235294118, - "Spanish,Vietnamese": 0.8621848739495799 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8453781512605042, - "English,Chinese,Vietnamese": 0.8067226890756303, - "English,Spanish,Vietnamese": 0.826890756302521, - "Chinese,Spanish,Vietnamese": 0.8042016806722689 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7840336134453781 - } - }, - "AC3_2": 0.8868906368394847, - "AC3_3": 0.8580024947782111, - "AC3_4": 0.8374771031510632 - }, - "prompt_3": { - "overall_acc": 0.896218487394958, - "language_acc": { - "English": 0.934453781512605, - "Chinese": 0.8697478991596639, - "Spanish": 0.9243697478991597, - "Vietnamese": 0.8563025210084033 - }, - "consistency_score_2": 0.868767507002801, - "consistency_score_3": 0.8105042016806723, - "consistency_score_4": 0.7714285714285715, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8789915966386554, - "English,Spanish": 0.9260504201680673, - "English,Vietnamese": 0.8504201680672269, - "Chinese,Spanish": 0.8680672268907563, - "Chinese,Vietnamese": 0.8285714285714286, - "Spanish,Vietnamese": 0.8605042016806723 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8394957983193277, - "English,Chinese,Vietnamese": 0.788235294117647, - "English,Spanish,Vietnamese": 0.8252100840336134, - "Chinese,Spanish,Vietnamese": 0.7890756302521008 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7714285714285715 - } - }, - "AC3_2": 0.8822795234083399, - "AC3_3": 0.8512089916708911, - "AC3_4": 0.8291545189007173 - }, - "prompt_4": { - "overall_acc": 0.9021008403361345, - "language_acc": { - "English": 0.9352941176470588, - "Chinese": 0.8890756302521008, - "Spanish": 0.9201680672268907, - "Vietnamese": 0.8638655462184874 - }, - "consistency_score_2": 0.8771708683473388, - "consistency_score_3": 0.8218487394957983, - "consistency_score_4": 0.7823529411764706, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8899159663865546, - "English,Spanish": 0.9310924369747899, - "English,Vietnamese": 0.8579831932773109, - "Chinese,Spanish": 0.8840336134453781, - "Chinese,Vietnamese": 0.838655462184874, - "Spanish,Vietnamese": 0.8613445378151261 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8546218487394958, - "English,Chinese,Vietnamese": 0.8008403361344538, - "English,Spanish,Vietnamese": 0.83109243697479, - "Chinese,Spanish,Vietnamese": 0.8008403361344538 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7823529411764706 - } - }, - "AC3_2": 0.8894612031970516, - "AC3_3": 0.8601068698976974, - "AC3_4": 0.8379704487932971 - }, - "prompt_5": { - "overall_acc": 0.8968487394957984, - "language_acc": { - "English": 0.9260504201680673, - "Chinese": 0.880672268907563, - "Spanish": 0.9184873949579831, - "Vietnamese": 0.8621848739495799 - }, - "consistency_score_2": 0.8729691876750699, - "consistency_score_3": 0.8170168067226891, - "consistency_score_4": 0.7798319327731092, - "detailed_consistency_score": { - "2_combine": { - "English,Chinese": 0.8781512605042017, - "English,Spanish": 0.9226890756302522, - "English,Vietnamese": 0.8554621848739495, - "Chinese,Spanish": 0.8756302521008403, - "Chinese,Vietnamese": 0.8394957983193277, - "Spanish,Vietnamese": 0.8663865546218488 - }, - "3_combine": { - "English,Chinese,Spanish": 0.8411764705882353, - "English,Chinese,Vietnamese": 0.795798319327731, - "English,Spanish,Vietnamese": 0.8285714285714286, - "Chinese,Spanish,Vietnamese": 0.8025210084033614 - }, - "4_combine": { - "English,Chinese,Spanish,Vietnamese": 0.7798319327731092 - } - }, - "AC3_2": 0.8847478641968038, - "AC3_3": 0.8550734855836362, - "AC3_4": 0.8342569905550308 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.5057142857142857, - "language_acc": { - "Spanish": 0.58, - "English": 0.62, - "Chinese": 0.44666666666666666, - "Vietnamese": 0.4066666666666667, - "Filipino": 0.5333333333333333, - "Indonesian": 0.5066666666666667, - "Malay": 0.44666666666666666 - }, - "consistency_score_2": 0.5663492063492064, - "consistency_score_3": 0.4015238095238096, - "consistency_score_4": 0.312, - "consistency_score_5": 0.2539682539682539, - "consistency_score_6": 0.21238095238095236, - "consistency_score_7": 0.18, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.7133333333333334, - "Spanish,Chinese": 0.56, - "Spanish,Vietnamese": 0.5133333333333333, - "Spanish,Filipino": 0.56, - "Spanish,Indonesian": 0.6533333333333333, - "Spanish,Malay": 0.56, - "English,Chinese": 0.5866666666666667, - "English,Vietnamese": 0.47333333333333333, - "English,Filipino": 0.54, - "English,Indonesian": 0.6666666666666666, - "English,Malay": 0.6133333333333333, - "Chinese,Vietnamese": 0.52, - "Chinese,Filipino": 0.48, - "Chinese,Indonesian": 0.62, - "Chinese,Malay": 0.5933333333333334, - "Vietnamese,Filipino": 0.47333333333333333, - "Vietnamese,Indonesian": 0.5733333333333334, - "Vietnamese,Malay": 0.56, - "Filipino,Indonesian": 0.54, - "Filipino,Malay": 0.44, - "Indonesian,Malay": 0.6533333333333333 - }, - "3_combine": { - "Spanish,English,Chinese": 0.4666666666666667, - "Spanish,English,Vietnamese": 0.38, - "Spanish,English,Filipino": 0.4666666666666667, - "Spanish,English,Indonesian": 0.56, - "Spanish,English,Malay": 0.47333333333333333, - "Spanish,Chinese,Vietnamese": 0.35333333333333333, - "Spanish,Chinese,Filipino": 0.36666666666666664, - "Spanish,Chinese,Indonesian": 0.46, - "Spanish,Chinese,Malay": 0.4066666666666667, - "Spanish,Vietnamese,Filipino": 0.3333333333333333, - "Spanish,Vietnamese,Indonesian": 0.43333333333333335, - "Spanish,Vietnamese,Malay": 0.38666666666666666, - "Spanish,Filipino,Indonesian": 0.44, - "Spanish,Filipino,Malay": 0.35333333333333333, - "Spanish,Indonesian,Malay": 0.4866666666666667, - "English,Chinese,Vietnamese": 0.34, - "English,Chinese,Filipino": 0.36, - "English,Chinese,Indonesian": 0.4533333333333333, - "English,Chinese,Malay": 0.44, - "English,Vietnamese,Filipino": 0.31333333333333335, - "English,Vietnamese,Indonesian": 0.4, - "English,Vietnamese,Malay": 0.37333333333333335, - "English,Filipino,Indonesian": 0.43333333333333335, - "English,Filipino,Malay": 0.3466666666666667, - "English,Indonesian,Malay": 0.49333333333333335, - "Chinese,Vietnamese,Filipino": 0.30666666666666664, - "Chinese,Vietnamese,Indonesian": 0.3933333333333333, - "Chinese,Vietnamese,Malay": 0.38, - "Chinese,Filipino,Indonesian": 0.37333333333333335, - "Chinese,Filipino,Malay": 0.3333333333333333, - "Chinese,Indonesian,Malay": 0.4666666666666667, - "Vietnamese,Filipino,Indonesian": 0.35333333333333333, - "Vietnamese,Filipino,Malay": 0.30666666666666664, - "Vietnamese,Indonesian,Malay": 0.44666666666666666, - "Filipino,Indonesian,Malay": 0.37333333333333335 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.29333333333333333, - "Spanish,English,Chinese,Filipino": 0.32666666666666666, - "Spanish,English,Chinese,Indonesian": 0.4, - "Spanish,English,Chinese,Malay": 0.35333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.36, - "Spanish,English,Vietnamese,Malay": 0.31333333333333335, - "Spanish,English,Filipino,Indonesian": 0.4, - "Spanish,English,Filipino,Malay": 0.32, - "Spanish,English,Indonesian,Malay": 0.42, - "Spanish,Chinese,Vietnamese,Filipino": 0.26, - "Spanish,Chinese,Vietnamese,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.29333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.32, - "Spanish,Chinese,Filipino,Malay": 0.29333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.38, - "Spanish,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,Vietnamese,Filipino,Malay": 0.26, - "Spanish,Vietnamese,Indonesian,Malay": 0.36, - "Spanish,Filipino,Indonesian,Malay": 0.32666666666666666, - "English,Chinese,Vietnamese,Filipino": 0.24666666666666667, - "English,Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "English,Chinese,Vietnamese,Malay": 0.26666666666666666, - "English,Chinese,Filipino,Indonesian": 0.31333333333333335, - "English,Chinese,Filipino,Malay": 0.29333333333333333, - "English,Chinese,Indonesian,Malay": 0.36, - "English,Vietnamese,Filipino,Indonesian": 0.28, - "English,Vietnamese,Filipino,Malay": 0.24, - "English,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "English,Filipino,Indonesian,Malay": 0.31333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Chinese,Vietnamese,Indonesian,Malay": 0.32, - "Chinese,Filipino,Indonesian,Malay": 0.3, - "Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.2733333333333333, - "Spanish,English,Chinese,Vietnamese,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,English,Chinese,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.32666666666666666, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.3, - "Spanish,English,Filipino,Indonesian,Malay": 0.3, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.24, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.28, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.22, - "English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.26, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18 - } - }, - "AC3_2": 0.5343170184365595, - "AC3_3": 0.4476362436146168, - "AC3_4": 0.38591194963834063, - "AC3_5": 0.33812906687686856, - "AC3_6": 0.2991360363357588, - "AC3_7": 0.2654999999612812 - }, - "prompt_2": { - "overall_acc": 0.5057142857142857, - "language_acc": { - "Spanish": 0.6133333333333333, - "English": 0.6, - "Chinese": 0.44666666666666666, - "Vietnamese": 0.42, - "Filipino": 0.5066666666666667, - "Indonesian": 0.5266666666666666, - "Malay": 0.4266666666666667 - }, - "consistency_score_2": 0.593968253968254, - "consistency_score_3": 0.4314285714285714, - "consistency_score_4": 0.33733333333333326, - "consistency_score_5": 0.2752380952380952, - "consistency_score_6": 0.23142857142857146, - "consistency_score_7": 0.2, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.74, - "Spanish,Chinese": 0.6133333333333333, - "Spanish,Vietnamese": 0.5933333333333334, - "Spanish,Filipino": 0.6133333333333333, - "Spanish,Indonesian": 0.7066666666666667, - "Spanish,Malay": 0.6266666666666667, - "English,Chinese": 0.6133333333333333, - "English,Vietnamese": 0.5133333333333333, - "English,Filipino": 0.5333333333333333, - "English,Indonesian": 0.66, - "English,Malay": 0.6066666666666667, - "Chinese,Vietnamese": 0.5, - "Chinese,Filipino": 0.5333333333333333, - "Chinese,Indonesian": 0.6266666666666667, - "Chinese,Malay": 0.5933333333333334, - "Vietnamese,Filipino": 0.56, - "Vietnamese,Indonesian": 0.5933333333333334, - "Vietnamese,Malay": 0.5866666666666667, - "Filipino,Indonesian": 0.54, - "Filipino,Malay": 0.4533333333333333, - "Indonesian,Malay": 0.6666666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.5066666666666667, - "Spanish,English,Vietnamese": 0.44666666666666666, - "Spanish,English,Filipino": 0.4666666666666667, - "Spanish,English,Indonesian": 0.5866666666666667, - "Spanish,English,Malay": 0.5133333333333333, - "Spanish,Chinese,Vietnamese": 0.4, - "Spanish,Chinese,Filipino": 0.42, - "Spanish,Chinese,Indonesian": 0.49333333333333335, - "Spanish,Chinese,Malay": 0.4533333333333333, - "Spanish,Vietnamese,Filipino": 0.4266666666666667, - "Spanish,Vietnamese,Indonesian": 0.48, - "Spanish,Vietnamese,Malay": 0.4533333333333333, - "Spanish,Filipino,Indonesian": 0.4533333333333333, - "Spanish,Filipino,Malay": 0.38666666666666666, - "Spanish,Indonesian,Malay": 0.5333333333333333, - "English,Chinese,Vietnamese": 0.36, - "English,Chinese,Filipino": 0.38666666666666666, - "English,Chinese,Indonesian": 0.47333333333333333, - "English,Chinese,Malay": 0.44666666666666666, - "English,Vietnamese,Filipino": 0.35333333333333333, - "English,Vietnamese,Indonesian": 0.4266666666666667, - "English,Vietnamese,Malay": 0.41333333333333333, - "English,Filipino,Indonesian": 0.41333333333333333, - "English,Filipino,Malay": 0.35333333333333333, - "English,Indonesian,Malay": 0.4866666666666667, - "Chinese,Vietnamese,Filipino": 0.35333333333333333, - "Chinese,Vietnamese,Indonesian": 0.3933333333333333, - "Chinese,Vietnamese,Malay": 0.3933333333333333, - "Chinese,Filipino,Indonesian": 0.38666666666666666, - "Chinese,Filipino,Malay": 0.36, - "Chinese,Indonesian,Malay": 0.47333333333333333, - "Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "Vietnamese,Filipino,Malay": 0.36, - "Vietnamese,Indonesian,Malay": 0.47333333333333333, - "Filipino,Indonesian,Malay": 0.38666666666666666 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.32, - "Spanish,English,Chinese,Filipino": 0.35333333333333333, - "Spanish,English,Chinese,Indonesian": 0.4266666666666667, - "Spanish,English,Chinese,Malay": 0.38, - "Spanish,English,Vietnamese,Filipino": 0.32, - "Spanish,English,Vietnamese,Indonesian": 0.3933333333333333, - "Spanish,English,Vietnamese,Malay": 0.37333333333333335, - "Spanish,English,Filipino,Indonesian": 0.3933333333333333, - "Spanish,English,Filipino,Malay": 0.32666666666666666, - "Spanish,English,Indonesian,Malay": 0.44, - "Spanish,Chinese,Vietnamese,Filipino": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Indonesian": 0.34, - "Spanish,Chinese,Vietnamese,Malay": 0.3333333333333333, - "Spanish,Chinese,Filipino,Indonesian": 0.3333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.31333333333333335, - "Spanish,Chinese,Indonesian,Malay": 0.41333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.34, - "Spanish,Vietnamese,Filipino,Malay": 0.31333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.4066666666666667, - "Spanish,Filipino,Indonesian,Malay": 0.34, - "English,Chinese,Vietnamese,Filipino": 0.2733333333333333, - "English,Chinese,Vietnamese,Indonesian": 0.3, - "English,Chinese,Vietnamese,Malay": 0.29333333333333333, - "English,Chinese,Filipino,Indonesian": 0.31333333333333335, - "English,Chinese,Filipino,Malay": 0.29333333333333333, - "English,Chinese,Indonesian,Malay": 0.37333333333333335, - "English,Vietnamese,Filipino,Indonesian": 0.30666666666666664, - "English,Vietnamese,Filipino,Malay": 0.2866666666666667, - "English,Vietnamese,Indonesian,Malay": 0.35333333333333333, - "English,Filipino,Indonesian,Malay": 0.30666666666666664, - "Chinese,Vietnamese,Filipino,Indonesian": 0.2866666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.2866666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.34, - "Chinese,Filipino,Indonesian,Malay": 0.30666666666666664, - "Vietnamese,Filipino,Indonesian,Malay": 0.32 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.25333333333333335, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.28, - "Spanish,English,Chinese,Vietnamese,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.3, - "Spanish,English,Chinese,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.3466666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.3333333333333333, - "Spanish,English,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.28, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.2866666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Filipino,Indonesian,Malay": 0.25333333333333335, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.22, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.24666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 - } - }, - "AC3_2": 0.5462999009730868, - "AC3_3": 0.4656271776506626, - "AC3_4": 0.4047085403948323, - "AC3_5": 0.35646689890905864, - "AC3_6": 0.31754152819612536, - "AC3_7": 0.2866396760727434 - }, - "prompt_3": { - "overall_acc": 0.49714285714285716, - "language_acc": { - "Spanish": 0.5866666666666667, - "English": 0.6266666666666667, - "Chinese": 0.4533333333333333, - "Vietnamese": 0.38666666666666666, - "Filipino": 0.48, - "Indonesian": 0.5066666666666667, - "Malay": 0.44 - }, - "consistency_score_2": 0.5253968253968254, - "consistency_score_3": 0.35580952380952385, - "consistency_score_4": 0.26799999999999996, - "consistency_score_5": 0.21333333333333335, - "consistency_score_6": 0.17523809523809522, - "consistency_score_7": 0.14666666666666667, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6533333333333333, - "Spanish,Chinese": 0.49333333333333335, - "Spanish,Vietnamese": 0.5333333333333333, - "Spanish,Filipino": 0.5733333333333334, - "Spanish,Indonesian": 0.5866666666666667, - "Spanish,Malay": 0.5533333333333333, - "English,Chinese": 0.54, - "English,Vietnamese": 0.49333333333333335, - "English,Filipino": 0.5066666666666667, - "English,Indonesian": 0.5933333333333334, - "English,Malay": 0.5666666666666667, - "Chinese,Vietnamese": 0.42, - "Chinese,Filipino": 0.44, - "Chinese,Indonesian": 0.5533333333333333, - "Chinese,Malay": 0.5066666666666667, - "Vietnamese,Filipino": 0.4533333333333333, - "Vietnamese,Indonesian": 0.5066666666666667, - "Vietnamese,Malay": 0.5266666666666666, - "Filipino,Indonesian": 0.4866666666666667, - "Filipino,Malay": 0.44666666666666666, - "Indonesian,Malay": 0.6 - }, - "3_combine": { - "Spanish,English,Chinese": 0.38666666666666666, - "Spanish,English,Vietnamese": 0.38666666666666666, - "Spanish,English,Filipino": 0.42, - "Spanish,English,Indonesian": 0.4666666666666667, - "Spanish,English,Malay": 0.44, - "Spanish,Chinese,Vietnamese": 0.29333333333333333, - "Spanish,Chinese,Filipino": 0.32666666666666666, - "Spanish,Chinese,Indonesian": 0.38, - "Spanish,Chinese,Malay": 0.34, - "Spanish,Vietnamese,Filipino": 0.34, - "Spanish,Vietnamese,Indonesian": 0.36666666666666664, - "Spanish,Vietnamese,Malay": 0.35333333333333333, - "Spanish,Filipino,Indonesian": 0.3933333333333333, - "Spanish,Filipino,Malay": 0.35333333333333333, - "Spanish,Indonesian,Malay": 0.44666666666666666, - "English,Chinese,Vietnamese": 0.31333333333333335, - "English,Chinese,Filipino": 0.32666666666666666, - "English,Chinese,Indonesian": 0.4, - "English,Chinese,Malay": 0.36, - "English,Vietnamese,Filipino": 0.31333333333333335, - "English,Vietnamese,Indonesian": 0.36, - "English,Vietnamese,Malay": 0.35333333333333333, - "English,Filipino,Indonesian": 0.37333333333333335, - "English,Filipino,Malay": 0.32, - "English,Indonesian,Malay": 0.43333333333333335, - "Chinese,Vietnamese,Filipino": 0.25333333333333335, - "Chinese,Vietnamese,Indonesian": 0.31333333333333335, - "Chinese,Vietnamese,Malay": 0.3, - "Chinese,Filipino,Indonesian": 0.32, - "Chinese,Filipino,Malay": 0.30666666666666664, - "Chinese,Indonesian,Malay": 0.38, - "Vietnamese,Filipino,Indonesian": 0.3, - "Vietnamese,Filipino,Malay": 0.28, - "Vietnamese,Indonesian,Malay": 0.3933333333333333, - "Filipino,Indonesian,Malay": 0.36 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.24666666666666667, - "Spanish,English,Chinese,Filipino": 0.2733333333333333, - "Spanish,English,Chinese,Indonesian": 0.31333333333333335, - "Spanish,English,Chinese,Malay": 0.28, - "Spanish,English,Vietnamese,Filipino": 0.2733333333333333, - "Spanish,English,Vietnamese,Indonesian": 0.31333333333333335, - "Spanish,English,Vietnamese,Malay": 0.3, - "Spanish,English,Filipino,Indonesian": 0.32666666666666666, - "Spanish,English,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Indonesian,Malay": 0.36666666666666664, - "Spanish,Chinese,Vietnamese,Filipino": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,Chinese,Vietnamese,Malay": 0.23333333333333334, - "Spanish,Chinese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,Chinese,Filipino,Malay": 0.25333333333333335, - "Spanish,Chinese,Indonesian,Malay": 0.29333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Spanish,Vietnamese,Filipino,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.32, - "English,Chinese,Vietnamese,Filipino": 0.22666666666666666, - "English,Chinese,Vietnamese,Indonesian": 0.26, - "English,Chinese,Vietnamese,Malay": 0.24, - "English,Chinese,Filipino,Indonesian": 0.28, - "English,Chinese,Filipino,Malay": 0.24, - "English,Chinese,Indonesian,Malay": 0.29333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.29333333333333333, - "English,Filipino,Indonesian,Malay": 0.2866666666666667, - "Chinese,Vietnamese,Filipino,Indonesian": 0.22, - "Chinese,Vietnamese,Filipino,Malay": 0.2, - "Chinese,Vietnamese,Indonesian,Malay": 0.26, - "Chinese,Filipino,Indonesian,Malay": 0.26, - "Vietnamese,Filipino,Indonesian,Malay": 0.24 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.19333333333333333, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.22, - "Spanish,English,Chinese,Vietnamese,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian": 0.24, - "Spanish,English,Chinese,Filipino,Malay": 0.20666666666666667, - "Spanish,English,Chinese,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.22666666666666666, - "Spanish,English,Vietnamese,Filipino,Malay": 0.2, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.26666666666666666, - "Spanish,English,Filipino,Indonesian,Malay": 0.26, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.17333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.2, - "English,Chinese,Vietnamese,Filipino,Malay": 0.18, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.21333333333333335, - "English,Chinese,Filipino,Indonesian,Malay": 0.22, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.2, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.17333333333333334, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.19333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.16 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.14666666666666667 - } - }, - "AC3_2": 0.5108794961136076, - "AC3_3": 0.4147667963528276, - "AC3_4": 0.3482598953988457, - "AC3_5": 0.29855227877835394, - "AC3_6": 0.2591339538262923, - "AC3_7": 0.2265088757044624 - }, - "prompt_4": { - "overall_acc": 0.5038095238095238, - "language_acc": { - "Spanish": 0.5733333333333334, - "English": 0.6133333333333333, - "Chinese": 0.46, - "Vietnamese": 0.3933333333333333, - "Filipino": 0.52, - "Indonesian": 0.5333333333333333, - "Malay": 0.43333333333333335 - }, - "consistency_score_2": 0.5765079365079366, - "consistency_score_3": 0.4100952380952381, - "consistency_score_4": 0.3156190476190476, - "consistency_score_5": 0.2542857142857142, - "consistency_score_6": 0.21142857142857138, - "consistency_score_7": 0.18, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.7133333333333334, - "Spanish,Chinese": 0.5866666666666667, - "Spanish,Vietnamese": 0.5533333333333333, - "Spanish,Filipino": 0.5666666666666667, - "Spanish,Indonesian": 0.6733333333333333, - "Spanish,Malay": 0.6, - "English,Chinese": 0.58, - "English,Vietnamese": 0.5333333333333333, - "English,Filipino": 0.54, - "English,Indonesian": 0.6533333333333333, - "English,Malay": 0.6133333333333333, - "Chinese,Vietnamese": 0.4666666666666667, - "Chinese,Filipino": 0.48, - "Chinese,Indonesian": 0.6066666666666667, - "Chinese,Malay": 0.6066666666666667, - "Vietnamese,Filipino": 0.5066666666666667, - "Vietnamese,Indonesian": 0.56, - "Vietnamese,Malay": 0.58, - "Filipino,Indonesian": 0.5466666666666666, - "Filipino,Malay": 0.4533333333333333, - "Indonesian,Malay": 0.6866666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.4666666666666667, - "Spanish,English,Vietnamese": 0.44, - "Spanish,English,Filipino": 0.44666666666666666, - "Spanish,English,Indonesian": 0.5533333333333333, - "Spanish,English,Malay": 0.49333333333333335, - "Spanish,Chinese,Vietnamese": 0.36666666666666664, - "Spanish,Chinese,Filipino": 0.37333333333333335, - "Spanish,Chinese,Indonesian": 0.4666666666666667, - "Spanish,Chinese,Malay": 0.43333333333333335, - "Spanish,Vietnamese,Filipino": 0.36666666666666664, - "Spanish,Vietnamese,Indonesian": 0.43333333333333335, - "Spanish,Vietnamese,Malay": 0.42, - "Spanish,Filipino,Indonesian": 0.44666666666666666, - "Spanish,Filipino,Malay": 0.36666666666666664, - "Spanish,Indonesian,Malay": 0.5133333333333333, - "English,Chinese,Vietnamese": 0.35333333333333333, - "English,Chinese,Filipino": 0.34, - "English,Chinese,Indonesian": 0.44666666666666666, - "English,Chinese,Malay": 0.43333333333333335, - "English,Vietnamese,Filipino": 0.34, - "English,Vietnamese,Indonesian": 0.42, - "English,Vietnamese,Malay": 0.41333333333333333, - "English,Filipino,Indonesian": 0.42, - "English,Filipino,Malay": 0.35333333333333333, - "English,Indonesian,Malay": 0.49333333333333335, - "Chinese,Vietnamese,Filipino": 0.30666666666666664, - "Chinese,Vietnamese,Indonesian": 0.36666666666666664, - "Chinese,Vietnamese,Malay": 0.36666666666666664, - "Chinese,Filipino,Indonesian": 0.37333333333333335, - "Chinese,Filipino,Malay": 0.3466666666666667, - "Chinese,Indonesian,Malay": 0.48, - "Vietnamese,Filipino,Indonesian": 0.3466666666666667, - "Vietnamese,Filipino,Malay": 0.32666666666666666, - "Vietnamese,Indonesian,Malay": 0.4533333333333333, - "Filipino,Indonesian,Malay": 0.38666666666666666 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.31333333333333335, - "Spanish,English,Chinese,Filipino": 0.3, - "Spanish,English,Chinese,Indonesian": 0.4, - "Spanish,English,Chinese,Malay": 0.36666666666666664, - "Spanish,English,Vietnamese,Filipino": 0.3, - "Spanish,English,Vietnamese,Indonesian": 0.37333333333333335, - "Spanish,English,Vietnamese,Malay": 0.35333333333333333, - "Spanish,English,Filipino,Indonesian": 0.38, - "Spanish,English,Filipino,Malay": 0.31333333333333335, - "Spanish,English,Indonesian,Malay": 0.4266666666666667, - "Spanish,Chinese,Vietnamese,Filipino": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian": 0.32, - "Spanish,Chinese,Vietnamese,Malay": 0.31333333333333335, - "Spanish,Chinese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Filipino,Malay": 0.29333333333333333, - "Spanish,Chinese,Indonesian,Malay": 0.3933333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Vietnamese,Filipino,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Indonesian,Malay": 0.36666666666666664, - "Spanish,Filipino,Indonesian,Malay": 0.3333333333333333, - "English,Chinese,Vietnamese,Filipino": 0.24, - "English,Chinese,Vietnamese,Indonesian": 0.3, - "English,Chinese,Vietnamese,Malay": 0.29333333333333333, - "English,Chinese,Filipino,Indonesian": 0.3, - "English,Chinese,Filipino,Malay": 0.2733333333333333, - "English,Chinese,Indonesian,Malay": 0.36, - "English,Vietnamese,Filipino,Indonesian": 0.28, - "English,Vietnamese,Filipino,Malay": 0.26666666666666666, - "English,Vietnamese,Indonesian,Malay": 0.34, - "English,Filipino,Indonesian,Malay": 0.3, - "Chinese,Vietnamese,Filipino,Indonesian": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Malay": 0.24666666666666667, - "Chinese,Vietnamese,Indonesian,Malay": 0.31333333333333335, - "Chinese,Filipino,Indonesian,Malay": 0.3, - "Vietnamese,Filipino,Indonesian,Malay": 0.2733333333333333 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.22666666666666666, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.28, - "Spanish,English,Chinese,Vietnamese,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Filipino,Indonesian": 0.28, - "Spanish,English,Chinese,Filipino,Malay": 0.25333333333333335, - "Spanish,English,Chinese,Indonesian,Malay": 0.3333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,English,Vietnamese,Filipino,Malay": 0.24, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Spanish,English,Filipino,Indonesian,Malay": 0.28, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.21333333333333335, - "English,Chinese,Vietnamese,Filipino,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "English,Chinese,Filipino,Indonesian,Malay": 0.24, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.24666666666666667, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.20666666666666667, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18 - } - }, - "AC3_2": 0.5377126624469031, - "AC3_3": 0.4521475217353964, - "AC3_4": 0.3881043146678306, - "AC3_5": 0.3379827709532631, - "AC3_6": 0.29785809392830015, - "AC3_7": 0.26523676876344027 - }, - "prompt_5": { - "overall_acc": 0.5057142857142857, - "language_acc": { - "Spanish": 0.56, - "English": 0.6333333333333333, - "Chinese": 0.48, - "Vietnamese": 0.4, - "Filipino": 0.5066666666666667, - "Indonesian": 0.54, - "Malay": 0.42 - }, - "consistency_score_2": 0.585079365079365, - "consistency_score_3": 0.41942857142857143, - "consistency_score_4": 0.32571428571428573, - "consistency_score_5": 0.2641269841269841, - "consistency_score_6": 0.22, - "consistency_score_7": 0.18666666666666668, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.6866666666666666, - "Spanish,Chinese": 0.5933333333333334, - "Spanish,Vietnamese": 0.5466666666666666, - "Spanish,Filipino": 0.5866666666666667, - "Spanish,Indonesian": 0.7, - "Spanish,Malay": 0.6533333333333333, - "English,Chinese": 0.6, - "English,Vietnamese": 0.54, - "English,Filipino": 0.5666666666666667, - "English,Indonesian": 0.66, - "English,Malay": 0.5733333333333334, - "Chinese,Vietnamese": 0.5133333333333333, - "Chinese,Filipino": 0.5, - "Chinese,Indonesian": 0.5933333333333334, - "Chinese,Malay": 0.5666666666666667, - "Vietnamese,Filipino": 0.52, - "Vietnamese,Indonesian": 0.56, - "Vietnamese,Malay": 0.5666666666666667, - "Filipino,Indonesian": 0.5866666666666667, - "Filipino,Malay": 0.4866666666666667, - "Indonesian,Malay": 0.6866666666666666 - }, - "3_combine": { - "Spanish,English,Chinese": 0.4666666666666667, - "Spanish,English,Vietnamese": 0.41333333333333333, - "Spanish,English,Filipino": 0.4666666666666667, - "Spanish,English,Indonesian": 0.5533333333333333, - "Spanish,English,Malay": 0.47333333333333333, - "Spanish,Chinese,Vietnamese": 0.37333333333333335, - "Spanish,Chinese,Filipino": 0.4066666666666667, - "Spanish,Chinese,Indonesian": 0.47333333333333333, - "Spanish,Chinese,Malay": 0.4533333333333333, - "Spanish,Vietnamese,Filipino": 0.37333333333333335, - "Spanish,Vietnamese,Indonesian": 0.44, - "Spanish,Vietnamese,Malay": 0.41333333333333333, - "Spanish,Filipino,Indonesian": 0.48, - "Spanish,Filipino,Malay": 0.4, - "Spanish,Indonesian,Malay": 0.54, - "English,Chinese,Vietnamese": 0.37333333333333335, - "English,Chinese,Filipino": 0.38, - "English,Chinese,Indonesian": 0.46, - "English,Chinese,Malay": 0.4066666666666667, - "English,Vietnamese,Filipino": 0.35333333333333333, - "English,Vietnamese,Indonesian": 0.42, - "English,Vietnamese,Malay": 0.38666666666666666, - "English,Filipino,Indonesian": 0.44666666666666666, - "English,Filipino,Malay": 0.36, - "English,Indonesian,Malay": 0.48, - "Chinese,Vietnamese,Filipino": 0.34, - "Chinese,Vietnamese,Indonesian": 0.37333333333333335, - "Chinese,Vietnamese,Malay": 0.36666666666666664, - "Chinese,Filipino,Indonesian": 0.3933333333333333, - "Chinese,Filipino,Malay": 0.36, - "Chinese,Indonesian,Malay": 0.4533333333333333, - "Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "Vietnamese,Filipino,Malay": 0.34, - "Vietnamese,Indonesian,Malay": 0.44666666666666666, - "Filipino,Indonesian,Malay": 0.4266666666666667 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.31333333333333335, - "Spanish,English,Chinese,Filipino": 0.34, - "Spanish,English,Chinese,Indonesian": 0.4, - "Spanish,English,Chinese,Malay": 0.35333333333333333, - "Spanish,English,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,English,Vietnamese,Indonesian": 0.35333333333333333, - "Spanish,English,Vietnamese,Malay": 0.32666666666666666, - "Spanish,English,Filipino,Indonesian": 0.4066666666666667, - "Spanish,English,Filipino,Malay": 0.32, - "Spanish,English,Indonesian,Malay": 0.43333333333333335, - "Spanish,Chinese,Vietnamese,Filipino": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Indonesian": 0.32666666666666666, - "Spanish,Chinese,Vietnamese,Malay": 0.30666666666666664, - "Spanish,Chinese,Filipino,Indonesian": 0.36, - "Spanish,Chinese,Filipino,Malay": 0.32666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.4, - "Spanish,Vietnamese,Filipino,Indonesian": 0.32666666666666666, - "Spanish,Vietnamese,Filipino,Malay": 0.28, - "Spanish,Vietnamese,Indonesian,Malay": 0.37333333333333335, - "Spanish,Filipino,Indonesian,Malay": 0.36666666666666664, - "English,Chinese,Vietnamese,Filipino": 0.26666666666666666, - "English,Chinese,Vietnamese,Indonesian": 0.29333333333333333, - "English,Chinese,Vietnamese,Malay": 0.28, - "English,Chinese,Filipino,Indonesian": 0.3333333333333333, - "English,Chinese,Filipino,Malay": 0.29333333333333333, - "English,Chinese,Indonesian,Malay": 0.35333333333333333, - "English,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "English,Vietnamese,Filipino,Malay": 0.26, - "English,Vietnamese,Indonesian,Malay": 0.32666666666666666, - "English,Filipino,Indonesian,Malay": 0.32666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian": 0.29333333333333333, - "Chinese,Vietnamese,Filipino,Malay": 0.26666666666666666, - "Chinese,Vietnamese,Indonesian,Malay": 0.30666666666666664, - "Chinese,Filipino,Indonesian,Malay": 0.32, - "Vietnamese,Filipino,Indonesian,Malay": 0.3 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.24, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.26666666666666666, - "Spanish,English,Chinese,Vietnamese,Malay": 0.25333333333333335, - "Spanish,English,Chinese,Filipino,Indonesian": 0.31333333333333335, - "Spanish,English,Chinese,Filipino,Malay": 0.2733333333333333, - "Spanish,English,Chinese,Indonesian,Malay": 0.3333333333333333, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.26, - "Spanish,English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.3, - "Spanish,English,Filipino,Indonesian,Malay": 0.30666666666666664, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.24, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.3, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.23333333333333334, - "English,Chinese,Vietnamese,Filipino,Malay": 0.21333333333333335, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "English,Chinese,Filipino,Indonesian,Malay": 0.26666666666666666, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.24 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.22, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.26, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.22666666666666666, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.18666666666666668 - } - }, - "AC3_2": 0.5425095625646003, - "AC3_3": 0.4585476042917391, - "AC3_4": 0.39622974958415497, - "AC3_5": 0.34701384384299466, - "AC3_6": 0.30661417318609646, - "AC3_7": 0.27268225580655897 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.44805194805194803, - "language_acc": { - "English": 0.5738636363636364, - "Filipino": 0.3352272727272727, - "Spanish": 0.5056818181818182, - "Malay": 0.39204545454545453, - "Chinese": 0.4943181818181818, - "Indonesian": 0.4943181818181818, - "Vietnamese": 0.3409090909090909 - }, - "consistency_score_2": 0.5411255411255411, - "consistency_score_3": 0.36266233766233763, - "consistency_score_4": 0.2647727272727272, - "consistency_score_5": 0.2010281385281385, - "consistency_score_6": 0.15665584415584416, - "consistency_score_7": 0.125, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3977272727272727, - "English,Spanish": 0.6818181818181818, - "English,Malay": 0.5625, - "English,Chinese": 0.6079545454545454, - "English,Indonesian": 0.6590909090909091, - "English,Vietnamese": 0.4318181818181818, - "Filipino,Spanish": 0.48863636363636365, - "Filipino,Malay": 0.5056818181818182, - "Filipino,Chinese": 0.44886363636363635, - "Filipino,Indonesian": 0.4772727272727273, - "Filipino,Vietnamese": 0.48295454545454547, - "Spanish,Malay": 0.6022727272727273, - "Spanish,Chinese": 0.6022727272727273, - "Spanish,Indonesian": 0.6534090909090909, - "Spanish,Vietnamese": 0.5113636363636364, - "Malay,Chinese": 0.5397727272727273, - "Malay,Indonesian": 0.6306818181818182, - "Malay,Vietnamese": 0.4772727272727273, - "Chinese,Indonesian": 0.6590909090909091, - "Chinese,Vietnamese": 0.4375, - "Indonesian,Vietnamese": 0.5056818181818182 - }, - "3_combine": { - "English,Filipino,Spanish": 0.32954545454545453, - "English,Filipino,Malay": 0.2897727272727273, - "English,Filipino,Chinese": 0.29545454545454547, - "English,Filipino,Indonesian": 0.3125, - "English,Filipino,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay": 0.45454545454545453, - "English,Spanish,Chinese": 0.48295454545454547, - "English,Spanish,Indonesian": 0.5113636363636364, - "English,Spanish,Vietnamese": 0.3522727272727273, - "English,Malay,Chinese": 0.3977272727272727, - "English,Malay,Indonesian": 0.4659090909090909, - "English,Malay,Vietnamese": 0.3068181818181818, - "English,Chinese,Indonesian": 0.4943181818181818, - "English,Chinese,Vietnamese": 0.29545454545454547, - "English,Indonesian,Vietnamese": 0.3522727272727273, - "Filipino,Spanish,Malay": 0.3352272727272727, - "Filipino,Spanish,Chinese": 0.32386363636363635, - "Filipino,Spanish,Indonesian": 0.35795454545454547, - "Filipino,Spanish,Vietnamese": 0.3068181818181818, - "Filipino,Malay,Chinese": 0.3181818181818182, - "Filipino,Malay,Indonesian": 0.36363636363636365, - "Filipino,Malay,Vietnamese": 0.29545454545454547, - "Filipino,Chinese,Indonesian": 0.3409090909090909, - "Filipino,Chinese,Vietnamese": 0.25, - "Filipino,Indonesian,Vietnamese": 0.2897727272727273, - "Spanish,Malay,Chinese": 0.4090909090909091, - "Spanish,Malay,Indonesian": 0.48863636363636365, - "Spanish,Malay,Vietnamese": 0.36363636363636365, - "Spanish,Chinese,Indonesian": 0.4943181818181818, - "Spanish,Chinese,Vietnamese": 0.3181818181818182, - "Spanish,Indonesian,Vietnamese": 0.38636363636363635, - "Malay,Chinese,Indonesian": 0.4602272727272727, - "Malay,Chinese,Vietnamese": 0.2897727272727273, - "Malay,Indonesian,Vietnamese": 0.3693181818181818, - "Chinese,Indonesian,Vietnamese": 0.3465909090909091 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.23295454545454544, - "English,Filipino,Spanish,Chinese": 0.26136363636363635, - "English,Filipino,Spanish,Indonesian": 0.26704545454545453, - "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, - "English,Filipino,Malay,Chinese": 0.23295454545454544, - "English,Filipino,Malay,Indonesian": 0.2556818181818182, - "English,Filipino,Malay,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian": 0.26136363636363635, - "English,Filipino,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Indonesian,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay,Chinese": 0.3465909090909091, - "English,Spanish,Malay,Indonesian": 0.39204545454545453, - "English,Spanish,Malay,Vietnamese": 0.26136363636363635, - "English,Spanish,Chinese,Indonesian": 0.42045454545454547, - "English,Spanish,Chinese,Vietnamese": 0.25, - "English,Spanish,Indonesian,Vietnamese": 0.29545454545454547, - "English,Malay,Chinese,Indonesian": 0.3693181818181818, - "English,Malay,Chinese,Vietnamese": 0.2215909090909091, - "English,Malay,Indonesian,Vietnamese": 0.2840909090909091, - "English,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "Filipino,Spanish,Malay,Chinese": 0.24431818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.2784090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Chinese,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.2840909090909091, - "Filipino,Malay,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.38636363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.25, - "Spanish,Malay,Indonesian,Vietnamese": 0.3125, - "Spanish,Chinese,Indonesian,Vietnamese": 0.2897727272727273, - "Malay,Chinese,Indonesian,Vietnamese": 0.26704545454545453 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.19886363636363635, - "English,Filipino,Spanish,Malay,Indonesian": 0.21022727272727273, - "English,Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Indonesian": 0.23295454545454544, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Chinese,Indonesian": 0.2159090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.14204545454545456, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Chinese,Indonesian": 0.32954545454545453, - "English,Spanish,Malay,Chinese,Vietnamese": 0.19886363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.24431818181818182, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.23863636363636365, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.22727272727272727, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1875, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.1875, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.125 - } - }, - "AC3_2": 0.49021000876001386, - "AC3_3": 0.40086025303347494, - "AC3_4": 0.3328502515008924, - "AC3_5": 0.2775344704887945, - "AC3_6": 0.2321450361334922, - "AC3_7": 0.19546742206220738 - }, - "prompt_2": { - "overall_acc": 0.439935064935065, - "language_acc": { - "English": 0.5681818181818182, - "Filipino": 0.3352272727272727, - "Spanish": 0.48863636363636365, - "Malay": 0.3522727272727273, - "Chinese": 0.5113636363636364, - "Indonesian": 0.4772727272727273, - "Vietnamese": 0.3465909090909091 - }, - "consistency_score_2": 0.5481601731601732, - "consistency_score_3": 0.3751623376623377, - "consistency_score_4": 0.2797077922077923, - "consistency_score_5": 0.21807359307359309, - "consistency_score_6": 0.17613636363636367, - "consistency_score_7": 0.14772727272727273, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.4147727272727273, - "English,Spanish": 0.6704545454545454, - "English,Malay": 0.5625, - "English,Chinese": 0.6306818181818182, - "English,Indonesian": 0.6647727272727273, - "English,Vietnamese": 0.4772727272727273, - "Filipino,Spanish": 0.4147727272727273, - "Filipino,Malay": 0.5227272727272727, - "Filipino,Chinese": 0.4602272727272727, - "Filipino,Indonesian": 0.4659090909090909, - "Filipino,Vietnamese": 0.4602272727272727, - "Spanish,Malay": 0.6022727272727273, - "Spanish,Chinese": 0.625, - "Spanish,Indonesian": 0.6818181818181818, - "Spanish,Vietnamese": 0.5056818181818182, - "Malay,Chinese": 0.5568181818181818, - "Malay,Indonesian": 0.6704545454545454, - "Malay,Vietnamese": 0.48863636363636365, - "Chinese,Indonesian": 0.6477272727272727, - "Chinese,Vietnamese": 0.4602272727272727, - "Indonesian,Vietnamese": 0.5284090909090909 - }, - "3_combine": { - "English,Filipino,Spanish": 0.3068181818181818, - "English,Filipino,Malay": 0.3125, - "English,Filipino,Chinese": 0.32386363636363635, - "English,Filipino,Indonesian": 0.3181818181818182, - "English,Filipino,Vietnamese": 0.26704545454545453, - "English,Spanish,Malay": 0.4431818181818182, - "English,Spanish,Chinese": 0.48863636363636365, - "English,Spanish,Indonesian": 0.5284090909090909, - "English,Spanish,Vietnamese": 0.375, - "English,Malay,Chinese": 0.4147727272727273, - "English,Malay,Indonesian": 0.4772727272727273, - "English,Malay,Vietnamese": 0.3465909090909091, - "English,Chinese,Indonesian": 0.5, - "English,Chinese,Vietnamese": 0.3409090909090909, - "English,Indonesian,Vietnamese": 0.38636363636363635, - "Filipino,Spanish,Malay": 0.3125, - "Filipino,Spanish,Chinese": 0.3068181818181818, - "Filipino,Spanish,Indonesian": 0.3352272727272727, - "Filipino,Spanish,Vietnamese": 0.2840909090909091, - "Filipino,Malay,Chinese": 0.32954545454545453, - "Filipino,Malay,Indonesian": 0.375, - "Filipino,Malay,Vietnamese": 0.3125, - "Filipino,Chinese,Indonesian": 0.3409090909090909, - "Filipino,Chinese,Vietnamese": 0.26704545454545453, - "Filipino,Indonesian,Vietnamese": 0.30113636363636365, - "Spanish,Malay,Chinese": 0.4375, - "Spanish,Malay,Indonesian": 0.5056818181818182, - "Spanish,Malay,Vietnamese": 0.375, - "Spanish,Chinese,Indonesian": 0.5170454545454546, - "Spanish,Chinese,Vietnamese": 0.3465909090909091, - "Spanish,Indonesian,Vietnamese": 0.4090909090909091, - "Malay,Chinese,Indonesian": 0.4659090909090909, - "Malay,Chinese,Vietnamese": 0.3181818181818182, - "Malay,Indonesian,Vietnamese": 0.39204545454545453, - "Chinese,Indonesian,Vietnamese": 0.3693181818181818 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.25, - "English,Filipino,Spanish,Chinese": 0.26136363636363635, - "English,Filipino,Spanish,Indonesian": 0.26136363636363635, - "English,Filipino,Spanish,Vietnamese": 0.2159090909090909, - "English,Filipino,Malay,Chinese": 0.25, - "English,Filipino,Malay,Indonesian": 0.26704545454545453, - "English,Filipino,Malay,Vietnamese": 0.2215909090909091, - "English,Filipino,Chinese,Indonesian": 0.26704545454545453, - "English,Filipino,Chinese,Vietnamese": 0.21022727272727273, - "English,Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay,Chinese": 0.35795454545454547, - "English,Spanish,Malay,Indonesian": 0.4034090909090909, - "English,Spanish,Malay,Vietnamese": 0.3068181818181818, - "English,Spanish,Chinese,Indonesian": 0.4375, - "English,Spanish,Chinese,Vietnamese": 0.2784090909090909, - "English,Spanish,Indonesian,Vietnamese": 0.32954545454545453, - "English,Malay,Chinese,Indonesian": 0.375, - "English,Malay,Chinese,Vietnamese": 0.2556818181818182, - "English,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "English,Chinese,Indonesian,Vietnamese": 0.29545454545454547, - "Filipino,Spanish,Malay,Chinese": 0.25, - "Filipino,Spanish,Malay,Indonesian": 0.2784090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Chinese,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.19886363636363635, - "Filipino,Spanish,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Malay,Chinese,Indonesian": 0.2784090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.23863636363636365, - "Filipino,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Chinese,Indonesian": 0.4034090909090909, - "Spanish,Malay,Chinese,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.3352272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.3125, - "Malay,Chinese,Indonesian,Vietnamese": 0.2727272727272727 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.2159090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.22727272727272727, - "English,Filipino,Spanish,Malay,Vietnamese": 0.1875, - "English,Filipino,Spanish,Chinese,Indonesian": 0.23863636363636365, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Malay,Chinese,Indonesian": 0.2215909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.1875, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "English,Spanish,Malay,Chinese,Indonesian": 0.3409090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.23295454545454544, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.26704545454545453, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.23295454545454544, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1875, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2556818181818182 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.20454545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.17045454545454544, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.16477272727272727 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273 - } - }, - "AC3_2": 0.4881207236956391, - "AC3_3": 0.4049750786457689, - "AC3_4": 0.3419842620247173, - "AC3_5": 0.29160169595705127, - "AC3_6": 0.25155707266248534, - "AC3_7": 0.22118282266452197 - }, - "prompt_3": { - "overall_acc": 0.4431818181818183, - "language_acc": { - "English": 0.5738636363636364, - "Filipino": 0.3181818181818182, - "Spanish": 0.5056818181818182, - "Malay": 0.3693181818181818, - "Chinese": 0.4943181818181818, - "Indonesian": 0.4772727272727273, - "Vietnamese": 0.36363636363636365 - }, - "consistency_score_2": 0.544642857142857, - "consistency_score_3": 0.3702922077922078, - "consistency_score_4": 0.2775974025974026, - "consistency_score_5": 0.21888528138528138, - "consistency_score_6": 0.17775974025974023, - "consistency_score_7": 0.14772727272727273, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3977272727272727, - "English,Spanish": 0.6931818181818182, - "English,Malay": 0.5568181818181818, - "English,Chinese": 0.6477272727272727, - "English,Indonesian": 0.6647727272727273, - "English,Vietnamese": 0.4772727272727273, - "Filipino,Spanish": 0.4034090909090909, - "Filipino,Malay": 0.5, - "Filipino,Chinese": 0.45454545454545453, - "Filipino,Indonesian": 0.4715909090909091, - "Filipino,Vietnamese": 0.4375, - "Spanish,Malay": 0.5852272727272727, - "Spanish,Chinese": 0.6477272727272727, - "Spanish,Indonesian": 0.6590909090909091, - "Spanish,Vietnamese": 0.5056818181818182, - "Malay,Chinese": 0.5397727272727273, - "Malay,Indonesian": 0.6022727272727273, - "Malay,Vietnamese": 0.5056818181818182, - "Chinese,Indonesian": 0.6647727272727273, - "Chinese,Vietnamese": 0.4772727272727273, - "Indonesian,Vietnamese": 0.5454545454545454 - }, - "3_combine": { - "English,Filipino,Spanish": 0.3068181818181818, - "English,Filipino,Malay": 0.2840909090909091, - "English,Filipino,Chinese": 0.3181818181818182, - "English,Filipino,Indonesian": 0.3181818181818182, - "English,Filipino,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay": 0.4431818181818182, - "English,Spanish,Chinese": 0.5170454545454546, - "English,Spanish,Indonesian": 0.5284090909090909, - "English,Spanish,Vietnamese": 0.38636363636363635, - "English,Malay,Chinese": 0.4147727272727273, - "English,Malay,Indonesian": 0.45454545454545453, - "English,Malay,Vietnamese": 0.3352272727272727, - "English,Chinese,Indonesian": 0.5170454545454546, - "English,Chinese,Vietnamese": 0.35795454545454547, - "English,Indonesian,Vietnamese": 0.3977272727272727, - "Filipino,Spanish,Malay": 0.2897727272727273, - "Filipino,Spanish,Chinese": 0.3125, - "Filipino,Spanish,Indonesian": 0.32954545454545453, - "Filipino,Spanish,Vietnamese": 0.25, - "Filipino,Malay,Chinese": 0.3125, - "Filipino,Malay,Indonesian": 0.3409090909090909, - "Filipino,Malay,Vietnamese": 0.2897727272727273, - "Filipino,Chinese,Indonesian": 0.3522727272727273, - "Filipino,Chinese,Vietnamese": 0.25, - "Filipino,Indonesian,Vietnamese": 0.29545454545454547, - "Spanish,Malay,Chinese": 0.4318181818181818, - "Spanish,Malay,Indonesian": 0.45454545454545453, - "Spanish,Malay,Vietnamese": 0.375, - "Spanish,Chinese,Indonesian": 0.5284090909090909, - "Spanish,Chinese,Vietnamese": 0.3693181818181818, - "Spanish,Indonesian,Vietnamese": 0.4090909090909091, - "Malay,Chinese,Indonesian": 0.4431818181818182, - "Malay,Chinese,Vietnamese": 0.32954545454545453, - "Malay,Indonesian,Vietnamese": 0.38636363636363635, - "Chinese,Indonesian,Vietnamese": 0.38636363636363635 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.23863636363636365, - "English,Filipino,Spanish,Chinese": 0.26704545454545453, - "English,Filipino,Spanish,Indonesian": 0.26704545454545453, - "English,Filipino,Spanish,Vietnamese": 0.20454545454545456, - "English,Filipino,Malay,Chinese": 0.23863636363636365, - "English,Filipino,Malay,Indonesian": 0.25, - "English,Filipino,Malay,Vietnamese": 0.19318181818181818, - "English,Filipino,Chinese,Indonesian": 0.2784090909090909, - "English,Filipino,Chinese,Vietnamese": 0.20454545454545456, - "English,Filipino,Indonesian,Vietnamese": 0.2159090909090909, - "English,Spanish,Malay,Chinese": 0.375, - "English,Spanish,Malay,Indonesian": 0.38636363636363635, - "English,Spanish,Malay,Vietnamese": 0.3068181818181818, - "English,Spanish,Chinese,Indonesian": 0.44886363636363635, - "English,Spanish,Chinese,Vietnamese": 0.3068181818181818, - "English,Spanish,Indonesian,Vietnamese": 0.3352272727272727, - "English,Malay,Chinese,Indonesian": 0.3806818181818182, - "English,Malay,Chinese,Vietnamese": 0.2784090909090909, - "English,Malay,Indonesian,Vietnamese": 0.3068181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.32386363636363635, - "Filipino,Spanish,Malay,Chinese": 0.23295454545454544, - "Filipino,Spanish,Malay,Indonesian": 0.25, - "Filipino,Spanish,Malay,Vietnamese": 0.20454545454545456, - "Filipino,Spanish,Chinese,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.26136363636363635, - "Filipino,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.3806818181818182, - "Spanish,Malay,Chinese,Vietnamese": 0.29545454545454547, - "Spanish,Malay,Indonesian,Vietnamese": 0.32954545454545453, - "Spanish,Chinese,Indonesian,Vietnamese": 0.32954545454545453, - "Malay,Chinese,Indonesian,Vietnamese": 0.29545454545454547 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.2159090909090909, - "English,Filipino,Spanish,Malay,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Chinese,Indonesian": 0.25, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Malay,Chinese,Indonesian": 0.2215909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.17045454545454544, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.17613636363636365, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.1875, - "English,Spanish,Malay,Chinese,Indonesian": 0.3465909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.26136363636363635, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2784090909090909, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.2159090909090909, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1875, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2727272727272727 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.20454545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273 - } - }, - "AC3_2": 0.48870172550517005, - "AC3_3": 0.40347145366132836, - "AC3_4": 0.3413697788224177, - "AC3_5": 0.2930397146341947, - "AC3_6": 0.25374331546715706, - "AC3_7": 0.2215909090534091 - }, - "prompt_4": { - "overall_acc": 0.45292207792207784, - "language_acc": { - "English": 0.5625, - "Filipino": 0.3409090909090909, - "Spanish": 0.5340909090909091, - "Malay": 0.4034090909090909, - "Chinese": 0.5, - "Indonesian": 0.4943181818181818, - "Vietnamese": 0.3352272727272727 - }, - "consistency_score_2": 0.5481601731601733, - "consistency_score_3": 0.3728896103896104, - "consistency_score_4": 0.2762987012987013, - "consistency_score_5": 0.21293290043290047, - "consistency_score_6": 0.16801948051948054, - "consistency_score_7": 0.13636363636363635, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.3977272727272727, - "English,Spanish": 0.6931818181818182, - "English,Malay": 0.5625, - "English,Chinese": 0.625, - "English,Indonesian": 0.6477272727272727, - "English,Vietnamese": 0.4431818181818182, - "Filipino,Spanish": 0.44886363636363635, - "Filipino,Malay": 0.5170454545454546, - "Filipino,Chinese": 0.4772727272727273, - "Filipino,Indonesian": 0.4772727272727273, - "Filipino,Vietnamese": 0.4715909090909091, - "Spanish,Malay": 0.5909090909090909, - "Spanish,Chinese": 0.6590909090909091, - "Spanish,Indonesian": 0.6647727272727273, - "Spanish,Vietnamese": 0.4943181818181818, - "Malay,Chinese": 0.5625, - "Malay,Indonesian": 0.6420454545454546, - "Malay,Vietnamese": 0.5113636363636364, - "Chinese,Indonesian": 0.6477272727272727, - "Chinese,Vietnamese": 0.44886363636363635, - "Indonesian,Vietnamese": 0.5284090909090909 - }, - "3_combine": { - "English,Filipino,Spanish": 0.32954545454545453, - "English,Filipino,Malay": 0.30113636363636365, - "English,Filipino,Chinese": 0.3125, - "English,Filipino,Indonesian": 0.32386363636363635, - "English,Filipino,Vietnamese": 0.24431818181818182, - "English,Spanish,Malay": 0.44886363636363635, - "English,Spanish,Chinese": 0.5113636363636364, - "English,Spanish,Indonesian": 0.5227272727272727, - "English,Spanish,Vietnamese": 0.3693181818181818, - "English,Malay,Chinese": 0.4090909090909091, - "English,Malay,Indonesian": 0.4715909090909091, - "English,Malay,Vietnamese": 0.3352272727272727, - "English,Chinese,Indonesian": 0.48295454545454547, - "English,Chinese,Vietnamese": 0.3068181818181818, - "English,Indonesian,Vietnamese": 0.36363636363636365, - "Filipino,Spanish,Malay": 0.32386363636363635, - "Filipino,Spanish,Chinese": 0.3465909090909091, - "Filipino,Spanish,Indonesian": 0.3522727272727273, - "Filipino,Spanish,Vietnamese": 0.2840909090909091, - "Filipino,Malay,Chinese": 0.3465909090909091, - "Filipino,Malay,Indonesian": 0.375, - "Filipino,Malay,Vietnamese": 0.3125, - "Filipino,Chinese,Indonesian": 0.3465909090909091, - "Filipino,Chinese,Vietnamese": 0.26704545454545453, - "Filipino,Indonesian,Vietnamese": 0.2897727272727273, - "Spanish,Malay,Chinese": 0.4375, - "Spanish,Malay,Indonesian": 0.48295454545454547, - "Spanish,Malay,Vietnamese": 0.35795454545454547, - "Spanish,Chinese,Indonesian": 0.5227272727272727, - "Spanish,Chinese,Vietnamese": 0.3522727272727273, - "Spanish,Indonesian,Vietnamese": 0.39204545454545453, - "Malay,Chinese,Indonesian": 0.4602272727272727, - "Malay,Chinese,Vietnamese": 0.32386363636363635, - "Malay,Indonesian,Vietnamese": 0.3806818181818182, - "Chinese,Indonesian,Vietnamese": 0.36363636363636365 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.26136363636363635, - "English,Filipino,Spanish,Chinese": 0.2840909090909091, - "English,Filipino,Spanish,Indonesian": 0.2840909090909091, - "English,Filipino,Spanish,Vietnamese": 0.21022727272727273, - "English,Filipino,Malay,Chinese": 0.24431818181818182, - "English,Filipino,Malay,Indonesian": 0.2727272727272727, - "English,Filipino,Malay,Vietnamese": 0.20454545454545456, - "English,Filipino,Chinese,Indonesian": 0.26704545454545453, - "English,Filipino,Chinese,Vietnamese": 0.1875, - "English,Filipino,Indonesian,Vietnamese": 0.20454545454545456, - "English,Spanish,Malay,Chinese": 0.3693181818181818, - "English,Spanish,Malay,Indonesian": 0.4034090909090909, - "English,Spanish,Malay,Vietnamese": 0.2840909090909091, - "English,Spanish,Chinese,Indonesian": 0.4318181818181818, - "English,Spanish,Chinese,Vietnamese": 0.2840909090909091, - "English,Spanish,Indonesian,Vietnamese": 0.3125, - "English,Malay,Chinese,Indonesian": 0.3693181818181818, - "English,Malay,Chinese,Vietnamese": 0.2556818181818182, - "English,Malay,Indonesian,Vietnamese": 0.30113636363636365, - "English,Chinese,Indonesian,Vietnamese": 0.2784090909090909, - "Filipino,Spanish,Malay,Chinese": 0.26136363636363635, - "Filipino,Spanish,Malay,Indonesian": 0.2784090909090909, - "Filipino,Spanish,Malay,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Chinese,Indonesian": 0.2840909090909091, - "Filipino,Spanish,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Chinese,Indonesian": 0.2784090909090909, - "Filipino,Malay,Chinese,Vietnamese": 0.2159090909090909, - "Filipino,Malay,Indonesian,Vietnamese": 0.22727272727272727, - "Filipino,Chinese,Indonesian,Vietnamese": 0.21022727272727273, - "Spanish,Malay,Chinese,Indonesian": 0.38636363636363635, - "Spanish,Malay,Chinese,Vietnamese": 0.2784090909090909, - "Spanish,Malay,Indonesian,Vietnamese": 0.3068181818181818, - "Spanish,Chinese,Indonesian,Vietnamese": 0.30113636363636365, - "Malay,Chinese,Indonesian,Vietnamese": 0.2727272727272727 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.22727272727272727, - "English,Filipino,Spanish,Malay,Indonesian": 0.23863636363636365, - "English,Filipino,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Chinese,Indonesian": 0.25, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.17613636363636365, - "English,Filipino,Malay,Chinese,Indonesian": 0.2215909090909091, - "English,Filipino,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Chinese,Indonesian": 0.3409090909090909, - "English,Spanish,Malay,Chinese,Vietnamese": 0.23863636363636365, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.26136363636363635, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2556818181818182, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.2215909090909091, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.23863636363636365 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.20454545454545456, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.1590909090909091, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14204545454545456 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.13636363636363635 - } - }, - "AC3_2": 0.4960108809613335, - "AC3_3": 0.40902772276824545, - "AC3_4": 0.3432205594468676, - "AC3_5": 0.28967872837963654, - "AC3_6": 0.24511077154188574, - "AC3_7": 0.20961682941596882 - }, - "prompt_5": { - "overall_acc": 0.4407467532467533, - "language_acc": { - "English": 0.5568181818181818, - "Filipino": 0.3352272727272727, - "Spanish": 0.5, - "Malay": 0.36363636363636365, - "Chinese": 0.4943181818181818, - "Indonesian": 0.48863636363636365, - "Vietnamese": 0.3465909090909091 - }, - "consistency_score_2": 0.5514069264069262, - "consistency_score_3": 0.3775974025974026, - "consistency_score_4": 0.2826298701298701, - "consistency_score_5": 0.22077922077922074, - "consistency_score_6": 0.17775974025974026, - "consistency_score_7": 0.14772727272727273, - "detailed_consistency_score": { - "2_combine": { - "English,Filipino": 0.39204545454545453, - "English,Spanish": 0.6818181818181818, - "English,Malay": 0.5738636363636364, - "English,Chinese": 0.6761363636363636, - "English,Indonesian": 0.6818181818181818, - "English,Vietnamese": 0.4715909090909091, - "Filipino,Spanish": 0.44886363636363635, - "Filipino,Malay": 0.48295454545454547, - "Filipino,Chinese": 0.4715909090909091, - "Filipino,Indonesian": 0.4602272727272727, - "Filipino,Vietnamese": 0.48863636363636365, - "Spanish,Malay": 0.5795454545454546, - "Spanish,Chinese": 0.6363636363636364, - "Spanish,Indonesian": 0.6534090909090909, - "Spanish,Vietnamese": 0.5681818181818182, - "Malay,Chinese": 0.5454545454545454, - "Malay,Indonesian": 0.6136363636363636, - "Malay,Vietnamese": 0.5340909090909091, - "Chinese,Indonesian": 0.625, - "Chinese,Vietnamese": 0.4772727272727273, - "Indonesian,Vietnamese": 0.5170454545454546 - }, - "3_combine": { - "English,Filipino,Spanish": 0.3181818181818182, - "English,Filipino,Malay": 0.2897727272727273, - "English,Filipino,Chinese": 0.3125, - "English,Filipino,Indonesian": 0.32386363636363635, - "English,Filipino,Vietnamese": 0.2556818181818182, - "English,Spanish,Malay": 0.45454545454545453, - "English,Spanish,Chinese": 0.5227272727272727, - "English,Spanish,Indonesian": 0.5284090909090909, - "English,Spanish,Vietnamese": 0.4034090909090909, - "English,Malay,Chinese": 0.4431818181818182, - "English,Malay,Indonesian": 0.4715909090909091, - "English,Malay,Vietnamese": 0.3522727272727273, - "English,Chinese,Indonesian": 0.5170454545454546, - "English,Chinese,Vietnamese": 0.35795454545454547, - "English,Indonesian,Vietnamese": 0.3806818181818182, - "Filipino,Spanish,Malay": 0.30113636363636365, - "Filipino,Spanish,Chinese": 0.32954545454545453, - "Filipino,Spanish,Indonesian": 0.3465909090909091, - "Filipino,Spanish,Vietnamese": 0.3181818181818182, - "Filipino,Malay,Chinese": 0.3125, - "Filipino,Malay,Indonesian": 0.32954545454545453, - "Filipino,Malay,Vietnamese": 0.3125, - "Filipino,Chinese,Indonesian": 0.3352272727272727, - "Filipino,Chinese,Vietnamese": 0.2727272727272727, - "Filipino,Indonesian,Vietnamese": 0.2840909090909091, - "Spanish,Malay,Chinese": 0.42613636363636365, - "Spanish,Malay,Indonesian": 0.4715909090909091, - "Spanish,Malay,Vietnamese": 0.3977272727272727, - "Spanish,Chinese,Indonesian": 0.4943181818181818, - "Spanish,Chinese,Vietnamese": 0.39204545454545453, - "Spanish,Indonesian,Vietnamese": 0.4318181818181818, - "Malay,Chinese,Indonesian": 0.4375, - "Malay,Chinese,Vietnamese": 0.3409090909090909, - "Malay,Indonesian,Vietnamese": 0.38636363636363635, - "Chinese,Indonesian,Vietnamese": 0.36363636363636365 - }, - "4_combine": { - "English,Filipino,Spanish,Malay": 0.24431818181818182, - "English,Filipino,Spanish,Chinese": 0.26136363636363635, - "English,Filipino,Spanish,Indonesian": 0.2784090909090909, - "English,Filipino,Spanish,Vietnamese": 0.23295454545454544, - "English,Filipino,Malay,Chinese": 0.23863636363636365, - "English,Filipino,Malay,Indonesian": 0.2556818181818182, - "English,Filipino,Malay,Vietnamese": 0.2159090909090909, - "English,Filipino,Chinese,Indonesian": 0.26704545454545453, - "English,Filipino,Chinese,Vietnamese": 0.19886363636363635, - "English,Filipino,Indonesian,Vietnamese": 0.21022727272727273, - "English,Spanish,Malay,Chinese": 0.3806818181818182, - "English,Spanish,Malay,Indonesian": 0.38636363636363635, - "English,Spanish,Malay,Vietnamese": 0.3125, - "English,Spanish,Chinese,Indonesian": 0.4431818181818182, - "English,Spanish,Chinese,Vietnamese": 0.32954545454545453, - "English,Spanish,Indonesian,Vietnamese": 0.3352272727272727, - "English,Malay,Chinese,Indonesian": 0.39204545454545453, - "English,Malay,Chinese,Vietnamese": 0.2840909090909091, - "English,Malay,Indonesian,Vietnamese": 0.3068181818181818, - "English,Chinese,Indonesian,Vietnamese": 0.29545454545454547, - "Filipino,Spanish,Malay,Chinese": 0.24431818181818182, - "Filipino,Spanish,Malay,Indonesian": 0.2556818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Chinese,Indonesian": 0.2727272727272727, - "Filipino,Spanish,Chinese,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Malay,Chinese,Indonesian": 0.25, - "Filipino,Malay,Chinese,Vietnamese": 0.2215909090909091, - "Filipino,Malay,Indonesian,Vietnamese": 0.2215909090909091, - "Filipino,Chinese,Indonesian,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Chinese,Indonesian": 0.375, - "Spanish,Malay,Chinese,Vietnamese": 0.3068181818181818, - "Spanish,Malay,Indonesian,Vietnamese": 0.3352272727272727, - "Spanish,Chinese,Indonesian,Vietnamese": 0.32386363636363635, - "Malay,Chinese,Indonesian,Vietnamese": 0.2784090909090909 - }, - "5_combine": { - "English,Filipino,Spanish,Malay,Chinese": 0.21022727272727273, - "English,Filipino,Spanish,Malay,Indonesian": 0.2159090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.19318181818181818, - "English,Filipino,Spanish,Chinese,Indonesian": 0.23863636363636365, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.1875, - "English,Filipino,Spanish,Indonesian,Vietnamese": 0.19318181818181818, - "English,Filipino,Malay,Chinese,Indonesian": 0.2159090909090909, - "English,Filipino,Malay,Chinese,Vietnamese": 0.17613636363636365, - "English,Filipino,Malay,Indonesian,Vietnamese": 0.18181818181818182, - "English,Filipino,Chinese,Indonesian,Vietnamese": 0.17045454545454544, - "English,Spanish,Malay,Chinese,Indonesian": 0.3465909090909091, - "English,Spanish,Malay,Chinese,Vietnamese": 0.2727272727272727, - "English,Spanish,Malay,Indonesian,Vietnamese": 0.2727272727272727, - "English,Spanish,Chinese,Indonesian,Vietnamese": 0.2840909090909091, - "English,Malay,Chinese,Indonesian,Vietnamese": 0.25, - "Filipino,Spanish,Malay,Chinese,Indonesian": 0.21022727272727273, - "Filipino,Spanish,Malay,Chinese,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.19318181818181818, - "Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.17613636363636365, - "Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.26136363636363635 - }, - "6_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian": 0.19318181818181818, - "English,Filipino,Spanish,Malay,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Malay,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Chinese,Indonesian,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Chinese,Indonesian,Vietnamese": 0.1534090909090909, - "English,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.1590909090909091 - }, - "7_combine": { - "English,Filipino,Spanish,Malay,Chinese,Indonesian,Vietnamese": 0.14772727272727273 - } - }, - "AC3_2": 0.4899055811433588, - "AC3_3": 0.40673554767969594, - "AC3_4": 0.34440758406486527, - "AC3_5": 0.29419169782820775, - "AC3_6": 0.2533426219040238, - "AC3_7": 0.22128526642007706 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.6310679611650486 - }, - "prompt_2": { - "accuracy": 0.6699029126213593 - }, - "prompt_3": { - "accuracy": 0.6601941747572816 - }, - "prompt_4": { - "accuracy": 0.7087378640776699 - }, - "prompt_5": { - "accuracy": 0.6504854368932039 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.3523809523809524 - }, - "prompt_2": { - "accuracy": 0.37142857142857144 - }, - "prompt_3": { - "accuracy": 0.4095238095238095 - }, - "prompt_4": { - "accuracy": 0.37142857142857144 - }, - "prompt_5": { - "accuracy": 0.3904761904761905 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.6822429906542056 - }, - "prompt_2": { - "accuracy": 0.6915887850467289 - }, - "prompt_3": { - "accuracy": 0.7009345794392523 - }, - "prompt_4": { - "accuracy": 0.7102803738317757 - }, - "prompt_5": { - "accuracy": 0.719626168224299 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.43, - "category_acc": { - "brand": 0.3, - "demographics": 0.0, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.6, - "culture": 0.9, - "film": 0.4, - "law": 0.3, - "geography": 0.7 - } - }, - "prompt_2": { - "accuracy": 0.45, - "category_acc": { - "brand": 0.3, - "demographics": 0.0, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.6, - "culture": 0.9, - "film": 0.4, - "law": 0.5, - "geography": 0.7 - } - }, - "prompt_3": { - "accuracy": 0.46, - "category_acc": { - "brand": 0.3, - "demographics": 0.0, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.7, - "culture": 0.8, - "film": 0.5, - "law": 0.4, - "geography": 0.7 - } - }, - "prompt_4": { - "accuracy": 0.49, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.5, - "history": 0.4, - "literature": 0.4, - "politics": 0.6, - "culture": 0.8, - "film": 0.4, - "law": 0.4, - "geography": 0.7 - } - }, - "prompt_5": { - "accuracy": 0.49, - "category_acc": { - "brand": 0.4, - "demographics": 0.4, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.7, - "culture": 0.8, - "film": 0.5, - "law": 0.4, - "geography": 0.7 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.2743027381303002 - }, - "prompt_2": { - "bleu_score": 0.2902011862633376 - }, - "prompt_3": { - "bleu_score": 0.2902579971261351 - }, - "prompt_4": { - "bleu_score": 0.2533932934493703 - }, - "prompt_5": { - "bleu_score": 0.21434722780078505 - } - }, - "indommlu": { - "prompt_1": -1, - "prompt_2": -1, - "prompt_3": -1, - "prompt_4": -1, - "prompt_5": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.3414669044850431 - }, - "prompt_2": { - "bleu_score": 0.36798299909556614 - }, - "prompt_3": { - "bleu_score": 0.36791973425015867 - }, - "prompt_4": { - "bleu_score": 0.3665825451023794 - }, - "prompt_5": { - "bleu_score": 0.35878786194273193 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.22198181301475597 - }, - "prompt_2": { - "bleu_score": 0.2412502760601626 - }, - "prompt_3": { - "bleu_score": 0.24063961650343155 - }, - "prompt_4": { - "bleu_score": 0.23664492839508175 - }, - "prompt_5": { - "bleu_score": 0.22507600559090932 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.18849441747894208 - }, - "prompt_2": { - "bleu_score": 0.21393676814518622 - }, - "prompt_3": { - "bleu_score": 0.2161097215694177 - }, - "prompt_4": { - "bleu_score": 0.20886026465469823 - }, - "prompt_5": { - "bleu_score": 0.19929565266587312 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.3123343913349507 - }, - "prompt_2": { - "bleu_score": 0.34391708786931385 - }, - "prompt_3": { - "bleu_score": 0.3430493411894306 - }, - "prompt_4": { - "bleu_score": 0.34096748318185466 - }, - "prompt_5": { - "bleu_score": 0.321466469251346 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.5939323220536756 - }, - "prompt_2": { - "accuracy": 0.5857642940490082 - }, - "prompt_3": { - "accuracy": 0.5857642940490082 - }, - "prompt_4": { - "accuracy": 0.5857642940490082 - }, - "prompt_5": { - "accuracy": 0.5857642940490082 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.5757597425813371, - "category_acc": { - "high_school_european_history": 0.7439024390243902, - "business_ethics": 0.5959595959595959, - "clinical_knowledge": 0.6666666666666666, - "medical_genetics": 0.6262626262626263, - "high_school_us_history": 0.7733990147783252, - "high_school_physics": 0.3333333333333333, - "high_school_world_history": 0.8177966101694916, - "virology": 0.4909090909090909, - "high_school_microeconomics": 0.6666666666666666, - "econometrics": 0.36283185840707965, - "college_computer_science": 0.5252525252525253, - "high_school_biology": 0.7346278317152104, - "abstract_algebra": 0.30303030303030304, - "professional_accounting": 0.4306049822064057, - "philosophy": 0.6193548387096774, - "professional_medicine": 0.6752767527675276, - "nutrition": 0.6786885245901639, - "global_facts": 0.2727272727272727, - "machine_learning": 0.36936936936936937, - "security_studies": 0.639344262295082, - "public_relations": 0.5871559633027523, - "professional_psychology": 0.5695581014729951, - "prehistory": 0.6439628482972136, - "anatomy": 0.5522388059701493, - "human_sexuality": 0.6692307692307692, - "college_medicine": 0.6162790697674418, - "high_school_government_and_politics": 0.8385416666666666, - "college_chemistry": 0.45454545454545453, - "logical_fallacies": 0.691358024691358, - "high_school_geography": 0.7411167512690355, - "elementary_mathematics": 0.38992042440318303, - "human_aging": 0.6486486486486487, - "college_mathematics": 0.31313131313131315, - "high_school_psychology": 0.8069852941176471, - "formal_logic": 0.456, - "high_school_statistics": 0.4418604651162791, - "international_law": 0.725, - "high_school_mathematics": 0.3159851301115242, - "high_school_computer_science": 0.5757575757575758, - "conceptual_physics": 0.5256410256410257, - "miscellaneous": 0.7966751918158568, - "high_school_chemistry": 0.4405940594059406, - "marketing": 0.7811158798283262, - "professional_law": 0.43052837573385516, - "management": 0.7549019607843137, - "college_physics": 0.36633663366336633, - "jurisprudence": 0.6635514018691588, - "world_religions": 0.8176470588235294, - "sociology": 0.79, - "us_foreign_policy": 0.8080808080808081, - "high_school_macroeconomics": 0.5604113110539846, - "computer_security": 0.696969696969697, - "moral_scenarios": 0.28187919463087246, - "moral_disputes": 0.5710144927536231, - "electrical_engineering": 0.4583333333333333, - "astronomy": 0.5827814569536424, - "college_biology": 0.6643356643356644 - } - }, - "prompt_2": { - "accuracy": 0.5769753307114766, - "category_acc": { - "high_school_european_history": 0.774390243902439, - "business_ethics": 0.5353535353535354, - "clinical_knowledge": 0.6893939393939394, - "medical_genetics": 0.5858585858585859, - "high_school_us_history": 0.7733990147783252, - "high_school_physics": 0.3466666666666667, - "high_school_world_history": 0.8135593220338984, - "virology": 0.49696969696969695, - "high_school_microeconomics": 0.6624472573839663, - "econometrics": 0.35398230088495575, - "college_computer_science": 0.5151515151515151, - "high_school_biology": 0.7216828478964401, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.4483985765124555, - "philosophy": 0.6193548387096774, - "professional_medicine": 0.6752767527675276, - "nutrition": 0.6622950819672131, - "global_facts": 0.32323232323232326, - "machine_learning": 0.3783783783783784, - "security_studies": 0.6147540983606558, - "public_relations": 0.6146788990825688, - "professional_psychology": 0.5695581014729951, - "prehistory": 0.6408668730650154, - "anatomy": 0.5522388059701493, - "human_sexuality": 0.6923076923076923, - "college_medicine": 0.627906976744186, - "high_school_government_and_politics": 0.8229166666666666, - "college_chemistry": 0.4444444444444444, - "logical_fallacies": 0.6728395061728395, - "high_school_geography": 0.751269035532995, - "elementary_mathematics": 0.3925729442970822, - "human_aging": 0.6486486486486487, - "college_mathematics": 0.2727272727272727, - "high_school_psychology": 0.8033088235294118, - "formal_logic": 0.432, - "high_school_statistics": 0.4558139534883721, - "international_law": 0.75, - "high_school_mathematics": 0.31226765799256506, - "high_school_computer_science": 0.5757575757575758, - "conceptual_physics": 0.5384615384615384, - "miscellaneous": 0.7979539641943734, - "high_school_chemistry": 0.4405940594059406, - "marketing": 0.7682403433476395, - "professional_law": 0.4390084801043705, - "management": 0.7647058823529411, - "college_physics": 0.37623762376237624, - "jurisprudence": 0.6915887850467289, - "world_religions": 0.8058823529411765, - "sociology": 0.81, - "us_foreign_policy": 0.797979797979798, - "high_school_macroeconomics": 0.5758354755784062, - "computer_security": 0.6868686868686869, - "moral_scenarios": 0.2740492170022371, - "moral_disputes": 0.5884057971014492, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.5960264900662252, - "college_biology": 0.6293706293706294 - } - }, - "prompt_3": { - "accuracy": 0.5730425455845549, - "category_acc": { - "high_school_european_history": 0.7439024390243902, - "business_ethics": 0.5757575757575758, - "clinical_knowledge": 0.6666666666666666, - "medical_genetics": 0.6060606060606061, - "high_school_us_history": 0.7635467980295566, - "high_school_physics": 0.32666666666666666, - "high_school_world_history": 0.8050847457627118, - "virology": 0.509090909090909, - "high_school_microeconomics": 0.6455696202531646, - "econometrics": 0.34513274336283184, - "college_computer_science": 0.5454545454545454, - "high_school_biology": 0.7152103559870551, - "abstract_algebra": 0.29292929292929293, - "professional_accounting": 0.45907473309608543, - "philosophy": 0.603225806451613, - "professional_medicine": 0.6752767527675276, - "nutrition": 0.6688524590163935, - "global_facts": 0.31313131313131315, - "machine_learning": 0.3783783783783784, - "security_studies": 0.6311475409836066, - "public_relations": 0.6330275229357798, - "professional_psychology": 0.5761047463175123, - "prehistory": 0.6470588235294118, - "anatomy": 0.5597014925373134, - "human_sexuality": 0.676923076923077, - "college_medicine": 0.622093023255814, - "high_school_government_and_politics": 0.84375, - "college_chemistry": 0.4444444444444444, - "logical_fallacies": 0.7160493827160493, - "high_school_geography": 0.7208121827411168, - "elementary_mathematics": 0.4005305039787798, - "human_aging": 0.6486486486486487, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.8051470588235294, - "formal_logic": 0.432, - "high_school_statistics": 0.4372093023255814, - "international_law": 0.7583333333333333, - "high_school_mathematics": 0.3308550185873606, - "high_school_computer_science": 0.5757575757575758, - "conceptual_physics": 0.5042735042735043, - "miscellaneous": 0.8043478260869565, - "high_school_chemistry": 0.39603960396039606, - "marketing": 0.7381974248927039, - "professional_law": 0.43509458577951726, - "management": 0.7745098039215687, - "college_physics": 0.36633663366336633, - "jurisprudence": 0.7102803738317757, - "world_religions": 0.8058823529411765, - "sociology": 0.81, - "us_foreign_policy": 0.8080808080808081, - "high_school_macroeconomics": 0.5629820051413882, - "computer_security": 0.6868686868686869, - "moral_scenarios": 0.24720357941834453, - "moral_disputes": 0.5507246376811594, - "electrical_engineering": 0.4722222222222222, - "astronomy": 0.5827814569536424, - "college_biology": 0.6433566433566433 - } - }, - "prompt_4": { - "accuracy": 0.5737575974258133, - "category_acc": { - "high_school_european_history": 0.7378048780487805, - "business_ethics": 0.5454545454545454, - "clinical_knowledge": 0.678030303030303, - "medical_genetics": 0.6262626262626263, - "high_school_us_history": 0.7635467980295566, - "high_school_physics": 0.36, - "high_school_world_history": 0.8135593220338984, - "virology": 0.503030303030303, - "high_school_microeconomics": 0.6708860759493671, - "econometrics": 0.36283185840707965, - "college_computer_science": 0.47474747474747475, - "high_school_biology": 0.7022653721682848, - "abstract_algebra": 0.2828282828282828, - "professional_accounting": 0.4234875444839858, - "philosophy": 0.6612903225806451, - "professional_medicine": 0.6678966789667896, - "nutrition": 0.6688524590163935, - "global_facts": 0.31313131313131315, - "machine_learning": 0.36036036036036034, - "security_studies": 0.5983606557377049, - "public_relations": 0.6146788990825688, - "professional_psychology": 0.5711947626841244, - "prehistory": 0.631578947368421, - "anatomy": 0.5746268656716418, - "human_sexuality": 0.6615384615384615, - "college_medicine": 0.622093023255814, - "high_school_government_and_politics": 0.8125, - "college_chemistry": 0.41414141414141414, - "logical_fallacies": 0.691358024691358, - "high_school_geography": 0.7411167512690355, - "elementary_mathematics": 0.44297082228116713, - "human_aging": 0.6351351351351351, - "college_mathematics": 0.2828282828282828, - "high_school_psychology": 0.7959558823529411, - "formal_logic": 0.392, - "high_school_statistics": 0.44651162790697674, - "international_law": 0.7166666666666667, - "high_school_mathematics": 0.2527881040892193, - "high_school_computer_science": 0.5353535353535354, - "conceptual_physics": 0.5, - "miscellaneous": 0.7953964194373402, - "high_school_chemistry": 0.4158415841584158, - "marketing": 0.7982832618025751, - "professional_law": 0.4227005870841487, - "management": 0.7549019607843137, - "college_physics": 0.4158415841584158, - "jurisprudence": 0.6822429906542056, - "world_religions": 0.8058823529411765, - "sociology": 0.79, - "us_foreign_policy": 0.8181818181818182, - "high_school_macroeconomics": 0.5732647814910026, - "computer_security": 0.7070707070707071, - "moral_scenarios": 0.3087248322147651, - "moral_disputes": 0.5739130434782609, - "electrical_engineering": 0.4444444444444444, - "astronomy": 0.5827814569536424, - "college_biology": 0.6713286713286714 - } - }, - "prompt_5": { - "accuracy": 0.5685377189846264, - "category_acc": { - "high_school_european_history": 0.7560975609756098, - "business_ethics": 0.5656565656565656, - "clinical_knowledge": 0.6212121212121212, - "medical_genetics": 0.5959595959595959, - "high_school_us_history": 0.7635467980295566, - "high_school_physics": 0.3466666666666667, - "high_school_world_history": 0.809322033898305, - "virology": 0.509090909090909, - "high_school_microeconomics": 0.6540084388185654, - "econometrics": 0.336283185840708, - "college_computer_science": 0.5050505050505051, - "high_school_biology": 0.7184466019417476, - "abstract_algebra": 0.32323232323232326, - "professional_accounting": 0.4128113879003559, - "philosophy": 0.6096774193548387, - "professional_medicine": 0.6715867158671587, - "nutrition": 0.6688524590163935, - "global_facts": 0.25252525252525254, - "machine_learning": 0.36036036036036034, - "security_studies": 0.6352459016393442, - "public_relations": 0.6422018348623854, - "professional_psychology": 0.5728314238952537, - "prehistory": 0.6346749226006192, - "anatomy": 0.5298507462686567, - "human_sexuality": 0.6615384615384615, - "college_medicine": 0.5988372093023255, - "high_school_government_and_politics": 0.8177083333333334, - "college_chemistry": 0.4444444444444444, - "logical_fallacies": 0.7222222222222222, - "high_school_geography": 0.7208121827411168, - "elementary_mathematics": 0.3819628647214854, - "human_aging": 0.6576576576576577, - "college_mathematics": 0.30303030303030304, - "high_school_psychology": 0.7977941176470589, - "formal_logic": 0.456, - "high_school_statistics": 0.4372093023255814, - "international_law": 0.725, - "high_school_mathematics": 0.34572490706319703, - "high_school_computer_science": 0.5555555555555556, - "conceptual_physics": 0.5128205128205128, - "miscellaneous": 0.80306905370844, - "high_school_chemistry": 0.42574257425742573, - "marketing": 0.703862660944206, - "professional_law": 0.4298760600130463, - "management": 0.7745098039215687, - "college_physics": 0.37623762376237624, - "jurisprudence": 0.7009345794392523, - "world_religions": 0.788235294117647, - "sociology": 0.805, - "us_foreign_policy": 0.7474747474747475, - "high_school_macroeconomics": 0.5526992287917738, - "computer_security": 0.6868686868686869, - "moral_scenarios": 0.26733780760626397, - "moral_disputes": 0.5420289855072464, - "electrical_engineering": 0.4513888888888889, - "astronomy": 0.5827814569536424, - "college_biology": 0.6503496503496503 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.4086181277860327 - }, - "prompt_2": { - "accuracy": 0.4078751857355126 - }, - "prompt_3": { - "accuracy": 0.4160475482912333 - }, - "prompt_4": { - "accuracy": 0.4145616641901932 - }, - "prompt_5": { - "accuracy": 0.39821693907875183 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.42092154420921546, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.23076923076923078, - "college_programming": 0.47619047619047616, - "college_physics": 0.5, - "college_chemistry": 0.13793103448275862, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.2619047619047619, - "metrology_engineer": 0.4827586206896552, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.5, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.6153846153846154, - "middle_school_physics": 0.5416666666666666, - "middle_school_chemistry": 0.56, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.4, - "business_administration": 0.3684210526315789, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.5294117647058824, - "teacher_qualification": 0.5102040816326531, - "high_school_politics": 0.375, - "high_school_geography": 0.5, - "middle_school_politics": 0.5384615384615384, - "middle_school_geography": 0.5294117647058824, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.5185185185185185, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.5, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.4642857142857143, - "high_school_chinese": 0.4583333333333333, - "high_school_history": 0.52, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.4807692307692308, - "sports_science": 0.20833333333333334, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.5416666666666666, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.43137254901960786, - "accountant": 0.4444444444444444, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.2777777777777778, - "physician": 0.46296296296296297 - } - }, - "prompt_2": { - "accuracy": 0.42341220423412207, - "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.5416666666666666, - "computer_architecture": 0.38461538461538464, - "college_programming": 0.42857142857142855, - "college_physics": 0.4166666666666667, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.38095238095238093, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.4482758620689655, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.4583333333333333, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.48, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.38333333333333336, - "business_administration": 0.3684210526315789, - "marxism": 0.5, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.5294117647058824, - "teacher_qualification": 0.5306122448979592, - "high_school_politics": 0.375, - "high_school_geography": 0.5, - "middle_school_politics": 0.5769230769230769, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.5, - "logic": 0.4444444444444444, - "law": 0.4482758620689655, - "chinese_language_and_literature": 0.5, - "art_studies": 0.47368421052631576, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.5, - "high_school_chinese": 0.3333333333333333, - "high_school_history": 0.52, - "middle_school_history": 0.5925925925925926, - "civil_servant": 0.46153846153846156, - "sports_science": 0.20833333333333334, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.5, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.4117647058823529, - "accountant": 0.4074074074074074, - "fire_engineer": 0.3611111111111111, - "environmental_impact_assessment_engineer": 0.4166666666666667, - "tax_accountant": 0.3148148148148148, - "physician": 0.46296296296296297 - } - }, - "prompt_3": { - "accuracy": 0.43212951432129515, - "category_acc": { - "computer_network": 0.4166666666666667, - "operating_system": 0.5833333333333334, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.4523809523809524, - "college_physics": 0.4583333333333333, - "college_chemistry": 0.13793103448275862, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.43478260869565216, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.5862068965517241, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.5416666666666666, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.6538461538461539, - "middle_school_physics": 0.5833333333333334, - "middle_school_chemistry": 0.56, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.4, - "business_administration": 0.39473684210526316, - "marxism": 0.4583333333333333, - "mao_zedong_thought": 0.6206896551724138, - "education_science": 0.5, - "teacher_qualification": 0.5510204081632653, - "high_school_politics": 0.4583333333333333, - "high_school_geography": 0.375, - "middle_school_politics": 0.5, - "middle_school_geography": 0.5294117647058824, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.5185185185185185, - "law": 0.4827586206896552, - "chinese_language_and_literature": 0.4642857142857143, - "art_studies": 0.5, - "professional_tour_guide": 0.4117647058823529, - "legal_professional": 0.5, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.52, - "middle_school_history": 0.5925925925925926, - "civil_servant": 0.4807692307692308, - "sports_science": 0.25, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.5, - "clinical_medicine": 0.37037037037037035, - "urban_and_rural_planner": 0.45098039215686275, - "accountant": 0.48148148148148145, - "fire_engineer": 0.3611111111111111, - "environmental_impact_assessment_engineer": 0.4444444444444444, - "tax_accountant": 0.2962962962962963, - "physician": 0.3888888888888889 - } - }, - "prompt_4": { - "accuracy": 0.4383561643835616, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.5, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.42857142857142855, - "college_physics": 0.4583333333333333, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.2916666666666667, - "probability_and_statistics": 0.5217391304347826, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.5, - "high_school_chemistry": 0.3333333333333333, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.25, - "middle_school_biology": 0.6538461538461539, - "middle_school_physics": 0.5416666666666666, - "middle_school_chemistry": 0.52, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.38333333333333336, - "business_administration": 0.39473684210526316, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.6896551724137931, - "education_science": 0.5294117647058824, - "teacher_qualification": 0.5306122448979592, - "high_school_politics": 0.5, - "high_school_geography": 0.5, - "middle_school_politics": 0.5769230769230769, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.5833333333333334, - "logic": 0.5185185185185185, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.42857142857142855, - "art_studies": 0.47368421052631576, - "professional_tour_guide": 0.3235294117647059, - "legal_professional": 0.5, - "high_school_chinese": 0.4583333333333333, - "high_school_history": 0.56, - "middle_school_history": 0.5925925925925926, - "civil_servant": 0.5192307692307693, - "sports_science": 0.25, - "plant_protection": 0.4074074074074074, - "basic_medicine": 0.5, - "clinical_medicine": 0.48148148148148145, - "urban_and_rural_planner": 0.5098039215686274, - "accountant": 0.42592592592592593, - "fire_engineer": 0.3055555555555556, - "environmental_impact_assessment_engineer": 0.4444444444444444, - "tax_accountant": 0.2962962962962963, - "physician": 0.48148148148148145 - } - }, - "prompt_5": { - "accuracy": 0.386052303860523, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.4583333333333333, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.35714285714285715, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.1724137931034483, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.391304347826087, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.5172413793103449, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.5, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.4166666666666667, - "business_administration": 0.34210526315789475, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.6551724137931034, - "education_science": 0.4117647058823529, - "teacher_qualification": 0.4897959183673469, - "high_school_politics": 0.375, - "high_school_geography": 0.4583333333333333, - "middle_school_politics": 0.5384615384615384, - "middle_school_geography": 0.47058823529411764, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.48148148148148145, - "law": 0.2413793103448276, - "chinese_language_and_literature": 0.42857142857142855, - "art_studies": 0.5, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.25, - "high_school_history": 0.52, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.5, - "sports_science": 0.25, - "plant_protection": 0.48148148148148145, - "basic_medicine": 0.4166666666666667, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.39215686274509803, - "accountant": 0.37037037037037035, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.3888888888888889, - "tax_accountant": 0.24074074074074073, - "physician": 0.4444444444444444 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.4121863799283154 - }, - "prompt_2": { - "accuracy": 0.41935483870967744 - }, - "prompt_3": { - "accuracy": 0.3942652329749104 - }, - "prompt_4": { - "accuracy": 0.4121863799283154 - }, - "prompt_5": { - "accuracy": 0.4050179211469534 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.41080987739595926, - "category_acc": { - "agronomy": 0.3076923076923077, - "anatomy": 0.30405405405405406, - "ancient_chinese": 0.23170731707317074, - "arts": 0.425, - "astronomy": 0.3393939393939394, - "business_ethics": 0.41148325358851673, - "chinese_civil_service_exam": 0.375, - "chinese_driving_rule": 0.5954198473282443, - "chinese_food_culture": 0.33088235294117646, - "chinese_foreign_policy": 0.5046728971962616, - "chinese_history": 0.4613003095975232, - "chinese_literature": 0.3431372549019608, - "chinese_teacher_qualification": 0.5083798882681564, - "clinical_knowledge": 0.38396624472573837, - "college_actuarial_science": 0.2358490566037736, - "college_education": 0.5327102803738317, - "college_engineering_hydrology": 0.46226415094339623, - "college_law": 0.4166666666666667, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.3584905660377358, - "college_medicine": 0.3772893772893773, - "computer_science": 0.4411764705882353, - "computer_security": 0.4853801169590643, - "conceptual_physics": 0.3741496598639456, - "construction_project_management": 0.3669064748201439, - "economics": 0.4716981132075472, - "education": 0.4539877300613497, - "electrical_engineering": 0.37790697674418605, - "elementary_chinese": 0.27380952380952384, - "elementary_commonsense": 0.35858585858585856, - "elementary_information_and_technology": 0.6638655462184874, - "elementary_mathematics": 0.2782608695652174, - "ethnology": 0.4, - "food_science": 0.44755244755244755, - "genetics": 0.32954545454545453, - "global_facts": 0.5167785234899329, - "high_school_biology": 0.3254437869822485, - "high_school_chemistry": 0.3181818181818182, - "high_school_geography": 0.4067796610169492, - "high_school_mathematics": 0.23170731707317074, - "high_school_physics": 0.2727272727272727, - "high_school_politics": 0.3706293706293706, - "human_sexuality": 0.5396825396825397, - "international_law": 0.3891891891891892, - "journalism": 0.46511627906976744, - "jurisprudence": 0.45255474452554745, - "legal_and_moral_basis": 0.6915887850467289, - "logical": 0.3983739837398374, - "machine_learning": 0.29508196721311475, - "management": 0.4857142857142857, - "marketing": 0.5277777777777778, - "marxist_theory": 0.4656084656084656, - "modern_chinese": 0.3275862068965517, - "nutrition": 0.47586206896551725, - "philosophy": 0.4095238095238095, - "professional_accounting": 0.4514285714285714, - "professional_law": 0.3222748815165877, - "professional_medicine": 0.3191489361702128, - "professional_psychology": 0.41810344827586204, - "public_relations": 0.4885057471264368, - "security_study": 0.45925925925925926, - "sociology": 0.5088495575221239, - "sports_science": 0.41818181818181815, - "traditional_chinese_medicine": 0.2864864864864865, - "virology": 0.44970414201183434, - "world_history": 0.4968944099378882, - "world_religions": 0.49375 - } - }, - "prompt_2": { - "accuracy": 0.4153859437057503, - "category_acc": { - "agronomy": 0.3609467455621302, - "anatomy": 0.2972972972972973, - "ancient_chinese": 0.2621951219512195, - "arts": 0.43125, - "astronomy": 0.32727272727272727, - "business_ethics": 0.41626794258373206, - "chinese_civil_service_exam": 0.425, - "chinese_driving_rule": 0.5725190839694656, - "chinese_food_culture": 0.3602941176470588, - "chinese_foreign_policy": 0.5046728971962616, - "chinese_history": 0.44891640866873067, - "chinese_literature": 0.31862745098039214, - "chinese_teacher_qualification": 0.5363128491620112, - "clinical_knowledge": 0.38396624472573837, - "college_actuarial_science": 0.22641509433962265, - "college_education": 0.514018691588785, - "college_engineering_hydrology": 0.46226415094339623, - "college_law": 0.39814814814814814, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.3490566037735849, - "college_medicine": 0.358974358974359, - "computer_science": 0.46568627450980393, - "computer_security": 0.4853801169590643, - "conceptual_physics": 0.42857142857142855, - "construction_project_management": 0.3237410071942446, - "economics": 0.4779874213836478, - "education": 0.4110429447852761, - "electrical_engineering": 0.38953488372093026, - "elementary_chinese": 0.29365079365079366, - "elementary_commonsense": 0.4090909090909091, - "elementary_information_and_technology": 0.6596638655462185, - "elementary_mathematics": 0.30434782608695654, - "ethnology": 0.3851851851851852, - "food_science": 0.44755244755244755, - "genetics": 0.3693181818181818, - "global_facts": 0.5100671140939598, - "high_school_biology": 0.3076923076923077, - "high_school_chemistry": 0.3181818181818182, - "high_school_geography": 0.4406779661016949, - "high_school_mathematics": 0.25, - "high_school_physics": 0.3090909090909091, - "high_school_politics": 0.3706293706293706, - "human_sexuality": 0.5952380952380952, - "international_law": 0.4, - "journalism": 0.47093023255813954, - "jurisprudence": 0.45012165450121655, - "legal_and_moral_basis": 0.7149532710280374, - "logical": 0.42276422764227645, - "machine_learning": 0.28688524590163933, - "management": 0.5047619047619047, - "marketing": 0.5055555555555555, - "marxist_theory": 0.5026455026455027, - "modern_chinese": 0.3275862068965517, - "nutrition": 0.46206896551724136, - "philosophy": 0.4095238095238095, - "professional_accounting": 0.45714285714285713, - "professional_law": 0.33175355450236965, - "professional_medicine": 0.29521276595744683, - "professional_psychology": 0.43103448275862066, - "public_relations": 0.47126436781609193, - "security_study": 0.4740740740740741, - "sociology": 0.4911504424778761, - "sports_science": 0.41818181818181815, - "traditional_chinese_medicine": 0.2864864864864865, - "virology": 0.4556213017751479, - "world_history": 0.484472049689441, - "world_religions": 0.50625 - } - }, - "prompt_3": { - "accuracy": 0.41547228457951996, - "category_acc": { - "agronomy": 0.3136094674556213, - "anatomy": 0.30405405405405406, - "ancient_chinese": 0.25609756097560976, - "arts": 0.41875, - "astronomy": 0.32727272727272727, - "business_ethics": 0.4258373205741627, - "chinese_civil_service_exam": 0.3875, - "chinese_driving_rule": 0.5801526717557252, - "chinese_food_culture": 0.33088235294117646, - "chinese_foreign_policy": 0.45794392523364486, - "chinese_history": 0.44272445820433437, - "chinese_literature": 0.35784313725490197, - "chinese_teacher_qualification": 0.5307262569832403, - "clinical_knowledge": 0.38396624472573837, - "college_actuarial_science": 0.27358490566037735, - "college_education": 0.5327102803738317, - "college_engineering_hydrology": 0.42452830188679247, - "college_law": 0.3611111111111111, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.37735849056603776, - "college_medicine": 0.3772893772893773, - "computer_science": 0.4803921568627451, - "computer_security": 0.5087719298245614, - "conceptual_physics": 0.41496598639455784, - "construction_project_management": 0.34532374100719426, - "economics": 0.4716981132075472, - "education": 0.39263803680981596, - "electrical_engineering": 0.3430232558139535, - "elementary_chinese": 0.32142857142857145, - "elementary_commonsense": 0.41919191919191917, - "elementary_information_and_technology": 0.6470588235294118, - "elementary_mathematics": 0.3391304347826087, - "ethnology": 0.37777777777777777, - "food_science": 0.4195804195804196, - "genetics": 0.3806818181818182, - "global_facts": 0.5033557046979866, - "high_school_biology": 0.3668639053254438, - "high_school_chemistry": 0.26515151515151514, - "high_school_geography": 0.423728813559322, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.3181818181818182, - "high_school_politics": 0.3986013986013986, - "human_sexuality": 0.5555555555555556, - "international_law": 0.3891891891891892, - "journalism": 0.45348837209302323, - "jurisprudence": 0.45742092457420924, - "legal_and_moral_basis": 0.6962616822429907, - "logical": 0.3821138211382114, - "machine_learning": 0.29508196721311475, - "management": 0.47619047619047616, - "marketing": 0.5055555555555555, - "marxist_theory": 0.5026455026455027, - "modern_chinese": 0.3448275862068966, - "nutrition": 0.47586206896551725, - "philosophy": 0.38095238095238093, - "professional_accounting": 0.42857142857142855, - "professional_law": 0.3412322274881517, - "professional_medicine": 0.32978723404255317, - "professional_psychology": 0.39655172413793105, - "public_relations": 0.5, - "security_study": 0.48148148148148145, - "sociology": 0.5088495575221239, - "sports_science": 0.3878787878787879, - "traditional_chinese_medicine": 0.3081081081081081, - "virology": 0.46745562130177515, - "world_history": 0.515527950310559, - "world_religions": 0.49375 - } - }, - "prompt_4": { - "accuracy": 0.41754446554999136, - "category_acc": { - "agronomy": 0.2958579881656805, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.2804878048780488, - "arts": 0.425, - "astronomy": 0.3333333333333333, - "business_ethics": 0.4258373205741627, - "chinese_civil_service_exam": 0.38125, - "chinese_driving_rule": 0.6259541984732825, - "chinese_food_culture": 0.3235294117647059, - "chinese_foreign_policy": 0.5420560747663551, - "chinese_history": 0.43962848297213625, - "chinese_literature": 0.35294117647058826, - "chinese_teacher_qualification": 0.5251396648044693, - "clinical_knowledge": 0.3755274261603376, - "college_actuarial_science": 0.3018867924528302, - "college_education": 0.5233644859813084, - "college_engineering_hydrology": 0.41509433962264153, - "college_law": 0.37037037037037035, - "college_mathematics": 0.29523809523809524, - "college_medical_statistics": 0.39622641509433965, - "college_medicine": 0.34798534798534797, - "computer_science": 0.47058823529411764, - "computer_security": 0.49707602339181284, - "conceptual_physics": 0.38095238095238093, - "construction_project_management": 0.3597122302158273, - "economics": 0.48427672955974843, - "education": 0.44785276073619634, - "electrical_engineering": 0.37790697674418605, - "elementary_chinese": 0.31746031746031744, - "elementary_commonsense": 0.3686868686868687, - "elementary_information_and_technology": 0.6554621848739496, - "elementary_mathematics": 0.29130434782608694, - "ethnology": 0.4148148148148148, - "food_science": 0.4195804195804196, - "genetics": 0.3693181818181818, - "global_facts": 0.4966442953020134, - "high_school_biology": 0.31952662721893493, - "high_school_chemistry": 0.29545454545454547, - "high_school_geography": 0.4152542372881356, - "high_school_mathematics": 0.25609756097560976, - "high_school_physics": 0.3181818181818182, - "high_school_politics": 0.40559440559440557, - "human_sexuality": 0.6031746031746031, - "international_law": 0.3837837837837838, - "journalism": 0.45930232558139533, - "jurisprudence": 0.43795620437956206, - "legal_and_moral_basis": 0.719626168224299, - "logical": 0.3983739837398374, - "machine_learning": 0.36065573770491804, - "management": 0.5, - "marketing": 0.5333333333333333, - "marxist_theory": 0.5026455026455027, - "modern_chinese": 0.31896551724137934, - "nutrition": 0.46206896551724136, - "philosophy": 0.4, - "professional_accounting": 0.44, - "professional_law": 0.3744075829383886, - "professional_medicine": 0.31648936170212766, - "professional_psychology": 0.40948275862068967, - "public_relations": 0.5344827586206896, - "security_study": 0.4666666666666667, - "sociology": 0.4778761061946903, - "sports_science": 0.43636363636363634, - "traditional_chinese_medicine": 0.2810810810810811, - "virology": 0.4911242603550296, - "world_history": 0.5279503105590062, - "world_religions": 0.46875 - } - }, - "prompt_5": { - "accuracy": 0.3919875669141772, - "category_acc": { - "agronomy": 0.31952662721893493, - "anatomy": 0.28378378378378377, - "ancient_chinese": 0.27439024390243905, - "arts": 0.40625, - "astronomy": 0.30303030303030304, - "business_ethics": 0.4019138755980861, - "chinese_civil_service_exam": 0.4, - "chinese_driving_rule": 0.6259541984732825, - "chinese_food_culture": 0.27205882352941174, - "chinese_foreign_policy": 0.40186915887850466, - "chinese_history": 0.38699690402476783, - "chinese_literature": 0.31862745098039214, - "chinese_teacher_qualification": 0.5307262569832403, - "clinical_knowledge": 0.35443037974683544, - "college_actuarial_science": 0.2641509433962264, - "college_education": 0.48598130841121495, - "college_engineering_hydrology": 0.39622641509433965, - "college_law": 0.3148148148148148, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.3584905660377358, - "college_medicine": 0.358974358974359, - "computer_science": 0.45098039215686275, - "computer_security": 0.4502923976608187, - "conceptual_physics": 0.3673469387755102, - "construction_project_management": 0.3237410071942446, - "economics": 0.4025157232704403, - "education": 0.44171779141104295, - "electrical_engineering": 0.3546511627906977, - "elementary_chinese": 0.2619047619047619, - "elementary_commonsense": 0.3838383838383838, - "elementary_information_and_technology": 0.5672268907563025, - "elementary_mathematics": 0.33043478260869563, - "ethnology": 0.3851851851851852, - "food_science": 0.43356643356643354, - "genetics": 0.375, - "global_facts": 0.4697986577181208, - "high_school_biology": 0.28402366863905326, - "high_school_chemistry": 0.29545454545454547, - "high_school_geography": 0.3983050847457627, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.35454545454545455, - "high_school_politics": 0.34965034965034963, - "human_sexuality": 0.5079365079365079, - "international_law": 0.3675675675675676, - "journalism": 0.48255813953488375, - "jurisprudence": 0.40145985401459855, - "legal_and_moral_basis": 0.6728971962616822, - "logical": 0.43089430894308944, - "machine_learning": 0.319672131147541, - "management": 0.47619047619047616, - "marketing": 0.5222222222222223, - "marxist_theory": 0.47619047619047616, - "modern_chinese": 0.31896551724137934, - "nutrition": 0.4689655172413793, - "philosophy": 0.34285714285714286, - "professional_accounting": 0.4228571428571429, - "professional_law": 0.3080568720379147, - "professional_medicine": 0.2872340425531915, - "professional_psychology": 0.38362068965517243, - "public_relations": 0.46551724137931033, - "security_study": 0.45925925925925926, - "sociology": 0.46017699115044247, - "sports_science": 0.4303030303030303, - "traditional_chinese_medicine": 0.2648648648648649, - "virology": 0.41420118343195267, - "world_history": 0.45962732919254656, - "world_religions": 0.41875 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.42424242424242425 - }, - "prompt_2": { - "accuracy": 0.45454545454545453 - }, - "prompt_3": { - "accuracy": 0.45454545454545453 - }, - "prompt_4": { - "accuracy": 0.36363636363636365 - }, - "prompt_5": { - "accuracy": 0.2727272727272727 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.7068181818181818 - }, - "prompt_2": { - "accuracy": 0.6636363636363637 - }, - "prompt_3": { - "accuracy": 0.6727272727272727 - }, - "prompt_4": { - "accuracy": 0.6522727272727272 - }, - "prompt_5": { - "accuracy": 0.6681818181818182 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.3813559322033898 - }, - "prompt_2": { - "accuracy": 0.3488135593220339 - }, - "prompt_3": { - "accuracy": 0.3383050847457627 - }, - "prompt_4": { - "accuracy": 0.36915254237288136 - }, - "prompt_5": { - "accuracy": 0.3342372881355932 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.8055347793567689 - }, - "prompt_2": { - "accuracy": 0.8006731488406881 - }, - "prompt_3": { - "accuracy": 0.793567688855647 - }, - "prompt_4": { - "accuracy": 0.8044128646222887 - }, - "prompt_5": { - "accuracy": 0.787584143605086 - } - }, - "dream": { - "prompt_1": { - "accuracy": 0.8912297893189612 - }, - "prompt_2": { - "accuracy": 0.8912297893189612 - }, - "prompt_3": { - "accuracy": 0.8868201861832435 - }, - "prompt_4": { - "accuracy": 0.8735913767760901 - }, - "prompt_5": { - "accuracy": 0.875061244487996 - } - }, - "samsum": { - "prompt_1": { - "rouge1": 0.3761105366007489, - "rouge2": 0.15079639629838312, - "rougeL": 0.2899358329569579, - "avg_rouge": 0.27228092195202996 - }, - "prompt_2": { - "rouge1": 0.4082133753752708, - "rouge2": 0.1683283090503273, - "rougeL": 0.3165570374351565, - "avg_rouge": 0.2976995739535849 - }, - "prompt_3": { - "rouge1": 0.3876251454822799, - "rouge2": 0.15437364784389704, - "rougeL": 0.30013656403104383, - "avg_rouge": 0.28071178578574024 - }, - "prompt_4": { - "rouge1": 0.38730795625328884, - "rouge2": 0.15260710312754794, - "rougeL": 0.2981038712596473, - "avg_rouge": 0.27933964354682805 - }, - "prompt_5": { - "rouge1": 0.40387359043761534, - "rouge2": 0.1584867599653843, - "rougeL": 0.31444958527378114, - "avg_rouge": 0.29226997855892695 - } - }, - "dialogsum": { - "prompt_1": { - "rouge1": 0.22896028596457765, - "rouge2": 0.0638223698311453, - "rougeL": 0.16876966868357382, - "avg_rouge": 0.15385077482643225 - }, - "prompt_2": { - "rouge1": 0.23224173160207423, - "rouge2": 0.06365552537846508, - "rougeL": 0.17136065037224038, - "avg_rouge": 0.1557526357842599 - }, - "prompt_3": { - "rouge1": 0.23222039229572256, - "rouge2": 0.06350375331031248, - "rougeL": 0.17137591413946984, - "avg_rouge": 0.1557000199151683 - }, - "prompt_4": { - "rouge1": 0.22968671201006294, - "rouge2": 0.06345905806860368, - "rougeL": 0.16942380065259544, - "avg_rouge": 0.15418985691042067 - }, - "prompt_5": { - "rouge1": 0.22252909774282864, - "rouge2": 0.05972702065874208, - "rougeL": 0.16664488143990033, - "avg_rouge": 0.14963366661382368 - } - }, - "sst2": { - "prompt_1": { - "accuracy": 0.9346330275229358 - }, - "prompt_2": { - "accuracy": 0.9403669724770642 - }, - "prompt_3": { - "accuracy": 0.9334862385321101 - }, - "prompt_4": { - "accuracy": 0.9094036697247706 - }, - "prompt_5": { - "accuracy": 0.9438073394495413 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.7909875359539789 - }, - "prompt_2": { - "accuracy": 0.7718120805369127 - }, - "prompt_3": { - "accuracy": 0.7996164908916586 - }, - "prompt_4": { - "accuracy": 0.8178331735378715 - }, - "prompt_5": { - "accuracy": 0.8015340364333653 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.7385 - }, - "prompt_2": { - "accuracy": 0.804 - }, - "prompt_3": { - "accuracy": 0.7565 - }, - "prompt_4": { - "accuracy": 0.7785 - }, - "prompt_5": { - "accuracy": 0.794 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.652 - }, - "prompt_2": { - "accuracy": 0.6525 - }, - "prompt_3": { - "accuracy": 0.649 - }, - "prompt_4": { - "accuracy": 0.647 - }, - "prompt_5": { - "accuracy": 0.6215 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.656 - }, - "prompt_2": { - "accuracy": 0.687 - }, - "prompt_3": { - "accuracy": 0.681 - }, - "prompt_4": { - "accuracy": 0.7025 - }, - "prompt_5": { - "accuracy": 0.7885 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.7323943661971831 - }, - "prompt_2": { - "accuracy": 0.704225352112676 - }, - "prompt_3": { - "accuracy": 0.6901408450704225 - }, - "prompt_4": { - "accuracy": 0.5070422535211268 - }, - "prompt_5": { - "accuracy": 0.6901408450704225 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.7906137184115524 - }, - "prompt_2": { - "accuracy": 0.8411552346570397 - }, - "prompt_3": { - "accuracy": 0.8086642599277978 - }, - "prompt_4": { - "accuracy": 0.6967509025270758 - }, - "prompt_5": { - "accuracy": 0.776173285198556 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.7083333333333334 - }, - "prompt_2": { - "accuracy": 0.7524509803921569 - }, - "prompt_3": { - "accuracy": 0.7181372549019608 - }, - "prompt_4": { - "accuracy": 0.7352941176470589 - }, - "prompt_5": { - "accuracy": 0.7818627450980392 - } - } - }, - "five_shot": { - "cross_xquad": { - "prompt_1": -1 - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.5219047619047619, - "language_acc": { - "Spanish": 0.62, - "English": 0.6666666666666666, - "Chinese": 0.5, - "Vietnamese": 0.43333333333333335, - "Filipino": 0.4533333333333333, - "Indonesian": 0.5333333333333333, - "Malay": 0.44666666666666666 - }, - "consistency_score_2": 0.5809523809523809, - "consistency_score_3": 0.40876190476190477, - "consistency_score_4": 0.3093333333333333, - "consistency_score_5": 0.24222222222222223, - "consistency_score_6": 0.19238095238095237, - "consistency_score_7": 0.15333333333333332, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.76, - "Spanish,Chinese": 0.5933333333333334, - "Spanish,Vietnamese": 0.5333333333333333, - "Spanish,Filipino": 0.5666666666666667, - "Spanish,Indonesian": 0.66, - "Spanish,Malay": 0.58, - "English,Chinese": 0.5866666666666667, - "English,Vietnamese": 0.56, - "English,Filipino": 0.58, - "English,Indonesian": 0.66, - "English,Malay": 0.58, - "Chinese,Vietnamese": 0.46, - "Chinese,Filipino": 0.5, - "Chinese,Indonesian": 0.56, - "Chinese,Malay": 0.5866666666666667, - "Vietnamese,Filipino": 0.5733333333333334, - "Vietnamese,Indonesian": 0.5533333333333333, - "Vietnamese,Malay": 0.5866666666666667, - "Filipino,Indonesian": 0.5733333333333334, - "Filipino,Malay": 0.4666666666666667, - "Indonesian,Malay": 0.68 - }, - "3_combine": { - "Spanish,English,Chinese": 0.4866666666666667, - "Spanish,English,Vietnamese": 0.46, - "Spanish,English,Filipino": 0.47333333333333333, - "Spanish,English,Indonesian": 0.5666666666666667, - "Spanish,English,Malay": 0.4866666666666667, - "Spanish,Chinese,Vietnamese": 0.3333333333333333, - "Spanish,Chinese,Filipino": 0.36, - "Spanish,Chinese,Indonesian": 0.43333333333333335, - "Spanish,Chinese,Malay": 0.41333333333333333, - "Spanish,Vietnamese,Filipino": 0.37333333333333335, - "Spanish,Vietnamese,Indonesian": 0.42, - "Spanish,Vietnamese,Malay": 0.3933333333333333, - "Spanish,Filipino,Indonesian": 0.4266666666666667, - "Spanish,Filipino,Malay": 0.3466666666666667, - "Spanish,Indonesian,Malay": 0.4866666666666667, - "English,Chinese,Vietnamese": 0.3466666666666667, - "English,Chinese,Filipino": 0.36, - "English,Chinese,Indonesian": 0.44, - "English,Chinese,Malay": 0.4066666666666667, - "English,Vietnamese,Filipino": 0.3933333333333333, - "English,Vietnamese,Indonesian": 0.4266666666666667, - "English,Vietnamese,Malay": 0.41333333333333333, - "English,Filipino,Indonesian": 0.4533333333333333, - "English,Filipino,Malay": 0.36666666666666664, - "English,Indonesian,Malay": 0.5, - "Chinese,Vietnamese,Filipino": 0.31333333333333335, - "Chinese,Vietnamese,Indonesian": 0.3333333333333333, - "Chinese,Vietnamese,Malay": 0.36666666666666664, - "Chinese,Filipino,Indonesian": 0.36, - "Chinese,Filipino,Malay": 0.3333333333333333, - "Chinese,Indonesian,Malay": 0.44, - "Vietnamese,Filipino,Indonesian": 0.38666666666666666, - "Vietnamese,Filipino,Malay": 0.3466666666666667, - "Vietnamese,Indonesian,Malay": 0.46, - "Filipino,Indonesian,Malay": 0.4 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.28, - "Spanish,English,Chinese,Filipino": 0.29333333333333333, - "Spanish,English,Chinese,Indonesian": 0.3933333333333333, - "Spanish,English,Chinese,Malay": 0.3466666666666667, - "Spanish,English,Vietnamese,Filipino": 0.32666666666666666, - "Spanish,English,Vietnamese,Indonesian": 0.37333333333333335, - "Spanish,English,Vietnamese,Malay": 0.3466666666666667, - "Spanish,English,Filipino,Indonesian": 0.38, - "Spanish,English,Filipino,Malay": 0.30666666666666664, - "Spanish,English,Indonesian,Malay": 0.44666666666666666, - "Spanish,Chinese,Vietnamese,Filipino": 0.23333333333333334, - "Spanish,Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "Spanish,Chinese,Vietnamese,Malay": 0.28, - "Spanish,Chinese,Filipino,Indonesian": 0.29333333333333333, - "Spanish,Chinese,Filipino,Malay": 0.26666666666666666, - "Spanish,Chinese,Indonesian,Malay": 0.35333333333333333, - "Spanish,Vietnamese,Filipino,Indonesian": 0.3, - "Spanish,Vietnamese,Filipino,Malay": 0.25333333333333335, - "Spanish,Vietnamese,Indonesian,Malay": 0.35333333333333333, - "Spanish,Filipino,Indonesian,Malay": 0.31333333333333335, - "English,Chinese,Vietnamese,Filipino": 0.25333333333333335, - "English,Chinese,Vietnamese,Indonesian": 0.2866666666666667, - "English,Chinese,Vietnamese,Malay": 0.28, - "English,Chinese,Filipino,Indonesian": 0.31333333333333335, - "English,Chinese,Filipino,Malay": 0.2733333333333333, - "English,Chinese,Indonesian,Malay": 0.3466666666666667, - "English,Vietnamese,Filipino,Indonesian": 0.32, - "English,Vietnamese,Filipino,Malay": 0.28, - "English,Vietnamese,Indonesian,Malay": 0.36, - "English,Filipino,Indonesian,Malay": 0.3333333333333333, - "Chinese,Vietnamese,Filipino,Indonesian": 0.24666666666666667, - "Chinese,Vietnamese,Filipino,Malay": 0.24, - "Chinese,Vietnamese,Indonesian,Malay": 0.2866666666666667, - "Chinese,Filipino,Indonesian,Malay": 0.2733333333333333, - "Vietnamese,Filipino,Indonesian,Malay": 0.30666666666666664 - }, - "5_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino": 0.2, - "Spanish,English,Chinese,Vietnamese,Indonesian": 0.25333333333333335, - "Spanish,English,Chinese,Vietnamese,Malay": 0.23333333333333334, - "Spanish,English,Chinese,Filipino,Indonesian": 0.26666666666666666, - "Spanish,English,Chinese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Chinese,Indonesian,Malay": 0.32, - "Spanish,English,Vietnamese,Filipino,Indonesian": 0.2733333333333333, - "Spanish,English,Vietnamese,Filipino,Malay": 0.22666666666666666, - "Spanish,English,Vietnamese,Indonesian,Malay": 0.32, - "Spanish,English,Filipino,Indonesian,Malay": 0.29333333333333333, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian": 0.20666666666666667, - "Spanish,Chinese,Vietnamese,Filipino,Malay": 0.18666666666666668, - "Spanish,Chinese,Vietnamese,Indonesian,Malay": 0.25333333333333335, - "Spanish,Chinese,Filipino,Indonesian,Malay": 0.23333333333333334, - "Spanish,Vietnamese,Filipino,Indonesian,Malay": 0.24, - "English,Chinese,Vietnamese,Filipino,Indonesian": 0.22, - "English,Chinese,Vietnamese,Filipino,Malay": 0.2, - "English,Chinese,Vietnamese,Indonesian,Malay": 0.24, - "English,Chinese,Filipino,Indonesian,Malay": 0.24, - "English,Vietnamese,Filipino,Indonesian,Malay": 0.25333333333333335, - "Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.2 - }, - "6_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian": 0.18666666666666668, - "Spanish,English,Chinese,Vietnamese,Filipino,Malay": 0.16, - "Spanish,English,Chinese,Vietnamese,Indonesian,Malay": 0.22, - "Spanish,English,Chinese,Filipino,Indonesian,Malay": 0.21333333333333335, - "Spanish,English,Vietnamese,Filipino,Indonesian,Malay": 0.22, - "Spanish,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334, - "English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.17333333333333334 - }, - "7_combine": { - "Spanish,English,Chinese,Vietnamese,Filipino,Indonesian,Malay": 0.15333333333333332 - } - }, - "AC3_2": 0.5498478492798579, - "AC3_3": 0.4584558407397767, - "AC3_4": 0.3884387411692796, - "AC3_5": 0.33087937954401314, - "AC3_6": 0.2811326983733399, - "AC3_7": 0.23702867885535306 - } - }, - "cross_logiqa": { - "prompt_1": -1 - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.7087378640776699 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.3904761904761905 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.6728971962616822 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.45, - "category_acc": { - "brand": 0.4, - "demographics": 0.2, - "biology": 0.4, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.7, - "culture": 0.8, - "film": 0.4, - "law": 0.4, - "geography": 0.6 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.31112761042206083 - } - }, - "indommlu": { - "prompt_1": -1 - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.3717513721371397 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.2477766461377027 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.22073066040769163 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.35080558543620877 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.6044340723453909 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.5844118698605649, - "category_acc": { - "high_school_european_history": 0.7439024390243902, - "business_ethics": 0.5151515151515151, - "clinical_knowledge": 0.6628787878787878, - "medical_genetics": 0.6666666666666666, - "high_school_us_history": 0.7783251231527094, - "high_school_physics": 0.3466666666666667, - "high_school_world_history": 0.8008474576271186, - "virology": 0.4727272727272727, - "high_school_microeconomics": 0.6329113924050633, - "econometrics": 0.3805309734513274, - "college_computer_science": 0.48484848484848486, - "high_school_biology": 0.7313915857605178, - "abstract_algebra": 0.31313131313131315, - "professional_accounting": 0.45195729537366547, - "philosophy": 0.6580645161290323, - "professional_medicine": 0.6826568265682657, - "nutrition": 0.6459016393442623, - "global_facts": 0.32323232323232326, - "machine_learning": 0.3333333333333333, - "security_studies": 0.6885245901639344, - "public_relations": 0.6146788990825688, - "professional_psychology": 0.5924713584288053, - "prehistory": 0.6656346749226006, - "anatomy": 0.582089552238806, - "human_sexuality": 0.7, - "college_medicine": 0.6104651162790697, - "high_school_government_and_politics": 0.8645833333333334, - "college_chemistry": 0.3939393939393939, - "logical_fallacies": 0.7222222222222222, - "high_school_geography": 0.8071065989847716, - "elementary_mathematics": 0.4005305039787798, - "human_aging": 0.6396396396396397, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.8069852941176471, - "formal_logic": 0.376, - "high_school_statistics": 0.4697674418604651, - "international_law": 0.7, - "high_school_mathematics": 0.3382899628252788, - "high_school_computer_science": 0.6161616161616161, - "conceptual_physics": 0.5341880341880342, - "miscellaneous": 0.809462915601023, - "high_school_chemistry": 0.48514851485148514, - "marketing": 0.8154506437768241, - "professional_law": 0.44031311154598823, - "management": 0.7843137254901961, - "college_physics": 0.42574257425742573, - "jurisprudence": 0.7102803738317757, - "world_religions": 0.8294117647058824, - "sociology": 0.825, - "us_foreign_policy": 0.797979797979798, - "high_school_macroeconomics": 0.5784061696658098, - "computer_security": 0.7272727272727273, - "moral_scenarios": 0.232662192393736, - "moral_disputes": 0.6347826086956522, - "electrical_engineering": 0.5, - "astronomy": 0.5960264900662252, - "college_biology": 0.6643356643356644 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.43164933135215455 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.42714819427148193, - "category_acc": { - "computer_network": 0.5833333333333334, - "operating_system": 0.5, - "computer_architecture": 0.4230769230769231, - "college_programming": 0.47619047619047616, - "college_physics": 0.375, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.2608695652173913, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.5172413793103449, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.6538461538461539, - "middle_school_physics": 0.625, - "middle_school_chemistry": 0.6, - "veterinary_medicine": 0.35714285714285715, - "college_economics": 0.43333333333333335, - "business_administration": 0.2894736842105263, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.5862068965517241, - "education_science": 0.5294117647058824, - "teacher_qualification": 0.6122448979591837, - "high_school_politics": 0.625, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.6538461538461539, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.4583333333333333, - "logic": 0.5185185185185185, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.42857142857142855, - "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.4117647058823529, - "legal_professional": 0.5, - "high_school_chinese": 0.375, - "high_school_history": 0.48, - "middle_school_history": 0.5185185185185185, - "civil_servant": 0.4807692307692308, - "sports_science": 0.3333333333333333, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.7083333333333334, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.5098039215686274, - "accountant": 0.42592592592592593, - "fire_engineer": 0.2777777777777778, - "environmental_impact_assessment_engineer": 0.4444444444444444, - "tax_accountant": 0.2962962962962963, - "physician": 0.3333333333333333 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.41935483870967744 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.43101364185805563, - "category_acc": { - "agronomy": 0.38461538461538464, - "anatomy": 0.28378378378378377, - "ancient_chinese": 0.21951219512195122, - "arts": 0.43125, - "astronomy": 0.28484848484848485, - "business_ethics": 0.44976076555023925, - "chinese_civil_service_exam": 0.45625, - "chinese_driving_rule": 0.5801526717557252, - "chinese_food_culture": 0.36764705882352944, - "chinese_foreign_policy": 0.5233644859813084, - "chinese_history": 0.4613003095975232, - "chinese_literature": 0.35784313725490197, - "chinese_teacher_qualification": 0.49162011173184356, - "clinical_knowledge": 0.3881856540084388, - "college_actuarial_science": 0.3584905660377358, - "college_education": 0.5233644859813084, - "college_engineering_hydrology": 0.42452830188679247, - "college_law": 0.37037037037037035, - "college_mathematics": 0.22857142857142856, - "college_medical_statistics": 0.39622641509433965, - "college_medicine": 0.3663003663003663, - "computer_science": 0.5049019607843137, - "computer_security": 0.5146198830409356, - "conceptual_physics": 0.41496598639455784, - "construction_project_management": 0.39568345323741005, - "economics": 0.49056603773584906, - "education": 0.4785276073619632, - "electrical_engineering": 0.4418604651162791, - "elementary_chinese": 0.3333333333333333, - "elementary_commonsense": 0.45454545454545453, - "elementary_information_and_technology": 0.6554621848739496, - "elementary_mathematics": 0.2826086956521739, - "ethnology": 0.37777777777777777, - "food_science": 0.4125874125874126, - "genetics": 0.3465909090909091, - "global_facts": 0.5369127516778524, - "high_school_biology": 0.34911242603550297, - "high_school_chemistry": 0.3106060606060606, - "high_school_geography": 0.4322033898305085, - "high_school_mathematics": 0.2804878048780488, - "high_school_physics": 0.3, - "high_school_politics": 0.43356643356643354, - "human_sexuality": 0.5634920634920635, - "international_law": 0.41081081081081083, - "journalism": 0.47674418604651164, - "jurisprudence": 0.45742092457420924, - "legal_and_moral_basis": 0.7289719626168224, - "logical": 0.37398373983739835, - "machine_learning": 0.3770491803278688, - "management": 0.5238095238095238, - "marketing": 0.5277777777777778, - "marxist_theory": 0.48677248677248675, - "modern_chinese": 0.3879310344827586, - "nutrition": 0.4827586206896552, - "philosophy": 0.45714285714285713, - "professional_accounting": 0.4857142857142857, - "professional_law": 0.35545023696682465, - "professional_medicine": 0.31117021276595747, - "professional_psychology": 0.4482758620689655, - "public_relations": 0.5057471264367817, - "security_study": 0.5925925925925926, - "sociology": 0.5442477876106194, - "sports_science": 0.44242424242424244, - "traditional_chinese_medicine": 0.33513513513513515, - "virology": 0.47337278106508873, - "world_history": 0.4409937888198758, - "world_religions": 0.48125 - } - } - }, - "zbench": { - "prompt_1": { - "accuracy": 0.45454545454545453 - } - }, - "ind_emotion": { - "prompt_1": { - "accuracy": 0.6863636363636364 - } - }, - "ocnli": { - "prompt_1": { - "accuracy": 0.4694915254237288 - } - }, - "c3": { - "prompt_1": { - "accuracy": 0.8152580403889305 - } - }, - "dream": { - "prompt_1": -1 - }, - "samsum": { - "prompt_1": { - "rouge1": 0.4492356350724478, - "rouge2": 0.19712865473047925, - "rougeL": 0.35237322687091155, - "avg_rouge": 0.3329125055579462 - } - }, - "dialogsum": { - "prompt_1": -1 - }, - "sst2": { - "prompt_1": { - "accuracy": 0.9311926605504587 - } - }, - "cola": { - "prompt_1": { - "accuracy": 0.7181208053691275 - } - }, - "qqp": { - "prompt_1": { - "accuracy": 0.779 - } - }, - "mnli": { - "prompt_1": { - "accuracy": 0.6315 - } - }, - "qnli": { - "prompt_1": { - "accuracy": 0.65 - } - }, - "wnli": { - "prompt_1": { - "accuracy": 0.647887323943662 - } - }, - "rte": { - "prompt_1": { - "accuracy": 0.7364620938628159 - } - }, - "mrpc": { - "prompt_1": { - "accuracy": 0.7377450980392157 - } - } - } - }, - "regional_sea_mistral_inst_128k_7b": { - "model_size": "7B", - "model_link": "https://seaeval.github.io/", - "zero_shot": { - "cross_xquad": { - "prompt_1": { - "overall_acc": 0.8210084033613445, - "language_acc": { - "Spanish": 0.8647058823529412, - "English": 0.8672268907563025, - "Chinese": 0.7857142857142857, - "Vietnamese": 0.7663865546218488 - }, - "consistency_score_2": 0.7826330532212885, - "consistency_score_3": 0.6899159663865546, - "consistency_score_4": 0.6277310924369748, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8621848739495799, - "Spanish,Chinese": 0.8, - "Spanish,Vietnamese": 0.7546218487394958, - "English,Chinese": 0.8092436974789916, - "English,Vietnamese": 0.746218487394958, - "Chinese,Vietnamese": 0.7235294117647059 - }, - "3_combine": { - "Spanish,English,Chinese": 0.7453781512605042, - "Spanish,English,Vietnamese": 0.6991596638655462, - "Spanish,Chinese,Vietnamese": 0.6588235294117647, - "English,Chinese,Vietnamese": 0.6563025210084034 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.6277310924369748 - } - }, - "AC3_2": 0.80136156466325, - "AC3_3": 0.749775193676365, - "AC3_4": 0.7114771198011502 - }, - "prompt_2": { - "overall_acc": 0.8163865546218487, - "language_acc": { - "Spanish": 0.8588235294117647, - "English": 0.8613445378151261, - "Chinese": 0.7857142857142857, - "Vietnamese": 0.7596638655462185 - }, - "consistency_score_2": 0.7838935574229692, - "consistency_score_3": 0.6911764705882353, - "consistency_score_4": 0.6294117647058823, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8680672268907563, - "Spanish,Chinese": 0.8033613445378152, - "Spanish,Vietnamese": 0.7621848739495798, - "English,Chinese": 0.8042016806722689, - "English,Vietnamese": 0.7470588235294118, - "Chinese,Vietnamese": 0.7184873949579832 - }, - "3_combine": { - "Spanish,English,Chinese": 0.7495798319327731, - "Spanish,English,Vietnamese": 0.7025210084033613, - "Spanish,Chinese,Vietnamese": 0.6605042016806723, - "English,Chinese,Vietnamese": 0.6521008403361345 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.6294117647058823 - } - }, - "AC3_2": 0.7998101778282783, - "AC3_3": 0.7485818741716053, - "AC3_4": 0.7108091012722715 - }, - "prompt_3": { - "overall_acc": 0.8245798319327731, - "language_acc": { - "Spanish": 0.8680672268907563, - "English": 0.8739495798319328, - "Chinese": 0.7932773109243697, - "Vietnamese": 0.7630252100840336 - }, - "consistency_score_2": 0.7866946778711484, - "consistency_score_3": 0.6978991596638655, - "consistency_score_4": 0.6378151260504201, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8764705882352941, - "Spanish,Chinese": 0.8042016806722689, - "Spanish,Vietnamese": 0.7579831932773109, - "English,Chinese": 0.8109243697478992, - "English,Vietnamese": 0.7495798319327731, - "Chinese,Vietnamese": 0.7210084033613445 - }, - "3_combine": { - "Spanish,English,Chinese": 0.753781512605042, - "Spanish,English,Vietnamese": 0.7100840336134454, - "Spanish,Chinese,Vietnamese": 0.6655462184873949, - "English,Chinese,Vietnamese": 0.6621848739495798 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.6378151260504201 - } - }, - "AC3_2": 0.8051918667789009, - "AC3_3": 0.7559691462676285, - "AC3_4": 0.7192714752411183 - }, - "prompt_4": { - "overall_acc": 0.8197478991596638, - "language_acc": { - "Spanish": 0.8647058823529412, - "English": 0.8605042016806723, - "Chinese": 0.7857142857142857, - "Vietnamese": 0.7680672268907563 - }, - "consistency_score_2": 0.7812324929971989, - "consistency_score_3": 0.6871848739495798, - "consistency_score_4": 0.626890756302521, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8638655462184874, - "Spanish,Chinese": 0.7966386554621848, - "Spanish,Vietnamese": 0.761344537815126, - "English,Chinese": 0.8008403361344538, - "English,Vietnamese": 0.7428571428571429, - "Chinese,Vietnamese": 0.7218487394957983 - }, - "3_combine": { - "Spanish,English,Chinese": 0.7378151260504202, - "Spanish,English,Vietnamese": 0.6991596638655462, - "Spanish,Chinese,Vietnamese": 0.6588235294117647, - "English,Chinese,Vietnamese": 0.6529411764705882 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.626890756302521 - } - }, - "AC3_2": 0.8000269059971997, - "AC3_3": 0.7476356832492188, - "AC3_4": 0.7104640519672804 - }, - "prompt_5": { - "overall_acc": 0.8245798319327731, - "language_acc": { - "Spanish": 0.8697478991596639, - "English": 0.8680672268907563, - "Chinese": 0.7966386554621848, - "Vietnamese": 0.7638655462184873 - }, - "consistency_score_2": 0.7837535014005602, - "consistency_score_3": 0.6911764705882353, - "consistency_score_4": 0.6294117647058823, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8630252100840337, - "Spanish,Chinese": 0.8058823529411765, - "Spanish,Vietnamese": 0.7579831932773109, - "English,Chinese": 0.8033613445378152, - "English,Vietnamese": 0.7453781512605042, - "Chinese,Vietnamese": 0.726890756302521 - }, - "3_combine": { - "Spanish,English,Chinese": 0.7453781512605042, - "Spanish,English,Vietnamese": 0.7, - "Spanish,Chinese,Vietnamese": 0.6638655462184874, - "English,Chinese,Vietnamese": 0.6554621848739496 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.6294117647058823 - } - }, - "AC3_2": 0.8036484937882867, - "AC3_3": 0.7520076637227568, - "AC3_4": 0.7138971756395551 - } - }, - "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.4523809523809524, - "language_acc": { - "Filipino": 0.36666666666666664, - "Vietnamese": 0.36, - "Chinese": 0.48, - "Spanish": 0.4866666666666667, - "Malay": 0.38, - "Indonesian": 0.46, - "English": 0.6333333333333333 - }, - "consistency_score_2": 0.5073015873015873, - "consistency_score_3": 0.3308571428571428, - "consistency_score_4": 0.24000000000000007, - "consistency_score_5": 0.18380952380952387, - "consistency_score_6": 0.14666666666666667, - "consistency_score_7": 0.12, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.47333333333333333, - "Filipino,Chinese": 0.4866666666666667, - "Filipino,Spanish": 0.48, - "Filipino,Malay": 0.46, - "Filipino,Indonesian": 0.46, - "Filipino,English": 0.47333333333333333, - "Vietnamese,Chinese": 0.38, - "Vietnamese,Spanish": 0.5, - "Vietnamese,Malay": 0.42, - "Vietnamese,Indonesian": 0.44, - "Vietnamese,English": 0.43333333333333335, - "Chinese,Spanish": 0.5533333333333333, - "Chinese,Malay": 0.49333333333333335, - "Chinese,Indonesian": 0.54, - "Chinese,English": 0.5733333333333334, - "Spanish,Malay": 0.5333333333333333, - "Spanish,Indonesian": 0.62, - "Spanish,English": 0.64, - "Malay,Indonesian": 0.6333333333333333, - "Malay,English": 0.5066666666666667, - "Indonesian,English": 0.5533333333333333 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.25333333333333335, - "Filipino,Vietnamese,Spanish": 0.30666666666666664, - "Filipino,Vietnamese,Malay": 0.2733333333333333, - "Filipino,Vietnamese,Indonesian": 0.28, - "Filipino,Vietnamese,English": 0.28, - "Filipino,Chinese,Spanish": 0.31333333333333335, - "Filipino,Chinese,Malay": 0.3, - "Filipino,Chinese,Indonesian": 0.32, - "Filipino,Chinese,English": 0.32, - "Filipino,Spanish,Malay": 0.31333333333333335, - "Filipino,Spanish,Indonesian": 0.3333333333333333, - "Filipino,Spanish,English": 0.3466666666666667, - "Filipino,Malay,Indonesian": 0.32666666666666666, - "Filipino,Malay,English": 0.32, - "Filipino,Indonesian,English": 0.32, - "Vietnamese,Chinese,Spanish": 0.29333333333333333, - "Vietnamese,Chinese,Malay": 0.25333333333333335, - "Vietnamese,Chinese,Indonesian": 0.26, - "Vietnamese,Chinese,English": 0.29333333333333333, - "Vietnamese,Spanish,Malay": 0.3, - "Vietnamese,Spanish,Indonesian": 0.3466666666666667, - "Vietnamese,Spanish,English": 0.35333333333333333, - "Vietnamese,Malay,Indonesian": 0.30666666666666664, - "Vietnamese,Malay,English": 0.2733333333333333, - "Vietnamese,Indonesian,English": 0.31333333333333335, - "Chinese,Spanish,Malay": 0.3466666666666667, - "Chinese,Spanish,Indonesian": 0.4, - "Chinese,Spanish,English": 0.41333333333333333, - "Chinese,Malay,Indonesian": 0.38666666666666666, - "Chinese,Malay,English": 0.34, - "Chinese,Indonesian,English": 0.3933333333333333, - "Spanish,Malay,Indonesian": 0.44, - "Spanish,Malay,English": 0.4066666666666667, - "Spanish,Indonesian,English": 0.4533333333333333, - "Malay,Indonesian,English": 0.4 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.2, - "Filipino,Vietnamese,Chinese,Malay": 0.18, - "Filipino,Vietnamese,Chinese,Indonesian": 0.18666666666666668, - "Filipino,Vietnamese,Chinese,English": 0.2, - "Filipino,Vietnamese,Spanish,Malay": 0.2, - "Filipino,Vietnamese,Spanish,Indonesian": 0.22, - "Filipino,Vietnamese,Spanish,English": 0.22666666666666666, - "Filipino,Vietnamese,Malay,Indonesian": 0.22, - "Filipino,Vietnamese,Malay,English": 0.20666666666666667, - "Filipino,Vietnamese,Indonesian,English": 0.22, - "Filipino,Chinese,Spanish,Malay": 0.23333333333333334, - "Filipino,Chinese,Spanish,Indonesian": 0.24, - "Filipino,Chinese,Spanish,English": 0.23333333333333334, - "Filipino,Chinese,Malay,Indonesian": 0.26, - "Filipino,Chinese,Malay,English": 0.22666666666666666, - "Filipino,Chinese,Indonesian,English": 0.24666666666666667, - "Filipino,Spanish,Malay,Indonesian": 0.26, - "Filipino,Spanish,Malay,English": 0.26, - "Filipino,Spanish,Indonesian,English": 0.2733333333333333, - "Filipino,Malay,Indonesian,English": 0.24, - "Vietnamese,Chinese,Spanish,Malay": 0.2, - "Vietnamese,Chinese,Spanish,Indonesian": 0.21333333333333335, - "Vietnamese,Chinese,Spanish,English": 0.24666666666666667, - "Vietnamese,Chinese,Malay,Indonesian": 0.2, - "Vietnamese,Chinese,Malay,English": 0.20666666666666667, - "Vietnamese,Chinese,Indonesian,English": 0.23333333333333334, - "Vietnamese,Spanish,Malay,Indonesian": 0.26, - "Vietnamese,Spanish,Malay,English": 0.23333333333333334, - "Vietnamese,Spanish,Indonesian,English": 0.2733333333333333, - "Vietnamese,Malay,Indonesian,English": 0.23333333333333334, - "Chinese,Spanish,Malay,Indonesian": 0.31333333333333335, - "Chinese,Spanish,Malay,English": 0.28, - "Chinese,Spanish,Indonesian,English": 0.3333333333333333, - "Chinese,Malay,Indonesian,English": 0.29333333333333333, - "Spanish,Malay,Indonesian,English": 0.3466666666666667 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.14666666666666667, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.14666666666666667, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.16, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.16, - "Filipino,Vietnamese,Chinese,Malay,English": 0.15333333333333332, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.16666666666666666, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, - "Filipino,Vietnamese,Spanish,Malay,English": 0.17333333333333334, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18666666666666668, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.18, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, - "Filipino,Chinese,Spanish,Malay,English": 0.18666666666666668, - "Filipino,Chinese,Spanish,Indonesian,English": 0.2, - "Filipino,Chinese,Malay,Indonesian,English": 0.19333333333333333, - "Filipino,Spanish,Malay,Indonesian,English": 0.22, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, - "Vietnamese,Chinese,Spanish,Malay,English": 0.17333333333333334, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.2, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.18, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, - "Chinese,Spanish,Malay,Indonesian,English": 0.26 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12666666666666668, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.14, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.16, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.17333333333333334, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 - } - }, - "AC3_2": 0.4782697729602609, - "AC3_3": 0.3821914952263565, - "AC3_4": 0.31361760655718046, - "AC3_5": 0.2614057598676648, - "AC3_6": 0.2215156332433612, - "AC3_7": 0.18968386019980563 - }, - "prompt_2": { - "overall_acc": 0.4685714285714285, - "language_acc": { - "Filipino": 0.38, - "Vietnamese": 0.36, - "Chinese": 0.4866666666666667, - "Spanish": 0.5266666666666666, - "Malay": 0.41333333333333333, - "Indonesian": 0.48, - "English": 0.6333333333333333 - }, - "consistency_score_2": 0.48857142857142855, - "consistency_score_3": 0.3135238095238095, - "consistency_score_4": 0.22838095238095238, - "consistency_score_5": 0.17619047619047618, - "consistency_score_6": 0.14, - "consistency_score_7": 0.11333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.44666666666666666, - "Filipino,Chinese": 0.48, - "Filipino,Spanish": 0.38666666666666666, - "Filipino,Malay": 0.4533333333333333, - "Filipino,Indonesian": 0.44, - "Filipino,English": 0.4066666666666667, - "Vietnamese,Chinese": 0.36666666666666664, - "Vietnamese,Spanish": 0.49333333333333335, - "Vietnamese,Malay": 0.4266666666666667, - "Vietnamese,Indonesian": 0.4666666666666667, - "Vietnamese,English": 0.41333333333333333, - "Chinese,Spanish": 0.49333333333333335, - "Chinese,Malay": 0.49333333333333335, - "Chinese,Indonesian": 0.56, - "Chinese,English": 0.5266666666666666, - "Spanish,Malay": 0.52, - "Spanish,Indonesian": 0.6066666666666667, - "Spanish,English": 0.62, - "Malay,Indonesian": 0.6066666666666667, - "Malay,English": 0.5066666666666667, - "Indonesian,English": 0.5466666666666666 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.25333333333333335, - "Filipino,Vietnamese,Spanish": 0.24, - "Filipino,Vietnamese,Malay": 0.26666666666666666, - "Filipino,Vietnamese,Indonesian": 0.26666666666666666, - "Filipino,Vietnamese,English": 0.22666666666666666, - "Filipino,Chinese,Spanish": 0.26666666666666666, - "Filipino,Chinese,Malay": 0.3, - "Filipino,Chinese,Indonesian": 0.31333333333333335, - "Filipino,Chinese,English": 0.2866666666666667, - "Filipino,Spanish,Malay": 0.28, - "Filipino,Spanish,Indonesian": 0.3, - "Filipino,Spanish,English": 0.2866666666666667, - "Filipino,Malay,Indonesian": 0.31333333333333335, - "Filipino,Malay,English": 0.3, - "Filipino,Indonesian,English": 0.29333333333333333, - "Vietnamese,Chinese,Spanish": 0.24666666666666667, - "Vietnamese,Chinese,Malay": 0.25333333333333335, - "Vietnamese,Chinese,Indonesian": 0.28, - "Vietnamese,Chinese,English": 0.24666666666666667, - "Vietnamese,Spanish,Malay": 0.3, - "Vietnamese,Spanish,Indonesian": 0.34, - "Vietnamese,Spanish,English": 0.32, - "Vietnamese,Malay,Indonesian": 0.32, - "Vietnamese,Malay,English": 0.26, - "Vietnamese,Indonesian,English": 0.3, - "Chinese,Spanish,Malay": 0.3333333333333333, - "Chinese,Spanish,Indonesian": 0.38666666666666666, - "Chinese,Spanish,English": 0.3933333333333333, - "Chinese,Malay,Indonesian": 0.4, - "Chinese,Malay,English": 0.35333333333333333, - "Chinese,Indonesian,English": 0.38666666666666666, - "Spanish,Malay,Indonesian": 0.4266666666666667, - "Spanish,Malay,English": 0.3933333333333333, - "Spanish,Indonesian,English": 0.44666666666666666, - "Malay,Indonesian,English": 0.3933333333333333 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,Malay": 0.19333333333333333, - "Filipino,Vietnamese,Chinese,Indonesian": 0.19333333333333333, - "Filipino,Vietnamese,Chinese,English": 0.16, - "Filipino,Vietnamese,Spanish,Malay": 0.18, - "Filipino,Vietnamese,Spanish,Indonesian": 0.2, - "Filipino,Vietnamese,Spanish,English": 0.18666666666666668, - "Filipino,Vietnamese,Malay,Indonesian": 0.21333333333333335, - "Filipino,Vietnamese,Malay,English": 0.17333333333333334, - "Filipino,Vietnamese,Indonesian,English": 0.18, - "Filipino,Chinese,Spanish,Malay": 0.22, - "Filipino,Chinese,Spanish,Indonesian": 0.22666666666666666, - "Filipino,Chinese,Spanish,English": 0.21333333333333335, - "Filipino,Chinese,Malay,Indonesian": 0.25333333333333335, - "Filipino,Chinese,Malay,English": 0.22666666666666666, - "Filipino,Chinese,Indonesian,English": 0.23333333333333334, - "Filipino,Spanish,Malay,Indonesian": 0.26, - "Filipino,Spanish,Malay,English": 0.23333333333333334, - "Filipino,Spanish,Indonesian,English": 0.24, - "Filipino,Malay,Indonesian,English": 0.24, - "Vietnamese,Chinese,Spanish,Malay": 0.19333333333333333, - "Vietnamese,Chinese,Spanish,Indonesian": 0.22666666666666666, - "Vietnamese,Chinese,Spanish,English": 0.21333333333333335, - "Vietnamese,Chinese,Malay,Indonesian": 0.22, - "Vietnamese,Chinese,Malay,English": 0.19333333333333333, - "Vietnamese,Chinese,Indonesian,English": 0.21333333333333335, - "Vietnamese,Spanish,Malay,Indonesian": 0.26666666666666666, - "Vietnamese,Spanish,Malay,English": 0.22666666666666666, - "Vietnamese,Spanish,Indonesian,English": 0.26666666666666666, - "Vietnamese,Malay,Indonesian,English": 0.22, - "Chinese,Spanish,Malay,Indonesian": 0.30666666666666664, - "Chinese,Spanish,Malay,English": 0.2866666666666667, - "Chinese,Spanish,Indonesian,English": 0.32, - "Chinese,Malay,Indonesian,English": 0.30666666666666664, - "Spanish,Malay,Indonesian,English": 0.34 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.14, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.15333333333333332, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.17333333333333334, - "Filipino,Vietnamese,Chinese,Malay,English": 0.13333333333333333, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.14, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.17333333333333334, - "Filipino,Vietnamese,Spanish,Malay,English": 0.14666666666666667, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.16, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.15333333333333332, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.21333333333333335, - "Filipino,Chinese,Spanish,Malay,English": 0.18666666666666668, - "Filipino,Chinese,Spanish,Indonesian,English": 0.18666666666666668, - "Filipino,Chinese,Malay,Indonesian,English": 0.2, - "Filipino,Spanish,Malay,Indonesian,English": 0.21333333333333335, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, - "Vietnamese,Chinese,Spanish,Malay,English": 0.16666666666666666, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.19333333333333333, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.17333333333333334, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.20666666666666667, - "Chinese,Spanish,Malay,Indonesian,English": 0.26666666666666666 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.14, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.12, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.14, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.18, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.16 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 - } - }, - "AC3_2": 0.47836247329756976, - "AC3_3": 0.37567879753042166, - "AC3_4": 0.30708780692896, - "AC3_5": 0.2560877821928203, - "AC3_6": 0.21558685442466885, - "AC3_7": 0.18252045823377308 - }, - "prompt_3": { - "overall_acc": 0.4571428571428572, - "language_acc": { - "Filipino": 0.37333333333333335, - "Vietnamese": 0.34, - "Chinese": 0.4666666666666667, - "Spanish": 0.49333333333333335, - "Malay": 0.36666666666666664, - "Indonesian": 0.5, - "English": 0.66 - }, - "consistency_score_2": 0.4685714285714286, - "consistency_score_3": 0.285904761904762, - "consistency_score_4": 0.19523809523809518, - "consistency_score_5": 0.13904761904761903, - "consistency_score_6": 0.10095238095238095, - "consistency_score_7": 0.07333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.44666666666666666, - "Filipino,Chinese": 0.44, - "Filipino,Spanish": 0.42, - "Filipino,Malay": 0.38, - "Filipino,Indonesian": 0.4266666666666667, - "Filipino,English": 0.43333333333333335, - "Vietnamese,Chinese": 0.30666666666666664, - "Vietnamese,Spanish": 0.44666666666666666, - "Vietnamese,Malay": 0.43333333333333335, - "Vietnamese,Indonesian": 0.44, - "Vietnamese,English": 0.3933333333333333, - "Chinese,Spanish": 0.4533333333333333, - "Chinese,Malay": 0.42, - "Chinese,Indonesian": 0.52, - "Chinese,English": 0.5066666666666667, - "Spanish,Malay": 0.48, - "Spanish,Indonesian": 0.6133333333333333, - "Spanish,English": 0.6333333333333333, - "Malay,Indonesian": 0.5866666666666667, - "Malay,English": 0.48, - "Indonesian,English": 0.58 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.2, - "Filipino,Vietnamese,Spanish": 0.22666666666666666, - "Filipino,Vietnamese,Malay": 0.23333333333333334, - "Filipino,Vietnamese,Indonesian": 0.25333333333333335, - "Filipino,Vietnamese,English": 0.24666666666666667, - "Filipino,Chinese,Spanish": 0.23333333333333334, - "Filipino,Chinese,Malay": 0.22666666666666666, - "Filipino,Chinese,Indonesian": 0.26666666666666666, - "Filipino,Chinese,English": 0.26666666666666666, - "Filipino,Spanish,Malay": 0.24666666666666667, - "Filipino,Spanish,Indonesian": 0.3, - "Filipino,Spanish,English": 0.32666666666666666, - "Filipino,Malay,Indonesian": 0.26, - "Filipino,Malay,English": 0.26666666666666666, - "Filipino,Indonesian,English": 0.2866666666666667, - "Vietnamese,Chinese,Spanish": 0.20666666666666667, - "Vietnamese,Chinese,Malay": 0.19333333333333333, - "Vietnamese,Chinese,Indonesian": 0.23333333333333334, - "Vietnamese,Chinese,English": 0.20666666666666667, - "Vietnamese,Spanish,Malay": 0.28, - "Vietnamese,Spanish,Indonesian": 0.32666666666666666, - "Vietnamese,Spanish,English": 0.32, - "Vietnamese,Malay,Indonesian": 0.30666666666666664, - "Vietnamese,Malay,English": 0.24666666666666667, - "Vietnamese,Indonesian,English": 0.29333333333333333, - "Chinese,Spanish,Malay": 0.26666666666666666, - "Chinese,Spanish,Indonesian": 0.34, - "Chinese,Spanish,English": 0.36, - "Chinese,Malay,Indonesian": 0.32666666666666666, - "Chinese,Malay,English": 0.2866666666666667, - "Chinese,Indonesian,English": 0.38, - "Spanish,Malay,Indonesian": 0.3933333333333333, - "Spanish,Malay,English": 0.36666666666666664, - "Spanish,Indonesian,English": 0.46, - "Malay,Indonesian,English": 0.37333333333333335 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.12666666666666668, - "Filipino,Vietnamese,Chinese,Malay": 0.13333333333333333, - "Filipino,Vietnamese,Chinese,Indonesian": 0.15333333333333332, - "Filipino,Vietnamese,Chinese,English": 0.14, - "Filipino,Vietnamese,Spanish,Malay": 0.16666666666666666, - "Filipino,Vietnamese,Spanish,Indonesian": 0.18, - "Filipino,Vietnamese,Spanish,English": 0.19333333333333333, - "Filipino,Vietnamese,Malay,Indonesian": 0.18666666666666668, - "Filipino,Vietnamese,Malay,English": 0.16666666666666666, - "Filipino,Vietnamese,Indonesian,English": 0.18, - "Filipino,Chinese,Spanish,Malay": 0.16666666666666666, - "Filipino,Chinese,Spanish,Indonesian": 0.18666666666666668, - "Filipino,Chinese,Spanish,English": 0.2, - "Filipino,Chinese,Malay,Indonesian": 0.18666666666666668, - "Filipino,Chinese,Malay,English": 0.17333333333333334, - "Filipino,Chinese,Indonesian,English": 0.2, - "Filipino,Spanish,Malay,Indonesian": 0.20666666666666667, - "Filipino,Spanish,Malay,English": 0.21333333333333335, - "Filipino,Spanish,Indonesian,English": 0.24666666666666667, - "Filipino,Malay,Indonesian,English": 0.19333333333333333, - "Vietnamese,Chinese,Spanish,Malay": 0.14666666666666667, - "Vietnamese,Chinese,Spanish,Indonesian": 0.17333333333333334, - "Vietnamese,Chinese,Spanish,English": 0.17333333333333334, - "Vietnamese,Chinese,Malay,Indonesian": 0.16, - "Vietnamese,Chinese,Malay,English": 0.14, - "Vietnamese,Chinese,Indonesian,English": 0.18, - "Vietnamese,Spanish,Malay,Indonesian": 0.26, - "Vietnamese,Spanish,Malay,English": 0.21333333333333335, - "Vietnamese,Spanish,Indonesian,English": 0.26666666666666666, - "Vietnamese,Malay,Indonesian,English": 0.21333333333333335, - "Chinese,Spanish,Malay,Indonesian": 0.23333333333333334, - "Chinese,Spanish,Malay,English": 0.22, - "Chinese,Spanish,Indonesian,English": 0.29333333333333333, - "Chinese,Malay,Indonesian,English": 0.25333333333333335, - "Spanish,Malay,Indonesian,English": 0.30666666666666664 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.1, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.10666666666666667, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.10666666666666667, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.11333333333333333, - "Filipino,Vietnamese,Chinese,Malay,English": 0.1, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.11333333333333333, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.15333333333333332, - "Filipino,Vietnamese,Spanish,Malay,English": 0.14, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.15333333333333332, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.14, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.14666666666666667, - "Filipino,Chinese,Spanish,Malay,English": 0.14, - "Filipino,Chinese,Spanish,Indonesian,English": 0.16, - "Filipino,Chinese,Malay,Indonesian,English": 0.14, - "Filipino,Spanish,Malay,Indonesian,English": 0.17333333333333334, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, - "Vietnamese,Chinese,Spanish,Malay,English": 0.12, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.15333333333333332, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.12666666666666668, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.2, - "Chinese,Spanish,Malay,Indonesian,English": 0.2 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.09333333333333334, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.08666666666666667, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.08666666666666667, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12666666666666668, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.12, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.07333333333333333 - } - }, - "AC3_2": 0.4627865960699371, - "AC3_3": 0.3517925806096299, - "AC3_4": 0.27361835240852767, - "AC3_5": 0.21323596527687622, - "AC3_6": 0.165382740097134, - "AC3_7": 0.12639138238191902 - }, - "prompt_4": { - "overall_acc": 0.45999999999999996, - "language_acc": { - "Filipino": 0.38, - "Vietnamese": 0.38666666666666666, - "Chinese": 0.47333333333333333, - "Spanish": 0.48, - "Malay": 0.38, - "Indonesian": 0.49333333333333335, - "English": 0.6266666666666667 - }, - "consistency_score_2": 0.47650793650793655, - "consistency_score_3": 0.2994285714285714, - "consistency_score_4": 0.2167619047619047, - "consistency_score_5": 0.1688888888888889, - "consistency_score_6": 0.13714285714285715, - "consistency_score_7": 0.11333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.5333333333333333, - "Filipino,Chinese": 0.46, - "Filipino,Spanish": 0.4266666666666667, - "Filipino,Malay": 0.44, - "Filipino,Indonesian": 0.43333333333333335, - "Filipino,English": 0.38666666666666666, - "Vietnamese,Chinese": 0.36, - "Vietnamese,Spanish": 0.4866666666666667, - "Vietnamese,Malay": 0.43333333333333335, - "Vietnamese,Indonesian": 0.44666666666666666, - "Vietnamese,English": 0.3933333333333333, - "Chinese,Spanish": 0.5, - "Chinese,Malay": 0.44666666666666666, - "Chinese,Indonesian": 0.48, - "Chinese,English": 0.4666666666666667, - "Spanish,Malay": 0.52, - "Spanish,Indonesian": 0.6266666666666667, - "Spanish,English": 0.5533333333333333, - "Malay,Indonesian": 0.56, - "Malay,English": 0.48, - "Indonesian,English": 0.5733333333333334 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.25333333333333335, - "Filipino,Vietnamese,Spanish": 0.3, - "Filipino,Vietnamese,Malay": 0.3, - "Filipino,Vietnamese,Indonesian": 0.2866666666666667, - "Filipino,Vietnamese,English": 0.24666666666666667, - "Filipino,Chinese,Spanish": 0.26666666666666666, - "Filipino,Chinese,Malay": 0.26666666666666666, - "Filipino,Chinese,Indonesian": 0.26, - "Filipino,Chinese,English": 0.24, - "Filipino,Spanish,Malay": 0.2866666666666667, - "Filipino,Spanish,Indonesian": 0.31333333333333335, - "Filipino,Spanish,English": 0.2733333333333333, - "Filipino,Malay,Indonesian": 0.2866666666666667, - "Filipino,Malay,English": 0.28, - "Filipino,Indonesian,English": 0.2866666666666667, - "Vietnamese,Chinese,Spanish": 0.26, - "Vietnamese,Chinese,Malay": 0.24, - "Vietnamese,Chinese,Indonesian": 0.23333333333333334, - "Vietnamese,Chinese,English": 0.22, - "Vietnamese,Spanish,Malay": 0.3, - "Vietnamese,Spanish,Indonesian": 0.3466666666666667, - "Vietnamese,Spanish,English": 0.29333333333333333, - "Vietnamese,Malay,Indonesian": 0.30666666666666664, - "Vietnamese,Malay,English": 0.25333333333333335, - "Vietnamese,Indonesian,English": 0.3, - "Chinese,Spanish,Malay": 0.32, - "Chinese,Spanish,Indonesian": 0.37333333333333335, - "Chinese,Spanish,English": 0.34, - "Chinese,Malay,Indonesian": 0.31333333333333335, - "Chinese,Malay,English": 0.29333333333333333, - "Chinese,Indonesian,English": 0.34, - "Spanish,Malay,Indonesian": 0.41333333333333333, - "Spanish,Malay,English": 0.37333333333333335, - "Spanish,Indonesian,English": 0.44, - "Malay,Indonesian,English": 0.37333333333333335 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.18, - "Filipino,Vietnamese,Chinese,Malay": 0.18666666666666668, - "Filipino,Vietnamese,Chinese,Indonesian": 0.15333333333333332, - "Filipino,Vietnamese,Chinese,English": 0.15333333333333332, - "Filipino,Vietnamese,Spanish,Malay": 0.21333333333333335, - "Filipino,Vietnamese,Spanish,Indonesian": 0.22666666666666666, - "Filipino,Vietnamese,Spanish,English": 0.2, - "Filipino,Vietnamese,Malay,Indonesian": 0.22666666666666666, - "Filipino,Vietnamese,Malay,English": 0.18666666666666668, - "Filipino,Vietnamese,Indonesian,English": 0.2, - "Filipino,Chinese,Spanish,Malay": 0.21333333333333335, - "Filipino,Chinese,Spanish,Indonesian": 0.20666666666666667, - "Filipino,Chinese,Spanish,English": 0.18666666666666668, - "Filipino,Chinese,Malay,Indonesian": 0.2, - "Filipino,Chinese,Malay,English": 0.18666666666666668, - "Filipino,Chinese,Indonesian,English": 0.2, - "Filipino,Spanish,Malay,Indonesian": 0.24666666666666667, - "Filipino,Spanish,Malay,English": 0.22666666666666666, - "Filipino,Spanish,Indonesian,English": 0.24, - "Filipino,Malay,Indonesian,English": 0.22666666666666666, - "Vietnamese,Chinese,Spanish,Malay": 0.19333333333333333, - "Vietnamese,Chinese,Spanish,Indonesian": 0.21333333333333335, - "Vietnamese,Chinese,Spanish,English": 0.19333333333333333, - "Vietnamese,Chinese,Malay,Indonesian": 0.18666666666666668, - "Vietnamese,Chinese,Malay,English": 0.17333333333333334, - "Vietnamese,Chinese,Indonesian,English": 0.18666666666666668, - "Vietnamese,Spanish,Malay,Indonesian": 0.26, - "Vietnamese,Spanish,Malay,English": 0.22, - "Vietnamese,Spanish,Indonesian,English": 0.26666666666666666, - "Vietnamese,Malay,Indonesian,English": 0.22666666666666666, - "Chinese,Spanish,Malay,Indonesian": 0.2733333333333333, - "Chinese,Spanish,Malay,English": 0.24666666666666667, - "Chinese,Spanish,Indonesian,English": 0.29333333333333333, - "Chinese,Malay,Indonesian,English": 0.25333333333333335, - "Spanish,Malay,Indonesian,English": 0.34 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.14666666666666667, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.14, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.13333333333333333, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.14666666666666667, - "Filipino,Vietnamese,Chinese,Malay,English": 0.12666666666666668, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.12666666666666668, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.18666666666666668, - "Filipino,Vietnamese,Spanish,Malay,English": 0.16666666666666666, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.18, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.17333333333333334, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.18666666666666668, - "Filipino,Chinese,Spanish,Malay,English": 0.16666666666666666, - "Filipino,Chinese,Spanish,Indonesian,English": 0.16666666666666666, - "Filipino,Chinese,Malay,Indonesian,English": 0.16666666666666666, - "Filipino,Spanish,Malay,Indonesian,English": 0.21333333333333335, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, - "Vietnamese,Chinese,Spanish,Malay,English": 0.15333333333333332, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.18, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.16, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, - "Chinese,Spanish,Malay,Indonesian,English": 0.24 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.13333333333333333, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.11333333333333333, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.12, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.12, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.16, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.16, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.15333333333333332 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 - } - }, - "AC3_2": 0.46810847452628673, - "AC3_3": 0.36273890138188164, - "AC3_4": 0.29466929351120136, - "AC3_5": 0.24706713776990102, - "AC3_6": 0.21129186599332436, - "AC3_7": 0.18186046508455922 - }, - "prompt_5": { - "overall_acc": 0.4571428571428572, - "language_acc": { - "Filipino": 0.37333333333333335, - "Vietnamese": 0.38666666666666666, - "Chinese": 0.48, - "Spanish": 0.4866666666666667, - "Malay": 0.38666666666666666, - "Indonesian": 0.4866666666666667, - "English": 0.6 - }, - "consistency_score_2": 0.473015873015873, - "consistency_score_3": 0.2942857142857143, - "consistency_score_4": 0.2038095238095238, - "consistency_score_5": 0.1473015873015873, - "consistency_score_6": 0.10857142857142855, - "consistency_score_7": 0.08, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.4533333333333333, - "Filipino,Chinese": 0.43333333333333335, - "Filipino,Spanish": 0.4, - "Filipino,Malay": 0.42, - "Filipino,Indonesian": 0.43333333333333335, - "Filipino,English": 0.38666666666666666, - "Vietnamese,Chinese": 0.34, - "Vietnamese,Spanish": 0.46, - "Vietnamese,Malay": 0.42, - "Vietnamese,Indonesian": 0.5133333333333333, - "Vietnamese,English": 0.38666666666666666, - "Chinese,Spanish": 0.4533333333333333, - "Chinese,Malay": 0.47333333333333333, - "Chinese,Indonesian": 0.4866666666666667, - "Chinese,English": 0.5066666666666667, - "Spanish,Malay": 0.5533333333333333, - "Spanish,Indonesian": 0.6133333333333333, - "Spanish,English": 0.58, - "Malay,Indonesian": 0.6, - "Malay,English": 0.48, - "Indonesian,English": 0.54 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.22, - "Filipino,Vietnamese,Spanish": 0.25333333333333335, - "Filipino,Vietnamese,Malay": 0.26, - "Filipino,Vietnamese,Indonesian": 0.29333333333333333, - "Filipino,Vietnamese,English": 0.22, - "Filipino,Chinese,Spanish": 0.22, - "Filipino,Chinese,Malay": 0.26, - "Filipino,Chinese,Indonesian": 0.26, - "Filipino,Chinese,English": 0.24666666666666667, - "Filipino,Spanish,Malay": 0.2866666666666667, - "Filipino,Spanish,Indonesian": 0.29333333333333333, - "Filipino,Spanish,English": 0.28, - "Filipino,Malay,Indonesian": 0.3, - "Filipino,Malay,English": 0.26, - "Filipino,Indonesian,English": 0.2733333333333333, - "Vietnamese,Chinese,Spanish": 0.22, - "Vietnamese,Chinese,Malay": 0.23333333333333334, - "Vietnamese,Chinese,Indonesian": 0.26, - "Vietnamese,Chinese,English": 0.22666666666666666, - "Vietnamese,Spanish,Malay": 0.2866666666666667, - "Vietnamese,Spanish,Indonesian": 0.35333333333333333, - "Vietnamese,Spanish,English": 0.3, - "Vietnamese,Malay,Indonesian": 0.3333333333333333, - "Vietnamese,Malay,English": 0.23333333333333334, - "Vietnamese,Indonesian,English": 0.29333333333333333, - "Chinese,Spanish,Malay": 0.32, - "Chinese,Spanish,Indonesian": 0.3466666666666667, - "Chinese,Spanish,English": 0.35333333333333333, - "Chinese,Malay,Indonesian": 0.34, - "Chinese,Malay,English": 0.3333333333333333, - "Chinese,Indonesian,English": 0.3466666666666667, - "Spanish,Malay,Indonesian": 0.42, - "Spanish,Malay,English": 0.38, - "Spanish,Indonesian,English": 0.4266666666666667, - "Malay,Indonesian,English": 0.36666666666666664 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.14, - "Filipino,Vietnamese,Chinese,Malay": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,Indonesian": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,English": 0.13333333333333333, - "Filipino,Vietnamese,Spanish,Malay": 0.19333333333333333, - "Filipino,Vietnamese,Spanish,Indonesian": 0.20666666666666667, - "Filipino,Vietnamese,Spanish,English": 0.18, - "Filipino,Vietnamese,Malay,Indonesian": 0.21333333333333335, - "Filipino,Vietnamese,Malay,English": 0.15333333333333332, - "Filipino,Vietnamese,Indonesian,English": 0.18, - "Filipino,Chinese,Spanish,Malay": 0.18666666666666668, - "Filipino,Chinese,Spanish,Indonesian": 0.19333333333333333, - "Filipino,Chinese,Spanish,English": 0.17333333333333334, - "Filipino,Chinese,Malay,Indonesian": 0.20666666666666667, - "Filipino,Chinese,Malay,English": 0.18666666666666668, - "Filipino,Chinese,Indonesian,English": 0.18666666666666668, - "Filipino,Spanish,Malay,Indonesian": 0.24, - "Filipino,Spanish,Malay,English": 0.21333333333333335, - "Filipino,Spanish,Indonesian,English": 0.22666666666666666, - "Filipino,Malay,Indonesian,English": 0.2, - "Vietnamese,Chinese,Spanish,Malay": 0.16666666666666666, - "Vietnamese,Chinese,Spanish,Indonesian": 0.2, - "Vietnamese,Chinese,Spanish,English": 0.18666666666666668, - "Vietnamese,Chinese,Malay,Indonesian": 0.2, - "Vietnamese,Chinese,Malay,English": 0.16666666666666666, - "Vietnamese,Chinese,Indonesian,English": 0.2, - "Vietnamese,Spanish,Malay,Indonesian": 0.25333333333333335, - "Vietnamese,Spanish,Malay,English": 0.2, - "Vietnamese,Spanish,Indonesian,English": 0.25333333333333335, - "Vietnamese,Malay,Indonesian,English": 0.20666666666666667, - "Chinese,Spanish,Malay,Indonesian": 0.26666666666666666, - "Chinese,Spanish,Malay,English": 0.25333333333333335, - "Chinese,Spanish,Indonesian,English": 0.28, - "Chinese,Malay,Indonesian,English": 0.25333333333333335, - "Spanish,Malay,Indonesian,English": 0.3 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.12, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.12666666666666668, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.10666666666666667, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.14666666666666667, - "Filipino,Vietnamese,Chinese,Malay,English": 0.10666666666666667, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.11333333333333333, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.16666666666666666, - "Filipino,Vietnamese,Spanish,Malay,English": 0.13333333333333333, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.15333333333333332, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.13333333333333333, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.17333333333333334, - "Filipino,Chinese,Spanish,Malay,English": 0.14666666666666667, - "Filipino,Chinese,Spanish,Indonesian,English": 0.15333333333333332, - "Filipino,Chinese,Malay,Indonesian,English": 0.14666666666666667, - "Filipino,Spanish,Malay,Indonesian,English": 0.18, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.15333333333333332, - "Vietnamese,Chinese,Spanish,Malay,English": 0.13333333333333333, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.16666666666666666, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.14666666666666667, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.18, - "Chinese,Spanish,Malay,Indonesian,English": 0.20666666666666667 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.11333333333333333, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.08666666666666667, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.09333333333333334, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.09333333333333334, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.12, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.13333333333333333, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.12 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.08 - } - }, - "AC3_2": 0.46494392974036075, - "AC3_3": 0.3580662682847766, - "AC3_4": 0.2819267187716723, - "AC3_5": 0.22280912361259797, - "AC3_6": 0.17546897543795822, - "AC3_7": 0.13617021274060662 - } - }, - "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.39285714285714285, - "language_acc": { - "Indonesian": 0.4431818181818182, - "English": 0.5284090909090909, - "Filipino": 0.2784090909090909, - "Spanish": 0.42045454545454547, - "Chinese": 0.42613636363636365, - "Malay": 0.3693181818181818, - "Vietnamese": 0.2840909090909091 - }, - "consistency_score_2": 0.44859307359307354, - "consistency_score_3": 0.2545454545454546, - "consistency_score_4": 0.16298701298701293, - "consistency_score_5": 0.1130952380952381, - "consistency_score_6": 0.08279220779220779, - "consistency_score_7": 0.0625, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6022727272727273, - "Indonesian,Filipino": 0.375, - "Indonesian,Spanish": 0.5965909090909091, - "Indonesian,Chinese": 0.4147727272727273, - "Indonesian,Malay": 0.5227272727272727, - "Indonesian,Vietnamese": 0.3806818181818182, - "English,Filipino": 0.39204545454545453, - "English,Spanish": 0.5965909090909091, - "English,Chinese": 0.5227272727272727, - "English,Malay": 0.5, - "English,Vietnamese": 0.3352272727272727, - "Filipino,Spanish": 0.4034090909090909, - "Filipino,Chinese": 0.3465909090909091, - "Filipino,Malay": 0.3806818181818182, - "Filipino,Vietnamese": 0.39204545454545453, - "Spanish,Chinese": 0.5056818181818182, - "Spanish,Malay": 0.5568181818181818, - "Spanish,Vietnamese": 0.3465909090909091, - "Chinese,Malay": 0.4431818181818182, - "Chinese,Vietnamese": 0.3806818181818182, - "Malay,Vietnamese": 0.42613636363636365 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.2556818181818182, - "Indonesian,English,Spanish": 0.4431818181818182, - "Indonesian,English,Chinese": 0.3465909090909091, - "Indonesian,English,Malay": 0.35795454545454547, - "Indonesian,English,Vietnamese": 0.23863636363636365, - "Indonesian,Filipino,Spanish": 0.2556818181818182, - "Indonesian,Filipino,Chinese": 0.18181818181818182, - "Indonesian,Filipino,Malay": 0.2159090909090909, - "Indonesian,Filipino,Vietnamese": 0.18181818181818182, - "Indonesian,Spanish,Chinese": 0.3352272727272727, - "Indonesian,Spanish,Malay": 0.39204545454545453, - "Indonesian,Spanish,Vietnamese": 0.25, - "Indonesian,Chinese,Malay": 0.26704545454545453, - "Indonesian,Chinese,Vietnamese": 0.1875, - "Indonesian,Malay,Vietnamese": 0.23863636363636365, - "English,Filipino,Spanish": 0.26136363636363635, - "English,Filipino,Chinese": 0.2215909090909091, - "English,Filipino,Malay": 0.2159090909090909, - "English,Filipino,Vietnamese": 0.17613636363636365, - "English,Spanish,Chinese": 0.35795454545454547, - "English,Spanish,Malay": 0.3693181818181818, - "English,Spanish,Vietnamese": 0.2215909090909091, - "English,Chinese,Malay": 0.3068181818181818, - "English,Chinese,Vietnamese": 0.2215909090909091, - "English,Malay,Vietnamese": 0.2215909090909091, - "Filipino,Spanish,Chinese": 0.2215909090909091, - "Filipino,Spanish,Malay": 0.24431818181818182, - "Filipino,Spanish,Vietnamese": 0.17613636363636365, - "Filipino,Chinese,Malay": 0.19886363636363635, - "Filipino,Chinese,Vietnamese": 0.17045454545454544, - "Filipino,Malay,Vietnamese": 0.1875, - "Spanish,Chinese,Malay": 0.3181818181818182, - "Spanish,Chinese,Vietnamese": 0.21022727272727273, - "Spanish,Malay,Vietnamese": 0.24431818181818182, - "Chinese,Malay,Vietnamese": 0.2159090909090909 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.19886363636363635, - "Indonesian,English,Filipino,Chinese": 0.1534090909090909, - "Indonesian,English,Filipino,Malay": 0.14772727272727273, - "Indonesian,English,Filipino,Vietnamese": 0.125, - "Indonesian,English,Spanish,Chinese": 0.2784090909090909, - "Indonesian,English,Spanish,Malay": 0.30113636363636365, - "Indonesian,English,Spanish,Vietnamese": 0.1875, - "Indonesian,English,Chinese,Malay": 0.22727272727272727, - "Indonesian,English,Chinese,Vietnamese": 0.1534090909090909, - "Indonesian,English,Malay,Vietnamese": 0.17045454545454544, - "Indonesian,Filipino,Spanish,Chinese": 0.16477272727272727, - "Indonesian,Filipino,Spanish,Malay": 0.16477272727272727, - "Indonesian,Filipino,Spanish,Vietnamese": 0.125, - "Indonesian,Filipino,Chinese,Malay": 0.125, - "Indonesian,Filipino,Chinese,Vietnamese": 0.09659090909090909, - "Indonesian,Filipino,Malay,Vietnamese": 0.10795454545454546, - "Indonesian,Spanish,Chinese,Malay": 0.23863636363636365, - "Indonesian,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "Indonesian,Spanish,Malay,Vietnamese": 0.19318181818181818, - "Indonesian,Chinese,Malay,Vietnamese": 0.13636363636363635, - "English,Filipino,Spanish,Chinese": 0.17045454545454544, - "English,Filipino,Spanish,Malay": 0.16477272727272727, - "English,Filipino,Spanish,Vietnamese": 0.11931818181818182, - "English,Filipino,Chinese,Malay": 0.14772727272727273, - "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Malay": 0.25, - "English,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Chinese,Malay,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Malay": 0.1534090909090909, - "Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, - "Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, - "Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.13636363636363635, - "Indonesian,English,Filipino,Spanish,Malay": 0.125, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.09659090909090909, - "Indonesian,English,Filipino,Chinese,Malay": 0.10227272727272728, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07954545454545454, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.07954545454545454, - "Indonesian,English,Spanish,Chinese,Malay": 0.20454545454545456, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.1534090909090909, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.11931818181818182, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Malay": 0.125, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, - "English,Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, - "English,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.09659090909090909, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.07386363636363637, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.07386363636363637, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0625, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 - } - }, - "AC3_2": 0.41887919149822117, - "AC3_3": 0.30892678029330517, - "AC3_4": 0.23039052065281018, - "AC3_5": 0.17563025206612753, - "AC3_6": 0.13676255482253924, - "AC3_7": 0.10784313723121876 - }, - "prompt_2": { - "overall_acc": 0.3920454545454546, - "language_acc": { - "Indonesian": 0.4318181818181818, - "English": 0.5284090909090909, - "Filipino": 0.2727272727272727, - "Spanish": 0.4147727272727273, - "Chinese": 0.42045454545454547, - "Malay": 0.3806818181818182, - "Vietnamese": 0.29545454545454547 - }, - "consistency_score_2": 0.4586038961038961, - "consistency_score_3": 0.2646103896103896, - "consistency_score_4": 0.17224025974025975, - "consistency_score_5": 0.12067099567099568, - "consistency_score_6": 0.08928571428571429, - "consistency_score_7": 0.06818181818181818, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.5738636363636364, - "Indonesian,Filipino": 0.3977272727272727, - "Indonesian,Spanish": 0.6193181818181818, - "Indonesian,Chinese": 0.4431818181818182, - "Indonesian,Malay": 0.5511363636363636, - "Indonesian,Vietnamese": 0.375, - "English,Filipino": 0.39204545454545453, - "English,Spanish": 0.5568181818181818, - "English,Chinese": 0.5738636363636364, - "English,Malay": 0.4943181818181818, - "English,Vietnamese": 0.39204545454545453, - "Filipino,Spanish": 0.38636363636363635, - "Filipino,Chinese": 0.3693181818181818, - "Filipino,Malay": 0.3806818181818182, - "Filipino,Vietnamese": 0.4318181818181818, - "Spanish,Chinese": 0.4943181818181818, - "Spanish,Malay": 0.5511363636363636, - "Spanish,Vietnamese": 0.39204545454545453, - "Chinese,Malay": 0.4375, - "Chinese,Vietnamese": 0.39204545454545453, - "Malay,Vietnamese": 0.42613636363636365 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.26704545454545453, - "Indonesian,English,Spanish": 0.42613636363636365, - "Indonesian,English,Chinese": 0.36363636363636365, - "Indonesian,English,Malay": 0.3522727272727273, - "Indonesian,English,Vietnamese": 0.25, - "Indonesian,Filipino,Spanish": 0.2727272727272727, - "Indonesian,Filipino,Chinese": 0.19886363636363635, - "Indonesian,Filipino,Malay": 0.26136363636363635, - "Indonesian,Filipino,Vietnamese": 0.21022727272727273, - "Indonesian,Spanish,Chinese": 0.3352272727272727, - "Indonesian,Spanish,Malay": 0.4034090909090909, - "Indonesian,Spanish,Vietnamese": 0.2727272727272727, - "Indonesian,Chinese,Malay": 0.2784090909090909, - "Indonesian,Chinese,Vietnamese": 0.21022727272727273, - "Indonesian,Malay,Vietnamese": 0.26136363636363635, - "English,Filipino,Spanish": 0.23295454545454544, - "English,Filipino,Chinese": 0.23295454545454544, - "English,Filipino,Malay": 0.2159090909090909, - "English,Filipino,Vietnamese": 0.21022727272727273, - "English,Spanish,Chinese": 0.36363636363636365, - "English,Spanish,Malay": 0.35795454545454547, - "English,Spanish,Vietnamese": 0.23295454545454544, - "English,Chinese,Malay": 0.3068181818181818, - "English,Chinese,Vietnamese": 0.25, - "English,Malay,Vietnamese": 0.23295454545454544, - "Filipino,Spanish,Chinese": 0.2215909090909091, - "Filipino,Spanish,Malay": 0.24431818181818182, - "Filipino,Spanish,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Malay": 0.17613636363636365, - "Filipino,Chinese,Vietnamese": 0.18181818181818182, - "Filipino,Malay,Vietnamese": 0.20454545454545456, - "Spanish,Chinese,Malay": 0.29545454545454547, - "Spanish,Chinese,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Vietnamese": 0.2840909090909091, - "Chinese,Malay,Vietnamese": 0.2215909090909091 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.19318181818181818, - "Indonesian,English,Filipino,Chinese": 0.17045454545454544, - "Indonesian,English,Filipino,Malay": 0.18181818181818182, - "Indonesian,English,Filipino,Vietnamese": 0.1590909090909091, - "Indonesian,English,Spanish,Chinese": 0.2897727272727273, - "Indonesian,English,Spanish,Malay": 0.2897727272727273, - "Indonesian,English,Spanish,Vietnamese": 0.19318181818181818, - "Indonesian,English,Chinese,Malay": 0.24431818181818182, - "Indonesian,English,Chinese,Vietnamese": 0.16477272727272727, - "Indonesian,English,Malay,Vietnamese": 0.18181818181818182, - "Indonesian,Filipino,Spanish,Chinese": 0.16477272727272727, - "Indonesian,Filipino,Spanish,Malay": 0.19886363636363635, - "Indonesian,Filipino,Spanish,Vietnamese": 0.1590909090909091, - "Indonesian,Filipino,Chinese,Malay": 0.13068181818181818, - "Indonesian,Filipino,Chinese,Vietnamese": 0.10227272727272728, - "Indonesian,Filipino,Malay,Vietnamese": 0.1590909090909091, - "Indonesian,Spanish,Chinese,Malay": 0.23295454545454544, - "Indonesian,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "Indonesian,Spanish,Malay,Vietnamese": 0.2159090909090909, - "Indonesian,Chinese,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Chinese": 0.1590909090909091, - "English,Filipino,Spanish,Malay": 0.16477272727272727, - "English,Filipino,Spanish,Vietnamese": 0.13068181818181818, - "English,Filipino,Chinese,Malay": 0.13068181818181818, - "English,Filipino,Chinese,Vietnamese": 0.125, - "English,Filipino,Malay,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Malay": 0.24431818181818182, - "English,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Vietnamese": 0.17613636363636365, - "English,Chinese,Malay,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Malay": 0.125, - "Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, - "Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.13636363636363635, - "Indonesian,English,Filipino,Spanish,Malay": 0.14204545454545456, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.11931818181818182, - "Indonesian,English,Filipino,Chinese,Malay": 0.11363636363636363, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09090909090909091, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.11931818181818182, - "Indonesian,English,Spanish,Chinese,Malay": 0.20454545454545456, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.1534090909090909, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.13068181818181818, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese,Malay": 0.10795454545454546, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, - "English,Spanish,Chinese,Malay,Vietnamese": 0.13636363636363635, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.09659090909090909, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.11931818181818182, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 - } - }, - "AC3_2": 0.42272076677889747, - "AC3_3": 0.31596246764488234, - "AC3_4": 0.2393326968924125, - "AC3_5": 0.18454065719595736, - "AC3_6": 0.14544688023959687, - "AC3_7": 0.1161616161363761 - }, - "prompt_3": { - "overall_acc": 0.3871753246753246, - "language_acc": { - "Indonesian": 0.42045454545454547, - "English": 0.5227272727272727, - "Filipino": 0.2727272727272727, - "Spanish": 0.42613636363636365, - "Chinese": 0.4147727272727273, - "Malay": 0.3693181818181818, - "Vietnamese": 0.2840909090909091 - }, - "consistency_score_2": 0.4548160173160173, - "consistency_score_3": 0.26282467532467546, - "consistency_score_4": 0.17029220779220777, - "consistency_score_5": 0.11742424242424242, - "consistency_score_6": 0.0844155844155844, - "consistency_score_7": 0.0625, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.5625, - "Indonesian,Filipino": 0.3977272727272727, - "Indonesian,Spanish": 0.5965909090909091, - "Indonesian,Chinese": 0.44886363636363635, - "Indonesian,Malay": 0.5568181818181818, - "Indonesian,Vietnamese": 0.3693181818181818, - "English,Filipino": 0.4034090909090909, - "English,Spanish": 0.5568181818181818, - "English,Chinese": 0.5511363636363636, - "English,Malay": 0.5056818181818182, - "English,Vietnamese": 0.38636363636363635, - "Filipino,Spanish": 0.3806818181818182, - "Filipino,Chinese": 0.375, - "Filipino,Malay": 0.3693181818181818, - "Filipino,Vietnamese": 0.4147727272727273, - "Spanish,Chinese": 0.5, - "Spanish,Malay": 0.5454545454545454, - "Spanish,Vietnamese": 0.39204545454545453, - "Chinese,Malay": 0.4431818181818182, - "Chinese,Vietnamese": 0.36363636363636365, - "Malay,Vietnamese": 0.4318181818181818 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.26704545454545453, - "Indonesian,English,Spanish": 0.4147727272727273, - "Indonesian,English,Chinese": 0.3693181818181818, - "Indonesian,English,Malay": 0.35795454545454547, - "Indonesian,English,Vietnamese": 0.2556818181818182, - "Indonesian,Filipino,Spanish": 0.2556818181818182, - "Indonesian,Filipino,Chinese": 0.20454545454545456, - "Indonesian,Filipino,Malay": 0.25, - "Indonesian,Filipino,Vietnamese": 0.19886363636363635, - "Indonesian,Spanish,Chinese": 0.3352272727272727, - "Indonesian,Spanish,Malay": 0.3977272727272727, - "Indonesian,Spanish,Vietnamese": 0.26704545454545453, - "Indonesian,Chinese,Malay": 0.29545454545454547, - "Indonesian,Chinese,Vietnamese": 0.19886363636363635, - "Indonesian,Malay,Vietnamese": 0.26136363636363635, - "English,Filipino,Spanish": 0.22727272727272727, - "English,Filipino,Chinese": 0.24431818181818182, - "English,Filipino,Malay": 0.2215909090909091, - "English,Filipino,Vietnamese": 0.22727272727272727, - "English,Spanish,Chinese": 0.35795454545454547, - "English,Spanish,Malay": 0.35795454545454547, - "English,Spanish,Vietnamese": 0.23295454545454544, - "English,Chinese,Malay": 0.3125, - "English,Chinese,Vietnamese": 0.22727272727272727, - "English,Malay,Vietnamese": 0.24431818181818182, - "Filipino,Spanish,Chinese": 0.22727272727272727, - "Filipino,Spanish,Malay": 0.22727272727272727, - "Filipino,Spanish,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Malay": 0.1875, - "Filipino,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Vietnamese": 0.19886363636363635, - "Spanish,Chinese,Malay": 0.30113636363636365, - "Spanish,Chinese,Vietnamese": 0.2215909090909091, - "Spanish,Malay,Vietnamese": 0.2784090909090909, - "Chinese,Malay,Vietnamese": 0.21022727272727273 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.1875, - "Indonesian,English,Filipino,Chinese": 0.17613636363636365, - "Indonesian,English,Filipino,Malay": 0.18181818181818182, - "Indonesian,English,Filipino,Vietnamese": 0.16477272727272727, - "Indonesian,English,Spanish,Chinese": 0.2897727272727273, - "Indonesian,English,Spanish,Malay": 0.2897727272727273, - "Indonesian,English,Spanish,Vietnamese": 0.19886363636363635, - "Indonesian,English,Chinese,Malay": 0.25, - "Indonesian,English,Chinese,Vietnamese": 0.1590909090909091, - "Indonesian,English,Malay,Vietnamese": 0.1875, - "Indonesian,Filipino,Spanish,Chinese": 0.1590909090909091, - "Indonesian,Filipino,Spanish,Malay": 0.18181818181818182, - "Indonesian,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "Indonesian,Filipino,Chinese,Malay": 0.14204545454545456, - "Indonesian,Filipino,Chinese,Vietnamese": 0.10227272727272728, - "Indonesian,Filipino,Malay,Vietnamese": 0.14204545454545456, - "Indonesian,Spanish,Chinese,Malay": 0.23863636363636365, - "Indonesian,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "Indonesian,Spanish,Malay,Vietnamese": 0.21022727272727273, - "Indonesian,Chinese,Malay,Vietnamese": 0.14204545454545456, - "English,Filipino,Spanish,Chinese": 0.1534090909090909, - "English,Filipino,Spanish,Malay": 0.1534090909090909, - "English,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "English,Filipino,Chinese,Malay": 0.14772727272727273, - "English,Filipino,Chinese,Vietnamese": 0.13068181818181818, - "English,Filipino,Malay,Vietnamese": 0.14204545454545456, - "English,Spanish,Chinese,Malay": 0.24431818181818182, - "English,Spanish,Chinese,Vietnamese": 0.16477272727272727, - "English,Spanish,Malay,Vietnamese": 0.18181818181818182, - "English,Chinese,Malay,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Chinese,Malay": 0.13068181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.11931818181818182, - "Filipino,Spanish,Malay,Vietnamese": 0.13068181818181818, - "Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, - "Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.13068181818181818, - "Indonesian,English,Filipino,Spanish,Malay": 0.13636363636363635, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.125, - "Indonesian,English,Filipino,Chinese,Malay": 0.11931818181818182, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09659090909090909, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.11931818181818182, - "Indonesian,English,Spanish,Chinese,Malay": 0.20454545454545456, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.14204545454545456, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.1590909090909091, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.11931818181818182, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.07954545454545454, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.125, - "English,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Malay,Vietnamese": 0.10227272727272728, - "English,Filipino,Chinese,Malay,Vietnamese": 0.09090909090909091, - "English,Spanish,Chinese,Malay,Vietnamese": 0.125, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.09090909090909091, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 - } - }, - "AC3_2": 0.41827874081100436, - "AC3_3": 0.31310531995650065, - "AC3_4": 0.23654450524843318, - "AC3_5": 0.1801974164887833, - "AC3_6": 0.1386100989937879, - "AC3_7": 0.10762635376667948 - }, - "prompt_4": { - "overall_acc": 0.39529220779220775, - "language_acc": { - "Indonesian": 0.45454545454545453, - "English": 0.5227272727272727, - "Filipino": 0.2727272727272727, - "Spanish": 0.42613636363636365, - "Chinese": 0.4318181818181818, - "Malay": 0.3693181818181818, - "Vietnamese": 0.2897727272727273 - }, - "consistency_score_2": 0.4558982683982683, - "consistency_score_3": 0.26185064935064933, - "consistency_score_4": 0.16899350649350647, - "consistency_score_5": 0.11742424242424244, - "consistency_score_6": 0.08522727272727273, - "consistency_score_7": 0.0625, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6022727272727273, - "Indonesian,Filipino": 0.39204545454545453, - "Indonesian,Spanish": 0.5965909090909091, - "Indonesian,Chinese": 0.44886363636363635, - "Indonesian,Malay": 0.5340909090909091, - "Indonesian,Vietnamese": 0.38636363636363635, - "English,Filipino": 0.3806818181818182, - "English,Spanish": 0.6136363636363636, - "English,Chinese": 0.5227272727272727, - "English,Malay": 0.5, - "English,Vietnamese": 0.39204545454545453, - "Filipino,Spanish": 0.4147727272727273, - "Filipino,Chinese": 0.36363636363636365, - "Filipino,Malay": 0.3806818181818182, - "Filipino,Vietnamese": 0.39204545454545453, - "Spanish,Chinese": 0.5, - "Spanish,Malay": 0.5625, - "Spanish,Vietnamese": 0.3693181818181818, - "Chinese,Malay": 0.4375, - "Chinese,Vietnamese": 0.36363636363636365, - "Malay,Vietnamese": 0.42045454545454547 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.26136363636363635, - "Indonesian,English,Spanish": 0.45454545454545453, - "Indonesian,English,Chinese": 0.3693181818181818, - "Indonesian,English,Malay": 0.35795454545454547, - "Indonesian,English,Vietnamese": 0.26136363636363635, - "Indonesian,Filipino,Spanish": 0.26136363636363635, - "Indonesian,Filipino,Chinese": 0.1875, - "Indonesian,Filipino,Malay": 0.22727272727272727, - "Indonesian,Filipino,Vietnamese": 0.19318181818181818, - "Indonesian,Spanish,Chinese": 0.3465909090909091, - "Indonesian,Spanish,Malay": 0.4034090909090909, - "Indonesian,Spanish,Vietnamese": 0.26704545454545453, - "Indonesian,Chinese,Malay": 0.2784090909090909, - "Indonesian,Chinese,Vietnamese": 0.2159090909090909, - "Indonesian,Malay,Vietnamese": 0.2556818181818182, - "English,Filipino,Spanish": 0.25, - "English,Filipino,Chinese": 0.23295454545454544, - "English,Filipino,Malay": 0.20454545454545456, - "English,Filipino,Vietnamese": 0.18181818181818182, - "English,Spanish,Chinese": 0.375, - "English,Spanish,Malay": 0.38636363636363635, - "English,Spanish,Vietnamese": 0.24431818181818182, - "English,Chinese,Malay": 0.29545454545454547, - "English,Chinese,Vietnamese": 0.23295454545454544, - "English,Malay,Vietnamese": 0.23863636363636365, - "Filipino,Spanish,Chinese": 0.2215909090909091, - "Filipino,Spanish,Malay": 0.23863636363636365, - "Filipino,Spanish,Vietnamese": 0.18181818181818182, - "Filipino,Chinese,Malay": 0.18181818181818182, - "Filipino,Chinese,Vietnamese": 0.16477272727272727, - "Filipino,Malay,Vietnamese": 0.19318181818181818, - "Spanish,Chinese,Malay": 0.29545454545454547, - "Spanish,Chinese,Vietnamese": 0.22727272727272727, - "Spanish,Malay,Vietnamese": 0.26704545454545453, - "Chinese,Malay,Vietnamese": 0.21022727272727273 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.20454545454545456, - "Indonesian,English,Filipino,Chinese": 0.17045454545454544, - "Indonesian,English,Filipino,Malay": 0.14772727272727273, - "Indonesian,English,Filipino,Vietnamese": 0.13068181818181818, - "Indonesian,English,Spanish,Chinese": 0.30113636363636365, - "Indonesian,English,Spanish,Malay": 0.3125, - "Indonesian,English,Spanish,Vietnamese": 0.20454545454545456, - "Indonesian,English,Chinese,Malay": 0.23863636363636365, - "Indonesian,English,Chinese,Vietnamese": 0.16477272727272727, - "Indonesian,English,Malay,Vietnamese": 0.18181818181818182, - "Indonesian,Filipino,Spanish,Chinese": 0.16477272727272727, - "Indonesian,Filipino,Spanish,Malay": 0.17045454545454544, - "Indonesian,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "Indonesian,Filipino,Chinese,Malay": 0.11931818181818182, - "Indonesian,Filipino,Chinese,Vietnamese": 0.09659090909090909, - "Indonesian,Filipino,Malay,Vietnamese": 0.13068181818181818, - "Indonesian,Spanish,Chinese,Malay": 0.24431818181818182, - "Indonesian,Spanish,Chinese,Vietnamese": 0.1875, - "Indonesian,Spanish,Malay,Vietnamese": 0.21022727272727273, - "Indonesian,Chinese,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Chinese": 0.16477272727272727, - "English,Filipino,Spanish,Malay": 0.1590909090909091, - "English,Filipino,Spanish,Vietnamese": 0.11931818181818182, - "English,Filipino,Chinese,Malay": 0.14772727272727273, - "English,Filipino,Chinese,Vietnamese": 0.11363636363636363, - "English,Filipino,Malay,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Malay": 0.25, - "English,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Vietnamese": 0.1875, - "English,Chinese,Malay,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Malay": 0.13636363636363635, - "Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "Filipino,Spanish,Malay,Vietnamese": 0.125, - "Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, - "Spanish,Chinese,Malay,Vietnamese": 0.16477272727272727 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.14772727272727273, - "Indonesian,English,Filipino,Spanish,Malay": 0.13068181818181818, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.10795454545454546, - "Indonesian,English,Filipino,Chinese,Malay": 0.10795454545454546, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.07954545454545454, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.09090909090909091, - "Indonesian,English,Spanish,Chinese,Malay": 0.2159090909090909, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.1534090909090909, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.16477272727272727, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.125, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.11363636363636363, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.09659090909090909, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.10795454545454546, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.07386363636363637, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.14772727272727273, - "English,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.08522727272727272, - "English,Filipino,Spanish,Malay,Vietnamese": 0.09090909090909091, - "English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, - "English,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.10227272727272728, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.07954545454545454, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.08522727272727272, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.0625, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.125, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07386363636363637, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.0625 - } - }, - "AC3_2": 0.4234376160531587, - "AC3_3": 0.315022890845753, - "AC3_4": 0.23676593109249963, - "AC3_5": 0.18106260490591855, - "AC3_6": 0.14022189800521673, - "AC3_7": 0.1079343971395434 - }, - "prompt_5": { - "overall_acc": 0.4090909090909091, - "language_acc": { - "Indonesian": 0.4943181818181818, - "English": 0.5397727272727273, - "Filipino": 0.29545454545454547, - "Spanish": 0.4318181818181818, - "Chinese": 0.4375, - "Malay": 0.3693181818181818, - "Vietnamese": 0.29545454545454547 - }, - "consistency_score_2": 0.4632034632034633, - "consistency_score_3": 0.27256493506493495, - "consistency_score_4": 0.18035714285714288, - "consistency_score_5": 0.12743506493506493, - "consistency_score_6": 0.09334415584415585, - "consistency_score_7": 0.06818181818181818, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.6193181818181818, - "Indonesian,Filipino": 0.39204545454545453, - "Indonesian,Spanish": 0.6079545454545454, - "Indonesian,Chinese": 0.44886363636363635, - "Indonesian,Malay": 0.5965909090909091, - "Indonesian,Vietnamese": 0.38636363636363635, - "English,Filipino": 0.39204545454545453, - "English,Spanish": 0.5909090909090909, - "English,Chinese": 0.5397727272727273, - "English,Malay": 0.5284090909090909, - "English,Vietnamese": 0.4147727272727273, - "Filipino,Spanish": 0.42045454545454547, - "Filipino,Chinese": 0.375, - "Filipino,Malay": 0.35795454545454547, - "Filipino,Vietnamese": 0.4090909090909091, - "Spanish,Chinese": 0.4715909090909091, - "Spanish,Malay": 0.5681818181818182, - "Spanish,Vietnamese": 0.3977272727272727, - "Chinese,Malay": 0.42045454545454547, - "Chinese,Vietnamese": 0.35795454545454547, - "Malay,Vietnamese": 0.4318181818181818 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.26704545454545453, - "Indonesian,English,Spanish": 0.45454545454545453, - "Indonesian,English,Chinese": 0.38636363636363635, - "Indonesian,English,Malay": 0.4034090909090909, - "Indonesian,English,Vietnamese": 0.2897727272727273, - "Indonesian,Filipino,Spanish": 0.2840909090909091, - "Indonesian,Filipino,Chinese": 0.18181818181818182, - "Indonesian,Filipino,Malay": 0.23863636363636365, - "Indonesian,Filipino,Vietnamese": 0.1875, - "Indonesian,Spanish,Chinese": 0.3352272727272727, - "Indonesian,Spanish,Malay": 0.4431818181818182, - "Indonesian,Spanish,Vietnamese": 0.2727272727272727, - "Indonesian,Chinese,Malay": 0.30113636363636365, - "Indonesian,Chinese,Vietnamese": 0.2159090909090909, - "Indonesian,Malay,Vietnamese": 0.2727272727272727, - "English,Filipino,Spanish": 0.2556818181818182, - "English,Filipino,Chinese": 0.24431818181818182, - "English,Filipino,Malay": 0.20454545454545456, - "English,Filipino,Vietnamese": 0.21022727272727273, - "English,Spanish,Chinese": 0.36363636363636365, - "English,Spanish,Malay": 0.4090909090909091, - "English,Spanish,Vietnamese": 0.26704545454545453, - "English,Chinese,Malay": 0.3068181818181818, - "English,Chinese,Vietnamese": 0.25, - "English,Malay,Vietnamese": 0.2727272727272727, - "Filipino,Spanish,Chinese": 0.2215909090909091, - "Filipino,Spanish,Malay": 0.2556818181818182, - "Filipino,Spanish,Vietnamese": 0.20454545454545456, - "Filipino,Chinese,Malay": 0.16477272727272727, - "Filipino,Chinese,Vietnamese": 0.17613636363636365, - "Filipino,Malay,Vietnamese": 0.19318181818181818, - "Spanish,Chinese,Malay": 0.2840909090909091, - "Spanish,Chinese,Vietnamese": 0.23295454545454544, - "Spanish,Malay,Vietnamese": 0.2840909090909091, - "Chinese,Malay,Vietnamese": 0.20454545454545456 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.20454545454545456, - "Indonesian,English,Filipino,Chinese": 0.16477272727272727, - "Indonesian,English,Filipino,Malay": 0.1590909090909091, - "Indonesian,English,Filipino,Vietnamese": 0.1590909090909091, - "Indonesian,English,Spanish,Chinese": 0.3125, - "Indonesian,English,Spanish,Malay": 0.3352272727272727, - "Indonesian,English,Spanish,Vietnamese": 0.2159090909090909, - "Indonesian,English,Chinese,Malay": 0.26136363636363635, - "Indonesian,English,Chinese,Vietnamese": 0.1875, - "Indonesian,English,Malay,Vietnamese": 0.2159090909090909, - "Indonesian,Filipino,Spanish,Chinese": 0.1534090909090909, - "Indonesian,Filipino,Spanish,Malay": 0.19886363636363635, - "Indonesian,Filipino,Spanish,Vietnamese": 0.14772727272727273, - "Indonesian,Filipino,Chinese,Malay": 0.11931818181818182, - "Indonesian,Filipino,Chinese,Vietnamese": 0.10795454545454546, - "Indonesian,Filipino,Malay,Vietnamese": 0.13636363636363635, - "Indonesian,Spanish,Chinese,Malay": 0.24431818181818182, - "Indonesian,Spanish,Chinese,Vietnamese": 0.1875, - "Indonesian,Spanish,Malay,Vietnamese": 0.22727272727272727, - "Indonesian,Chinese,Malay,Vietnamese": 0.16477272727272727, - "English,Filipino,Spanish,Chinese": 0.17045454545454544, - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Vietnamese": 0.14772727272727273, - "English,Filipino,Chinese,Malay": 0.13636363636363635, - "English,Filipino,Chinese,Vietnamese": 0.125, - "English,Filipino,Malay,Vietnamese": 0.13068181818181818, - "English,Spanish,Chinese,Malay": 0.26136363636363635, - "English,Spanish,Chinese,Vietnamese": 0.19318181818181818, - "English,Spanish,Malay,Vietnamese": 0.2159090909090909, - "English,Chinese,Malay,Vietnamese": 0.17045454545454544, - "Filipino,Spanish,Chinese,Malay": 0.13068181818181818, - "Filipino,Spanish,Chinese,Vietnamese": 0.125, - "Filipino,Spanish,Malay,Vietnamese": 0.14772727272727273, - "Filipino,Chinese,Malay,Vietnamese": 0.10795454545454546, - "Spanish,Chinese,Malay,Vietnamese": 0.17045454545454544 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.14204545454545456, - "Indonesian,English,Filipino,Spanish,Malay": 0.14204545454545456, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.125, - "Indonesian,English,Filipino,Chinese,Malay": 0.10227272727272728, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.09090909090909091, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.10795454545454546, - "Indonesian,English,Spanish,Chinese,Malay": 0.22727272727272727, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.17045454545454544, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.18181818181818182, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.14772727272727273, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.10795454545454546, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.11931818181818182, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.10227272727272728, - "English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, - "English,Filipino,Chinese,Malay,Vietnamese": 0.08522727272727272, - "English,Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09090909090909091 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.09659090909090909, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.09090909090909091, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.09659090909090909, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.06818181818181818, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.14204545454545456, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.07954545454545454 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.06818181818181818 - } - }, - "AC3_2": 0.4344687569995947, - "AC3_3": 0.3271558162719087, - "AC3_4": 0.25034425773775487, - "AC3_5": 0.19433365420666196, - "AC3_6": 0.1520046996319582, - "AC3_7": 0.11688311685862708 - } - }, - "sg_eval": { - "prompt_1": { - "accuracy": 0.6310679611650486 - }, - "prompt_2": { - "accuracy": 0.6310679611650486 - }, - "prompt_3": { - "accuracy": 0.6504854368932039 - }, - "prompt_4": { - "accuracy": 0.6310679611650486 - }, - "prompt_5": { - "accuracy": 0.6407766990291263 - } - }, - "cn_eval": { - "prompt_1": { - "accuracy": 0.3523809523809524 - }, - "prompt_2": { - "accuracy": 0.34285714285714286 - }, - "prompt_3": { - "accuracy": 0.3333333333333333 - }, - "prompt_4": { - "accuracy": 0.3523809523809524 - }, - "prompt_5": { - "accuracy": 0.4 - } - }, - "us_eval": { - "prompt_1": { - "accuracy": 0.616822429906542 - }, - "prompt_2": { - "accuracy": 0.5981308411214953 - }, - "prompt_3": { - "accuracy": 0.6074766355140186 - }, - "prompt_4": { - "accuracy": 0.6448598130841121 - }, - "prompt_5": { - "accuracy": 0.6355140186915887 - } - }, - "ph_eval": { - "prompt_1": { - "accuracy": 0.46, - "category_acc": { - "brand": 0.3, - "demographics": 0.0, - "biology": 0.5, - "history": 0.3333333333333333, - "literature": 0.6, - "politics": 0.6, - "culture": 0.6, - "film": 0.6, - "law": 0.2, - "geography": 0.7 - } - }, - "prompt_2": { - "accuracy": 0.46, - "category_acc": { - "brand": 0.3, - "demographics": 0.0, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.5, - "politics": 0.6, - "culture": 0.6, - "film": 0.6, - "law": 0.3, - "geography": 0.8 - } - }, - "prompt_3": { - "accuracy": 0.44, - "category_acc": { - "brand": 0.2, - "demographics": 0.0, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.5, - "politics": 0.6, - "culture": 0.6, - "film": 0.6, - "law": 0.3, - "geography": 0.7 - } - }, - "prompt_4": { - "accuracy": 0.49, - "category_acc": { - "brand": 0.3, - "demographics": 0.2, - "biology": 0.5, - "history": 0.3333333333333333, - "literature": 0.6, - "politics": 0.6, - "culture": 0.7, - "film": 0.6, - "law": 0.3, - "geography": 0.7 - } - }, - "prompt_5": { - "accuracy": 0.47, - "category_acc": { - "brand": 0.2, - "demographics": 0.2, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.5, - "politics": 0.6, - "culture": 0.8, - "film": 0.6, - "law": 0.3, - "geography": 0.7 - } - } - }, - "sing2eng": { - "prompt_1": { - "bleu_score": 0.21849742595272317 - }, - "prompt_2": { - "bleu_score": 0.22760683632659018 - }, - "prompt_3": { - "bleu_score": 0.2170682835782222 - }, - "prompt_4": { - "bleu_score": 0.19995124300308736 - }, - "prompt_5": { - "bleu_score": 0.12598005546495297 - } - }, - "indommlu": { - "prompt_1": { - "accuracy": 0.4386808198144068, - "category_acc": { - "History": 0.41967871485943775, - "Geography": 0.41020408163265304, - "Lampungic": 0.29931972789115646, - "Social science": 0.666110183639399, - "Balinese": 0.3354564755838641, - "Makassarese": 0.3279569892473118, - "Banjarese": 0.3888888888888889, - "Chemistry": 0.2846715328467153, - "Biology": 0.39289940828402364, - "Science": 0.5510835913312694, - "Christian religion": 0.5920398009950248, - "Art": 0.5207986688851913, - "Islam religion": 0.534850640113798, - "Hindu religion": 0.4066666666666667, - "Madurese": 0.3016949152542373, - "Sport": 0.5067567567567568, - "Indonesian language": 0.4965753424657534, - "Physics": 0.3434343434343434, - "Minangkabau culture": 0.3316582914572864, - "Dayak language": 0.3211009174311927, - "Sociology": 0.4415322580645161, - "Economy": 0.4016393442622951, - "Sundanese": 0.32843560933448573, - "Javanese": 0.3215725806451613, - "Civic education": 0.5278969957081545 - } - }, - "prompt_2": { - "accuracy": 0.4384137792910074, - "category_acc": { - "History": 0.40963855421686746, - "Geography": 0.4163265306122449, - "Lampungic": 0.3129251700680272, - "Social science": 0.676126878130217, - "Balinese": 0.3184713375796178, - "Makassarese": 0.34946236559139787, - "Banjarese": 0.375, - "Chemistry": 0.27883211678832115, - "Biology": 0.3940828402366864, - "Science": 0.5376676986584107, - "Christian religion": 0.582089552238806, - "Art": 0.5141430948419301, - "Islam religion": 0.5234708392603129, - "Hindu religion": 0.36, - "Madurese": 0.2847457627118644, - "Sport": 0.49324324324324326, - "Indonesian language": 0.5043586550435866, - "Physics": 0.3373737373737374, - "Minangkabau culture": 0.36180904522613067, - "Dayak language": 0.3211009174311927, - "Sociology": 0.4334677419354839, - "Economy": 0.39139344262295084, - "Sundanese": 0.331028522039758, - "Javanese": 0.3326612903225806, - "Civic education": 0.5379113018597997 - } - }, - "prompt_3": { - "accuracy": 0.4380132185059083, - "category_acc": { - "History": 0.41365461847389556, - "Geography": 0.4163265306122449, - "Lampungic": 0.32653061224489793, - "Social science": 0.672787979966611, - "Balinese": 0.3375796178343949, - "Makassarese": 0.3010752688172043, - "Banjarese": 0.3819444444444444, - "Chemistry": 0.27883211678832115, - "Biology": 0.4, - "Science": 0.544891640866873, - "Christian religion": 0.582089552238806, - "Art": 0.5074875207986689, - "Islam religion": 0.534850640113798, - "Hindu religion": 0.38666666666666666, - "Madurese": 0.30847457627118646, - "Sport": 0.5067567567567568, - "Indonesian language": 0.49564134495641343, - "Physics": 0.3333333333333333, - "Minangkabau culture": 0.36180904522613067, - "Dayak language": 0.3394495412844037, - "Sociology": 0.4254032258064516, - "Economy": 0.38934426229508196, - "Sundanese": 0.34485738980121, - "Javanese": 0.3165322580645161, - "Civic education": 0.530758226037196 - } - }, - "prompt_4": { - "accuracy": 0.4370785766740103, - "category_acc": { - "History": 0.41767068273092367, - "Geography": 0.42448979591836733, - "Lampungic": 0.2857142857142857, - "Social science": 0.676126878130217, - "Balinese": 0.3354564755838641, - "Makassarese": 0.3225806451612903, - "Banjarese": 0.3402777777777778, - "Chemistry": 0.2832116788321168, - "Biology": 0.39763313609467454, - "Science": 0.544891640866873, - "Christian religion": 0.5621890547263682, - "Art": 0.5041597337770383, - "Islam religion": 0.5277382645803699, - "Hindu religion": 0.38666666666666666, - "Madurese": 0.3152542372881356, - "Sport": 0.5067567567567568, - "Indonesian language": 0.4922166874221669, - "Physics": 0.32727272727272727, - "Minangkabau culture": 0.35175879396984927, - "Dayak language": 0.3394495412844037, - "Sociology": 0.4314516129032258, - "Economy": 0.39139344262295084, - "Sundanese": 0.34053586862575624, - "Javanese": 0.3276209677419355, - "Civic education": 0.5321888412017167 - } - }, - "prompt_5": { - "accuracy": 0.43834701916015756, - "category_acc": { - "History": 0.42168674698795183, - "Geography": 0.42448979591836733, - "Lampungic": 0.2789115646258503, - "Social science": 0.6878130217028381, - "Balinese": 0.32696390658174096, - "Makassarese": 0.3333333333333333, - "Banjarese": 0.3263888888888889, - "Chemistry": 0.27883211678832115, - "Biology": 0.39644970414201186, - "Science": 0.5479876160990712, - "Christian religion": 0.5671641791044776, - "Art": 0.5058236272878536, - "Islam religion": 0.519203413940256, - "Hindu religion": 0.38, - "Madurese": 0.29491525423728815, - "Sport": 0.5, - "Indonesian language": 0.4943960149439601, - "Physics": 0.35353535353535354, - "Minangkabau culture": 0.35678391959798994, - "Dayak language": 0.3302752293577982, - "Sociology": 0.42338709677419356, - "Economy": 0.38729508196721313, - "Sundanese": 0.3500432152117545, - "Javanese": 0.3245967741935484, - "Civic education": 0.5407725321888412 - } - } - }, - "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.32927886761360825 - }, - "prompt_2": { - "bleu_score": 0.33761429719094244 - }, - "prompt_3": { - "bleu_score": 0.34208487542761046 - }, - "prompt_4": { - "bleu_score": 0.3179353474928239 - }, - "prompt_5": { - "bleu_score": 0.23820091224231876 - } - }, - "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.11730529445992 - }, - "prompt_2": { - "bleu_score": 0.12454156485658088 - }, - "prompt_3": { - "bleu_score": 0.12587191532740058 - }, - "prompt_4": { - "bleu_score": 0.11031968694773317 - }, - "prompt_5": { - "bleu_score": 0.07173318237582105 - } - }, - "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.19463118603359678 - }, - "prompt_2": { - "bleu_score": 0.2027260711564121 - }, - "prompt_3": { - "bleu_score": 0.2048728607157833 - }, - "prompt_4": { - "bleu_score": 0.1919153335634948 - }, - "prompt_5": { - "bleu_score": 0.14814809576841048 - } - }, - "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.2916010251816113 - }, - "prompt_2": { - "bleu_score": 0.2948552922694405 - }, - "prompt_3": { - "bleu_score": 0.30262589404937634 - }, - "prompt_4": { - "bleu_score": 0.277180839314322 - }, - "prompt_5": { - "bleu_score": 0.20153881980627356 - } - }, - "mmlu": { - "prompt_1": { - "accuracy": 0.5390898483080513 - }, - "prompt_2": { - "accuracy": 0.5472578763127188 - }, - "prompt_3": { - "accuracy": 0.5519253208868145 - }, - "prompt_4": { - "accuracy": 0.5460910151691949 - }, - "prompt_5": { - "accuracy": 0.543757292882147 - } - }, - "mmlu_full": { - "prompt_1": { - "accuracy": 0.5219878441186986, - "category_acc": { - "high_school_european_history": 0.7012195121951219, - "business_ethics": 0.5353535353535354, - "clinical_knowledge": 0.5984848484848485, - "medical_genetics": 0.5353535353535354, - "high_school_us_history": 0.7438423645320197, - "high_school_physics": 0.37333333333333335, - "high_school_world_history": 0.7542372881355932, - "virology": 0.41818181818181815, - "high_school_microeconomics": 0.5569620253164557, - "econometrics": 0.3008849557522124, - "college_computer_science": 0.3939393939393939, - "high_school_biology": 0.6407766990291263, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.3807829181494662, - "philosophy": 0.5935483870967742, - "professional_medicine": 0.6014760147601476, - "nutrition": 0.5836065573770491, - "global_facts": 0.2222222222222222, - "machine_learning": 0.34234234234234234, - "security_studies": 0.6311475409836066, - "public_relations": 0.5412844036697247, - "professional_psychology": 0.5008183306055647, - "prehistory": 0.5510835913312694, - "anatomy": 0.4701492537313433, - "human_sexuality": 0.6307692307692307, - "college_medicine": 0.47674418604651164, - "high_school_government_and_politics": 0.765625, - "college_chemistry": 0.35353535353535354, - "logical_fallacies": 0.6049382716049383, - "high_school_geography": 0.6852791878172588, - "elementary_mathematics": 0.4270557029177719, - "human_aging": 0.6171171171171171, - "college_mathematics": 0.25252525252525254, - "high_school_psychology": 0.7518382352941176, - "formal_logic": 0.336, - "high_school_statistics": 0.4232558139534884, - "international_law": 0.6666666666666666, - "high_school_mathematics": 0.2379182156133829, - "high_school_computer_science": 0.5656565656565656, - "conceptual_physics": 0.4017094017094017, - "miscellaneous": 0.731457800511509, - "high_school_chemistry": 0.3613861386138614, - "marketing": 0.7467811158798283, - "professional_law": 0.4090019569471624, - "management": 0.6862745098039216, - "college_physics": 0.25742574257425743, - "jurisprudence": 0.6822429906542056, - "world_religions": 0.6941176470588235, - "sociology": 0.755, - "us_foreign_policy": 0.7878787878787878, - "high_school_macroeconomics": 0.5115681233933161, - "computer_security": 0.6060606060606061, - "moral_scenarios": 0.21476510067114093, - "moral_disputes": 0.591304347826087, - "electrical_engineering": 0.4652777777777778, - "astronomy": 0.5298013245033113, - "college_biology": 0.6013986013986014 - } - }, - "prompt_2": { - "accuracy": 0.5278512692170182, - "category_acc": { - "high_school_european_history": 0.7073170731707317, - "business_ethics": 0.5353535353535354, - "clinical_knowledge": 0.6022727272727273, - "medical_genetics": 0.5858585858585859, - "high_school_us_history": 0.7339901477832512, - "high_school_physics": 0.29333333333333333, - "high_school_world_history": 0.7203389830508474, - "virology": 0.43636363636363634, - "high_school_microeconomics": 0.5738396624472574, - "econometrics": 0.3185840707964602, - "college_computer_science": 0.37373737373737376, - "high_school_biology": 0.6472491909385113, - "abstract_algebra": 0.25252525252525254, - "professional_accounting": 0.40213523131672596, - "philosophy": 0.603225806451613, - "professional_medicine": 0.6125461254612546, - "nutrition": 0.5901639344262295, - "global_facts": 0.20202020202020202, - "machine_learning": 0.32432432432432434, - "security_studies": 0.6557377049180327, - "public_relations": 0.5596330275229358, - "professional_psychology": 0.5188216039279869, - "prehistory": 0.5758513931888545, - "anatomy": 0.4701492537313433, - "human_sexuality": 0.6076923076923076, - "college_medicine": 0.5232558139534884, - "high_school_government_and_politics": 0.7552083333333334, - "college_chemistry": 0.37373737373737376, - "logical_fallacies": 0.6111111111111112, - "high_school_geography": 0.6903553299492385, - "elementary_mathematics": 0.4350132625994695, - "human_aging": 0.6171171171171171, - "college_mathematics": 0.24242424242424243, - "high_school_psychology": 0.7536764705882353, - "formal_logic": 0.352, - "high_school_statistics": 0.40930232558139534, - "international_law": 0.6666666666666666, - "high_school_mathematics": 0.2527881040892193, - "high_school_computer_science": 0.5656565656565656, - "conceptual_physics": 0.42735042735042733, - "miscellaneous": 0.7225063938618926, - "high_school_chemistry": 0.38613861386138615, - "marketing": 0.7725321888412017, - "professional_law": 0.4018264840182648, - "management": 0.7156862745098039, - "college_physics": 0.2376237623762376, - "jurisprudence": 0.7009345794392523, - "world_religions": 0.6941176470588235, - "sociology": 0.76, - "us_foreign_policy": 0.7878787878787878, - "high_school_macroeconomics": 0.5218508997429306, - "computer_security": 0.6060606060606061, - "moral_scenarios": 0.24161073825503357, - "moral_disputes": 0.6144927536231884, - "electrical_engineering": 0.4791666666666667, - "astronomy": 0.5099337748344371, - "college_biology": 0.5944055944055944 - } - }, - "prompt_3": { - "accuracy": 0.5283518055058992, - "category_acc": { - "high_school_european_history": 0.7073170731707317, - "business_ethics": 0.5252525252525253, - "clinical_knowledge": 0.5946969696969697, - "medical_genetics": 0.6161616161616161, - "high_school_us_history": 0.7389162561576355, - "high_school_physics": 0.3333333333333333, - "high_school_world_history": 0.711864406779661, - "virology": 0.4121212121212121, - "high_school_microeconomics": 0.5907172995780591, - "econometrics": 0.3008849557522124, - "college_computer_science": 0.40404040404040403, - "high_school_biology": 0.6634304207119741, - "abstract_algebra": 0.2222222222222222, - "professional_accounting": 0.39501779359430605, - "philosophy": 0.6193548387096774, - "professional_medicine": 0.6051660516605166, - "nutrition": 0.5901639344262295, - "global_facts": 0.24242424242424243, - "machine_learning": 0.3333333333333333, - "security_studies": 0.6516393442622951, - "public_relations": 0.5504587155963303, - "professional_psychology": 0.5204582651391162, - "prehistory": 0.5820433436532507, - "anatomy": 0.47761194029850745, - "human_sexuality": 0.6, - "college_medicine": 0.5116279069767442, - "high_school_government_and_politics": 0.7760416666666666, - "college_chemistry": 0.35353535353535354, - "logical_fallacies": 0.5987654320987654, - "high_school_geography": 0.6903553299492385, - "elementary_mathematics": 0.4217506631299735, - "human_aging": 0.6216216216216216, - "college_mathematics": 0.2222222222222222, - "high_school_psychology": 0.7481617647058824, - "formal_logic": 0.352, - "high_school_statistics": 0.4232558139534884, - "international_law": 0.6416666666666667, - "high_school_mathematics": 0.26022304832713755, - "high_school_computer_science": 0.5757575757575758, - "conceptual_physics": 0.4188034188034188, - "miscellaneous": 0.7186700767263428, - "high_school_chemistry": 0.39603960396039606, - "marketing": 0.7553648068669528, - "professional_law": 0.41682974559686886, - "management": 0.7156862745098039, - "college_physics": 0.19801980198019803, - "jurisprudence": 0.6728971962616822, - "world_religions": 0.7058823529411765, - "sociology": 0.77, - "us_foreign_policy": 0.797979797979798, - "high_school_macroeconomics": 0.5295629820051414, - "computer_security": 0.6363636363636364, - "moral_scenarios": 0.22706935123042504, - "moral_disputes": 0.6028985507246377, - "electrical_engineering": 0.4652777777777778, - "astronomy": 0.5165562913907285, - "college_biology": 0.5804195804195804 - } - }, - "prompt_4": { - "accuracy": 0.5241329996424741, - "category_acc": { - "high_school_european_history": 0.7073170731707317, - "business_ethics": 0.5454545454545454, - "clinical_knowledge": 0.5909090909090909, - "medical_genetics": 0.5555555555555556, - "high_school_us_history": 0.7339901477832512, - "high_school_physics": 0.38, - "high_school_world_history": 0.7372881355932204, - "virology": 0.41818181818181815, - "high_school_microeconomics": 0.5569620253164557, - "econometrics": 0.3185840707964602, - "college_computer_science": 0.36363636363636365, - "high_school_biology": 0.6634304207119741, - "abstract_algebra": 0.1919191919191919, - "professional_accounting": 0.38434163701067614, - "philosophy": 0.5838709677419355, - "professional_medicine": 0.5867158671586716, - "nutrition": 0.5737704918032787, - "global_facts": 0.24242424242424243, - "machine_learning": 0.34234234234234234, - "security_studies": 0.6352459016393442, - "public_relations": 0.5688073394495413, - "professional_psychology": 0.5155482815057283, - "prehistory": 0.5603715170278638, - "anatomy": 0.48507462686567165, - "human_sexuality": 0.6076923076923076, - "college_medicine": 0.5058139534883721, - "high_school_government_and_politics": 0.7604166666666666, - "college_chemistry": 0.43434343434343436, - "logical_fallacies": 0.6234567901234568, - "high_school_geography": 0.6751269035532995, - "elementary_mathematics": 0.46153846153846156, - "human_aging": 0.5990990990990991, - "college_mathematics": 0.15151515151515152, - "high_school_psychology": 0.75, - "formal_logic": 0.312, - "high_school_statistics": 0.4418604651162791, - "international_law": 0.6583333333333333, - "high_school_mathematics": 0.21561338289962825, - "high_school_computer_science": 0.5858585858585859, - "conceptual_physics": 0.3974358974358974, - "miscellaneous": 0.7161125319693095, - "high_school_chemistry": 0.38613861386138615, - "marketing": 0.7682403433476395, - "professional_law": 0.4116112198303979, - "management": 0.696078431372549, - "college_physics": 0.24752475247524752, - "jurisprudence": 0.6728971962616822, - "world_religions": 0.6882352941176471, - "sociology": 0.74, - "us_foreign_policy": 0.7878787878787878, - "high_school_macroeconomics": 0.506426735218509, - "computer_security": 0.5959595959595959, - "moral_scenarios": 0.24049217002237136, - "moral_disputes": 0.5971014492753624, - "electrical_engineering": 0.4791666666666667, - "astronomy": 0.5099337748344371, - "college_biology": 0.6013986013986014 - } - }, - "prompt_5": { - "accuracy": 0.5198426885949231, - "category_acc": { - "high_school_european_history": 0.6890243902439024, - "business_ethics": 0.5454545454545454, - "clinical_knowledge": 0.6098484848484849, - "medical_genetics": 0.5050505050505051, - "high_school_us_history": 0.7044334975369458, - "high_school_physics": 0.32, - "high_school_world_history": 0.75, - "virology": 0.42424242424242425, - "high_school_microeconomics": 0.5569620253164557, - "econometrics": 0.26548672566371684, - "college_computer_science": 0.3434343434343434, - "high_school_biology": 0.656957928802589, - "abstract_algebra": 0.18181818181818182, - "professional_accounting": 0.398576512455516, - "philosophy": 0.5967741935483871, - "professional_medicine": 0.6088560885608856, - "nutrition": 0.5901639344262295, - "global_facts": 0.21212121212121213, - "machine_learning": 0.35135135135135137, - "security_studies": 0.6229508196721312, - "public_relations": 0.5321100917431193, - "professional_psychology": 0.4959083469721768, - "prehistory": 0.5294117647058824, - "anatomy": 0.48507462686567165, - "human_sexuality": 0.5461538461538461, - "college_medicine": 0.47093023255813954, - "high_school_government_and_politics": 0.7604166666666666, - "college_chemistry": 0.3939393939393939, - "logical_fallacies": 0.6049382716049383, - "high_school_geography": 0.6954314720812182, - "elementary_mathematics": 0.46153846153846156, - "human_aging": 0.6126126126126126, - "college_mathematics": 0.21212121212121213, - "high_school_psychology": 0.75, - "formal_logic": 0.328, - "high_school_statistics": 0.4, - "international_law": 0.625, - "high_school_mathematics": 0.22676579925650558, - "high_school_computer_science": 0.5757575757575758, - "conceptual_physics": 0.44017094017094016, - "miscellaneous": 0.7148337595907929, - "high_school_chemistry": 0.3712871287128713, - "marketing": 0.7510729613733905, - "professional_law": 0.41030658838878015, - "management": 0.696078431372549, - "college_physics": 0.26732673267326734, - "jurisprudence": 0.6728971962616822, - "world_religions": 0.6764705882352942, - "sociology": 0.735, - "us_foreign_policy": 0.8181818181818182, - "high_school_macroeconomics": 0.506426735218509, - "computer_security": 0.5959595959595959, - "moral_scenarios": 0.22930648769574943, - "moral_disputes": 0.6086956521739131, - "electrical_engineering": 0.4375, - "astronomy": 0.5231788079470199, - "college_biology": 0.6153846153846154 - } - } - }, - "c_eval": { - "prompt_1": { - "accuracy": 0.3640416047548291 - }, - "prompt_2": { - "accuracy": 0.37518573551263 - }, - "prompt_3": { - "accuracy": 0.3885586924219911 - }, - "prompt_4": { - "accuracy": 0.36924219910846956 - }, - "prompt_5": { - "accuracy": 0.35661218424962854 - } - }, - "c_eval_full": { - "prompt_1": { - "accuracy": 0.3605230386052304, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.25, - "computer_architecture": 0.2692307692307692, - "college_programming": 0.3333333333333333, - "college_physics": 0.375, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.375, - "probability_and_statistics": 0.13043478260869565, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.41379310344827586, - "high_school_mathematics": 0.30434782608695654, - "high_school_physics": 0.3333333333333333, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.20833333333333334, - "middle_school_mathematics": 0.2916666666666667, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.25, - "college_economics": 0.45, - "business_administration": 0.2894736842105263, - "marxism": 0.375, - "mao_zedong_thought": 0.41379310344827586, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.3877551020408163, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.375, - "middle_school_politics": 0.5384615384615384, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.35714285714285715, - "ideological_and_moral_cultivation": 0.5416666666666666, - "logic": 0.37037037037037035, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.2894736842105263, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.25, - "high_school_history": 0.48, - "middle_school_history": 0.4074074074074074, - "civil_servant": 0.40384615384615385, - "sports_science": 0.3333333333333333, - "plant_protection": 0.2962962962962963, - "basic_medicine": 0.25, - "clinical_medicine": 0.2962962962962963, - "urban_and_rural_planner": 0.5294117647058824, - "accountant": 0.4074074074074074, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.3148148148148148, - "physician": 0.35185185185185186 - } - }, - "prompt_2": { - "accuracy": 0.3667496886674969, - "category_acc": { - "computer_network": 0.25, - "operating_system": 0.2916666666666667, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.40476190476190477, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.17391304347826086, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.2857142857142857, - "metrology_engineer": 0.4827586206896552, - "high_school_mathematics": 0.34782608695652173, - "high_school_physics": 0.25, - "high_school_chemistry": 0.375, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.16666666666666666, - "middle_school_biology": 0.5, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.4, - "veterinary_medicine": 0.2857142857142857, - "college_economics": 0.48333333333333334, - "business_administration": 0.21052631578947367, - "marxism": 0.375, - "mao_zedong_thought": 0.3103448275862069, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.5102040816326531, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.5769230769230769, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.625, - "logic": 0.25925925925925924, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.3157894736842105, - "professional_tour_guide": 0.29411764705882354, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.52, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.40384615384615385, - "sports_science": 0.25, - "plant_protection": 0.37037037037037035, - "basic_medicine": 0.25, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.45098039215686275, - "accountant": 0.3888888888888889, - "fire_engineer": 0.3888888888888889, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.3148148148148148, - "physician": 0.4444444444444444 - } - }, - "prompt_3": { - "accuracy": 0.363013698630137, - "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.20833333333333334, - "computer_architecture": 0.3076923076923077, - "college_programming": 0.42857142857142855, - "college_physics": 0.4583333333333333, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.3333333333333333, - "probability_and_statistics": 0.13043478260869565, - "discrete_mathematics": 0.3333333333333333, - "electrical_engineer": 0.3333333333333333, - "metrology_engineer": 0.5517241379310345, - "high_school_mathematics": 0.17391304347826086, - "high_school_physics": 0.25, - "high_school_chemistry": 0.375, - "high_school_biology": 0.25, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.46153846153846156, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.4, - "veterinary_medicine": 0.25, - "college_economics": 0.36666666666666664, - "business_administration": 0.34210526315789475, - "marxism": 0.4583333333333333, - "mao_zedong_thought": 0.3793103448275862, - "education_science": 0.47058823529411764, - "teacher_qualification": 0.4489795918367347, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.375, - "middle_school_politics": 0.5769230769230769, - "middle_school_geography": 0.29411764705882354, - "modern_chinese_history": 0.39285714285714285, - "ideological_and_moral_cultivation": 0.5416666666666666, - "logic": 0.2962962962962963, - "law": 0.3448275862068966, - "chinese_language_and_literature": 0.35714285714285715, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.35294117647058826, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.48, - "middle_school_history": 0.2962962962962963, - "civil_servant": 0.3269230769230769, - "sports_science": 0.20833333333333334, - "plant_protection": 0.37037037037037035, - "basic_medicine": 0.2916666666666667, - "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.47058823529411764, - "accountant": 0.3888888888888889, - "fire_engineer": 0.4166666666666667, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.3333333333333333, - "physician": 0.37037037037037035 - } - }, - "prompt_4": { - "accuracy": 0.35554171855541716, - "category_acc": { - "computer_network": 0.3333333333333333, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.42857142857142855, - "college_physics": 0.3333333333333333, - "college_chemistry": 0.27586206896551724, - "advanced_mathematics": 0.25, - "probability_and_statistics": 0.043478260869565216, - "discrete_mathematics": 0.23809523809523808, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.3103448275862069, - "high_school_mathematics": 0.391304347826087, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.25, - "high_school_biology": 0.375, - "middle_school_mathematics": 0.20833333333333334, - "middle_school_biology": 0.5384615384615384, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.36, - "veterinary_medicine": 0.25, - "college_economics": 0.38333333333333336, - "business_administration": 0.3157894736842105, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.4482758620689655, - "education_science": 0.5294117647058824, - "teacher_qualification": 0.5306122448979592, - "high_school_politics": 0.3333333333333333, - "high_school_geography": 0.3333333333333333, - "middle_school_politics": 0.5384615384615384, - "middle_school_geography": 0.23529411764705882, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.5, - "logic": 0.2962962962962963, - "law": 0.41379310344827586, - "chinese_language_and_literature": 0.2857142857142857, - "art_studies": 0.3684210526315789, - "professional_tour_guide": 0.23529411764705882, - "legal_professional": 0.42857142857142855, - "high_school_chinese": 0.2916666666666667, - "high_school_history": 0.52, - "middle_school_history": 0.4444444444444444, - "civil_servant": 0.36538461538461536, - "sports_science": 0.3333333333333333, - "plant_protection": 0.37037037037037035, - "basic_medicine": 0.25, - "clinical_medicine": 0.3333333333333333, - "urban_and_rural_planner": 0.39215686274509803, - "accountant": 0.2962962962962963, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.3333333333333333, - "tax_accountant": 0.3148148148148148, - "physician": 0.3333333333333333 - } - }, - "prompt_5": { - "accuracy": 0.3642590286425903, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.2916666666666667, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.3333333333333333, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.20833333333333334, - "probability_and_statistics": 0.21739130434782608, - "discrete_mathematics": 0.2857142857142857, - "electrical_engineer": 0.30952380952380953, - "metrology_engineer": 0.3793103448275862, - "high_school_mathematics": 0.43478260869565216, - "high_school_physics": 0.2916666666666667, - "high_school_chemistry": 0.2916666666666667, - "high_school_biology": 0.3333333333333333, - "middle_school_mathematics": 0.3333333333333333, - "middle_school_biology": 0.4230769230769231, - "middle_school_physics": 0.375, - "middle_school_chemistry": 0.32, - "veterinary_medicine": 0.25, - "college_economics": 0.4, - "business_administration": 0.2894736842105263, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.4827586206896552, - "education_science": 0.4411764705882353, - "teacher_qualification": 0.46938775510204084, - "high_school_politics": 0.5, - "high_school_geography": 0.4166666666666667, - "middle_school_politics": 0.5769230769230769, - "middle_school_geography": 0.17647058823529413, - "modern_chinese_history": 0.42857142857142855, - "ideological_and_moral_cultivation": 0.4166666666666667, - "logic": 0.5185185185185185, - "law": 0.3793103448275862, - "chinese_language_and_literature": 0.42857142857142855, - "art_studies": 0.34210526315789475, - "professional_tour_guide": 0.23529411764705882, - "legal_professional": 0.39285714285714285, - "high_school_chinese": 0.4166666666666667, - "high_school_history": 0.48, - "middle_school_history": 0.37037037037037035, - "civil_servant": 0.38461538461538464, - "sports_science": 0.25, - "plant_protection": 0.3333333333333333, - "basic_medicine": 0.20833333333333334, - "clinical_medicine": 0.4444444444444444, - "urban_and_rural_planner": 0.39215686274509803, - "accountant": 0.3148148148148148, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.3611111111111111, - "tax_accountant": 0.35185185185185186, - "physician": 0.5 - } - } - }, - "cmmlu": { - "prompt_1": { - "accuracy": 0.32974910394265233 - }, - "prompt_2": { - "accuracy": 0.34408602150537637 - }, - "prompt_3": { - "accuracy": 0.37992831541218636 - }, - "prompt_4": { - "accuracy": 0.34767025089605735 - }, - "prompt_5": { - "accuracy": 0.34408602150537637 - } - }, - "cmmlu_full": { - "prompt_1": { - "accuracy": 0.3853393196339147, - "category_acc": { - "agronomy": 0.378698224852071, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.25, - "arts": 0.425, - "astronomy": 0.2727272727272727, - "business_ethics": 0.39712918660287083, - "chinese_civil_service_exam": 0.38125, - "chinese_driving_rule": 0.5801526717557252, - "chinese_food_culture": 0.3602941176470588, - "chinese_foreign_policy": 0.4672897196261682, - "chinese_history": 0.4551083591331269, - "chinese_literature": 0.29411764705882354, - "chinese_teacher_qualification": 0.4245810055865922, - "clinical_knowledge": 0.3333333333333333, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.4766355140186916, - "college_engineering_hydrology": 0.4339622641509434, - "college_law": 0.2777777777777778, - "college_mathematics": 0.2761904761904762, - "college_medical_statistics": 0.27358490566037735, - "college_medicine": 0.31868131868131866, - "computer_science": 0.35294117647058826, - "computer_security": 0.4444444444444444, - "conceptual_physics": 0.3741496598639456, - "construction_project_management": 0.37410071942446044, - "economics": 0.4276729559748428, - "education": 0.4294478527607362, - "electrical_engineering": 0.37790697674418605, - "elementary_chinese": 0.2857142857142857, - "elementary_commonsense": 0.3888888888888889, - "elementary_information_and_technology": 0.592436974789916, - "elementary_mathematics": 0.28695652173913044, - "ethnology": 0.3851851851851852, - "food_science": 0.36363636363636365, - "genetics": 0.3522727272727273, - "global_facts": 0.4429530201342282, - "high_school_biology": 0.378698224852071, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.3559322033898305, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.22727272727272727, - "high_school_politics": 0.3356643356643357, - "human_sexuality": 0.47619047619047616, - "international_law": 0.3945945945945946, - "journalism": 0.46511627906976744, - "jurisprudence": 0.40632603406326034, - "legal_and_moral_basis": 0.6261682242990654, - "logical": 0.34146341463414637, - "machine_learning": 0.29508196721311475, - "management": 0.4238095238095238, - "marketing": 0.5055555555555555, - "marxist_theory": 0.48677248677248675, - "modern_chinese": 0.31896551724137934, - "nutrition": 0.41379310344827586, - "philosophy": 0.4095238095238095, - "professional_accounting": 0.37714285714285717, - "professional_law": 0.2843601895734597, - "professional_medicine": 0.300531914893617, - "professional_psychology": 0.41379310344827586, - "public_relations": 0.4425287356321839, - "security_study": 0.42962962962962964, - "sociology": 0.42035398230088494, - "sports_science": 0.3878787878787879, - "traditional_chinese_medicine": 0.3621621621621622, - "virology": 0.4378698224852071, - "world_history": 0.5652173913043478, - "world_religions": 0.425 - } + "AC3_2": 0.5237839281043418, + "AC3_3": 0.43618697474052426, + "AC3_4": 0.37914481938983285, + "AC3_5": 0.3409581778458603, + "AC3_6": 0.31433976592725005, + "AC3_7": 0.29426946166661816 + } + }, + "sg_eval": { + "prompt_1": { + "accuracy": 0.6504854368932039 }, "prompt_2": { - "accuracy": 0.3890519772060093, - "category_acc": { - "agronomy": 0.3609467455621302, - "anatomy": 0.25675675675675674, - "ancient_chinese": 0.2073170731707317, - "arts": 0.38125, - "astronomy": 0.2787878787878788, - "business_ethics": 0.4449760765550239, - "chinese_civil_service_exam": 0.39375, - "chinese_driving_rule": 0.5954198473282443, - "chinese_food_culture": 0.36764705882352944, - "chinese_foreign_policy": 0.48598130841121495, - "chinese_history": 0.4613003095975232, - "chinese_literature": 0.31862745098039214, - "chinese_teacher_qualification": 0.45251396648044695, - "clinical_knowledge": 0.31223628691983124, - "college_actuarial_science": 0.25471698113207547, - "college_education": 0.48598130841121495, - "college_engineering_hydrology": 0.4339622641509434, - "college_law": 0.2962962962962963, - "college_mathematics": 0.2571428571428571, - "college_medical_statistics": 0.330188679245283, - "college_medicine": 0.3333333333333333, - "computer_science": 0.38235294117647056, - "computer_security": 0.43859649122807015, - "conceptual_physics": 0.32653061224489793, - "construction_project_management": 0.381294964028777, - "economics": 0.4528301886792453, - "education": 0.4539877300613497, - "electrical_engineering": 0.38953488372093026, - "elementary_chinese": 0.2777777777777778, - "elementary_commonsense": 0.3888888888888889, - "elementary_information_and_technology": 0.592436974789916, - "elementary_mathematics": 0.30869565217391304, - "ethnology": 0.37777777777777777, - "food_science": 0.35664335664335667, - "genetics": 0.32954545454545453, - "global_facts": 0.44966442953020136, - "high_school_biology": 0.3668639053254438, - "high_school_chemistry": 0.2878787878787879, - "high_school_geography": 0.3898305084745763, - "high_school_mathematics": 0.2621951219512195, - "high_school_physics": 0.38181818181818183, - "high_school_politics": 0.3706293706293706, - "human_sexuality": 0.42857142857142855, - "international_law": 0.40540540540540543, - "journalism": 0.4186046511627907, - "jurisprudence": 0.41362530413625304, - "legal_and_moral_basis": 0.6401869158878505, - "logical": 0.3333333333333333, - "machine_learning": 0.3114754098360656, - "management": 0.38571428571428573, - "marketing": 0.5222222222222223, - "marxist_theory": 0.49206349206349204, - "modern_chinese": 0.27586206896551724, - "nutrition": 0.41379310344827586, - "philosophy": 0.3523809523809524, - "professional_accounting": 0.3657142857142857, - "professional_law": 0.3175355450236967, - "professional_medicine": 0.2845744680851064, - "professional_psychology": 0.4525862068965517, - "public_relations": 0.46551724137931033, - "security_study": 0.43703703703703706, - "sociology": 0.3938053097345133, - "sports_science": 0.37575757575757573, - "traditional_chinese_medicine": 0.372972972972973, - "virology": 0.40236686390532544, - "world_history": 0.546583850931677, - "world_religions": 0.44375 - } + "accuracy": 0.6213592233009708 }, "prompt_3": { - "accuracy": 0.38749784147815575, - "category_acc": { - "agronomy": 0.3727810650887574, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.23780487804878048, - "arts": 0.41875, - "astronomy": 0.24242424242424243, - "business_ethics": 0.4258373205741627, - "chinese_civil_service_exam": 0.35, - "chinese_driving_rule": 0.5572519083969466, - "chinese_food_culture": 0.3602941176470588, - "chinese_foreign_policy": 0.4953271028037383, - "chinese_history": 0.4520123839009288, - "chinese_literature": 0.3235294117647059, - "chinese_teacher_qualification": 0.45251396648044695, - "clinical_knowledge": 0.3206751054852321, - "college_actuarial_science": 0.16981132075471697, - "college_education": 0.5046728971962616, - "college_engineering_hydrology": 0.33962264150943394, - "college_law": 0.3148148148148148, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.3584905660377358, - "college_medicine": 0.3443223443223443, - "computer_science": 0.3627450980392157, - "computer_security": 0.4444444444444444, - "conceptual_physics": 0.3469387755102041, - "construction_project_management": 0.37410071942446044, - "economics": 0.44025157232704404, - "education": 0.4785276073619632, - "electrical_engineering": 0.3953488372093023, - "elementary_chinese": 0.25396825396825395, - "elementary_commonsense": 0.40404040404040403, - "elementary_information_and_technology": 0.5588235294117647, - "elementary_mathematics": 0.2826086956521739, - "ethnology": 0.37777777777777777, - "food_science": 0.38461538461538464, - "genetics": 0.35795454545454547, - "global_facts": 0.44966442953020136, - "high_school_biology": 0.34911242603550297, - "high_school_chemistry": 0.2727272727272727, - "high_school_geography": 0.3983050847457627, - "high_school_mathematics": 0.27439024390243905, - "high_school_physics": 0.36363636363636365, - "high_school_politics": 0.3706293706293706, - "human_sexuality": 0.4365079365079365, - "international_law": 0.3891891891891892, - "journalism": 0.45348837209302323, - "jurisprudence": 0.39659367396593675, - "legal_and_moral_basis": 0.6355140186915887, - "logical": 0.42276422764227645, - "machine_learning": 0.29508196721311475, - "management": 0.41904761904761906, - "marketing": 0.5166666666666667, - "marxist_theory": 0.48148148148148145, - "modern_chinese": 0.21551724137931033, - "nutrition": 0.4, - "philosophy": 0.3523809523809524, - "professional_accounting": 0.41714285714285715, - "professional_law": 0.3175355450236967, - "professional_medicine": 0.30319148936170215, - "professional_psychology": 0.4051724137931034, - "public_relations": 0.47126436781609193, - "security_study": 0.4888888888888889, - "sociology": 0.3938053097345133, - "sports_science": 0.3878787878787879, - "traditional_chinese_medicine": 0.3675675675675676, - "virology": 0.42011834319526625, - "world_history": 0.5279503105590062, - "world_religions": 0.41875 - } + "accuracy": 0.6407766990291263 }, "prompt_4": { - "accuracy": 0.36902089449145226, - "category_acc": { - "agronomy": 0.35502958579881655, - "anatomy": 0.22297297297297297, - "ancient_chinese": 0.24390243902439024, - "arts": 0.35, - "astronomy": 0.296969696969697, - "business_ethics": 0.40669856459330145, - "chinese_civil_service_exam": 0.33125, - "chinese_driving_rule": 0.5114503816793893, - "chinese_food_culture": 0.3161764705882353, - "chinese_foreign_policy": 0.4766355140186916, - "chinese_history": 0.4148606811145511, - "chinese_literature": 0.25980392156862747, - "chinese_teacher_qualification": 0.441340782122905, - "clinical_knowledge": 0.350210970464135, - "college_actuarial_science": 0.19811320754716982, - "college_education": 0.4766355140186916, - "college_engineering_hydrology": 0.3584905660377358, - "college_law": 0.3333333333333333, - "college_mathematics": 0.21904761904761905, - "college_medical_statistics": 0.29245283018867924, - "college_medicine": 0.2967032967032967, - "computer_science": 0.3627450980392157, - "computer_security": 0.38011695906432746, - "conceptual_physics": 0.3673469387755102, - "construction_project_management": 0.3597122302158273, - "economics": 0.4591194968553459, - "education": 0.43558282208588955, - "electrical_engineering": 0.3546511627906977, - "elementary_chinese": 0.3055555555555556, - "elementary_commonsense": 0.3686868686868687, - "elementary_information_and_technology": 0.5672268907563025, - "elementary_mathematics": 0.3130434782608696, - "ethnology": 0.35555555555555557, - "food_science": 0.34965034965034963, - "genetics": 0.3409090909090909, - "global_facts": 0.38926174496644295, - "high_school_biology": 0.34911242603550297, - "high_school_chemistry": 0.24242424242424243, - "high_school_geography": 0.3813559322033898, - "high_school_mathematics": 0.23780487804878048, - "high_school_physics": 0.2818181818181818, - "high_school_politics": 0.3706293706293706, - "human_sexuality": 0.42063492063492064, - "international_law": 0.34054054054054056, - "journalism": 0.4069767441860465, - "jurisprudence": 0.38686131386861317, - "legal_and_moral_basis": 0.5887850467289719, - "logical": 0.3333333333333333, - "machine_learning": 0.27049180327868855, - "management": 0.4142857142857143, - "marketing": 0.5111111111111111, - "marxist_theory": 0.48148148148148145, - "modern_chinese": 0.25862068965517243, - "nutrition": 0.3931034482758621, - "philosophy": 0.38095238095238093, - "professional_accounting": 0.37714285714285717, - "professional_law": 0.2985781990521327, - "professional_medicine": 0.2872340425531915, - "professional_psychology": 0.41379310344827586, - "public_relations": 0.4540229885057471, - "security_study": 0.4222222222222222, - "sociology": 0.4026548672566372, - "sports_science": 0.3575757575757576, - "traditional_chinese_medicine": 0.3027027027027027, - "virology": 0.40236686390532544, - "world_history": 0.4906832298136646, - "world_religions": 0.39375 - } + "accuracy": 0.6699029126213593 }, "prompt_5": { - "accuracy": 0.3750647556553272, - "category_acc": { - "agronomy": 0.3668639053254438, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.21341463414634146, - "arts": 0.40625, - "astronomy": 0.24242424242424243, - "business_ethics": 0.3923444976076555, - "chinese_civil_service_exam": 0.375, - "chinese_driving_rule": 0.549618320610687, - "chinese_food_culture": 0.3161764705882353, - "chinese_foreign_policy": 0.45794392523364486, - "chinese_history": 0.43343653250773995, - "chinese_literature": 0.28431372549019607, - "chinese_teacher_qualification": 0.4134078212290503, - "clinical_knowledge": 0.350210970464135, - "college_actuarial_science": 0.1509433962264151, - "college_education": 0.5046728971962616, - "college_engineering_hydrology": 0.4056603773584906, - "college_law": 0.3055555555555556, - "college_mathematics": 0.23809523809523808, - "college_medical_statistics": 0.27358490566037735, - "college_medicine": 0.2967032967032967, - "computer_science": 0.35294117647058826, - "computer_security": 0.38596491228070173, - "conceptual_physics": 0.3469387755102041, - "construction_project_management": 0.3597122302158273, - "economics": 0.4779874213836478, - "education": 0.44785276073619634, - "electrical_engineering": 0.3488372093023256, - "elementary_chinese": 0.2857142857142857, - "elementary_commonsense": 0.41919191919191917, - "elementary_information_and_technology": 0.5630252100840336, - "elementary_mathematics": 0.2826086956521739, - "ethnology": 0.37777777777777777, - "food_science": 0.3356643356643357, - "genetics": 0.3352272727272727, - "global_facts": 0.44966442953020136, - "high_school_biology": 0.378698224852071, - "high_school_chemistry": 0.23484848484848486, - "high_school_geography": 0.3474576271186441, - "high_school_mathematics": 0.27439024390243905, - "high_school_physics": 0.3181818181818182, - "high_school_politics": 0.3986013986013986, - "human_sexuality": 0.47619047619047616, - "international_law": 0.35135135135135137, - "journalism": 0.436046511627907, - "jurisprudence": 0.40145985401459855, - "legal_and_moral_basis": 0.6448598130841121, - "logical": 0.3008130081300813, - "machine_learning": 0.30327868852459017, - "management": 0.42857142857142855, - "marketing": 0.5055555555555555, - "marxist_theory": 0.4973544973544973, - "modern_chinese": 0.29310344827586204, - "nutrition": 0.3793103448275862, - "philosophy": 0.3904761904761905, - "professional_accounting": 0.37714285714285717, - "professional_law": 0.3222748815165877, - "professional_medicine": 0.2978723404255319, - "professional_psychology": 0.35344827586206895, - "public_relations": 0.4540229885057471, - "security_study": 0.4222222222222222, - "sociology": 0.39823008849557523, - "sports_science": 0.3515151515151515, - "traditional_chinese_medicine": 0.2810810810810811, - "virology": 0.38461538461538464, - "world_history": 0.5279503105590062, - "world_religions": 0.4 - } + "accuracy": 0.6407766990291263 } }, - "zbench": { + "cn_eval": { "prompt_1": { - "accuracy": 0.24242424242424243 + "accuracy": 0.4666666666666667 }, "prompt_2": { - "accuracy": 0.3333333333333333 + "accuracy": 0.4666666666666667 }, "prompt_3": { - "accuracy": 0.30303030303030304 + "accuracy": 0.47619047619047616 }, "prompt_4": { - "accuracy": 0.18181818181818182 + "accuracy": 0.49523809523809526 }, "prompt_5": { - "accuracy": 0.21212121212121213 + "accuracy": 0.5142857142857142 } }, - "ind_emotion": { + "us_eval": { "prompt_1": { - "accuracy": 0.6272727272727273 + "accuracy": 0.7570093457943925 }, "prompt_2": { - "accuracy": 0.6068181818181818 + "accuracy": 0.7476635514018691 }, "prompt_3": { - "accuracy": 0.6 + "accuracy": 0.7570093457943925 }, "prompt_4": { - "accuracy": 0.6045454545454545 + "accuracy": 0.7476635514018691 }, "prompt_5": { - "accuracy": 0.6022727272727273 + "accuracy": 0.7383177570093458 } }, - "ocnli": { + "ph_eval": { "prompt_1": { - "accuracy": 0.45152542372881355 + "accuracy": 0.61, + "category_acc": { + "brand": 0.5, + "demographics": 0.2, + "biology": 0.7, + "history": 0.4666666666666667, + "literature": 0.6, + "politics": 0.9, + "culture": 0.8, + "film": 0.5, + "law": 0.6, + "geography": 0.7 + } }, "prompt_2": { - "accuracy": 0.44745762711864406 + "accuracy": 0.58, + "category_acc": { + "brand": 0.4, + "demographics": 0.0, + "biology": 0.8, + "history": 0.4666666666666667, + "literature": 0.5, + "politics": 0.9, + "culture": 0.8, + "film": 0.4, + "law": 0.6, + "geography": 0.7 + } }, "prompt_3": { - "accuracy": 0.4206779661016949 + "accuracy": 0.55, + "category_acc": { + "brand": 0.3, + "demographics": 0.2, + "biology": 0.7, + "history": 0.4, + "literature": 0.5, + "politics": 0.8, + "culture": 0.7, + "film": 0.6, + "law": 0.6, + "geography": 0.6 + } }, "prompt_4": { - "accuracy": 0.4277966101694915 + "accuracy": 0.6, + "category_acc": { + "brand": 0.4, + "demographics": 0.4, + "biology": 0.7, + "history": 0.5333333333333333, + "literature": 0.4, + "politics": 0.8, + "culture": 0.8, + "film": 0.6, + "law": 0.7, + "geography": 0.6 + } }, "prompt_5": { - "accuracy": 0.4386440677966102 + "accuracy": 0.58, + "category_acc": { + "brand": 0.4, + "demographics": 0.2, + "biology": 0.8, + "history": 0.5333333333333333, + "literature": 0.5, + "politics": 0.8, + "culture": 0.6, + "film": 0.4, + "law": 0.6, + "geography": 0.8 + } } }, - "c3": { + "sing2eng": { "prompt_1": { - "accuracy": 0.7456993268511594 + "bleu_score": 0.2794452574404052 }, "prompt_2": { - "accuracy": 0.7385938668661182 + "bleu_score": 0.2674775026836445 }, "prompt_3": { - "accuracy": 0.7262528047868362 + "bleu_score": 0.2722339627774794 }, "prompt_4": { - "accuracy": 0.7456993268511594 + "bleu_score": 0.26992687523049974 }, "prompt_5": { - "accuracy": 0.7322363500373972 + "bleu_score": 0.2339828457497402 } }, - "dream": { + "indommlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_ind2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_vie2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_zho2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "flores_zsm2eng": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "mmlu": { "prompt_1": { - "accuracy": 0.8549730524252818 + "accuracy": 0.6721120186697783 }, "prompt_2": { - "accuracy": 0.8515433610975012 + "accuracy": 0.6779463243873979 }, "prompt_3": { - "accuracy": 0.8539931406173444 + "accuracy": 0.661610268378063 }, "prompt_4": { - "accuracy": 0.8535031847133758 + "accuracy": 0.6779463243873979 }, "prompt_5": { - "accuracy": 0.8554630083292504 + "accuracy": 0.6837806301050176 } }, + "mmlu_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c_eval": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c_eval_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cmmlu": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "cmmlu_full": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "zbench": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ind_emotion": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "ocnli": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "c3": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, + "dream": { + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 + }, "samsum": { "prompt_1": -1, "prompt_2": -1, @@ -137313,824 +63542,134 @@ "prompt_5": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.8899082568807339 - }, - "prompt_2": { - "accuracy": 0.9288990825688074 - }, - "prompt_3": { - "accuracy": 0.9311926605504587 - }, - "prompt_4": { - "accuracy": 0.8600917431192661 - }, - "prompt_5": { - "accuracy": 0.926605504587156 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.7813998082454459 - }, - "prompt_2": { - "accuracy": 0.7622243528283796 - }, - "prompt_3": { - "accuracy": 0.7660594439117929 - }, - "prompt_4": { - "accuracy": 0.7833173537871524 - }, - "prompt_5": { - "accuracy": 0.788111217641419 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.8 - }, - "prompt_2": { - "accuracy": 0.7695 - }, - "prompt_3": { - "accuracy": 0.7795 - }, - "prompt_4": { - "accuracy": 0.795 - }, - "prompt_5": { - "accuracy": 0.806 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.6855 - }, - "prompt_2": { - "accuracy": 0.677 - }, - "prompt_3": { - "accuracy": 0.6705 - }, - "prompt_4": { - "accuracy": 0.672 - }, - "prompt_5": { - "accuracy": 0.6745 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.68 - }, - "prompt_2": { - "accuracy": 0.6965 - }, - "prompt_3": { - "accuracy": 0.689 - }, - "prompt_4": { - "accuracy": 0.677 - }, - "prompt_5": { - "accuracy": 0.646 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.7183098591549296 - }, - "prompt_2": { - "accuracy": 0.7464788732394366 - }, - "prompt_3": { - "accuracy": 0.6901408450704225 - }, - "prompt_4": { - "accuracy": 0.676056338028169 - }, - "prompt_5": { - "accuracy": 0.6901408450704225 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.779783393501805 - }, - "prompt_2": { - "accuracy": 0.8014440433212996 - }, - "prompt_3": { - "accuracy": 0.7870036101083032 - }, - "prompt_4": { - "accuracy": 0.7292418772563177 - }, - "prompt_5": { - "accuracy": 0.8194945848375451 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.7745098039215687 - }, - "prompt_2": { - "accuracy": 0.7647058823529411 - }, - "prompt_3": { - "accuracy": 0.7769607843137255 - }, - "prompt_4": { - "accuracy": 0.7769607843137255 - }, - "prompt_5": { - "accuracy": 0.7818627450980392 - } + "prompt_1": -1, + "prompt_2": -1, + "prompt_3": -1, + "prompt_4": -1, + "prompt_5": -1 } }, "five_shot": { "cross_xquad": { - "prompt_1": { - "overall_acc": 0.8378151260504201, - "language_acc": { - "Spanish": 0.8873949579831932, - "English": 0.9100840336134454, - "Chinese": 0.8033613445378152, - "Vietnamese": 0.7504201680672269 - }, - "consistency_score_2": 0.7822128851540616, - "consistency_score_3": 0.6857142857142856, - "consistency_score_4": 0.6184873949579832, - "detailed_consistency_score": { - "2_combine": { - "Spanish,English": 0.8949579831932774, - "Spanish,Chinese": 0.7966386554621848, - "Spanish,Vietnamese": 0.7445378151260504, - "English,Chinese": 0.8092436974789916, - "English,Vietnamese": 0.7445378151260504, - "Chinese,Vietnamese": 0.7033613445378152 - }, - "3_combine": { - "Spanish,English,Chinese": 0.7571428571428571, - "Spanish,English,Vietnamese": 0.7016806722689075, - "Spanish,Chinese,Vietnamese": 0.6403361344537815, - "English,Chinese,Vietnamese": 0.6436974789915967 - }, - "4_combine": { - "Spanish,English,Chinese,Vietnamese": 0.6184873949579832 - } - }, - "AC3_2": 0.8090598216828497, - "AC3_3": 0.7541722480002973, - "AC3_4": 0.7116352368502793 - } + "prompt_1": -1 }, "cross_mmlu": { - "prompt_1": { - "overall_acc": 0.47619047619047616, - "language_acc": { - "Filipino": 0.4, - "Vietnamese": 0.3466666666666667, - "Chinese": 0.48, - "Spanish": 0.56, - "Malay": 0.4, - "Indonesian": 0.5266666666666666, - "English": 0.62 - }, - "consistency_score_2": 0.5047619047619047, - "consistency_score_3": 0.3232380952380953, - "consistency_score_4": 0.23085714285714287, - "consistency_score_5": 0.17523809523809528, - "consistency_score_6": 0.13809523809523808, - "consistency_score_7": 0.11333333333333333, - "detailed_consistency_score": { - "2_combine": { - "Filipino,Vietnamese": 0.42, - "Filipino,Chinese": 0.4266666666666667, - "Filipino,Spanish": 0.44, - "Filipino,Malay": 0.43333333333333335, - "Filipino,Indonesian": 0.44666666666666666, - "Filipino,English": 0.5066666666666667, - "Vietnamese,Chinese": 0.4066666666666667, - "Vietnamese,Spanish": 0.47333333333333333, - "Vietnamese,Malay": 0.43333333333333335, - "Vietnamese,Indonesian": 0.4666666666666667, - "Vietnamese,English": 0.4266666666666667, - "Chinese,Spanish": 0.58, - "Chinese,Malay": 0.4666666666666667, - "Chinese,Indonesian": 0.5533333333333333, - "Chinese,English": 0.56, - "Spanish,Malay": 0.49333333333333335, - "Spanish,Indonesian": 0.6733333333333333, - "Spanish,English": 0.7, - "Malay,Indonesian": 0.5866666666666667, - "Malay,English": 0.48, - "Indonesian,English": 0.6266666666666667 - }, - "3_combine": { - "Filipino,Vietnamese,Chinese": 0.22666666666666666, - "Filipino,Vietnamese,Spanish": 0.24, - "Filipino,Vietnamese,Malay": 0.22666666666666666, - "Filipino,Vietnamese,Indonesian": 0.24, - "Filipino,Vietnamese,English": 0.26, - "Filipino,Chinese,Spanish": 0.29333333333333333, - "Filipino,Chinese,Malay": 0.26666666666666666, - "Filipino,Chinese,Indonesian": 0.30666666666666664, - "Filipino,Chinese,English": 0.31333333333333335, - "Filipino,Spanish,Malay": 0.2733333333333333, - "Filipino,Spanish,Indonesian": 0.3466666666666667, - "Filipino,Spanish,English": 0.36666666666666664, - "Filipino,Malay,Indonesian": 0.3, - "Filipino,Malay,English": 0.2866666666666667, - "Filipino,Indonesian,English": 0.36666666666666664, - "Vietnamese,Chinese,Spanish": 0.28, - "Vietnamese,Chinese,Malay": 0.24, - "Vietnamese,Chinese,Indonesian": 0.28, - "Vietnamese,Chinese,English": 0.25333333333333335, - "Vietnamese,Spanish,Malay": 0.26666666666666666, - "Vietnamese,Spanish,Indonesian": 0.36666666666666664, - "Vietnamese,Spanish,English": 0.35333333333333333, - "Vietnamese,Malay,Indonesian": 0.3, - "Vietnamese,Malay,English": 0.26, - "Vietnamese,Indonesian,English": 0.32, - "Chinese,Spanish,Malay": 0.32666666666666666, - "Chinese,Spanish,Indonesian": 0.44666666666666666, - "Chinese,Spanish,English": 0.46, - "Chinese,Malay,Indonesian": 0.36666666666666664, - "Chinese,Malay,English": 0.31333333333333335, - "Chinese,Indonesian,English": 0.41333333333333333, - "Spanish,Malay,Indonesian": 0.42, - "Spanish,Malay,English": 0.4066666666666667, - "Spanish,Indonesian,English": 0.54, - "Malay,Indonesian,English": 0.38666666666666666 - }, - "4_combine": { - "Filipino,Vietnamese,Chinese,Spanish": 0.16, - "Filipino,Vietnamese,Chinese,Malay": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,Indonesian": 0.16666666666666666, - "Filipino,Vietnamese,Chinese,English": 0.16, - "Filipino,Vietnamese,Spanish,Malay": 0.17333333333333334, - "Filipino,Vietnamese,Spanish,Indonesian": 0.20666666666666667, - "Filipino,Vietnamese,Spanish,English": 0.21333333333333335, - "Filipino,Vietnamese,Malay,Indonesian": 0.18, - "Filipino,Vietnamese,Malay,English": 0.16666666666666666, - "Filipino,Vietnamese,Indonesian,English": 0.2, - "Filipino,Chinese,Spanish,Malay": 0.20666666666666667, - "Filipino,Chinese,Spanish,Indonesian": 0.24666666666666667, - "Filipino,Chinese,Spanish,English": 0.26, - "Filipino,Chinese,Malay,Indonesian": 0.23333333333333334, - "Filipino,Chinese,Malay,English": 0.20666666666666667, - "Filipino,Chinese,Indonesian,English": 0.26, - "Filipino,Spanish,Malay,Indonesian": 0.24666666666666667, - "Filipino,Spanish,Malay,English": 0.24, - "Filipino,Spanish,Indonesian,English": 0.32666666666666666, - "Filipino,Malay,Indonesian,English": 0.24666666666666667, - "Vietnamese,Chinese,Spanish,Malay": 0.18, - "Vietnamese,Chinese,Spanish,Indonesian": 0.23333333333333334, - "Vietnamese,Chinese,Spanish,English": 0.23333333333333334, - "Vietnamese,Chinese,Malay,Indonesian": 0.19333333333333333, - "Vietnamese,Chinese,Malay,English": 0.16666666666666666, - "Vietnamese,Chinese,Indonesian,English": 0.22666666666666666, - "Vietnamese,Spanish,Malay,Indonesian": 0.23333333333333334, - "Vietnamese,Spanish,Malay,English": 0.24, - "Vietnamese,Spanish,Indonesian,English": 0.3, - "Vietnamese,Malay,Indonesian,English": 0.22, - "Chinese,Spanish,Malay,Indonesian": 0.29333333333333333, - "Chinese,Spanish,Malay,English": 0.2866666666666667, - "Chinese,Spanish,Indonesian,English": 0.38, - "Chinese,Malay,Indonesian,English": 0.26666666666666666, - "Spanish,Malay,Indonesian,English": 0.36 - }, - "5_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay": 0.13333333333333333, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian": 0.14, - "Filipino,Vietnamese,Chinese,Spanish,English": 0.14666666666666667, - "Filipino,Vietnamese,Chinese,Malay,Indonesian": 0.14, - "Filipino,Vietnamese,Chinese,Malay,English": 0.12666666666666668, - "Filipino,Vietnamese,Chinese,Indonesian,English": 0.14, - "Filipino,Vietnamese,Spanish,Malay,Indonesian": 0.16, - "Filipino,Vietnamese,Spanish,Malay,English": 0.16, - "Filipino,Vietnamese,Spanish,Indonesian,English": 0.19333333333333333, - "Filipino,Vietnamese,Malay,Indonesian,English": 0.15333333333333332, - "Filipino,Chinese,Spanish,Malay,Indonesian": 0.19333333333333333, - "Filipino,Chinese,Spanish,Malay,English": 0.18666666666666668, - "Filipino,Chinese,Spanish,Indonesian,English": 0.23333333333333334, - "Filipino,Chinese,Malay,Indonesian,English": 0.18666666666666668, - "Filipino,Spanish,Malay,Indonesian,English": 0.23333333333333334, - "Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.16, - "Vietnamese,Chinese,Spanish,Malay,English": 0.16, - "Vietnamese,Chinese,Spanish,Indonesian,English": 0.21333333333333335, - "Vietnamese,Chinese,Malay,Indonesian,English": 0.14666666666666667, - "Vietnamese,Spanish,Malay,Indonesian,English": 0.21333333333333335, - "Chinese,Spanish,Malay,Indonesian,English": 0.26 - }, - "6_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian": 0.12, - "Filipino,Vietnamese,Chinese,Spanish,Malay,English": 0.12, - "Filipino,Vietnamese,Chinese,Spanish,Indonesian,English": 0.13333333333333333, - "Filipino,Vietnamese,Chinese,Malay,Indonesian,English": 0.11333333333333333, - "Filipino,Vietnamese,Spanish,Malay,Indonesian,English": 0.15333333333333332, - "Filipino,Chinese,Spanish,Malay,Indonesian,English": 0.18, - "Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.14666666666666667 - }, - "7_combine": { - "Filipino,Vietnamese,Chinese,Spanish,Malay,Indonesian,English": 0.11333333333333333 - } - }, - "AC3_2": 0.4900601016606295, - "AC3_3": 0.38508231498410966, - "AC3_4": 0.3109605910890248, - "AC3_5": 0.25619604563040155, - "AC3_6": 0.21410114429884874, - "AC3_7": 0.18309100696948108 - } + "prompt_1": -1 }, "cross_logiqa": { - "prompt_1": { - "overall_acc": 0.41883116883116883, - "language_acc": { - "Indonesian": 0.4318181818181818, - "English": 0.5568181818181818, - "Filipino": 0.32386363636363635, - "Spanish": 0.4602272727272727, - "Chinese": 0.4602272727272727, - "Malay": 0.38636363636363635, - "Vietnamese": 0.3125 - }, - "consistency_score_2": 0.45481601731601723, - "consistency_score_3": 0.27094155844155837, - "consistency_score_4": 0.1862012987012987, - "consistency_score_5": 0.13825757575757575, - "consistency_score_6": 0.10714285714285714, - "consistency_score_7": 0.08522727272727272, - "detailed_consistency_score": { - "2_combine": { - "Indonesian,English": 0.5852272727272727, - "Indonesian,Filipino": 0.4318181818181818, - "Indonesian,Spanish": 0.6022727272727273, - "Indonesian,Chinese": 0.5, - "Indonesian,Malay": 0.5170454545454546, - "Indonesian,Vietnamese": 0.4034090909090909, - "English,Filipino": 0.4034090909090909, - "English,Spanish": 0.5852272727272727, - "English,Chinese": 0.5454545454545454, - "English,Malay": 0.45454545454545453, - "English,Vietnamese": 0.375, - "Filipino,Spanish": 0.36363636363636365, - "Filipino,Chinese": 0.4034090909090909, - "Filipino,Malay": 0.39204545454545453, - "Filipino,Vietnamese": 0.45454545454545453, - "Spanish,Chinese": 0.5170454545454546, - "Spanish,Malay": 0.4659090909090909, - "Spanish,Vietnamese": 0.3522727272727273, - "Chinese,Malay": 0.42613636363636365, - "Chinese,Vietnamese": 0.36363636363636365, - "Malay,Vietnamese": 0.4090909090909091 - }, - "3_combine": { - "Indonesian,English,Filipino": 0.2897727272727273, - "Indonesian,English,Spanish": 0.4318181818181818, - "Indonesian,English,Chinese": 0.39204545454545453, - "Indonesian,English,Malay": 0.3409090909090909, - "Indonesian,English,Vietnamese": 0.26704545454545453, - "Indonesian,Filipino,Spanish": 0.26704545454545453, - "Indonesian,Filipino,Chinese": 0.25, - "Indonesian,Filipino,Malay": 0.26704545454545453, - "Indonesian,Filipino,Vietnamese": 0.25, - "Indonesian,Spanish,Chinese": 0.3693181818181818, - "Indonesian,Spanish,Malay": 0.3522727272727273, - "Indonesian,Spanish,Vietnamese": 0.2556818181818182, - "Indonesian,Chinese,Malay": 0.3068181818181818, - "Indonesian,Chinese,Vietnamese": 0.2215909090909091, - "Indonesian,Malay,Vietnamese": 0.25, - "English,Filipino,Spanish": 0.24431818181818182, - "English,Filipino,Chinese": 0.26704545454545453, - "English,Filipino,Malay": 0.2159090909090909, - "English,Filipino,Vietnamese": 0.23863636363636365, - "English,Spanish,Chinese": 0.39204545454545453, - "English,Spanish,Malay": 0.32386363636363635, - "English,Spanish,Vietnamese": 0.22727272727272727, - "English,Chinese,Malay": 0.3068181818181818, - "English,Chinese,Vietnamese": 0.22727272727272727, - "English,Malay,Vietnamese": 0.22727272727272727, - "Filipino,Spanish,Chinese": 0.23863636363636365, - "Filipino,Spanish,Malay": 0.2215909090909091, - "Filipino,Spanish,Vietnamese": 0.19886363636363635, - "Filipino,Chinese,Malay": 0.20454545454545456, - "Filipino,Chinese,Vietnamese": 0.23295454545454544, - "Filipino,Malay,Vietnamese": 0.23295454545454544, - "Spanish,Chinese,Malay": 0.3125, - "Spanish,Chinese,Vietnamese": 0.2159090909090909, - "Spanish,Malay,Vietnamese": 0.2215909090909091, - "Chinese,Malay,Vietnamese": 0.2215909090909091 - }, - "4_combine": { - "Indonesian,English,Filipino,Spanish": 0.20454545454545456, - "Indonesian,English,Filipino,Chinese": 0.21022727272727273, - "Indonesian,English,Filipino,Malay": 0.19318181818181818, - "Indonesian,English,Filipino,Vietnamese": 0.19886363636363635, - "Indonesian,English,Spanish,Chinese": 0.3125, - "Indonesian,English,Spanish,Malay": 0.26704545454545453, - "Indonesian,English,Spanish,Vietnamese": 0.1875, - "Indonesian,English,Chinese,Malay": 0.2556818181818182, - "Indonesian,English,Chinese,Vietnamese": 0.18181818181818182, - "Indonesian,English,Malay,Vietnamese": 0.17613636363636365, - "Indonesian,Filipino,Spanish,Chinese": 0.1875, - "Indonesian,Filipino,Spanish,Malay": 0.1875, - "Indonesian,Filipino,Spanish,Vietnamese": 0.17045454545454544, - "Indonesian,Filipino,Chinese,Malay": 0.17045454545454544, - "Indonesian,Filipino,Chinese,Vietnamese": 0.16477272727272727, - "Indonesian,Filipino,Malay,Vietnamese": 0.17613636363636365, - "Indonesian,Spanish,Chinese,Malay": 0.25, - "Indonesian,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "Indonesian,Spanish,Malay,Vietnamese": 0.1875, - "Indonesian,Chinese,Malay,Vietnamese": 0.1534090909090909, - "English,Filipino,Spanish,Chinese": 0.1875, - "English,Filipino,Spanish,Malay": 0.17613636363636365, - "English,Filipino,Spanish,Vietnamese": 0.1534090909090909, - "English,Filipino,Chinese,Malay": 0.16477272727272727, - "English,Filipino,Chinese,Vietnamese": 0.16477272727272727, - "English,Filipino,Malay,Vietnamese": 0.1534090909090909, - "English,Spanish,Chinese,Malay": 0.2556818181818182, - "English,Spanish,Chinese,Vietnamese": 0.17613636363636365, - "English,Spanish,Malay,Vietnamese": 0.16477272727272727, - "English,Chinese,Malay,Vietnamese": 0.1534090909090909, - "Filipino,Spanish,Chinese,Malay": 0.14772727272727273, - "Filipino,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "Filipino,Spanish,Malay,Vietnamese": 0.1534090909090909, - "Filipino,Chinese,Malay,Vietnamese": 0.14772727272727273, - "Spanish,Chinese,Malay,Vietnamese": 0.1590909090909091 - }, - "5_combine": { - "Indonesian,English,Filipino,Spanish,Chinese": 0.1590909090909091, - "Indonesian,English,Filipino,Spanish,Malay": 0.1534090909090909, - "Indonesian,English,Filipino,Spanish,Vietnamese": 0.14204545454545456, - "Indonesian,English,Filipino,Chinese,Malay": 0.14772727272727273, - "Indonesian,English,Filipino,Chinese,Vietnamese": 0.14204545454545456, - "Indonesian,English,Filipino,Malay,Vietnamese": 0.14204545454545456, - "Indonesian,English,Spanish,Chinese,Malay": 0.2159090909090909, - "Indonesian,English,Spanish,Chinese,Vietnamese": 0.14772727272727273, - "Indonesian,English,Spanish,Malay,Vietnamese": 0.14204545454545456, - "Indonesian,English,Chinese,Malay,Vietnamese": 0.125, - "Indonesian,Filipino,Spanish,Chinese,Malay": 0.13068181818181818, - "Indonesian,Filipino,Spanish,Chinese,Vietnamese": 0.13068181818181818, - "Indonesian,Filipino,Spanish,Malay,Vietnamese": 0.13636363636363635, - "Indonesian,Filipino,Chinese,Malay,Vietnamese": 0.11931818181818182, - "Indonesian,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818, - "English,Filipino,Spanish,Chinese,Malay": 0.13636363636363635, - "English,Filipino,Spanish,Chinese,Vietnamese": 0.125, - "English,Filipino,Spanish,Malay,Vietnamese": 0.125, - "English,Filipino,Chinese,Malay,Vietnamese": 0.11363636363636363, - "English,Spanish,Chinese,Malay,Vietnamese": 0.13068181818181818, - "Filipino,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546 - }, - "6_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay": 0.11931818181818182, - "Indonesian,English,Filipino,Spanish,Chinese,Vietnamese": 0.11363636363636363, - "Indonesian,English,Filipino,Spanish,Malay,Vietnamese": 0.11363636363636363, - "Indonesian,English,Filipino,Chinese,Malay,Vietnamese": 0.10227272727272728, - "Indonesian,English,Spanish,Chinese,Malay,Vietnamese": 0.10795454545454546, - "Indonesian,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909, - "English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.09659090909090909 - }, - "7_combine": { - "Indonesian,English,Filipino,Spanish,Chinese,Malay,Vietnamese": 0.08522727272727272 - } - }, - "AC3_2": 0.43608249905519153, - "AC3_3": 0.329032346753532, - "AC3_4": 0.257794124266139, - "AC3_5": 0.20788997302992634, - "AC3_6": 0.17063492060247892, - "AC3_7": 0.14163372856215165 - } + "prompt_1": -1 }, "sg_eval": { - "prompt_1": { - "accuracy": 0.6796116504854369 - } + "prompt_1": -1 }, "cn_eval": { - "prompt_1": { - "accuracy": 0.45714285714285713 - } + "prompt_1": -1 }, "us_eval": { - "prompt_1": { - "accuracy": 0.5794392523364486 - } + "prompt_1": -1 }, "ph_eval": { - "prompt_1": { - "accuracy": 0.4, - "category_acc": { - "brand": 0.4, - "demographics": 0.0, - "biology": 0.5, - "history": 0.26666666666666666, - "literature": 0.3, - "politics": 0.6, - "culture": 0.4, - "film": 0.5, - "law": 0.3, - "geography": 0.6 - } - } + "prompt_1": -1 }, "sing2eng": { - "prompt_1": { - "bleu_score": 0.27211252439428596 - } + "prompt_1": -1 }, "indommlu": { - "prompt_1": { - "accuracy": 0.4392149008612057, - "category_acc": { - "History": 0.43172690763052207, - "Geography": 0.4, - "Lampungic": 0.32653061224489793, - "Social science": 0.662771285475793, - "Balinese": 0.32908704883227174, - "Makassarese": 0.3709677419354839, - "Banjarese": 0.3819444444444444, - "Chemistry": 0.2846715328467153, - "Biology": 0.36449704142011835, - "Science": 0.5263157894736842, - "Christian religion": 0.5223880597014925, - "Art": 0.5174708818635607, - "Islam religion": 0.5220483641536273, - "Hindu religion": 0.43333333333333335, - "Madurese": 0.3254237288135593, - "Sport": 0.527027027027027, - "Indonesian language": 0.5071606475716065, - "Physics": 0.3292929292929293, - "Minangkabau culture": 0.36180904522613067, - "Dayak language": 0.22935779816513763, - "Sociology": 0.4415322580645161, - "Economy": 0.4036885245901639, - "Sundanese": 0.35695764909248057, - "Javanese": 0.3074596774193548, - "Civic education": 0.5522174535050072 - } - } + "prompt_1": -1 }, "flores_ind2eng": { - "prompt_1": { - "bleu_score": 0.344889923274345 - } + "prompt_1": -1 }, "flores_vie2eng": { - "prompt_1": { - "bleu_score": 0.14552500173871624 - } + "prompt_1": -1 }, "flores_zho2eng": { - "prompt_1": { - "bleu_score": 0.2045158077566893 - } + "prompt_1": -1 }, "flores_zsm2eng": { - "prompt_1": { - "bleu_score": 0.315854425021981 - } + "prompt_1": -1 }, "mmlu": { - "prompt_1": { - "accuracy": 0.5344224037339557 - } + "prompt_1": -1 }, "mmlu_full": { - "prompt_1": { - "accuracy": 0.5263496603503754, - "category_acc": { - "high_school_european_history": 0.6829268292682927, - "business_ethics": 0.5656565656565656, - "clinical_knowledge": 0.6098484848484849, - "medical_genetics": 0.5757575757575758, - "high_school_us_history": 0.7142857142857143, - "high_school_physics": 0.32, - "high_school_world_history": 0.7415254237288136, - "virology": 0.43636363636363634, - "high_school_microeconomics": 0.510548523206751, - "econometrics": 0.36283185840707965, - "college_computer_science": 0.4444444444444444, - "high_school_biology": 0.6537216828478964, - "abstract_algebra": 0.24242424242424243, - "professional_accounting": 0.40213523131672596, - "philosophy": 0.5838709677419355, - "professional_medicine": 0.5645756457564576, - "nutrition": 0.5475409836065573, - "global_facts": 0.18181818181818182, - "machine_learning": 0.36036036036036034, - "security_studies": 0.6311475409836066, - "public_relations": 0.5871559633027523, - "professional_psychology": 0.5155482815057283, - "prehistory": 0.5820433436532507, - "anatomy": 0.4552238805970149, - "human_sexuality": 0.5692307692307692, - "college_medicine": 0.5232558139534884, - "high_school_government_and_politics": 0.7447916666666666, - "college_chemistry": 0.3434343434343434, - "logical_fallacies": 0.6419753086419753, - "high_school_geography": 0.700507614213198, - "elementary_mathematics": 0.35543766578249336, - "human_aging": 0.6396396396396397, - "college_mathematics": 0.3333333333333333, - "high_school_psychology": 0.7536764705882353, - "formal_logic": 0.368, - "high_school_statistics": 0.4, - "international_law": 0.6583333333333333, - "high_school_mathematics": 0.2899628252788104, - "high_school_computer_science": 0.5353535353535354, - "conceptual_physics": 0.41452991452991456, - "miscellaneous": 0.7263427109974424, - "high_school_chemistry": 0.4603960396039604, - "marketing": 0.7682403433476395, - "professional_law": 0.4279191128506197, - "management": 0.6470588235294118, - "college_physics": 0.297029702970297, - "jurisprudence": 0.7009345794392523, - "world_religions": 0.7352941176470589, - "sociology": 0.745, - "us_foreign_policy": 0.7676767676767676, - "high_school_macroeconomics": 0.5115681233933161, - "computer_security": 0.6262626262626263, - "moral_scenarios": 0.21029082774049218, - "moral_disputes": 0.6144927536231884, - "electrical_engineering": 0.4930555555555556, - "astronomy": 0.5099337748344371, - "college_biology": 0.5734265734265734 - } - } + "prompt_1": -1 }, "c_eval": { - "prompt_1": { - "accuracy": 0.4086181277860327 - } + "prompt_1": -1 }, "c_eval_full": { - "prompt_1": { - "accuracy": 0.37982565379825656, - "category_acc": { - "computer_network": 0.2916666666666667, - "operating_system": 0.3333333333333333, - "computer_architecture": 0.34615384615384615, - "college_programming": 0.3333333333333333, - "college_physics": 0.2916666666666667, - "college_chemistry": 0.20689655172413793, - "advanced_mathematics": 0.4166666666666667, - "probability_and_statistics": 0.34782608695652173, - "discrete_mathematics": 0.14285714285714285, - "electrical_engineer": 0.35714285714285715, - "metrology_engineer": 0.41379310344827586, - "high_school_mathematics": 0.21739130434782608, - "high_school_physics": 0.16666666666666666, - "high_school_chemistry": 0.375, - "high_school_biology": 0.2916666666666667, - "middle_school_mathematics": 0.375, - "middle_school_biology": 0.6538461538461539, - "middle_school_physics": 0.4583333333333333, - "middle_school_chemistry": 0.48, - "veterinary_medicine": 0.32142857142857145, - "college_economics": 0.5333333333333333, - "business_administration": 0.34210526315789475, - "marxism": 0.4166666666666667, - "mao_zedong_thought": 0.5172413793103449, - "education_science": 0.5, - "teacher_qualification": 0.4489795918367347, - "high_school_politics": 0.16666666666666666, - "high_school_geography": 0.375, - "middle_school_politics": 0.5, - "middle_school_geography": 0.4117647058823529, - "modern_chinese_history": 0.2857142857142857, - "ideological_and_moral_cultivation": 0.5416666666666666, - "logic": 0.5185185185185185, - "law": 0.27586206896551724, - "chinese_language_and_literature": 0.39285714285714285, - "art_studies": 0.4473684210526316, - "professional_tour_guide": 0.38235294117647056, - "legal_professional": 0.5, - "high_school_chinese": 0.25, - "high_school_history": 0.48, - "middle_school_history": 0.5555555555555556, - "civil_servant": 0.36538461538461536, - "sports_science": 0.20833333333333334, - "plant_protection": 0.4444444444444444, - "basic_medicine": 0.25, - "clinical_medicine": 0.4074074074074074, - "urban_and_rural_planner": 0.4117647058823529, - "accountant": 0.42592592592592593, - "fire_engineer": 0.3333333333333333, - "environmental_impact_assessment_engineer": 0.3055555555555556, - "tax_accountant": 0.2962962962962963, - "physician": 0.35185185185185186 - } - } + "prompt_1": -1 }, "cmmlu": { - "prompt_1": { - "accuracy": 0.3978494623655914 - } + "prompt_1": -1 }, "cmmlu_full": { - "prompt_1": { - "accuracy": 0.40701087895009497, - "category_acc": { - "agronomy": 0.4260355029585799, - "anatomy": 0.2702702702702703, - "ancient_chinese": 0.23170731707317074, - "arts": 0.4125, - "astronomy": 0.296969696969697, - "business_ethics": 0.4258373205741627, - "chinese_civil_service_exam": 0.45625, - "chinese_driving_rule": 0.5267175572519084, - "chinese_food_culture": 0.375, - "chinese_foreign_policy": 0.514018691588785, - "chinese_history": 0.4582043343653251, - "chinese_literature": 0.35294117647058826, - "chinese_teacher_qualification": 0.45251396648044695, - "clinical_knowledge": 0.37130801687763715, - "college_actuarial_science": 0.24528301886792453, - "college_education": 0.48598130841121495, - "college_engineering_hydrology": 0.37735849056603776, - "college_law": 0.3055555555555556, - "college_mathematics": 0.29523809523809524, - "college_medical_statistics": 0.3867924528301887, - "college_medicine": 0.358974358974359, - "computer_science": 0.39215686274509803, - "computer_security": 0.4619883040935672, - "conceptual_physics": 0.3673469387755102, - "construction_project_management": 0.37410071942446044, - "economics": 0.4716981132075472, - "education": 0.48466257668711654, - "electrical_engineering": 0.31976744186046513, - "elementary_chinese": 0.29365079365079366, - "elementary_commonsense": 0.3686868686868687, - "elementary_information_and_technology": 0.5504201680672269, - "elementary_mathematics": 0.28695652173913044, - "ethnology": 0.34074074074074073, - "food_science": 0.43356643356643354, - "genetics": 0.3352272727272727, - "global_facts": 0.44966442953020136, - "high_school_biology": 0.35502958579881655, - "high_school_chemistry": 0.3106060606060606, - "high_school_geography": 0.3728813559322034, - "high_school_mathematics": 0.2865853658536585, - "high_school_physics": 0.33636363636363636, - "high_school_politics": 0.34965034965034963, - "human_sexuality": 0.5, - "international_law": 0.4486486486486487, - "journalism": 0.4883720930232558, - "jurisprudence": 0.4282238442822384, - "legal_and_moral_basis": 0.6915887850467289, - "logical": 0.3821138211382114, - "machine_learning": 0.28688524590163933, - "management": 0.4523809523809524, - "marketing": 0.5222222222222223, - "marxist_theory": 0.5396825396825397, - "modern_chinese": 0.3103448275862069, - "nutrition": 0.41379310344827586, - "philosophy": 0.42857142857142855, - "professional_accounting": 0.5028571428571429, - "professional_law": 0.35545023696682465, - "professional_medicine": 0.3404255319148936, - "professional_psychology": 0.4396551724137931, - "public_relations": 0.46551724137931033, - "security_study": 0.48148148148148145, - "sociology": 0.4469026548672566, - "sports_science": 0.40606060606060607, - "traditional_chinese_medicine": 0.33513513513513515, - "virology": 0.44970414201183434, - "world_history": 0.515527950310559, - "world_religions": 0.46875 - } - } + "prompt_1": -1 }, "zbench": { - "prompt_1": { - "accuracy": 0.3333333333333333 - } + "prompt_1": -1 }, "ind_emotion": { - "prompt_1": { - "accuracy": 0.5295454545454545 - } + "prompt_1": -1 }, "ocnli": { - "prompt_1": { - "accuracy": 0.3976271186440678 - } + "prompt_1": -1 }, "c3": { - "prompt_1": { - "accuracy": 0.7673896783844428 - } + "prompt_1": -1 }, "dream": { - "prompt_1": { - "accuracy": 0.8530132288094071 - } + "prompt_1": -1 }, "samsum": { "prompt_1": -1 @@ -138139,44 +63678,28 @@ "prompt_1": -1 }, "sst2": { - "prompt_1": { - "accuracy": 0.8830275229357798 - } + "prompt_1": -1 }, "cola": { - "prompt_1": { - "accuracy": 0.6174496644295302 - } + "prompt_1": -1 }, "qqp": { - "prompt_1": { - "accuracy": 0.759 - } + "prompt_1": -1 }, "mnli": { - "prompt_1": { - "accuracy": 0.591 - } + "prompt_1": -1 }, "qnli": { - "prompt_1": { - "accuracy": 0.5865 - } + "prompt_1": -1 }, "wnli": { - "prompt_1": { - "accuracy": 0.5774647887323944 - } + "prompt_1": -1 }, "rte": { - "prompt_1": { - "accuracy": 0.703971119133574 - } + "prompt_1": -1 }, "mrpc": { - "prompt_1": { - "accuracy": 0.7083333333333334 - } + "prompt_1": -1 } } }