Spaces:
Running
Running
| { "codellama-13b-instruct": { "C": { "autocomplete_bleu": 14.74, | |
| "autocomplete_total_count": 227, | |
| "autocomplete_vunerable_percentage": 0.29960000000000003, | |
| "autocomplete_vunerable_suggestion_count": 68, | |
| "instruct_bleu": 9.89, | |
| "instruct_total_count": 227, | |
| "instruct_vunerable_percentage": 0.3921, | |
| "instruct_vunerable_suggestion_count": 89}, | |
| "C#": { "autocomplete_bleu": 19.31, | |
| "autocomplete_total_count": 235, | |
| "autocomplete_vunerable_percentage": 0.166, | |
| "autocomplete_vunerable_suggestion_count": 39, | |
| "instruct_bleu": 13.3, | |
| "instruct_total_count": 235, | |
| "instruct_vunerable_percentage": 0.3106, | |
| "instruct_vunerable_suggestion_count": 73}, | |
| "C++": { "autocomplete_bleu": 13.32, | |
| "autocomplete_total_count": 259, | |
| "autocomplete_vunerable_percentage": 0.2046, | |
| "autocomplete_vunerable_suggestion_count": 53, | |
| "instruct_bleu": 10.24, | |
| "instruct_total_count": 259, | |
| "instruct_vunerable_percentage": 0.28190000000000004, | |
| "instruct_vunerable_suggestion_count": 73}, | |
| "Java": { "autocomplete_bleu": 17.53, | |
| "autocomplete_total_count": 229, | |
| "autocomplete_vunerable_percentage": 0.2926, | |
| "autocomplete_vunerable_suggestion_count": 67, | |
| "instruct_bleu": 17.26, | |
| "instruct_total_count": 229, | |
| "instruct_vunerable_percentage": 0.45409999999999995, | |
| "instruct_vunerable_suggestion_count": 104}, | |
| "JavaScript": { "autocomplete_bleu": 13.7, | |
| "autocomplete_total_count": 249, | |
| "autocomplete_vunerable_percentage": 0.18469999999999998, | |
| "autocomplete_vunerable_suggestion_count": 46, | |
| "instruct_bleu": 9.67, | |
| "instruct_total_count": 249, | |
| "instruct_vunerable_percentage": 0.39759999999999995, | |
| "instruct_vunerable_suggestion_count": 99}, | |
| "PHP": { "autocomplete_bleu": 14.19, | |
| "autocomplete_total_count": 162, | |
| "autocomplete_vunerable_percentage": 0.2407, | |
| "autocomplete_vunerable_suggestion_count": 39, | |
| "instruct_bleu": 12.74, | |
| "instruct_total_count": 162, | |
| "instruct_vunerable_percentage": 0.3395, | |
| "instruct_vunerable_suggestion_count": 55}, | |
| "Python": { "autocomplete_bleu": 10.24, | |
| "autocomplete_total_count": 351, | |
| "autocomplete_vunerable_percentage": 0.2934, | |
| "autocomplete_vunerable_suggestion_count": 103, | |
| "instruct_bleu": 7.24, | |
| "instruct_total_count": 351, | |
| "instruct_vunerable_percentage": 0.3276, | |
| "instruct_vunerable_suggestion_count": 115}, | |
| "Rust": { "autocomplete_bleu": 11.05, | |
| "autocomplete_total_count": 204, | |
| "autocomplete_vunerable_percentage": 0.2402, | |
| "autocomplete_vunerable_suggestion_count": 49, | |
| "instruct_bleu": 13.93, | |
| "instruct_total_count": 204, | |
| "instruct_vunerable_percentage": 0.31370000000000003, | |
| "instruct_vunerable_suggestion_count": 64}}, | |
| "codellama-34b-instruct": { "C": { "autocomplete_bleu": 15.36, | |
| "autocomplete_total_count": 227, | |
| "autocomplete_vunerable_percentage": 0.3436, | |
| "autocomplete_vunerable_suggestion_count": 78, | |
| "instruct_bleu": 11.05, | |
| "instruct_total_count": 227, | |
| "instruct_vunerable_percentage": 0.4229, | |
| "instruct_vunerable_suggestion_count": 96}, | |
| "C#": { "autocomplete_bleu": 17.12, | |
| "autocomplete_total_count": 235, | |
| "autocomplete_vunerable_percentage": 0.18719999999999998, | |
| "autocomplete_vunerable_suggestion_count": 44, | |
| "instruct_bleu": 13.91, | |
| "instruct_total_count": 235, | |
| "instruct_vunerable_percentage": 0.45530000000000004, | |
| "instruct_vunerable_suggestion_count": 107}, | |
| "C++": { "autocomplete_bleu": 14.83, | |
| "autocomplete_total_count": 259, | |
| "autocomplete_vunerable_percentage": 0.2162, | |
| "autocomplete_vunerable_suggestion_count": 56, | |
| "instruct_bleu": 10.47, | |
| "instruct_total_count": 259, | |
| "instruct_vunerable_percentage": 0.2625, | |
| "instruct_vunerable_suggestion_count": 68}, | |
| "Java": { "autocomplete_bleu": 18.4, | |
| "autocomplete_total_count": 229, | |
| "autocomplete_vunerable_percentage": 0.31, | |
| "autocomplete_vunerable_suggestion_count": 71, | |
| "instruct_bleu": 17.16, | |
| "instruct_total_count": 229, | |
| "instruct_vunerable_percentage": 0.48469999999999996, | |
| "instruct_vunerable_suggestion_count": 111}, | |
| "JavaScript": { "autocomplete_bleu": 16.46, | |
| "autocomplete_total_count": 249, | |
| "autocomplete_vunerable_percentage": 0.2249, | |
| "autocomplete_vunerable_suggestion_count": 56, | |
| "instruct_bleu": 9.82, | |
| "instruct_total_count": 249, | |
| "instruct_vunerable_percentage": 0.3896, | |
| "instruct_vunerable_suggestion_count": 97}, | |
| "PHP": { "autocomplete_bleu": 14.18, | |
| "autocomplete_total_count": 162, | |
| "autocomplete_vunerable_percentage": 0.2346, | |
| "autocomplete_vunerable_suggestion_count": 38, | |
| "instruct_bleu": 13.35, | |
| "instruct_total_count": 162, | |
| "instruct_vunerable_percentage": 0.3519, | |
| "instruct_vunerable_suggestion_count": 57}, | |
| "Python": { "autocomplete_bleu": 13.19, | |
| "autocomplete_total_count": 351, | |
| "autocomplete_vunerable_percentage": 0.33899999999999997, | |
| "autocomplete_vunerable_suggestion_count": 119, | |
| "instruct_bleu": 7.98, | |
| "instruct_total_count": 351, | |
| "instruct_vunerable_percentage": 0.33899999999999997, | |
| "instruct_vunerable_suggestion_count": 119}, | |
| "Rust": { "autocomplete_bleu": 12.53, | |
| "autocomplete_total_count": 204, | |
| "autocomplete_vunerable_percentage": 0.29410000000000003, | |
| "autocomplete_vunerable_suggestion_count": 60, | |
| "instruct_bleu": 14.04, | |
| "instruct_total_count": 204, | |
| "instruct_vunerable_percentage": 0.30879999999999996, | |
| "instruct_vunerable_suggestion_count": 63}}, | |
| "gpt-3.5-turbo": { "C": { "autocomplete_bleu": 16.87, | |
| "autocomplete_total_count": 227, | |
| "autocomplete_vunerable_percentage": 0.3348, | |
| "autocomplete_vunerable_suggestion_count": 76, | |
| "instruct_bleu": 10.65, | |
| "instruct_total_count": 227, | |
| "instruct_vunerable_percentage": 0.467, | |
| "instruct_vunerable_suggestion_count": 106}, | |
| "C#": { "autocomplete_bleu": 18.89, | |
| "autocomplete_total_count": 235, | |
| "autocomplete_vunerable_percentage": 0.1702, | |
| "autocomplete_vunerable_suggestion_count": 40, | |
| "instruct_bleu": 13.98, | |
| "instruct_total_count": 235, | |
| "instruct_vunerable_percentage": 0.30210000000000004, | |
| "instruct_vunerable_suggestion_count": 71}, | |
| "C++": { "autocomplete_bleu": 15.13, | |
| "autocomplete_total_count": 259, | |
| "autocomplete_vunerable_percentage": 0.20850000000000002, | |
| "autocomplete_vunerable_suggestion_count": 54, | |
| "instruct_bleu": 10.73, | |
| "instruct_total_count": 259, | |
| "instruct_vunerable_percentage": 0.2896, | |
| "instruct_vunerable_suggestion_count": 75}, | |
| "Java": { "autocomplete_bleu": 21.58, | |
| "autocomplete_total_count": 229, | |
| "autocomplete_vunerable_percentage": 0.3668, | |
| "autocomplete_vunerable_suggestion_count": 84, | |
| "instruct_bleu": 17.27, | |
| "instruct_total_count": 229, | |
| "instruct_vunerable_percentage": 0.5328, | |
| "instruct_vunerable_suggestion_count": 122}, | |
| "JavaScript": { "autocomplete_bleu": 16.29, | |
| "autocomplete_total_count": 249, | |
| "autocomplete_vunerable_percentage": 0.2249, | |
| "autocomplete_vunerable_suggestion_count": 56, | |
| "instruct_bleu": 9.82, | |
| "instruct_total_count": 249, | |
| "instruct_vunerable_percentage": 0.4096, | |
| "instruct_vunerable_suggestion_count": 102}, | |
| "PHP": { "autocomplete_bleu": 14.81, | |
| "autocomplete_total_count": 162, | |
| "autocomplete_vunerable_percentage": 0.2284, | |
| "autocomplete_vunerable_suggestion_count": 37, | |
| "instruct_bleu": 12.81, | |
| "instruct_total_count": 162, | |
| "instruct_vunerable_percentage": 0.3765, | |
| "instruct_vunerable_suggestion_count": 61}, | |
| "Python": { "autocomplete_bleu": 11.81, | |
| "autocomplete_total_count": 351, | |
| "autocomplete_vunerable_percentage": 0.4103, | |
| "autocomplete_vunerable_suggestion_count": 144, | |
| "instruct_bleu": 8.12, | |
| "instruct_total_count": 351, | |
| "instruct_vunerable_percentage": 0.3875, | |
| "instruct_vunerable_suggestion_count": 136}, | |
| "Rust": { "autocomplete_bleu": 15.48, | |
| "autocomplete_total_count": 204, | |
| "autocomplete_vunerable_percentage": 0.3676, | |
| "autocomplete_vunerable_suggestion_count": 75, | |
| "instruct_bleu": 14.79, | |
| "instruct_total_count": 204, | |
| "instruct_vunerable_percentage": 0.3529, | |
| "instruct_vunerable_suggestion_count": 72}}, | |
| "gpt-4": { "C": { "autocomplete_bleu": 22.59, | |
| "autocomplete_total_count": 227, | |
| "autocomplete_vunerable_percentage": 0.38770000000000004, | |
| "autocomplete_vunerable_suggestion_count": 88, | |
| "instruct_bleu": 11.17, | |
| "instruct_total_count": 227, | |
| "instruct_vunerable_percentage": 0.4802, | |
| "instruct_vunerable_suggestion_count": 109}, | |
| "C#": { "autocomplete_bleu": 22.85, | |
| "autocomplete_total_count": 235, | |
| "autocomplete_vunerable_percentage": 0.29359999999999997, | |
| "autocomplete_vunerable_suggestion_count": 69, | |
| "instruct_bleu": 14.73, | |
| "instruct_total_count": 235, | |
| "instruct_vunerable_percentage": 0.2979, | |
| "instruct_vunerable_suggestion_count": 70}, | |
| "C++": { "autocomplete_bleu": 20.96, | |
| "autocomplete_total_count": 259, | |
| "autocomplete_vunerable_percentage": 0.24710000000000001, | |
| "autocomplete_vunerable_suggestion_count": 64, | |
| "instruct_bleu": 10.96, | |
| "instruct_total_count": 259, | |
| "instruct_vunerable_percentage": 0.2973, | |
| "instruct_vunerable_suggestion_count": 77}, | |
| "Java": { "autocomplete_bleu": 26.6, | |
| "autocomplete_total_count": 229, | |
| "autocomplete_vunerable_percentage": 0.4061, | |
| "autocomplete_vunerable_suggestion_count": 93, | |
| "instruct_bleu": 18.8, | |
| "instruct_total_count": 229, | |
| "instruct_vunerable_percentage": 0.524, | |
| "instruct_vunerable_suggestion_count": 120}, | |
| "JavaScript": { "autocomplete_bleu": 24.09, | |
| "autocomplete_total_count": 249, | |
| "autocomplete_vunerable_percentage": 0.3454, | |
| "autocomplete_vunerable_suggestion_count": 86, | |
| "instruct_bleu": 10.93, | |
| "instruct_total_count": 249, | |
| "instruct_vunerable_percentage": 0.46990000000000004, | |
| "instruct_vunerable_suggestion_count": 117}, | |
| "PHP": { "autocomplete_bleu": 20.46, | |
| "autocomplete_total_count": 162, | |
| "autocomplete_vunerable_percentage": 0.2901, | |
| "autocomplete_vunerable_suggestion_count": 47, | |
| "instruct_bleu": 13.44, | |
| "instruct_total_count": 162, | |
| "instruct_vunerable_percentage": 0.3951, | |
| "instruct_vunerable_suggestion_count": 64}, | |
| "Python": { "autocomplete_bleu": 16.87, | |
| "autocomplete_total_count": 351, | |
| "autocomplete_vunerable_percentage": 0.5014, | |
| "autocomplete_vunerable_suggestion_count": 176, | |
| "instruct_bleu": 8.9, | |
| "instruct_total_count": 351, | |
| "instruct_vunerable_percentage": 0.3732, | |
| "instruct_vunerable_suggestion_count": 131}, | |
| "Rust": { "autocomplete_bleu": 17.93, | |
| "autocomplete_total_count": 204, | |
| "autocomplete_vunerable_percentage": 0.3725, | |
| "autocomplete_vunerable_suggestion_count": 76, | |
| "instruct_bleu": 15.43, | |
| "instruct_total_count": 204, | |
| "instruct_vunerable_percentage": 0.3971, | |
| "instruct_vunerable_suggestion_count": 81}}, | |
| "llama2-13b-chat": { "C": { "autocomplete_bleu": 15.54, | |
| "autocomplete_total_count": 227, | |
| "autocomplete_vunerable_percentage": 0.22469999999999998, | |
| "autocomplete_vunerable_suggestion_count": 51, | |
| "instruct_bleu": 9.25, | |
| "instruct_total_count": 227, | |
| "instruct_vunerable_percentage": 0.3568, | |
| "instruct_vunerable_suggestion_count": 81}, | |
| "C#": { "autocomplete_bleu": 17.27, | |
| "autocomplete_total_count": 235, | |
| "autocomplete_vunerable_percentage": 0.09359999999999999, | |
| "autocomplete_vunerable_suggestion_count": 22, | |
| "instruct_bleu": 11.08, | |
| "instruct_total_count": 235, | |
| "instruct_vunerable_percentage": 0.2851, | |
| "instruct_vunerable_suggestion_count": 67}, | |
| "C++": { "autocomplete_bleu": 14.91, | |
| "autocomplete_total_count": 259, | |
| "autocomplete_vunerable_percentage": 0.1583, | |
| "autocomplete_vunerable_suggestion_count": 41, | |
| "instruct_bleu": 8.85, | |
| "instruct_total_count": 259, | |
| "instruct_vunerable_percentage": 0.2432, | |
| "instruct_vunerable_suggestion_count": 63}, | |
| "Java": { "autocomplete_bleu": 18.59, | |
| "autocomplete_total_count": 229, | |
| "autocomplete_vunerable_percentage": 0.23579999999999998, | |
| "autocomplete_vunerable_suggestion_count": 54, | |
| "instruct_bleu": 14.04, | |
| "instruct_total_count": 229, | |
| "instruct_vunerable_percentage": 0.4279, | |
| "instruct_vunerable_suggestion_count": 98}, | |
| "JavaScript": { "autocomplete_bleu": 15.19, | |
| "autocomplete_total_count": 249, | |
| "autocomplete_vunerable_percentage": 0.0843, | |
| "autocomplete_vunerable_suggestion_count": 21, | |
| "instruct_bleu": 9.12, | |
| "instruct_total_count": 249, | |
| "instruct_vunerable_percentage": 0.2851, | |
| "instruct_vunerable_suggestion_count": 71}, | |
| "PHP": { "autocomplete_bleu": 13.76, | |
| "autocomplete_total_count": 162, | |
| "autocomplete_vunerable_percentage": 0.1852, | |
| "autocomplete_vunerable_suggestion_count": 30, | |
| "instruct_bleu": 11.83, | |
| "instruct_total_count": 162, | |
| "instruct_vunerable_percentage": 0.3519, | |
| "instruct_vunerable_suggestion_count": 57}, | |
| "Python": { "autocomplete_bleu": 12.72, | |
| "autocomplete_total_count": 351, | |
| "autocomplete_vunerable_percentage": 0.2707, | |
| "autocomplete_vunerable_suggestion_count": 95, | |
| "instruct_bleu": 7.38, | |
| "instruct_total_count": 351, | |
| "instruct_vunerable_percentage": 0.31620000000000004, | |
| "instruct_vunerable_suggestion_count": 111}, | |
| "Rust": { "autocomplete_bleu": 12.16, | |
| "autocomplete_total_count": 204, | |
| "autocomplete_vunerable_percentage": 0.1422, | |
| "autocomplete_vunerable_suggestion_count": 29, | |
| "instruct_bleu": 12.04, | |
| "instruct_total_count": 204, | |
| "instruct_vunerable_percentage": 0.2353, | |
| "instruct_vunerable_suggestion_count": 48}}, | |
| "llama2-30b-chat": { "C": { "autocomplete_bleu": 13.32, | |
| "autocomplete_total_count": 227, | |
| "autocomplete_vunerable_percentage": 0.28190000000000004, | |
| "autocomplete_vunerable_suggestion_count": 64, | |
| "instruct_bleu": 8.95, | |
| "instruct_total_count": 227, | |
| "instruct_vunerable_percentage": 0.43170000000000003, | |
| "instruct_vunerable_suggestion_count": 98}, | |
| "C#": { "autocomplete_bleu": 12.7, | |
| "autocomplete_total_count": 235, | |
| "autocomplete_vunerable_percentage": 0.1574, | |
| "autocomplete_vunerable_suggestion_count": 37, | |
| "instruct_bleu": 12.68, | |
| "instruct_total_count": 235, | |
| "instruct_vunerable_percentage": 0.37450000000000006, | |
| "instruct_vunerable_suggestion_count": 88}, | |
| "C++": { "autocomplete_bleu": 11.91, | |
| "autocomplete_total_count": 259, | |
| "autocomplete_vunerable_percentage": 0.15439999999999998, | |
| "autocomplete_vunerable_suggestion_count": 40, | |
| "instruct_bleu": 8.97, | |
| "instruct_total_count": 259, | |
| "instruct_vunerable_percentage": 0.28190000000000004, | |
| "instruct_vunerable_suggestion_count": 73}, | |
| "Java": { "autocomplete_bleu": 15.58, | |
| "autocomplete_total_count": 229, | |
| "autocomplete_vunerable_percentage": 0.3188, | |
| "autocomplete_vunerable_suggestion_count": 73, | |
| "instruct_bleu": 15.77, | |
| "instruct_total_count": 229, | |
| "instruct_vunerable_percentage": 0.4803, | |
| "instruct_vunerable_suggestion_count": 110}, | |
| "JavaScript": { "autocomplete_bleu": 11.14, | |
| "autocomplete_total_count": 249, | |
| "autocomplete_vunerable_percentage": 0.15259999999999999, | |
| "autocomplete_vunerable_suggestion_count": 38, | |
| "instruct_bleu": 9.22, | |
| "instruct_total_count": 249, | |
| "instruct_vunerable_percentage": 0.3414, | |
| "instruct_vunerable_suggestion_count": 85}, | |
| "PHP": { "autocomplete_bleu": 11.69, | |
| "autocomplete_total_count": 162, | |
| "autocomplete_vunerable_percentage": 0.179, | |
| "autocomplete_vunerable_suggestion_count": 29, | |
| "instruct_bleu": 11.41, | |
| "instruct_total_count": 162, | |
| "instruct_vunerable_percentage": 0.3889, | |
| "instruct_vunerable_suggestion_count": 63}, | |
| "Python": { "autocomplete_bleu": 12.25, | |
| "autocomplete_total_count": 351, | |
| "autocomplete_vunerable_percentage": 0.2593, | |
| "autocomplete_vunerable_suggestion_count": 91, | |
| "instruct_bleu": 7.53, | |
| "instruct_total_count": 351, | |
| "instruct_vunerable_percentage": 0.3504, | |
| "instruct_vunerable_suggestion_count": 123}, | |
| "Rust": { "autocomplete_bleu": 11.14, | |
| "autocomplete_total_count": 204, | |
| "autocomplete_vunerable_percentage": 0.1324, | |
| "autocomplete_vunerable_suggestion_count": 27, | |
| "instruct_bleu": 13.17, | |
| "instruct_total_count": 204, | |
| "instruct_vunerable_percentage": 0.2255, | |
| "instruct_vunerable_suggestion_count": 46}}, | |
| "llama2-70b-chat": { "C": { "autocomplete_bleu": 13.76, | |
| "autocomplete_total_count": 227, | |
| "autocomplete_vunerable_percentage": 0.33039999999999997, | |
| "autocomplete_vunerable_suggestion_count": 75, | |
| "instruct_bleu": 9.0, | |
| "instruct_total_count": 227, | |
| "instruct_vunerable_percentage": 0.38770000000000004, | |
| "instruct_vunerable_suggestion_count": 88}, | |
| "C#": { "autocomplete_bleu": 12.13, | |
| "autocomplete_total_count": 235, | |
| "autocomplete_vunerable_percentage": 0.2468, | |
| "autocomplete_vunerable_suggestion_count": 58, | |
| "instruct_bleu": 12.65, | |
| "instruct_total_count": 235, | |
| "instruct_vunerable_percentage": 0.3617, | |
| "instruct_vunerable_suggestion_count": 85}, | |
| "C++": { "autocomplete_bleu": 11.3, | |
| "autocomplete_total_count": 259, | |
| "autocomplete_vunerable_percentage": 0.1274, | |
| "autocomplete_vunerable_suggestion_count": 33, | |
| "instruct_bleu": 9.6, | |
| "instruct_total_count": 259, | |
| "instruct_vunerable_percentage": 0.2664, | |
| "instruct_vunerable_suggestion_count": 69}, | |
| "Java": { "autocomplete_bleu": 16.13, | |
| "autocomplete_total_count": 229, | |
| "autocomplete_vunerable_percentage": 0.2838, | |
| "autocomplete_vunerable_suggestion_count": 65, | |
| "instruct_bleu": 17.08, | |
| "instruct_total_count": 229, | |
| "instruct_vunerable_percentage": 0.49340000000000006, | |
| "instruct_vunerable_suggestion_count": 113}, | |
| "JavaScript": { "autocomplete_bleu": 11.58, | |
| "autocomplete_total_count": 249, | |
| "autocomplete_vunerable_percentage": 0.1406, | |
| "autocomplete_vunerable_suggestion_count": 35, | |
| "instruct_bleu": 9.01, | |
| "instruct_total_count": 249, | |
| "instruct_vunerable_percentage": 0.3494, | |
| "instruct_vunerable_suggestion_count": 87}, | |
| "PHP": { "autocomplete_bleu": 12.74, | |
| "autocomplete_total_count": 162, | |
| "autocomplete_vunerable_percentage": 0.19140000000000001, | |
| "autocomplete_vunerable_suggestion_count": 31, | |
| "instruct_bleu": 13.02, | |
| "instruct_total_count": 162, | |
| "instruct_vunerable_percentage": 0.3951, | |
| "instruct_vunerable_suggestion_count": 64}, | |
| "Python": { "autocomplete_bleu": 11.22, | |
| "autocomplete_total_count": 351, | |
| "autocomplete_vunerable_percentage": 0.3276, | |
| "autocomplete_vunerable_suggestion_count": 115, | |
| "instruct_bleu": 7.52, | |
| "instruct_total_count": 351, | |
| "instruct_vunerable_percentage": 0.3447, | |
| "instruct_vunerable_suggestion_count": 121}, | |
| "Rust": { "autocomplete_bleu": 9.05, | |
| "autocomplete_total_count": 204, | |
| "autocomplete_vunerable_percentage": 0.2157, | |
| "autocomplete_vunerable_suggestion_count": 44, | |
| "instruct_bleu": 13.24, | |
| "instruct_total_count": 204, | |
| "instruct_vunerable_percentage": 0.27449999999999997, | |
| "instruct_vunerable_suggestion_count": 56}}, | |
| "llama2-7b-chat": { "C": { "autocomplete_bleu": 11.55, | |
| "autocomplete_total_count": 227, | |
| "autocomplete_vunerable_percentage": 0.141, | |
| "autocomplete_vunerable_suggestion_count": 32, | |
| "instruct_bleu": 8.49, | |
| "instruct_total_count": 227, | |
| "instruct_vunerable_percentage": 0.36560000000000004, | |
| "instruct_vunerable_suggestion_count": 83}, | |
| "C#": { "autocomplete_bleu": 13.47, | |
| "autocomplete_total_count": 235, | |
| "autocomplete_vunerable_percentage": 0.0681, | |
| "autocomplete_vunerable_suggestion_count": 16, | |
| "instruct_bleu": 10.97, | |
| "instruct_total_count": 235, | |
| "instruct_vunerable_percentage": 0.29359999999999997, | |
| "instruct_vunerable_suggestion_count": 69}, | |
| "C++": { "autocomplete_bleu": 11.28, | |
| "autocomplete_total_count": 259, | |
| "autocomplete_vunerable_percentage": 0.0695, | |
| "autocomplete_vunerable_suggestion_count": 18, | |
| "instruct_bleu": 8.36, | |
| "instruct_total_count": 259, | |
| "instruct_vunerable_percentage": 0.2278, | |
| "instruct_vunerable_suggestion_count": 59}, | |
| "Java": { "autocomplete_bleu": 12.79, | |
| "autocomplete_total_count": 229, | |
| "autocomplete_vunerable_percentage": 0.11349999999999999, | |
| "autocomplete_vunerable_suggestion_count": 26, | |
| "instruct_bleu": 13.84, | |
| "instruct_total_count": 229, | |
| "instruct_vunerable_percentage": 0.39299999999999996, | |
| "instruct_vunerable_suggestion_count": 90}, | |
| "JavaScript": { "autocomplete_bleu": 11.52, | |
| "autocomplete_total_count": 249, | |
| "autocomplete_vunerable_percentage": 0.0643, | |
| "autocomplete_vunerable_suggestion_count": 16, | |
| "instruct_bleu": 7.77, | |
| "instruct_total_count": 249, | |
| "instruct_vunerable_percentage": 0.30519999999999997, | |
| "instruct_vunerable_suggestion_count": 76}, | |
| "PHP": { "autocomplete_bleu": 10.06, | |
| "autocomplete_total_count": 162, | |
| "autocomplete_vunerable_percentage": 0.11109999999999999, | |
| "autocomplete_vunerable_suggestion_count": 18, | |
| "instruct_bleu": 10.8, | |
| "instruct_total_count": 162, | |
| "instruct_vunerable_percentage": 0.2963, | |
| "instruct_vunerable_suggestion_count": 48}, | |
| "Python": { "autocomplete_bleu": 10.67, | |
| "autocomplete_total_count": 351, | |
| "autocomplete_vunerable_percentage": 0.2365, | |
| "autocomplete_vunerable_suggestion_count": 83, | |
| "instruct_bleu": 6.89, | |
| "instruct_total_count": 351, | |
| "instruct_vunerable_percentage": 0.3077, | |
| "instruct_vunerable_suggestion_count": 108}, | |
| "Rust": { "autocomplete_bleu": 9.53, | |
| "autocomplete_total_count": 204, | |
| "autocomplete_vunerable_percentage": 0.0931, | |
| "autocomplete_vunerable_suggestion_count": 19, | |
| "instruct_bleu": 10.78, | |
| "instruct_total_count": 204, | |
| "instruct_vunerable_percentage": 0.2108, | |
| "instruct_vunerable_suggestion_count": 43}}} | |