Spaces:
Running
Running
| { | |
| "gpt-4o": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 70.1, | |
| "cpp_input": 64.6, | |
| "d_input": 71.6, | |
| "go_input": 75.4, | |
| "java_input": 69.8, | |
| "js_input": 73.2, | |
| "jl_input": 67.0, | |
| "lua_input": 73.0, | |
| "pl_input": 70.1, | |
| "php_input": 74.8, | |
| "py_input": 70.6, | |
| "r_input": 74.4, | |
| "rkt_input": 67.4, | |
| "rb_input": 72.0, | |
| "rs_input": 73.6, | |
| "scala_input": 65.4, | |
| "sh_input": 70.6, | |
| "swift_input": 74.2, | |
| "ts_input": 74.0, | |
| "cs_output": 75.0, | |
| "cpp_output": 74.8, | |
| "d_output": 71.3, | |
| "go_output": 77.0, | |
| "java_output": 73.2, | |
| "js_output": 77.6, | |
| "jl_output": 73.6, | |
| "lua_output": 74.8, | |
| "pl_output": 74.0, | |
| "php_output": 75.4, | |
| "py_output": 75.4, | |
| "r_output": 72.0, | |
| "rkt_output": 70.8, | |
| "rb_output": 74.0, | |
| "rs_output": 74.4, | |
| "scala_output": 71.8, | |
| "sh_output": 71.6, | |
| "swift_output": 76.0, | |
| "ts_output": 76.4 | |
| }, | |
| "prompted": false, | |
| "size": null | |
| }, | |
| "gpt-4o-mini": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 58.8, | |
| "cpp_input": 52.2, | |
| "d_input": 60.6, | |
| "go_input": 62.0, | |
| "java_input": 57.2, | |
| "js_input": 59.6, | |
| "jl_input": 56.2, | |
| "lua_input": 63.4, | |
| "pl_input": 57.4, | |
| "php_input": 61.0, | |
| "py_input": 59.6, | |
| "r_input": 60.4, | |
| "rkt_input": 51.2, | |
| "rb_input": 61.6, | |
| "rs_input": 61.2, | |
| "scala_input": 52.6, | |
| "sh_input": 57.2, | |
| "swift_input": 63.4, | |
| "ts_input": 61.2, | |
| "cs_output": 63.0, | |
| "cpp_output": 63.0, | |
| "d_output": 61.4, | |
| "go_output": 63.4, | |
| "java_output": 54.0, | |
| "js_output": 61.8, | |
| "jl_output": 57.8, | |
| "lua_output": 60.0, | |
| "pl_output": 57.4, | |
| "php_output": 64.2, | |
| "py_output": 61.6, | |
| "r_output": 59.6, | |
| "rkt_output": 56.6, | |
| "rb_output": 61.2, | |
| "rs_output": 61.8, | |
| "scala_output": 61.2, | |
| "sh_output": 56.2, | |
| "swift_output": 63.0, | |
| "ts_output": 61.2 | |
| }, | |
| "prompted": true, | |
| "size": null | |
| }, | |
| "gpt-3.5-turbo": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 52.2, | |
| "cpp_input": 39.2, | |
| "d_input": 50.2, | |
| "go_input": 53.4, | |
| "java_input": 55.4, | |
| "js_input": 50.0, | |
| "jl_input": 47.0, | |
| "lua_input": 53.2, | |
| "pl_input": 47.6, | |
| "php_input": 52.2, | |
| "py_input": 51.6, | |
| "r_input": 48.6, | |
| "rkt_input": 45.4, | |
| "rb_input": 49.6, | |
| "rs_input": 53.0, | |
| "scala_input": 54.2, | |
| "sh_input": 47.6, | |
| "swift_input": 58.2, | |
| "ts_input": 48.4, | |
| "cs_output": 54.2, | |
| "cpp_output": 43.2, | |
| "d_output": 56.0, | |
| "go_output": 53.2, | |
| "java_output": 43.6, | |
| "js_output": 56.2, | |
| "jl_output": 54.2, | |
| "lua_output": 54.6, | |
| "pl_output": 51.8, | |
| "php_output": 55.2, | |
| "py_output": 57.2, | |
| "r_output": 49.4, | |
| "rkt_output": 48.0, | |
| "rb_output": 56.4, | |
| "rs_output": 54.6, | |
| "scala_output": 56.4, | |
| "sh_output": 51.0, | |
| "swift_output": 57.8, | |
| "ts_output": 53.6 | |
| }, | |
| "prompted": true, | |
| "size": null | |
| }, | |
| "DeepSeek-Coder-V2-0724": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 63.8, | |
| "cpp_input": 57.0, | |
| "d_input": 66.6, | |
| "go_input": 64.0, | |
| "java_input": 64.8, | |
| "js_input": 67.0, | |
| "jl_input": 58.4, | |
| "lua_input": 62.0, | |
| "pl_input": 61.4, | |
| "php_input": 64.2, | |
| "py_input": 64.0, | |
| "r_input": 65.8, | |
| "rkt_input": 58.0, | |
| "rb_input": 63.2, | |
| "rs_input": 63.6, | |
| "scala_input": 58.2, | |
| "sh_input": 62.4, | |
| "swift_input": 62.6, | |
| "ts_input": 66.6, | |
| "cs_output": 66.6, | |
| "cpp_output": 66.2, | |
| "d_output": 63.4, | |
| "go_output": 68.0, | |
| "java_output": 67.6, | |
| "js_output": 65.4, | |
| "jl_output": 64.8, | |
| "lua_output": 63.6, | |
| "pl_output": 63.0, | |
| "php_output": 67.4, | |
| "py_output": 66.8, | |
| "r_output": 63.0, | |
| "rkt_output": 62.2, | |
| "rb_output": 65.2, | |
| "rs_output": 65.8, | |
| "scala_output": 63.2, | |
| "sh_output": 58.8, | |
| "swift_output": 67.8, | |
| "ts_output": 66.4 | |
| }, | |
| "prompted": true, | |
| "size": 23 | |
| }, | |
| "Qwen2-72B-Instruct-GPTQ-Int4": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 52.0, | |
| "cpp_input": 54.2, | |
| "d_input": 49.6, | |
| "go_input": 55.4, | |
| "java_input": 50.0, | |
| "js_input": 51.6, | |
| "jl_input": 51.0, | |
| "lua_input": 51.2, | |
| "pl_input": 47.8, | |
| "php_input": 55.2, | |
| "py_input": 52.4, | |
| "r_input": 53.2, | |
| "rkt_input": 47.8, | |
| "rb_input": 54.4, | |
| "rs_input": 57.2, | |
| "scala_input": 50.6, | |
| "sh_input": 52.4, | |
| "swift_input": 51.6, | |
| "ts_input": 52.0, | |
| "cs_output": 51.2, | |
| "cpp_output": 50.2, | |
| "d_output": 51.6, | |
| "go_output": 53.6, | |
| "java_output": 38.2, | |
| "js_output": 52.0, | |
| "jl_output": 51.0, | |
| "lua_output": 49.0, | |
| "pl_output": 45.8, | |
| "php_output": 50.8, | |
| "py_output": 51.2, | |
| "r_output": 45.0, | |
| "rkt_output": 46.8, | |
| "rb_output": 50.8, | |
| "rs_output": 51.0, | |
| "scala_output": 51.0, | |
| "sh_output": 45.6, | |
| "swift_output": 50.4, | |
| "ts_output": 53.2 | |
| }, | |
| "prompted": true, | |
| "size": 72 | |
| }, | |
| "CodeLlama-34b-Python-hf": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 38.8, | |
| "cpp_input": 40.0, | |
| "d_input": 39.2, | |
| "go_input": 39.0, | |
| "java_input": 41.4, | |
| "js_input": 45.8, | |
| "jl_input": 44.8, | |
| "lua_input": 45.0, | |
| "pl_input": 43.2, | |
| "php_input": 48.0, | |
| "py_input": 46.8, | |
| "r_input": 42.2, | |
| "rkt_input": 38.8, | |
| "rb_input": 44.0, | |
| "rs_input": 44.2, | |
| "scala_input": 43.0, | |
| "sh_input": 44.6, | |
| "swift_input": 45.0, | |
| "ts_input": 44.0, | |
| "cs_output": 41.4, | |
| "cpp_output": 44.8, | |
| "d_output": 45.6, | |
| "go_output": 41.8, | |
| "java_output": 41.4, | |
| "js_output": 45.4, | |
| "jl_output": 45.2, | |
| "lua_output": 42.8, | |
| "pl_output": 43.6, | |
| "php_output": 43.8, | |
| "py_output": 43.8, | |
| "r_output": 42.4, | |
| "rkt_output": 38.6, | |
| "rb_output": 42.8, | |
| "rs_output": 46.6, | |
| "scala_output": 43.8, | |
| "sh_output": 42.0, | |
| "swift_output": 44.4, | |
| "ts_output": 44.8 | |
| }, | |
| "prompted": true, | |
| "size": 34 | |
| }, | |
| "CodeLlama-34b-Instruct-hf": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 44.6, | |
| "cpp_input": 48.4, | |
| "d_input": 43.8, | |
| "go_input": 46.0, | |
| "java_input": 44.4, | |
| "js_input": 52.6, | |
| "jl_input": 50.4, | |
| "lua_input": 49.4, | |
| "pl_input": 46.0, | |
| "php_input": 52.0, | |
| "py_input": 51.2, | |
| "r_input": 48.4, | |
| "rkt_input": 42.4, | |
| "rb_input": 48.2, | |
| "rs_input": 48.6, | |
| "scala_input": 48.0, | |
| "sh_input": 46.2, | |
| "swift_input": 49.4, | |
| "ts_input": 53.2, | |
| "cs_output": 44.4, | |
| "cpp_output": 46.2, | |
| "d_output": 45.8, | |
| "go_output": 46.8, | |
| "java_output": 40.6, | |
| "js_output": 47.4, | |
| "jl_output": 45.6, | |
| "lua_output": 42.8, | |
| "pl_output": 44.0, | |
| "php_output": 44.8, | |
| "py_output": 44.0, | |
| "r_output": 40.2, | |
| "rkt_output": 38.2, | |
| "rb_output": 44.2, | |
| "rs_output": 46.4, | |
| "scala_output": 43.8, | |
| "sh_output": 40.6, | |
| "swift_output": 45.2, | |
| "ts_output": 45.0 | |
| }, | |
| "prompted": true, | |
| "size": 34 | |
| }, | |
| "CodeLlama-34b-hf": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 40.4, | |
| "cpp_input": 44.6, | |
| "d_input": 45.6, | |
| "go_input": 41.2, | |
| "java_input": 39.0, | |
| "js_input": 50.0, | |
| "jl_input": 49.0, | |
| "lua_input": 47.0, | |
| "pl_input": 46.6, | |
| "php_input": 48.8, | |
| "py_input": 49.8, | |
| "r_input": 47.6, | |
| "rkt_input": 39.8, | |
| "rb_input": 46.6, | |
| "rs_input": 46.8, | |
| "scala_input": 44.6, | |
| "sh_input": 44.4, | |
| "swift_input": 50.0, | |
| "ts_input": 48.6, | |
| "cs_output": 44.6, | |
| "cpp_output": 47.8, | |
| "d_output": 44.2, | |
| "go_output": 45.2, | |
| "java_output": 38.4, | |
| "js_output": 47.0, | |
| "jl_output": 45.8, | |
| "lua_output": 42.8, | |
| "pl_output": 43.8, | |
| "php_output": 46.4, | |
| "py_output": 46.4, | |
| "r_output": 38.8, | |
| "rkt_output": 38.4, | |
| "rb_output": 45.4, | |
| "rs_output": 47.2, | |
| "scala_output": 47.4, | |
| "sh_output": 43.8, | |
| "swift_output": 47.6, | |
| "ts_output": 47.4 | |
| }, | |
| "prompted": true, | |
| "size": 34 | |
| }, | |
| "WizardCoder-33B-V1.1": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 44.8, | |
| "cpp_input": 25.4, | |
| "d_input": 46.4, | |
| "go_input": 47.6, | |
| "java_input": 48.4, | |
| "js_input": 45.6, | |
| "jl_input": 49.2, | |
| "lua_input": 48.8, | |
| "pl_input": 44.6, | |
| "php_input": 50.0, | |
| "py_input": 50.0, | |
| "r_input": 45.0, | |
| "rkt_input": 42.4, | |
| "rb_input": 49.2, | |
| "rs_input": 48.2, | |
| "scala_input": 48.2, | |
| "sh_input": 45.4, | |
| "swift_input": 51.0, | |
| "ts_input": 46.4, | |
| "cs_output": 47.0, | |
| "cpp_output": 46.8, | |
| "d_output": 45.8, | |
| "go_output": 44.2, | |
| "java_output": 50.8, | |
| "js_output": 50.0, | |
| "jl_output": 47.0, | |
| "lua_output": 46.0, | |
| "pl_output": 45.2, | |
| "php_output": 51.4, | |
| "py_output": 49.6, | |
| "r_output": 44.0, | |
| "rkt_output": 42.4, | |
| "rb_output": 48.2, | |
| "rs_output": 47.8, | |
| "scala_output": 45.0, | |
| "sh_output": 44.4, | |
| "swift_output": 48.0, | |
| "ts_output": 49.8 | |
| }, | |
| "prompted": true, | |
| "size": 33 | |
| }, | |
| "deepseek-coder-33b-instruct": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 46.0, | |
| "cpp_input": 43.6, | |
| "d_input": 49.8, | |
| "go_input": 49.0, | |
| "java_input": 46.8, | |
| "js_input": 48.8, | |
| "jl_input": 47.0, | |
| "lua_input": 50.0, | |
| "pl_input": 46.8, | |
| "php_input": 52.0, | |
| "py_input": 51.8, | |
| "r_input": 48.2, | |
| "rkt_input": 41.6, | |
| "rb_input": 52.0, | |
| "rs_input": 48.4, | |
| "scala_input": 47.0, | |
| "sh_input": 48.2, | |
| "swift_input": 52.2, | |
| "ts_input": 49.6, | |
| "cs_output": 52.0, | |
| "cpp_output": 51.4, | |
| "d_output": 49.0, | |
| "go_output": 48.8, | |
| "java_output": 53.2, | |
| "js_output": 55.0, | |
| "jl_output": 50.4, | |
| "lua_output": 50.4, | |
| "pl_output": 50.0, | |
| "php_output": 53.0, | |
| "py_output": 52.2, | |
| "r_output": 48.2, | |
| "rkt_output": 46.6, | |
| "rb_output": 52.8, | |
| "rs_output": 50.6, | |
| "scala_output": 48.0, | |
| "sh_output": 49.4, | |
| "swift_output": 52.8, | |
| "ts_output": 53.6 | |
| }, | |
| "prompted": true, | |
| "size": 33 | |
| }, | |
| "deepseek-coder-33b-base": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 41.2, | |
| "cpp_input": 42.8, | |
| "d_input": 43.2, | |
| "go_input": 45.6, | |
| "java_input": 43.8, | |
| "js_input": 46.0, | |
| "jl_input": 47.6, | |
| "lua_input": 47.4, | |
| "pl_input": 47.2, | |
| "php_input": 48.6, | |
| "py_input": 49.2, | |
| "r_input": 50.6, | |
| "rkt_input": 42.8, | |
| "rb_input": 47.4, | |
| "rs_input": 46.8, | |
| "scala_input": 44.0, | |
| "sh_input": 46.4, | |
| "swift_input": 48.2, | |
| "ts_input": 45.0, | |
| "cs_output": 48.2, | |
| "cpp_output": 50.0, | |
| "d_output": 46.0, | |
| "go_output": 48.6, | |
| "java_output": 49.2, | |
| "js_output": 51.4, | |
| "jl_output": 46.8, | |
| "lua_output": 48.0, | |
| "pl_output": 48.4, | |
| "php_output": 52.0, | |
| "py_output": 49.8, | |
| "r_output": 45.2, | |
| "rkt_output": 46.4, | |
| "rb_output": 49.0, | |
| "rs_output": 46.2, | |
| "scala_output": 47.6, | |
| "sh_output": 46.0, | |
| "swift_output": 49.2, | |
| "ts_output": 51.2 | |
| }, | |
| "prompted": true, | |
| "size": 33 | |
| }, | |
| "starcoder2-15b": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 41.4, | |
| "cpp_input": 43.8, | |
| "d_input": 51.6, | |
| "go_input": 45.2, | |
| "java_input": 42.6, | |
| "js_input": 44.0, | |
| "jl_input": 48.2, | |
| "lua_input": 44.6, | |
| "pl_input": 44.8, | |
| "php_input": 49.8, | |
| "py_input": 46.6, | |
| "r_input": 45.8, | |
| "rkt_input": 45.0, | |
| "rb_input": 49.0, | |
| "rs_input": 46.6, | |
| "scala_input": 37.0, | |
| "sh_input": 47.4, | |
| "swift_input": 52.2, | |
| "ts_input": 46.2, | |
| "cs_output": 46.0, | |
| "cpp_output": 47.4, | |
| "d_output": 47.2, | |
| "go_output": 49.0, | |
| "java_output": 48.4, | |
| "js_output": 50.0, | |
| "jl_output": 49.2, | |
| "lua_output": 44.8, | |
| "pl_output": 49.4, | |
| "php_output": 48.4, | |
| "py_output": 48.4, | |
| "r_output": 47.2, | |
| "rkt_output": 45.0, | |
| "rb_output": 51.0, | |
| "rs_output": 48.8, | |
| "scala_output": 45.2, | |
| "sh_output": 45.8, | |
| "swift_output": 49.6, | |
| "ts_output": 48.6 | |
| }, | |
| "prompted": true, | |
| "size": 15 | |
| }, | |
| "WizardCoder-15B-V1.0": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 29.2, | |
| "cpp_input": 30.0, | |
| "d_input": 30.6, | |
| "go_input": 28.6, | |
| "java_input": 29.6, | |
| "js_input": 33.0, | |
| "jl_input": 34.8, | |
| "lua_input": 33.6, | |
| "pl_input": 36.2, | |
| "php_input": 36.8, | |
| "py_input": 33.2, | |
| "r_input": 33.4, | |
| "rkt_input": 36.4, | |
| "rb_input": 33.6, | |
| "rs_input": 33.0, | |
| "scala_input": 29.0, | |
| "sh_input": 35.0, | |
| "swift_input": 34.0, | |
| "ts_input": 32.4, | |
| "cs_output": 25.2, | |
| "cpp_output": 30.0, | |
| "d_output": 30.6, | |
| "go_output": 33.2, | |
| "java_output": 26.8, | |
| "js_output": 33.6, | |
| "jl_output": 30.2, | |
| "lua_output": 30.2, | |
| "pl_output": 31.0, | |
| "php_output": 33.0, | |
| "py_output": 34.0, | |
| "r_output": 31.6, | |
| "rkt_output": 29.6, | |
| "rb_output": 32.8, | |
| "rs_output": 31.2, | |
| "scala_output": 31.2, | |
| "sh_output": 29.8, | |
| "swift_output": 34.2, | |
| "ts_output": 34.0 | |
| }, | |
| "prompted": true, | |
| "size": 15 | |
| }, | |
| "starcoder": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 28.2, | |
| "cpp_input": 30.0, | |
| "d_input": 33.0, | |
| "go_input": 33.2, | |
| "java_input": 33.4, | |
| "js_input": 35.2, | |
| "jl_input": 34.4, | |
| "lua_input": 31.6, | |
| "pl_input": 34.0, | |
| "php_input": 36.4, | |
| "py_input": 34.8, | |
| "r_input": 33.4, | |
| "rkt_input": 36.6, | |
| "rb_input": 35.0, | |
| "rs_input": 34.8, | |
| "scala_input": 27.4, | |
| "sh_input": 37.0, | |
| "swift_input": 30.8, | |
| "ts_input": 33.2, | |
| "cs_output": 20.4, | |
| "cpp_output": 31.6, | |
| "d_output": 31.8, | |
| "go_output": 31.0, | |
| "java_output": 18.4, | |
| "js_output": 33.4, | |
| "jl_output": 32.2, | |
| "lua_output": 31.8, | |
| "pl_output": 29.8, | |
| "php_output": 32.6, | |
| "py_output": 32.6, | |
| "r_output": 30.0, | |
| "rkt_output": 29.2, | |
| "rb_output": 33.4, | |
| "rs_output": 32.6, | |
| "scala_output": 30.0, | |
| "sh_output": 30.2, | |
| "swift_output": 33.0, | |
| "ts_output": 33.0 | |
| }, | |
| "prompted": true, | |
| "size": 15 | |
| }, | |
| "Phi-3-medium-4k-instruct": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 31.8, | |
| "cpp_input": 26.0, | |
| "d_input": 38.8, | |
| "go_input": 36.4, | |
| "java_input": 37.2, | |
| "js_input": 42.4, | |
| "jl_input": 36.2, | |
| "lua_input": 37.2, | |
| "pl_input": 35.6, | |
| "php_input": 41.2, | |
| "py_input": 43.4, | |
| "r_input": 39.2, | |
| "rkt_input": 24.4, | |
| "rb_input": 36.0, | |
| "rs_input": 36.8, | |
| "scala_input": 38.0, | |
| "sh_input": 33.6, | |
| "swift_input": 41.2, | |
| "ts_input": 42.8, | |
| "cs_output": 34.2, | |
| "cpp_output": 37.6, | |
| "d_output": 39.0, | |
| "go_output": 31.0, | |
| "java_output": 34.2, | |
| "js_output": 41.6, | |
| "jl_output": 41.2, | |
| "lua_output": 34.4, | |
| "pl_output": 35.8, | |
| "php_output": 37.8, | |
| "py_output": 42.4, | |
| "r_output": 36.6, | |
| "rkt_output": 24.6, | |
| "rb_output": 42.2, | |
| "rs_output": 37.4, | |
| "scala_output": 36.2, | |
| "sh_output": 37.2, | |
| "swift_output": 41.4, | |
| "ts_output": 43.0 | |
| }, | |
| "prompted": true, | |
| "size": 14 | |
| }, | |
| "Meta-Llama-3-8B-Instruct-hf": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 37.0, | |
| "cpp_input": 36.4, | |
| "d_input": 35.0, | |
| "go_input": 38.6, | |
| "java_input": 36.2, | |
| "js_input": 38.4, | |
| "jl_input": 39.6, | |
| "lua_input": 40.0, | |
| "pl_input": 36.2, | |
| "php_input": 36.6, | |
| "py_input": 38.4, | |
| "r_input": 42.2, | |
| "rkt_input": 24.2, | |
| "rb_input": 35.8, | |
| "rs_input": 37.6, | |
| "scala_input": 38.0, | |
| "sh_input": 31.6, | |
| "swift_input": 42.2, | |
| "ts_input": 38.2, | |
| "cs_output": 32.0, | |
| "cpp_output": 30.8, | |
| "d_output": 31.2, | |
| "go_output": 31.4, | |
| "java_output": 25.0, | |
| "js_output": 35.0, | |
| "jl_output": 31.4, | |
| "lua_output": 34.0, | |
| "pl_output": 29.6, | |
| "php_output": 27.0, | |
| "py_output": 33.6, | |
| "r_output": 27.2, | |
| "rkt_output": 28.0, | |
| "rb_output": 31.8, | |
| "rs_output": 34.4, | |
| "scala_output": 33.8, | |
| "sh_output": 32.0, | |
| "swift_output": 36.4, | |
| "ts_output": 33.8 | |
| }, | |
| "prompted": true, | |
| "size": 8 | |
| }, | |
| "CodeQwen1.5-7B-Chat": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 42.8, | |
| "cpp_input": 42.0, | |
| "d_input": 43.0, | |
| "go_input": 46.4, | |
| "java_input": 44.6, | |
| "js_input": 43.8, | |
| "jl_input": 42.2, | |
| "lua_input": 42.8, | |
| "pl_input": 41.6, | |
| "php_input": 44.8, | |
| "py_input": 43.0, | |
| "r_input": 43.4, | |
| "rkt_input": 38.2, | |
| "rb_input": 43.6, | |
| "rs_input": 42.0, | |
| "scala_input": 39.4, | |
| "sh_input": 46.6, | |
| "swift_input": 45.8, | |
| "ts_input": 43.6, | |
| "cs_output": 37.8, | |
| "cpp_output": 40.2, | |
| "d_output": 40.2, | |
| "go_output": 40.6, | |
| "java_output": 35.4, | |
| "js_output": 43.6, | |
| "jl_output": 42.6, | |
| "lua_output": 40.4, | |
| "pl_output": 39.6, | |
| "php_output": 43.0, | |
| "py_output": 41.4, | |
| "r_output": 38.2, | |
| "rkt_output": 39.0, | |
| "rb_output": 44.6, | |
| "rs_output": 42.0, | |
| "scala_output": 35.0, | |
| "sh_output": 38.2, | |
| "swift_output": 43.8, | |
| "ts_output": 42.2 | |
| }, | |
| "prompted": true, | |
| "size": 7 | |
| }, | |
| "CodeLlama-7b-Instruct-hf": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 38.6, | |
| "cpp_input": 36.0, | |
| "d_input": 38.4, | |
| "go_input": 38.4, | |
| "java_input": 38.2, | |
| "js_input": 39.6, | |
| "jl_input": 42.2, | |
| "lua_input": 43.4, | |
| "pl_input": 36.4, | |
| "php_input": 40.4, | |
| "py_input": 41.0, | |
| "r_input": 41.0, | |
| "rkt_input": 38.8, | |
| "rb_input": 41.6, | |
| "rs_input": 37.6, | |
| "scala_input": 42.6, | |
| "sh_input": 39.6, | |
| "swift_input": 40.2, | |
| "ts_input": 41.0, | |
| "cs_output": 32.2, | |
| "cpp_output": 35.6, | |
| "d_output": 34.4, | |
| "go_output": 35.0, | |
| "java_output": 24.4, | |
| "js_output": 38.2, | |
| "jl_output": 35.2, | |
| "lua_output": 32.2, | |
| "pl_output": 34.2, | |
| "php_output": 36.0, | |
| "py_output": 35.4, | |
| "r_output": 32.0, | |
| "rkt_output": 29.6, | |
| "rb_output": 37.0, | |
| "rs_output": 37.4, | |
| "scala_output": 33.0, | |
| "sh_output": 33.0, | |
| "swift_output": 34.6, | |
| "ts_output": 38.8 | |
| }, | |
| "prompted": true, | |
| "size": 7 | |
| }, | |
| "CodeLlama-7b-hf": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 36.4, | |
| "cpp_input": 36.2, | |
| "d_input": 36.8, | |
| "go_input": 34.6, | |
| "java_input": 36.4, | |
| "js_input": 36.6, | |
| "jl_input": 40.2, | |
| "lua_input": 39.6, | |
| "pl_input": 36.0, | |
| "php_input": 39.4, | |
| "py_input": 40.2, | |
| "r_input": 40.0, | |
| "rkt_input": 36.6, | |
| "rb_input": 39.2, | |
| "rs_input": 35.4, | |
| "scala_input": 37.8, | |
| "sh_input": 36.8, | |
| "swift_input": 39.2, | |
| "ts_input": 38.8, | |
| "cs_output": 32.6, | |
| "cpp_output": 34.4, | |
| "d_output": 33.8, | |
| "go_output": 33.4, | |
| "java_output": 28.4, | |
| "js_output": 38.0, | |
| "jl_output": 35.2, | |
| "lua_output": 34.4, | |
| "pl_output": 35.2, | |
| "php_output": 38.0, | |
| "py_output": 34.4, | |
| "r_output": 32.6, | |
| "rkt_output": 30.8, | |
| "rb_output": 34.8, | |
| "rs_output": 36.8, | |
| "scala_output": 33.4, | |
| "sh_output": 31.0, | |
| "swift_output": 35.0, | |
| "ts_output": 38.2 | |
| }, | |
| "prompted": true, | |
| "size": 7 | |
| }, | |
| "deepseek-coder-6.7b-instruct": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 35.0, | |
| "cpp_input": 37.0, | |
| "d_input": 35.6, | |
| "go_input": 40.4, | |
| "java_input": 35.0, | |
| "js_input": 36.6, | |
| "jl_input": 39.2, | |
| "lua_input": 38.8, | |
| "pl_input": 39.4, | |
| "php_input": 42.2, | |
| "py_input": 38.2, | |
| "r_input": 42.0, | |
| "rkt_input": 37.2, | |
| "rb_input": 40.2, | |
| "rs_input": 37.4, | |
| "scala_input": 36.8, | |
| "sh_input": 42.8, | |
| "swift_input": 40.8, | |
| "ts_input": 34.2, | |
| "cs_output": 34.8, | |
| "cpp_output": 41.8, | |
| "d_output": 40.4, | |
| "go_output": 39.4, | |
| "java_output": 32.8, | |
| "js_output": 47.6, | |
| "jl_output": 42.6, | |
| "lua_output": 38.8, | |
| "pl_output": 42.0, | |
| "php_output": 43.8, | |
| "py_output": 43.6, | |
| "r_output": 40.8, | |
| "rkt_output": 39.2, | |
| "rb_output": 43.2, | |
| "rs_output": 41.8, | |
| "scala_output": 40.6, | |
| "sh_output": 37.8, | |
| "swift_output": 43.2, | |
| "ts_output": 44.0 | |
| }, | |
| "prompted": true, | |
| "size": 6.7 | |
| }, | |
| "deepseek-coder-6.7b-base": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 38.8, | |
| "cpp_input": 42.4, | |
| "d_input": 41.2, | |
| "go_input": 43.2, | |
| "java_input": 40.4, | |
| "js_input": 43.6, | |
| "jl_input": 42.6, | |
| "lua_input": 42.8, | |
| "pl_input": 41.6, | |
| "php_input": 46.4, | |
| "py_input": 41.4, | |
| "r_input": 46.2, | |
| "rkt_input": 43.0, | |
| "rb_input": 44.6, | |
| "rs_input": 41.6, | |
| "scala_input": 40.8, | |
| "sh_input": 44.8, | |
| "swift_input": 43.4, | |
| "ts_input": 41.8, | |
| "cs_output": 41.2, | |
| "cpp_output": 46.2, | |
| "d_output": 43.2, | |
| "go_output": 42.8, | |
| "java_output": 42.6, | |
| "js_output": 44.8, | |
| "jl_output": 46.0, | |
| "lua_output": 41.0, | |
| "pl_output": 40.4, | |
| "php_output": 41.8, | |
| "py_output": 44.8, | |
| "r_output": 42.8, | |
| "rkt_output": 43.0, | |
| "rb_output": 42.6, | |
| "rs_output": 42.0, | |
| "scala_output": 43.2, | |
| "sh_output": 40.6, | |
| "swift_output": 47.6, | |
| "ts_output": 45.4 | |
| }, | |
| "prompted": true, | |
| "size": 6.7 | |
| }, | |
| "codegen-6B-multi": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 28.8, | |
| "cpp_input": 25.4, | |
| "d_input": 6.2, | |
| "go_input": 25.6, | |
| "java_input": 36.2, | |
| "js_input": 25.2, | |
| "jl_input": 17.4, | |
| "lua_input": 24.4, | |
| "pl_input": 38.4, | |
| "php_input": 22.8, | |
| "py_input": 22.6, | |
| "r_input": 27.2, | |
| "rkt_input": 16.2, | |
| "rb_input": 6.4, | |
| "rs_input": 18.8, | |
| "scala_input": 31.0, | |
| "sh_input": 48.6, | |
| "swift_input": 32.4, | |
| "ts_input": 25.2, | |
| "cs_output": 21.4, | |
| "cpp_output": 23.6, | |
| "d_output": 25.0, | |
| "go_output": 26.4, | |
| "java_output": 21.6, | |
| "js_output": 22.8, | |
| "jl_output": 22.8, | |
| "lua_output": 23.8, | |
| "pl_output": 20.4, | |
| "php_output": 25.2, | |
| "py_output": 24.8, | |
| "r_output": 23.4, | |
| "rkt_output": 17.8, | |
| "rb_output": 24.0, | |
| "rs_output": 25.2, | |
| "scala_output": 22.0, | |
| "sh_output": 22.2, | |
| "swift_output": 25.0, | |
| "ts_output": 21.4 | |
| }, | |
| "prompted": true, | |
| "size": 6 | |
| }, | |
| "phi-1_5": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 29.2, | |
| "cpp_input": 16.0, | |
| "d_input": 13.2, | |
| "go_input": 25.8, | |
| "java_input": 26.8, | |
| "js_input": 9.8, | |
| "jl_input": 30.4, | |
| "lua_input": 26.6, | |
| "pl_input": 17.8, | |
| "php_input": 26.6, | |
| "py_input": 25.8, | |
| "r_input": 8.4, | |
| "rkt_input": 6.6, | |
| "rb_input": 1.4, | |
| "rs_input": 25.2, | |
| "scala_input": 30.4, | |
| "sh_input": 34.4, | |
| "swift_input": 26.6, | |
| "ts_input": 30.8, | |
| "cs_output": 16.0, | |
| "cpp_output": 26.0, | |
| "d_output": 24.8, | |
| "go_output": 22.6, | |
| "java_output": 15.8, | |
| "js_output": 23.0, | |
| "jl_output": 23.6, | |
| "lua_output": 21.2, | |
| "pl_output": 22.0, | |
| "php_output": 22.2, | |
| "py_output": 25.6, | |
| "r_output": 21.8, | |
| "rkt_output": 16.8, | |
| "rb_output": 19.6, | |
| "rs_output": 22.0, | |
| "scala_output": 21.6, | |
| "sh_output": 17.6, | |
| "swift_output": 25.6, | |
| "ts_output": 25.2 | |
| }, | |
| "prompted": true, | |
| "size": 1.3 | |
| }, | |
| "phi-1": { | |
| "link": "", | |
| "open-data": "NONE", | |
| "pass@1": { | |
| "cs_input": 0.2, | |
| "cpp_input": 7.0, | |
| "d_input": 9.6, | |
| "go_input": 3.6, | |
| "java_input": 2.8, | |
| "js_input": 17.0, | |
| "jl_input": 19.0, | |
| "lua_input": 17.4, | |
| "pl_input": 23.6, | |
| "php_input": 9.2, | |
| "py_input": 11.8, | |
| "r_input": 9.4, | |
| "rkt_input": 11.2, | |
| "rb_input": 6.8, | |
| "rs_input": 5.4, | |
| "scala_input": 1.8, | |
| "sh_input": 19.8, | |
| "swift_input": 14.0, | |
| "ts_input": 14.0, | |
| "cs_output": 5.8, | |
| "cpp_output": 9.0, | |
| "d_output": 13.2, | |
| "go_output": 14.8, | |
| "java_output": 4.6, | |
| "js_output": 20.8, | |
| "jl_output": 19.2, | |
| "lua_output": 15.8, | |
| "pl_output": 15.6, | |
| "php_output": 18.6, | |
| "py_output": 22.4, | |
| "r_output": 17.6, | |
| "rkt_output": 10.4, | |
| "rb_output": 18.0, | |
| "rs_output": 16.4, | |
| "scala_output": 11.0, | |
| "sh_output": 16.4, | |
| "swift_output": 19.2, | |
| "ts_output": 19.0 | |
| }, | |
| "prompted": true, | |
| "size": 1.3 | |
| } | |
| } | |