Spaces:
Running
Running
openchat, nous hermes results
Browse files- IFEval.csv +2 -2
- app.py +1 -1
- results/AlphaMonarch-7B.json +11 -11
- results/Hermes-2-Pro-Llama-3-8B-GGUF.json +11 -11
IFEval.csv
CHANGED
|
@@ -18,5 +18,5 @@ Mistral 7B v0.1,7,nousresearch/nous-hermes-2-mistral-7b-dpo (Openrouter),FP16,0.
|
|
| 18 |
Mixtral 8x7B v0.1,56,mixtral-8x7b-32768 (Groq),FP16,0.5887,0.5028,0.6247,0.5545,0.6727,https://console.groq.com/docs/models
|
| 19 |
Llama 3 8B,8,bartowski/openchat-3.6-8b-20240522-GGUF,Q8_0,0.5791,0.5139,0.6139,0.5434,0.6451,https://huggingface.co/bartowski/openchat-3.6-8b-20240522-GGUF
|
| 20 |
Mistral 7B v0.3,7,MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF,Q8_0,0.5689,0.4972,0.5983,0.5397,0.6403,https://huggingface.co/MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF
|
| 21 |
-
Llama 3 8B,8,bartowski/Hermes-2-Pro-Llama-3-8B-GGUF,Q8_0,0.
|
| 22 |
-
Mistral 7B v0.1,7,mlabonne/AlphaMonarch-7B,Q8_0,0.
|
|
|
|
| 18 |
Mixtral 8x7B v0.1,56,mixtral-8x7b-32768 (Groq),FP16,0.5887,0.5028,0.6247,0.5545,0.6727,https://console.groq.com/docs/models
|
| 19 |
Llama 3 8B,8,bartowski/openchat-3.6-8b-20240522-GGUF,Q8_0,0.5791,0.5139,0.6139,0.5434,0.6451,https://huggingface.co/bartowski/openchat-3.6-8b-20240522-GGUF
|
| 20 |
Mistral 7B v0.3,7,MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF,Q8_0,0.5689,0.4972,0.5983,0.5397,0.6403,https://huggingface.co/MaziyarPanahi/Mistral-7B-Instruct-v0.3-GGUF
|
| 21 |
+
Llama 3 8B,8,bartowski/Hermes-2-Pro-Llama-3-8B-GGUF,Q8_0,0.5671,0.4917,0.5983,0.5379,0.6403,https://huggingface.co/bartowski/Hermes-2-Pro-Llama-3-8B-GGUF
|
| 22 |
+
Mistral 7B v0.1,7,mlabonne/AlphaMonarch-7B,Q8_0,0.5372,0.4529,0.5552,0.5194,0.6211,https://huggingface.co/mlabonne/AlphaMonarch-7B
|
app.py
CHANGED
|
@@ -80,7 +80,7 @@ def build_leaderboard_tab(leaderboard_table_file, show_plot=False):
|
|
| 80 |
# arena table
|
| 81 |
arena_table_vals = get_arena_table(model_table_df)
|
| 82 |
with gr.Tab("IFEval", id=0):
|
| 83 |
-
md = "Leaderboard for various Large Language Models measured with IFEval benchmark.\n\n[IFEval](https://github.com/google-research/google-research/tree/master/instruction_following_eval) is a straightforward and easy-to-reproduce evaluation benchmark. It focuses on a set of \"verifiable instructions\" such as \"write in more than 400 words\" and \"mention the keyword of AI at least 3 times\". We identified 25 types of those verifiable instructions and constructed around 500 prompts, with each prompt containing one or more verifiable instructions. \n\nTest ran with `lm-evaluation-harness`. Raw results can be found in the `results` directory. Made by [Kristian Polso](https://polso.info)\n\n**Changelog**\n\n8.6.2024 -
|
| 84 |
gr.Markdown(md, elem_id="leaderboard_markdown")
|
| 85 |
gr.Dataframe(
|
| 86 |
headers=[
|
|
|
|
| 80 |
# arena table
|
| 81 |
arena_table_vals = get_arena_table(model_table_df)
|
| 82 |
with gr.Tab("IFEval", id=0):
|
| 83 |
+
md = "Leaderboard for various Large Language Models measured with IFEval benchmark.\n\n[IFEval](https://github.com/google-research/google-research/tree/master/instruction_following_eval) is a straightforward and easy-to-reproduce evaluation benchmark. It focuses on a set of \"verifiable instructions\" such as \"write in more than 400 words\" and \"mention the keyword of AI at least 3 times\". We identified 25 types of those verifiable instructions and constructed around 500 prompts, with each prompt containing one or more verifiable instructions. \n\nTest ran with `lm-evaluation-harness`. Raw results can be found in the `results` directory. Made by [Kristian Polso](https://polso.info)\n\n**Changelog**\n\n8.6.2024 - Fixed CapybaraHermes, AlphaMonarch results, was using the wrong prompt template"
|
| 84 |
gr.Markdown(md, elem_id="leaderboard_markdown")
|
| 85 |
gr.Dataframe(
|
| 86 |
headers=[
|
results/AlphaMonarch-7B.json
CHANGED
|
@@ -1,13 +1,13 @@
|
|
| 1 |
{
|
| 2 |
"results": {
|
| 3 |
"ifeval": {
|
| 4 |
-
"prompt_level_strict_acc,none": 0.
|
| 5 |
-
"prompt_level_strict_acc_stderr,none": 0.
|
| 6 |
-
"inst_level_strict_acc,none": 0.
|
| 7 |
"inst_level_strict_acc_stderr,none": "N/A",
|
| 8 |
-
"prompt_level_loose_acc,none": 0.
|
| 9 |
-
"prompt_level_loose_acc_stderr,none": 0.
|
| 10 |
-
"inst_level_loose_acc,none": 0.
|
| 11 |
"inst_level_loose_acc_stderr,none": "N/A",
|
| 12 |
"alias": "ifeval"
|
| 13 |
}
|
|
@@ -99,8 +99,8 @@
|
|
| 99 |
"fewshot_seed": 1234
|
| 100 |
},
|
| 101 |
"git_hash": "e39df01c",
|
| 102 |
-
"date":
|
| 103 |
-
"pretty_env_info": "PyTorch version: 2.3.1+
|
| 104 |
"transformers_version": "4.41.2",
|
| 105 |
"upper_git_hash": null,
|
| 106 |
"task_hashes": {},
|
|
@@ -111,7 +111,7 @@
|
|
| 111 |
"system_instruction_sha": null,
|
| 112 |
"chat_template": null,
|
| 113 |
"chat_template_sha": null,
|
| 114 |
-
"start_time":
|
| 115 |
-
"end_time":
|
| 116 |
-
"total_evaluation_time_seconds": "
|
| 117 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"results": {
|
| 3 |
"ifeval": {
|
| 4 |
+
"prompt_level_strict_acc,none": 0.45286506469500926,
|
| 5 |
+
"prompt_level_strict_acc_stderr,none": 0.02142075394952955,
|
| 6 |
+
"inst_level_strict_acc,none": 0.5551558752997602,
|
| 7 |
"inst_level_strict_acc_stderr,none": "N/A",
|
| 8 |
+
"prompt_level_loose_acc,none": 0.5194085027726433,
|
| 9 |
+
"prompt_level_loose_acc_stderr,none": 0.021500357879025083,
|
| 10 |
+
"inst_level_loose_acc,none": 0.6211031175059952,
|
| 11 |
"inst_level_loose_acc_stderr,none": "N/A",
|
| 12 |
"alias": "ifeval"
|
| 13 |
}
|
|
|
|
| 99 |
"fewshot_seed": 1234
|
| 100 |
},
|
| 101 |
"git_hash": "e39df01c",
|
| 102 |
+
"date": 1717836507.5502021,
|
| 103 |
+
"pretty_env_info": "PyTorch version: 2.3.1+rocm6.0\nIs debug build: False\nCUDA used to build PyTorch: N/A\nROCM used to build PyTorch: 6.0.32830-d62f6a171\n\nOS: Arch Linux (x86_64)\nGCC version: (GCC) 14.1.1 20240522\nClang version: Could not collect\nCMake version: version 3.29.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Apr 23 2024, 09:16:07) [GCC 13.2.1 20240417] (64-bit runtime)\nPython platform: Linux-6.9.2-arch1-1-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: AMD Radeon RX 7900 XTX (gfx1100)\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nHIP runtime version: 6.0.32830\nMIOpen runtime version: 3.0.0\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: AuthenticAMD\nModel name: AMD Ryzen 7 3700X 8-Core Processor\nCPU family: 23\nModel: 113\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 0\nFrequency boost: enabled\nCPU(s) scaling MHz: 74%\nCPU max MHz: 4426.1709\nCPU min MHz: 2200.0000\nBogoMIPS: 7189.76\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization: AMD-V\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 4 MiB (8 instances)\nL3 cache: 32 MiB (2 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow: Mitigation; Safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.3\n[pip3] torch==2.3.1+rocm6.0\n[pip3] torchaudio==2.3.1+rocm6.0\n[pip3] torchvision==0.18.1+rocm6.0\n[conda] Could not collect",
|
| 104 |
"transformers_version": "4.41.2",
|
| 105 |
"upper_git_hash": null,
|
| 106 |
"task_hashes": {},
|
|
|
|
| 111 |
"system_instruction_sha": null,
|
| 112 |
"chat_template": null,
|
| 113 |
"chat_template_sha": null,
|
| 114 |
+
"start_time": 779763.040245179,
|
| 115 |
+
"end_time": 782737.886704555,
|
| 116 |
+
"total_evaluation_time_seconds": "2974.8464593760436"
|
| 117 |
}
|
results/Hermes-2-Pro-Llama-3-8B-GGUF.json
CHANGED
|
@@ -1,13 +1,13 @@
|
|
| 1 |
{
|
| 2 |
"results": {
|
| 3 |
"ifeval": {
|
| 4 |
-
"prompt_level_strict_acc,none": 0.
|
| 5 |
-
"prompt_level_strict_acc_stderr,none": 0.
|
| 6 |
-
"inst_level_strict_acc,none": 0.
|
| 7 |
"inst_level_strict_acc_stderr,none": "N/A",
|
| 8 |
-
"prompt_level_loose_acc,none": 0.
|
| 9 |
-
"prompt_level_loose_acc_stderr,none": 0.
|
| 10 |
-
"inst_level_loose_acc,none": 0.
|
| 11 |
"inst_level_loose_acc_stderr,none": "N/A",
|
| 12 |
"alias": "ifeval"
|
| 13 |
}
|
|
@@ -99,8 +99,8 @@
|
|
| 99 |
"fewshot_seed": 1234
|
| 100 |
},
|
| 101 |
"git_hash": "e39df01c",
|
| 102 |
-
"date":
|
| 103 |
-
"pretty_env_info": "PyTorch version: 2.3.1+
|
| 104 |
"transformers_version": "4.41.2",
|
| 105 |
"upper_git_hash": null,
|
| 106 |
"task_hashes": {},
|
|
@@ -111,7 +111,7 @@
|
|
| 111 |
"system_instruction_sha": null,
|
| 112 |
"chat_template": null,
|
| 113 |
"chat_template_sha": null,
|
| 114 |
-
"start_time":
|
| 115 |
-
"end_time":
|
| 116 |
-
"total_evaluation_time_seconds": "
|
| 117 |
}
|
|
|
|
| 1 |
{
|
| 2 |
"results": {
|
| 3 |
"ifeval": {
|
| 4 |
+
"prompt_level_strict_acc,none": 0.49168207024029575,
|
| 5 |
+
"prompt_level_strict_acc_stderr,none": 0.02151359656402127,
|
| 6 |
+
"inst_level_strict_acc,none": 0.5983213429256595,
|
| 7 |
"inst_level_strict_acc_stderr,none": "N/A",
|
| 8 |
+
"prompt_level_loose_acc,none": 0.5378927911275416,
|
| 9 |
+
"prompt_level_loose_acc_stderr,none": 0.021454695436204742,
|
| 10 |
+
"inst_level_loose_acc,none": 0.6402877697841727,
|
| 11 |
"inst_level_loose_acc_stderr,none": "N/A",
|
| 12 |
"alias": "ifeval"
|
| 13 |
}
|
|
|
|
| 99 |
"fewshot_seed": 1234
|
| 100 |
},
|
| 101 |
"git_hash": "e39df01c",
|
| 102 |
+
"date": 1717839960.303947,
|
| 103 |
+
"pretty_env_info": "PyTorch version: 2.3.1+rocm6.0\nIs debug build: False\nCUDA used to build PyTorch: N/A\nROCM used to build PyTorch: 6.0.32830-d62f6a171\n\nOS: Arch Linux (x86_64)\nGCC version: (GCC) 14.1.1 20240522\nClang version: Could not collect\nCMake version: version 3.29.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Apr 23 2024, 09:16:07) [GCC 13.2.1 20240417] (64-bit runtime)\nPython platform: Linux-6.9.2-arch1-1-x86_64-with-glibc2.39\nIs CUDA available: True\nCUDA runtime version: Could not collect\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: AMD Radeon RX 7900 XTX (gfx1100)\nNvidia driver version: Could not collect\ncuDNN version: Could not collect\nHIP runtime version: 6.0.32830\nMIOpen runtime version: 3.0.0\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: AuthenticAMD\nModel name: AMD Ryzen 7 3700X 8-Core Processor\nCPU family: 23\nModel: 113\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 0\nFrequency boost: enabled\nCPU(s) scaling MHz: 68%\nCPU max MHz: 4426.1709\nCPU min MHz: 2200.0000\nBogoMIPS: 7189.76\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization: AMD-V\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 4 MiB (8 instances)\nL3 cache: 32 MiB (2 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow: Mitigation; Safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.3\n[pip3] torch==2.3.1+rocm6.0\n[pip3] torchaudio==2.3.1+rocm6.0\n[pip3] torchvision==0.18.1+rocm6.0\n[conda] Could not collect",
|
| 104 |
"transformers_version": "4.41.2",
|
| 105 |
"upper_git_hash": null,
|
| 106 |
"task_hashes": {},
|
|
|
|
| 111 |
"system_instruction_sha": null,
|
| 112 |
"chat_template": null,
|
| 113 |
"chat_template_sha": null,
|
| 114 |
+
"start_time": 783215.870763237,
|
| 115 |
+
"end_time": 785823.803532197,
|
| 116 |
+
"total_evaluation_time_seconds": "2607.9327689600177"
|
| 117 |
}
|