MaziyarPanahi/TheTop-5x7B-Instruct-S5-v0.1

Merge of top 7B models and the SLERP of other 7B models

mergekit is a toolkit for merging pre-trained language models. mergekit uses an out-of-core approach to perform unreasonably elaborate merges in resource-constrained situations. Merges can be run entirely on CPU or accelerated with as little as 8 GB of VRAM. Many merging algorithms are supported, with more coming as they catch my attention.

Eval

{
    "all": {
        "acc": 0.6564118716978186,
        "acc_stderr": 0.03200912848183244,
        "acc_norm": 0.6553902167958241,
        "acc_norm_stderr": 0.03268788255929441,
        "mc1": 0.5312117503059975,
        "mc1_stderr": 0.01746936487457752,
        "mc2": 0.6758096547963126,
        "mc2_stderr": 0.015381620483561457
    },
    "harness|arc:challenge|25": {
        "acc": 0.6919795221843004,
        "acc_stderr": 0.013491429517292038,
        "acc_norm": 0.7252559726962458,
        "acc_norm_stderr": 0.013044617212771227
    },
    "harness|hellaswag|10": {
        "acc": 0.7234614618601872,
        "acc_stderr": 0.004463721071319078,
        "acc_norm": 0.8870742879904402,
        "acc_norm_stderr": 0.0031585512705264054
    },
    "harness|hendrycksTest-abstract_algebra|5": {
        "acc": 0.33,
        "acc_stderr": 0.047258156262526045,
        "acc_norm": 0.33,
        "acc_norm_stderr": 0.047258156262526045
    },
    "harness|hendrycksTest-anatomy|5": {
        "acc": 0.6518518518518519,
        "acc_stderr": 0.041153246103369526,
        "acc_norm": 0.6518518518518519,
        "acc_norm_stderr": 0.041153246103369526
    },
    "harness|hendrycksTest-astronomy|5": {
        "acc": 0.7039473684210527,
        "acc_stderr": 0.03715062154998904,
        "acc_norm": 0.7039473684210527,
        "acc_norm_stderr": 0.03715062154998904
    },
    "harness|hendrycksTest-business_ethics|5": {
        "acc": 0.65,
        "acc_stderr": 0.0479372485441102,
        "acc_norm": 0.65,
        "acc_norm_stderr": 0.0479372485441102
    },
    "harness|hendrycksTest-clinical_knowledge|5": {
        "acc": 0.6943396226415094,
        "acc_stderr": 0.028353298073322663,
        "acc_norm": 0.6943396226415094,
        "acc_norm_stderr": 0.028353298073322663
    },
    "harness|hendrycksTest-college_biology|5": {
        "acc": 0.7708333333333334,
        "acc_stderr": 0.03514697467862388,
        "acc_norm": 0.7708333333333334,
        "acc_norm_stderr": 0.03514697467862388
    },
    "harness|hendrycksTest-college_chemistry|5": {
        "acc": 0.49,
        "acc_stderr": 0.05024183937956912,
        "acc_norm": 0.49,
        "acc_norm_stderr": 0.05024183937956912
    },
    "harness|hendrycksTest-college_computer_science|5": {
        "acc": 0.52,
        "acc_stderr": 0.050211673156867795,
        "acc_norm": 0.52,
        "acc_norm_stderr": 0.050211673156867795
    },
    "harness|hendrycksTest-college_mathematics|5": {
        "acc": 0.28,
        "acc_stderr": 0.04512608598542126,
        "acc_norm": 0.28,
        "acc_norm_stderr": 0.04512608598542126
    },
    "harness|hendrycksTest-college_medicine|5": {
        "acc": 0.6820809248554913,
        "acc_stderr": 0.0355068398916558,
        "acc_norm": 0.6820809248554913,
        "acc_norm_stderr": 0.0355068398916558
    },
    "harness|hendrycksTest-college_physics|5": {
        "acc": 0.38235294117647056,
        "acc_stderr": 0.04835503696107224,
        "acc_norm": 0.38235294117647056,
        "acc_norm_stderr": 0.04835503696107224
    },
    "harness|hendrycksTest-computer_security|5": {
        "acc": 0.77,
        "acc_stderr": 0.04229525846816506,
        "acc_norm": 0.77,
        "acc_norm_stderr": 0.04229525846816506
    },
    "harness|hendrycksTest-conceptual_physics|5": {
        "acc": 0.5957446808510638,
        "acc_stderr": 0.03208115750788684,
        "acc_norm": 0.5957446808510638,
        "acc_norm_stderr": 0.03208115750788684
    },
    "harness|hendrycksTest-econometrics|5": {
        "acc": 0.5087719298245614,
        "acc_stderr": 0.04702880432049615,
        "acc_norm": 0.5087719298245614,
        "acc_norm_stderr": 0.04702880432049615
    },
    "harness|hendrycksTest-electrical_engineering|5": {
        "acc": 0.5724137931034483,
        "acc_stderr": 0.04122737111370332,
        "acc_norm": 0.5724137931034483,
        "acc_norm_stderr": 0.04122737111370332
    },
    "harness|hendrycksTest-elementary_mathematics|5": {
        "acc": 0.4312169312169312,
        "acc_stderr": 0.025506481698138208,
        "acc_norm": 0.4312169312169312,
        "acc_norm_stderr": 0.025506481698138208
    },
    "harness|hendrycksTest-formal_logic|5": {
        "acc": 0.5,
        "acc_stderr": 0.04472135954999579,
        "acc_norm": 0.5,
        "acc_norm_stderr": 0.04472135954999579
    },
    "harness|hendrycksTest-global_facts|5": {
        "acc": 0.37,
        "acc_stderr": 0.04852365870939099,
        "acc_norm": 0.37,
        "acc_norm_stderr": 0.04852365870939099
    },
    "harness|hendrycksTest-high_school_biology|5": {
        "acc": 0.7903225806451613,
        "acc_stderr": 0.023157879349083525,
        "acc_norm": 0.7903225806451613,
        "acc_norm_stderr": 0.023157879349083525
    },
    "harness|hendrycksTest-high_school_chemistry|5": {
        "acc": 0.4975369458128079,
        "acc_stderr": 0.03517945038691063,
        "acc_norm": 0.4975369458128079,
        "acc_norm_stderr": 0.03517945038691063
    },
    "harness|hendrycksTest-high_school_computer_science|5": {
        "acc": 0.66,
        "acc_stderr": 0.04760952285695237,
        "acc_norm": 0.66,
        "acc_norm_stderr": 0.04760952285695237
    },
    "harness|hendrycksTest-high_school_european_history|5": {
        "acc": 0.7696969696969697,
        "acc_stderr": 0.0328766675860349,
        "acc_norm": 0.7696969696969697,
        "acc_norm_stderr": 0.0328766675860349
    },
    "harness|hendrycksTest-high_school_geography|5": {
        "acc": 0.7878787878787878,
        "acc_stderr": 0.029126522834586818,
        "acc_norm": 0.7878787878787878,
        "acc_norm_stderr": 0.029126522834586818
    },
    "harness|hendrycksTest-high_school_government_and_politics|5": {
        "acc": 0.9067357512953368,
        "acc_stderr": 0.020986854593289733,
        "acc_norm": 0.9067357512953368,
        "acc_norm_stderr": 0.020986854593289733
    },
    "harness|hendrycksTest-high_school_macroeconomics|5": {
        "acc": 0.6641025641025641,
        "acc_stderr": 0.023946724741563976,
        "acc_norm": 0.6641025641025641,
        "acc_norm_stderr": 0.023946724741563976
    },
    "harness|hendrycksTest-high_school_mathematics|5": {
        "acc": 0.3592592592592593,
        "acc_stderr": 0.02925290592725197,
        "acc_norm": 0.3592592592592593,
        "acc_norm_stderr": 0.02925290592725197
    },
    "harness|hendrycksTest-high_school_microeconomics|5": {
        "acc": 0.6764705882352942,
        "acc_stderr": 0.03038835355188679,
        "acc_norm": 0.6764705882352942,
        "acc_norm_stderr": 0.03038835355188679
    },
    "harness|hendrycksTest-high_school_physics|5": {
        "acc": 0.36423841059602646,
        "acc_stderr": 0.03929111781242742,
        "acc_norm": 0.36423841059602646,
        "acc_norm_stderr": 0.03929111781242742
    },
    "harness|hendrycksTest-high_school_psychology|5": {
        "acc": 0.8385321100917431,
        "acc_stderr": 0.015776239256163224,
        "acc_norm": 0.8385321100917431,
        "acc_norm_stderr": 0.015776239256163224
    },
    "harness|hendrycksTest-high_school_statistics|5": {
        "acc": 0.5138888888888888,
        "acc_stderr": 0.03408655867977749,
        "acc_norm": 0.5138888888888888,
        "acc_norm_stderr": 0.03408655867977749
    },
    "harness|hendrycksTest-high_school_us_history|5": {
        "acc": 0.8529411764705882,
        "acc_stderr": 0.024857478080250447,
        "acc_norm": 0.8529411764705882,
        "acc_norm_stderr": 0.024857478080250447
    },
    "harness|hendrycksTest-high_school_world_history|5": {
        "acc": 0.8143459915611815,
        "acc_stderr": 0.025310495376944856,
        "acc_norm": 0.8143459915611815,
        "acc_norm_stderr": 0.025310495376944856
    },
    "harness|hendrycksTest-human_aging|5": {
        "acc": 0.6816143497757847,
        "acc_stderr": 0.03126580522513713,
        "acc_norm": 0.6816143497757847,
        "acc_norm_stderr": 0.03126580522513713
    },
    "harness|hendrycksTest-human_sexuality|5": {
        "acc": 0.816793893129771,
        "acc_stderr": 0.03392770926494733,
        "acc_norm": 0.816793893129771,
        "acc_norm_stderr": 0.03392770926494733
    },
    "harness|hendrycksTest-international_law|5": {
        "acc": 0.7933884297520661,
        "acc_stderr": 0.03695980128098824,
        "acc_norm": 0.7933884297520661,
        "acc_norm_stderr": 0.03695980128098824
    },
    "harness|hendrycksTest-jurisprudence|5": {
        "acc": 0.7870370370370371,
        "acc_stderr": 0.0395783547198098,
        "acc_norm": 0.7870370370370371,
        "acc_norm_stderr": 0.0395783547198098
    },
    "harness|hendrycksTest-logical_fallacies|5": {
        "acc": 0.7607361963190185,
        "acc_stderr": 0.0335195387952127,
        "acc_norm": 0.7607361963190185,
        "acc_norm_stderr": 0.0335195387952127
    },
    "harness|hendrycksTest-machine_learning|5": {
        "acc": 0.48214285714285715,
        "acc_stderr": 0.047427623612430116,
        "acc_norm": 0.48214285714285715,
        "acc_norm_stderr": 0.047427623612430116
    },
    "harness|hendrycksTest-management|5": {
        "acc": 0.7864077669902912,
        "acc_stderr": 0.040580420156460344,
        "acc_norm": 0.7864077669902912,
        "acc_norm_stderr": 0.040580420156460344
    },
    "harness|hendrycksTest-marketing|5": {
        "acc": 0.8760683760683761,
        "acc_stderr": 0.021586494001281365,
        "acc_norm": 0.8760683760683761,
        "acc_norm_stderr": 0.021586494001281365
    },
    "harness|hendrycksTest-medical_genetics|5": {
        "acc": 0.72,
        "acc_stderr": 0.04512608598542128,
        "acc_norm": 0.72,
        "acc_norm_stderr": 0.04512608598542128
    },
    "harness|hendrycksTest-miscellaneous|5": {
        "acc": 0.8250319284802043,
        "acc_stderr": 0.013586619219903341,
        "acc_norm": 0.8250319284802043,
        "acc_norm_stderr": 0.013586619219903341
    },
    "harness|hendrycksTest-moral_disputes|5": {
        "acc": 0.7456647398843931,
        "acc_stderr": 0.02344582627654554,
        "acc_norm": 0.7456647398843931,
        "acc_norm_stderr": 0.02344582627654554
    },
    "harness|hendrycksTest-moral_scenarios|5": {
        "acc": 0.45251396648044695,
        "acc_stderr": 0.016646914804438778,
        "acc_norm": 0.45251396648044695,
        "acc_norm_stderr": 0.016646914804438778
    },
    "harness|hendrycksTest-nutrition|5": {
        "acc": 0.7254901960784313,
        "acc_stderr": 0.02555316999182652,
        "acc_norm": 0.7254901960784313,
        "acc_norm_stderr": 0.02555316999182652
    },
    "harness|hendrycksTest-philosophy|5": {
        "acc": 0.707395498392283,
        "acc_stderr": 0.02583989833487798,
        "acc_norm": 0.707395498392283,
        "acc_norm_stderr": 0.02583989833487798
    },
    "harness|hendrycksTest-prehistory|5": {
        "acc": 0.7561728395061729,
        "acc_stderr": 0.02389187954195961,
        "acc_norm": 0.7561728395061729,
        "acc_norm_stderr": 0.02389187954195961
    },
    "harness|hendrycksTest-professional_accounting|5": {
        "acc": 0.4645390070921986,
        "acc_stderr": 0.029752389657427047,
        "acc_norm": 0.4645390070921986,
        "acc_norm_stderr": 0.029752389657427047
    },
    "harness|hendrycksTest-professional_law|5": {
        "acc": 0.47327249022164275,
        "acc_stderr": 0.01275197796767601,
        "acc_norm": 0.47327249022164275,
        "acc_norm_stderr": 0.01275197796767601
    },
    "harness|hendrycksTest-professional_medicine|5": {
        "acc": 0.6838235294117647,
        "acc_stderr": 0.02824568739146292,
        "acc_norm": 0.6838235294117647,
        "acc_norm_stderr": 0.02824568739146292
    },
    "harness|hendrycksTest-professional_psychology|5": {
        "acc": 0.6715686274509803,
        "acc_stderr": 0.018999707383162673,
        "acc_norm": 0.6715686274509803,
        "acc_norm_stderr": 0.018999707383162673
    },
    "harness|hendrycksTest-public_relations|5": {
        "acc": 0.6545454545454545,
        "acc_stderr": 0.04554619617541054,
        "acc_norm": 0.6545454545454545,
        "acc_norm_stderr": 0.04554619617541054
    },
    "harness|hendrycksTest-security_studies|5": {
        "acc": 0.7306122448979592,
        "acc_stderr": 0.02840125202902294,
        "acc_norm": 0.7306122448979592,
        "acc_norm_stderr": 0.02840125202902294
    },
    "harness|hendrycksTest-sociology|5": {
        "acc": 0.8208955223880597,
        "acc_stderr": 0.027113286753111837,
        "acc_norm": 0.8208955223880597,
        "acc_norm_stderr": 0.027113286753111837
    },
    "harness|hendrycksTest-us_foreign_policy|5": {
        "acc": 0.85,
        "acc_stderr": 0.03588702812826371,
        "acc_norm": 0.85,
        "acc_norm_stderr": 0.03588702812826371
    },
    "harness|hendrycksTest-virology|5": {
        "acc": 0.5542168674698795,
        "acc_stderr": 0.038695433234721015,
        "acc_norm": 0.5542168674698795,
        "acc_norm_stderr": 0.038695433234721015
    },
    "harness|hendrycksTest-world_religions|5": {
        "acc": 0.8362573099415205,
        "acc_stderr": 0.028380919596145866,
        "acc_norm": 0.8362573099415205,
        "acc_norm_stderr": 0.028380919596145866
    },
    "harness|truthfulqa:mc|0": {
        "mc1": 0.5312117503059975,
        "mc1_stderr": 0.01746936487457752,
        "mc2": 0.6758096547963126,
        "mc2_stderr": 0.015381620483561457
    },
    "harness|winogrande|5": {
        "acc": 0.861878453038674,
        "acc_stderr": 0.00969698839367458
    },
    "harness|gsm8k|5": {
        "acc": 0.7081122062168309,
        "acc_stderr": 0.012522795894420867
    }
}

Open LLM Leaderboard Evaluation Results

Detailed results can be found here

Metric	Value
Avg.	75.14
AI2 Reasoning Challenge (25-Shot)	72.53
HellaSwag (10-Shot)	88.71
MMLU (5-Shot)	65.01
TruthfulQA (0-shot)	67.58
Winogrande (5-shot)	86.19
GSM8k (5-shot)	70.81