Spaces:
Running
Running
| Model,task,0K,1K,2K,4K,8K,16K,32K,64K,128K,512K,1M,10M | |
| GPT-2 (137M),avg,27,15,,,,,,,,,, | |
| mamba-2.8b-hf,avg,70,52,35,9,0,,,,,,, | |
| rwkv-6-world-7b,avg,56,55,48,35,7,,,,,,, | |
| v5-Eagle-7B-HF,avg,62,54,48,41,2,,,,,,, | |
| Meta-Llama-3-8B-Instruct,avg,64,60,58,50,44,,,,,,, | |
| LLaMA-2-7B-32K,avg,41,53,45,40,39,32,3,,,,, | |
| longchat-7b-v1.5-32k,avg,46,42,40,41,42,39,5,,,,, | |
| LongAlpaca-13B,avg,48,47,46,43,40,36,4,,,,, | |
| Llama-2-7B-32K-Instruct,avg,49,52,49,43,40,35,5,,,,, | |
| 01-ai/Yi-34B,avg,72,52,43,37,38,31,4,,,,, | |
| Mistral-7b-Instruct-v0.2,avg,60,56,52,49,45,42,37,,,,, | |
| Mixtral-8x7B-Instruct-v0.1,avg,65,63,60,55,50,46,40,,,,, | |
| 01-ai/Yi-34B-200k,avg,65,59,56,54,52,50,48,48,,,, | |
| Mixtral-8x22B-Instruct-v0.1,avg,75,73,70,65,58,51,43,35,,,, | |
| activation-beacon-llama2-7b-chat,avg,55,52,47,43,36,23,16,8,6,,, | |
| Yarn-Mistral-7b-128k,avg,51,52,43,40,38,30,16,10,9,,, | |
| chatglm3-6b-128k,avg,56,55,51,48,46,41,36,21,13,,, | |
| activation-beacon-mistral-7b,avg,59,56,51,48,43,37,36,27,14,,, | |
| 01-ai/Yi-9B-200k,avg,52,55,48,46,45,36,37,29,24,,, | |
| Phi-3-mini-128k-instruct,avg,64,57,55,51,50,46,42,37,7,,, | |
| ai21labs/Jamba-v0.1,avg,65,53,50,48,46,45,41,40,34,,, | |
| c4ai-command-r-v01,avg,64,64,63,61,59,52,51,46,38,,, | |
| Meta-Llama-3.1-8B-Instruct,avg,67,68,66,66,62,60,56,49,39,,, | |
| Phi-3-medium-128k-instruct,avg,72,70,67,62,60,57,53,45,30,,, | |
| GPT-4,avg,87,81,77,74,71,64,53,43,36,,, | |
| Meta-Llama-3.1-70B-Instruct,avg,85,81,78,74,70,65,59,53,45,,, | |
| ~ Mamba (130M) fine-tune,avg,,,,"98,7","98,5","98,5","98,1",97,"92,5",,, | |
| Llama3-ChatQA-1.5-8B + RAG,avg,48,48,47,46,45,45,44,42,45,42,39,37 | |
| ~ RMT (137M) fine-tune,avg,"99,36","97,4","94,66","92,32","89,9","85,62","77,88","69,86","58,52","46,36","42,84","33,78" | |
| ~ ARMT (137M) fine-tune,avg,"99,32",,,"98,1","98,2","98,1",98,"97,9","96,9","95,3","93,4","76,6" | |
| GPT-2 (137M),qa1,35,13,,,,,,,,,, | |
| mamba-2.8b-hf,qa1,65,56,40,7,1,,,,,,, | |
| rwkv-6-world-7b,qa1,65,62,53,39,5,,,,,,, | |
| v5-Eagle-7B-HF,qa1,68,58,52,36,3,,,,,,, | |
| Meta-Llama-3-8B-Instruct,qa1,98,93,90,79,62,,,,,,, | |
| LLaMA-2-7B-32K,qa1,54,57,33,26,34,32,3,,,,, | |
| longchat-7b-v1.5-32k,qa1,52,60,56,55,50,42,4,,,,, | |
| LongAlpaca-13B,qa1,58,55,58,50,37,23,2,,,,, | |
| Llama-2-7B-32K-Instruct,qa1,65,61,52,41,35,23,3,,,,, | |
| 01-ai/Yi-34B,qa1,99,59,51,34,46,31,4,,,,, | |
| Mistral-7b-Instruct-v0.2,qa1,92,86,75,64,63,57,45,,,,, | |
| Mixtral-8x7B-Instruct-v0.1,qa1,99,92,84,77,65,53,49,,,,, | |
| 01-ai/Yi-34B-200k,qa1,85,73,68,66,63,65,62,60,,,, | |
| Mixtral-8x22B-Instruct-v0.1,qa1,100,99,95,89,79,63,40,38,,,, | |
| activation-beacon-llama2-7b-chat,qa1,85,81,67,65,48,21,16,6,5,,, | |
| Yarn-Mistral-7b-128k,qa1,61,71,58,45,51,34,21,8,8,,, | |
| chatglm3-6b-128k,qa1,82,77,74,72,67,56,47,13,13,,, | |
| activation-beacon-mistral-7b,qa1,92,86,73,59,47,42,42,27,9,,, | |
| 01-ai/Yi-9B-200k,qa1,33,82,69,59,56,47,44,32,29,,, | |
| Phi-3-mini-128k-instruct,qa1,97,84,72,69,70,60,53,38,1,,, | |
| ai21labs/Jamba-v0.1,qa1,90,72,66,63,65,58,50,49,38,,, | |
| c4ai-command-r-v01,qa1,98,95,94,91,89,68,70,50,30,,, | |
| Phi-3-medium-128k-instruct,qa1,100,93,87,80,81,72,69,58,21,,, | |
| GPT-4,qa1,99,100,100,95,93,84,62,58,39,,, | |
| ~ Mamba (130M) fine-tune,qa1,100,100,100,100,100,100,100,100,100,92,, | |
| Llama3-ChatQA-1.5-8B + RAG,qa1,60,62,60,58,58,60,60,56,64,54,55,50 | |
| ~ RMT (137M) fine-tune,qa1,100,100,100,100,100,100,99,96,94,87,84,66 | |
| ~ ARMT (137M) fine-tune,qa1,100,,,100,100,100,100,100,100,99,99,89 | |
| GPT-2 (137M),qa2,21,17,,,,,,,,,, | |
| mamba-2.8b-hf,qa2,68,47,24,8,0,,,,,,, | |
| rwkv-6-world-7b,qa2,42,26,20,16,3,,,,,,, | |
| v5-Eagle-7B-HF,qa2,43,27,25,19,2,,,,,,, | |
| Meta-Llama-3-8B-Instruct,qa2,47,46,49,39,20,,,,,,, | |
| LLaMA-2-7B-32K,qa2,37,40,37,26,27,21,2,,,,, | |
| longchat-7b-v1.5-32k,qa2,31,19,14,13,18,14,2,,,,, | |
| LongAlpaca-13B,qa2,30,30,28,29,26,18,2,,,,, | |
| Llama-2-7B-32K-Instruct,qa2,41,40,36,27,20,16,3,,,,, | |
| 01-ai/Yi-34B,qa2,67,43,32,30,23,15,4,,,,, | |
| Mistral-7b-Instruct-v0.2,qa2,46,34,32,22,11,9,7,,,,, | |
| Mixtral-8x7B-Instruct-v0.1,qa2,58,51,46,35,27,18,14,,,,, | |
| 01-ai/Yi-34B-200k,qa2,62,51,46,42,35,32,27,30,,,, | |
| Mixtral-8x22B-Instruct-v0.1,qa2,77,65,61,56,48,41,33,11,,,, | |
| activation-beacon-llama2-7b-chat,qa2,52,37,28,25,23,11,9,4,2,,, | |
| Yarn-Mistral-7b-128k,qa2,47,48,37,30,34,21,12,5,3,,, | |
| chatglm3-6b-128k,qa2,51,46,45,39,38,31,24,6,5,,, | |
| activation-beacon-mistral-7b,qa2,45,35,32,28,22,14,12,10,2,,, | |
| 01-ai/Yi-9B-200k,qa2,67,52,43,39,31,25,22,12,8,,, | |
| Phi-3-mini-128k-instruct,qa2,57,38,38,36,34,23,22,15,2,,, | |
| ai21labs/Jamba-v0.1,qa2,57,43,42,39,37,29,26,20,16,,, | |
| c4ai-command-r-v01,qa2,64,58,56,54,50,39,37,32,16,,, | |
| Phi-3-medium-128k-instruct,qa2,76,62,58,51,44,41,27,14,11,,, | |
| GPT-4,qa2,88,79,72,68,65,59,42,25,25,,, | |
| ~ Mamba (130M) fine-tune,qa2,98,98,98,98,98,98,98,95,87,,, | |
| Llama3-ChatQA-1.5-8B + RAG,qa2,28,25,22,19,14,13,9,7,6,6,2,2 | |
| ~ RMT (137M) fine-tune,qa2,100,100,99,98,97,94,82,59,39,25,22,19 | |
| ~ ARMT (137M) fine-tune,qa2,100,,,100,100,100,100,100,100,99,99,84 | |
| GPT-2 (137M),qa3,6,8,,,,,,,,,, | |
| mamba-2.8b-hf,qa3,48,39,21,8,0,,,,,,, | |
| rwkv-6-world-7b,qa3,40,45,28,24,4,,,,,,, | |
| v5-Eagle-7B-HF,qa3,43,34,30,40,1,,,,,,, | |
| Meta-Llama-3-8B-Instruct,qa3,33,28,30,26,11,,,,,,, | |
| LLaMA-2-7B-32K,qa3,32,38,34,28,27,21,1,,,,, | |
| longchat-7b-v1.5-32k,qa3,22,16,15,17,21,22,4,,,,, | |
| LongAlpaca-13B,qa3,25,26,26,25,24,27,2,,,,, | |
| Llama-2-7B-32K-Instruct,qa3,35,36,34,26,23,20,2,,,,, | |
| 01-ai/Yi-34B,qa3,45,34,24,20,17,12,4,,,,, | |
| Mistral-7b-Instruct-v0.2,qa3,36,34,31,30,24,18,12,,,,, | |
| Mixtral-8x7B-Instruct-v0.1,qa3,34,32,31,30,27,29,24,,,,, | |
| 01-ai/Yi-34B-200k,qa3,35,30,27,24,24,22,22,26,,,, | |
| Mixtral-8x22B-Instruct-v0.1,qa3,53,56,49,39,31,27,26,26,,,, | |
| activation-beacon-llama2-7b-chat,qa3,33,25,25,21,20,17,13,5,5,,, | |
| Yarn-Mistral-7b-128k,qa3,31,36,33,32,27,25,9,13,7,,, | |
| chatglm3-6b-128k,qa3,33,37,31,31,27,25,23,17,9,,, | |
| activation-beacon-mistral-7b,qa3,36,33,25,21,18,15,15,15,16,,, | |
| 01-ai/Yi-9B-200k,qa3,34,33,29,24,25,21,20,20,8,,, | |
| Phi-3-mini-128k-instruct,qa3,32,41,31,27,26,24,21,22,4,,, | |
| ai21labs/Jamba-v0.1,qa3,32,31,29,26,24,22,22,21,26,,, | |
| c4ai-command-r-v01,qa3,25,28,26,28,26,30,28,33,24,,, | |
| Phi-3-medium-128k-instruct,qa3,53,51,45,35,30,30,27,25,17,,, | |
| GPT-4,qa3,56,63,57,56,53,45,31,31,32,,, | |
| ~ Mamba (130M) fine-tune,qa3,97,97,97,97,97,96,95,92,81,,, | |
| Llama3-ChatQA-1.5-8B + RAG,qa3,17,18,17,17,16,17,15,13,19,17,10,11 | |
| ~ RMT (137M) fine-tune,qa3,97,94,88,81,73,66,55,55,36,25,22,21 | |
| ~ ARMT (137M) fine-tune,qa3,97,,,92,92,92,91,90,86,80,72,37 | |
| GPT-2 (137M),qa4,29,18,,,,,,,,,, | |
| mamba-2.8b-hf,qa4,96,59,47,12,0,,,,,,, | |
| rwkv-6-world-7b,qa4,54,65,57,35,7,,,,,,, | |
| v5-Eagle-7B-HF,qa4,79,74,63,55,3,,,,,,, | |
| Meta-Llama-3-8B-Instruct,qa4,58,55,50,43,52,,,,,,, | |
| LLaMA-2-7B-32K,qa4,26,54,51,51,46,36,3,,,,, | |
| longchat-7b-v1.5-32k,qa4,60,55,52,57,57,49,4,,,,, | |
| LongAlpaca-13B,qa4,65,61,58,52,50,44,4,,,,, | |
| Llama-2-7B-32K-Instruct,qa4,39,52,54,56,55,52,6,,,,, | |
| 01-ai/Yi-34B,qa4,59,56,51,55,52,43,4,,,,, | |
| Mistral-7b-Instruct-v0.2,qa4,54,58,58,60,60,58,54,,,,, | |
| Mixtral-8x7B-Instruct-v0.1,qa4,55,60,59,61,63,61,58,,,,, | |
| 01-ai/Yi-34B-200k,qa4,64,65,64,63,61,56,54,44,,,, | |
| Mixtral-8x22B-Instruct-v0.1,qa4,56,62,59,62,62,60,54,39,,,, | |
| activation-beacon-llama2-7b-chat,qa4,40,50,52,43,34,22,14,9,10,,, | |
| Yarn-Mistral-7b-128k,qa4,60,56,43,45,32,31,16,7,8,,, | |
| chatglm3-6b-128k,qa4,45,48,42,38,35,32,27,13,11,,, | |
| activation-beacon-mistral-7b,qa4,53,58,60,57,53,50,45,29,15,,, | |
| 01-ai/Yi-9B-200k,qa4,49,47,50,50,54,43,45,36,33,,, | |
| Phi-3-mini-128k-instruct,qa4,54,56,56,50,49,50,45,47,5,,, | |
| ai21labs/Jamba-v0.1,qa4,64,50,49,49,48,52,46,49,38,,, | |
| c4ai-command-r-v01,qa4,46,58,59,54,56,46,46,47,52,,, | |
| Phi-3-medium-128k-instruct,qa4,54,61,63,64,64,61,59,52,33,,, | |
| GPT-4,qa4,98,70,63,60,52,47,46,40,32,,, | |
| ~ Mamba (130M) fine-tune,qa4,100,100,100,100,100,100,99,100,98,,, | |
| Llama3-ChatQA-1.5-8B + RAG,qa4,53,58,56,59,57,60,60,59,60,59,54,56 | |
| ~ RMT (137M) fine-tune,qa4,100,94,87,83,80,75,64,51,38,26,24,20 | |
| ~ ARMT (137M) fine-tune,qa4,100,,,100,100,100,100,100,100,100,100,92 | |
| GPT-2 (137M),qa5,45,19,,,,,,,,,, | |
| mamba-2.8b-hf,qa5,75,58,43,9,0,,,,,,, | |
| rwkv-6-world-7b,qa5,79,77,80,61,14,,,,,,, | |
| v5-Eagle-7B-HF,qa5,75,76,71,57,3,,,,,,, | |
| Meta-Llama-3-8B-Instruct,qa5,85,78,73,65,73,,,,,,, | |
| LLaMA-2-7B-32K,qa5,55,74,70,67,59,51,7,,,,, | |
| longchat-7b-v1.5-32k,qa5,63,62,62,65,66,67,9,,,,, | |
| LongAlpaca-13B,qa5,63,61,61,61,62,66,12,,,,, | |
| Llama-2-7B-32K-Instruct,qa5,63,69,69,67,66,63,9,,,,, | |
| 01-ai/Yi-34B,qa5,88,70,59,48,53,55,4,,,,, | |
| Mistral-7b-Instruct-v0.2,qa5,70,66,66,67,69,67,67,,,,, | |
| Mixtral-8x7B-Instruct-v0.1,qa5,80,79,80,73,66,67,56,,,,, | |
| 01-ai/Yi-34B-200k,qa5,78,77,77,76,76,75,76,80,,,, | |
| Mixtral-8x22B-Instruct-v0.1,qa5,87,84,84,79,69,64,63,63,,,, | |
| activation-beacon-llama2-7b-chat,qa5,65,67,64,63,57,45,29,17,7,,, | |
| Yarn-Mistral-7b-128k,qa5,58,47,45,47,47,38,23,17,19,,, | |
| chatglm3-6b-128k,qa5,70,69,64,60,61,61,58,55,26,,, | |
| activation-beacon-mistral-7b,qa5,68,66,66,74,74,66,67,55,28,,, | |
| 01-ai/Yi-9B-200k,qa5,76,59,50,57,57,45,52,47,40,,, | |
| Phi-3-mini-128k-instruct,qa5,79,66,76,72,72,73,71,64,23,,, | |
| ai21labs/Jamba-v0.1,qa5,83,70,64,62,58,64,63,60,50,,, | |
| c4ai-command-r-v01,qa5,86,82,81,78,75,79,72,70,66,,, | |
| Phi-3-medium-128k-instruct,qa5,77,85,84,81,82,82,81,78,69,,, | |
| GPT-4,qa5,96,95,92,90,93,85,82,60,51,,, | |
| ~ Mamba (130M) fine-tune,qa5,98,99,98,99,99,99,98,99,98,,, | |
| Llama3-ChatQA-1.5-8B + RAG,qa5,80,77,78,77,78,77,78,76,75,75,76,67 | |
| ~ RMT (137M) fine-tune,qa5,100,100,99,99,99,94,90,89,86,69,63,44 | |
| ~ ARMT (137M) fine-tune,qa5,"99,6",,,"98,1","98,2","98,1",98,"97,9","96,9","95,3","93,4","76,6" |