Spaces:

RMT-team
/

babilong

Running

App Files Files Community

Add results of Llama 3 8b, Llama 3 70b, Qwen 2.5 72b (all Instruct fp16)

by sm1rk - opened Oct 11, 2024

base: refs/heads/main

←

from: refs/pr/3

Discussion Files changed

+234

-171

Files changed (1) hide show

data/BABILong NeurIPS24 Figs - leaderboard.csv +234 -171

data/BABILong NeurIPS24 Figs - leaderboard.csv CHANGED Viewed

@@ -1,171 +1,234 @@
-Model,task,0K,1K,2K,4K,8K,16K,32K,64K,128K,512K,1M,10M
-GPT-2 (137M),avg,27,15,,,,,,,,,,
-mamba-2.8b-hf,avg,70,52,35,9,0,,,,,,,
-rwkv-6-world-7b,avg,56,55,48,35,7,,,,,,,
-v5-Eagle-7B-HF,avg,62,54,48,41,2,,,,,,,
-Meta-Llama-3-8B-Instruct,avg,64,60,58,50,44,,,,,,,
-LLaMA-2-7B-32K,avg,41,53,45,40,39,32,3,,,,,
-longchat-7b-v1.5-32k,avg,46,42,40,41,42,39,5,,,,,
-LongAlpaca-13B,avg,48,47,46,43,40,36,4,,,,,
-Llama-2-7B-32K-Instruct,avg,49,52,49,43,40,35,5,,,,,
-01-ai/Yi-34B,avg,72,52,43,37,38,31,4,,,,,
-Mistral-7b-Instruct-v0.2,avg,60,56,52,49,45,42,37,,,,,
-Mixtral-8x7B-Instruct-v0.1,avg,65,63,60,55,50,46,40,,,,,
-01-ai/Yi-34B-200k,avg,65,59,56,54,52,50,48,48,,,,
-Mixtral-8x22B-Instruct-v0.1,avg,75,73,70,65,58,51,43,35,,,,
-activation-beacon-llama2-7b-chat,avg,55,52,47,43,36,23,16,8,6,,,
-Yarn-Mistral-7b-128k,avg,51,52,43,40,38,30,16,10,9,,,
-chatglm3-6b-128k,avg,56,55,51,48,46,41,36,21,13,,,
-activation-beacon-mistral-7b,avg,59,56,51,48,43,37,36,27,14,,,
-01-ai/Yi-9B-200k,avg,52,55,48,46,45,36,37,29,24,,,
-Phi-3-mini-128k-instruct,avg,64,57,55,51,50,46,42,37,7,,,
-ai21labs/Jamba-v0.1,avg,65,53,50,48,46,45,41,40,34,,,
-c4ai-command-r-v01,avg,64,64,63,61,59,52,51,46,38,,,
-Meta-Llama-3.1-8B-Instruct,avg,67,68,66,66,62,60,56,49,39,,,
-Phi-3-medium-128k-instruct,avg,72,70,67,62,60,57,53,45,30,,,
-GPT-4,avg,87,81,77,74,71,64,53,43,36,,,
-Meta-Llama-3.1-70B-Instruct,avg,85,81,78,74,70,65,59,53,45,,,
-~ Mamba (130M) fine-tune,avg,,,,"98,7","98,5","98,5","98,1",97,"92,5",,,
-Llama3-ChatQA-1.5-8B + RAG,avg,48,48,47,46,45,45,44,42,45,42,39,37
-~ RMT (137M) fine-tune,avg,"99,36","97,4","94,66","92,32","89,9","85,62","77,88","69,86","58,52","46,36","42,84","33,78"
-~ ARMT (137M) fine-tune,avg,"99,32",,,"98,1","98,2","98,1",98,"97,9","96,9","95,3","93,4","76,6"
-GPT-2 (137M),qa1,35,13,,,,,,,,,,
-mamba-2.8b-hf,qa1,65,56,40,7,1,,,,,,,
-rwkv-6-world-7b,qa1,65,62,53,39,5,,,,,,,
-v5-Eagle-7B-HF,qa1,68,58,52,36,3,,,,,,,
-Meta-Llama-3-8B-Instruct,qa1,98,93,90,79,62,,,,,,,
-LLaMA-2-7B-32K,qa1,54,57,33,26,34,32,3,,,,,
-longchat-7b-v1.5-32k,qa1,52,60,56,55,50,42,4,,,,,
-LongAlpaca-13B,qa1,58,55,58,50,37,23,2,,,,,
-Llama-2-7B-32K-Instruct,qa1,65,61,52,41,35,23,3,,,,,
-01-ai/Yi-34B,qa1,99,59,51,34,46,31,4,,,,,
-Mistral-7b-Instruct-v0.2,qa1,92,86,75,64,63,57,45,,,,,
-Mixtral-8x7B-Instruct-v0.1,qa1,99,92,84,77,65,53,49,,,,,
-01-ai/Yi-34B-200k,qa1,85,73,68,66,63,65,62,60,,,,
-Mixtral-8x22B-Instruct-v0.1,qa1,100,99,95,89,79,63,40,38,,,,
-activation-beacon-llama2-7b-chat,qa1,85,81,67,65,48,21,16,6,5,,,
-Yarn-Mistral-7b-128k,qa1,61,71,58,45,51,34,21,8,8,,,
-chatglm3-6b-128k,qa1,82,77,74,72,67,56,47,13,13,,,
-activation-beacon-mistral-7b,qa1,92,86,73,59,47,42,42,27,9,,,
-01-ai/Yi-9B-200k,qa1,33,82,69,59,56,47,44,32,29,,,
-Phi-3-mini-128k-instruct,qa1,97,84,72,69,70,60,53,38,1,,,
-ai21labs/Jamba-v0.1,qa1,90,72,66,63,65,58,50,49,38,,,
-c4ai-command-r-v01,qa1,98,95,94,91,89,68,70,50,30,,,
-Phi-3-medium-128k-instruct,qa1,100,93,87,80,81,72,69,58,21,,,
-GPT-4,qa1,99,100,100,95,93,84,62,58,39,,,
-~ Mamba (130M) fine-tune,qa1,100,100,100,100,100,100,100,100,100,92,,
-Llama3-ChatQA-1.5-8B + RAG,qa1,60,62,60,58,58,60,60,56,64,54,55,50
-~ RMT (137M) fine-tune,qa1,100,100,100,100,100,100,99,96,94,87,84,66
-~ ARMT (137M) fine-tune,qa1,100,,,100,100,100,100,100,100,99,99,89
-GPT-2 (137M),qa2,21,17,,,,,,,,,,
-mamba-2.8b-hf,qa2,68,47,24,8,0,,,,,,,
-rwkv-6-world-7b,qa2,42,26,20,16,3,,,,,,,
-v5-Eagle-7B-HF,qa2,43,27,25,19,2,,,,,,,
-Meta-Llama-3-8B-Instruct,qa2,47,46,49,39,20,,,,,,,
-LLaMA-2-7B-32K,qa2,37,40,37,26,27,21,2,,,,,
-longchat-7b-v1.5-32k,qa2,31,19,14,13,18,14,2,,,,,
-LongAlpaca-13B,qa2,30,30,28,29,26,18,2,,,,,
-Llama-2-7B-32K-Instruct,qa2,41,40,36,27,20,16,3,,,,,
-01-ai/Yi-34B,qa2,67,43,32,30,23,15,4,,,,,
-Mistral-7b-Instruct-v0.2,qa2,46,34,32,22,11,9,7,,,,,
-Mixtral-8x7B-Instruct-v0.1,qa2,58,51,46,35,27,18,14,,,,,
-01-ai/Yi-34B-200k,qa2,62,51,46,42,35,32,27,30,,,,
-Mixtral-8x22B-Instruct-v0.1,qa2,77,65,61,56,48,41,33,11,,,,
-activation-beacon-llama2-7b-chat,qa2,52,37,28,25,23,11,9,4,2,,,
-Yarn-Mistral-7b-128k,qa2,47,48,37,30,34,21,12,5,3,,,
-chatglm3-6b-128k,qa2,51,46,45,39,38,31,24,6,5,,,
-activation-beacon-mistral-7b,qa2,45,35,32,28,22,14,12,10,2,,,
-01-ai/Yi-9B-200k,qa2,67,52,43,39,31,25,22,12,8,,,
-Phi-3-mini-128k-instruct,qa2,57,38,38,36,34,23,22,15,2,,,
-ai21labs/Jamba-v0.1,qa2,57,43,42,39,37,29,26,20,16,,,
-c4ai-command-r-v01,qa2,64,58,56,54,50,39,37,32,16,,,
-Phi-3-medium-128k-instruct,qa2,76,62,58,51,44,41,27,14,11,,,
-GPT-4,qa2,88,79,72,68,65,59,42,25,25,,,
-~ Mamba (130M) fine-tune,qa2,98,98,98,98,98,98,98,95,87,,,
-Llama3-ChatQA-1.5-8B + RAG,qa2,28,25,22,19,14,13,9,7,6,6,2,2
-~ RMT (137M) fine-tune,qa2,100,100,99,98,97,94,82,59,39,25,22,19
-~ ARMT (137M) fine-tune,qa2,100,,,100,100,100,100,100,100,99,99,84
-GPT-2 (137M),qa3,6,8,,,,,,,,,,
-mamba-2.8b-hf,qa3,48,39,21,8,0,,,,,,,
-rwkv-6-world-7b,qa3,40,45,28,24,4,,,,,,,
-v5-Eagle-7B-HF,qa3,43,34,30,40,1,,,,,,,
-Meta-Llama-3-8B-Instruct,qa3,33,28,30,26,11,,,,,,,
-LLaMA-2-7B-32K,qa3,32,38,34,28,27,21,1,,,,,
-longchat-7b-v1.5-32k,qa3,22,16,15,17,21,22,4,,,,,
-LongAlpaca-13B,qa3,25,26,26,25,24,27,2,,,,,
-Llama-2-7B-32K-Instruct,qa3,35,36,34,26,23,20,2,,,,,
-01-ai/Yi-34B,qa3,45,34,24,20,17,12,4,,,,,
-Mistral-7b-Instruct-v0.2,qa3,36,34,31,30,24,18,12,,,,,
-Mixtral-8x7B-Instruct-v0.1,qa3,34,32,31,30,27,29,24,,,,,
-01-ai/Yi-34B-200k,qa3,35,30,27,24,24,22,22,26,,,,
-Mixtral-8x22B-Instruct-v0.1,qa3,53,56,49,39,31,27,26,26,,,,
-activation-beacon-llama2-7b-chat,qa3,33,25,25,21,20,17,13,5,5,,,
-Yarn-Mistral-7b-128k,qa3,31,36,33,32,27,25,9,13,7,,,
-chatglm3-6b-128k,qa3,33,37,31,31,27,25,23,17,9,,,
-activation-beacon-mistral-7b,qa3,36,33,25,21,18,15,15,15,16,,,
-01-ai/Yi-9B-200k,qa3,34,33,29,24,25,21,20,20,8,,,
-Phi-3-mini-128k-instruct,qa3,32,41,31,27,26,24,21,22,4,,,
-ai21labs/Jamba-v0.1,qa3,32,31,29,26,24,22,22,21,26,,,
-c4ai-command-r-v01,qa3,25,28,26,28,26,30,28,33,24,,,
-Phi-3-medium-128k-instruct,qa3,53,51,45,35,30,30,27,25,17,,,
-GPT-4,qa3,56,63,57,56,53,45,31,31,32,,,
-~ Mamba (130M) fine-tune,qa3,97,97,97,97,97,96,95,92,81,,,
-Llama3-ChatQA-1.5-8B + RAG,qa3,17,18,17,17,16,17,15,13,19,17,10,11
-~ RMT (137M) fine-tune,qa3,97,94,88,81,73,66,55,55,36,25,22,21
-~ ARMT (137M) fine-tune,qa3,97,,,92,92,92,91,90,86,80,72,37
-GPT-2 (137M),qa4,29,18,,,,,,,,,,
-mamba-2.8b-hf,qa4,96,59,47,12,0,,,,,,,
-rwkv-6-world-7b,qa4,54,65,57,35,7,,,,,,,
-v5-Eagle-7B-HF,qa4,79,74,63,55,3,,,,,,,
-Meta-Llama-3-8B-Instruct,qa4,58,55,50,43,52,,,,,,,
-LLaMA-2-7B-32K,qa4,26,54,51,51,46,36,3,,,,,
-longchat-7b-v1.5-32k,qa4,60,55,52,57,57,49,4,,,,,
-LongAlpaca-13B,qa4,65,61,58,52,50,44,4,,,,,
-Llama-2-7B-32K-Instruct,qa4,39,52,54,56,55,52,6,,,,,
-01-ai/Yi-34B,qa4,59,56,51,55,52,43,4,,,,,
-Mistral-7b-Instruct-v0.2,qa4,54,58,58,60,60,58,54,,,,,
-Mixtral-8x7B-Instruct-v0.1,qa4,55,60,59,61,63,61,58,,,,,
-01-ai/Yi-34B-200k,qa4,64,65,64,63,61,56,54,44,,,,
-Mixtral-8x22B-Instruct-v0.1,qa4,56,62,59,62,62,60,54,39,,,,
-activation-beacon-llama2-7b-chat,qa4,40,50,52,43,34,22,14,9,10,,,
-Yarn-Mistral-7b-128k,qa4,60,56,43,45,32,31,16,7,8,,,
-chatglm3-6b-128k,qa4,45,48,42,38,35,32,27,13,11,,,
-activation-beacon-mistral-7b,qa4,53,58,60,57,53,50,45,29,15,,,
-01-ai/Yi-9B-200k,qa4,49,47,50,50,54,43,45,36,33,,,
-Phi-3-mini-128k-instruct,qa4,54,56,56,50,49,50,45,47,5,,,
-ai21labs/Jamba-v0.1,qa4,64,50,49,49,48,52,46,49,38,,,
-c4ai-command-r-v01,qa4,46,58,59,54,56,46,46,47,52,,,
-Phi-3-medium-128k-instruct,qa4,54,61,63,64,64,61,59,52,33,,,
-GPT-4,qa4,98,70,63,60,52,47,46,40,32,,,
-~ Mamba (130M) fine-tune,qa4,100,100,100,100,100,100,99,100,98,,,
-Llama3-ChatQA-1.5-8B + RAG,qa4,53,58,56,59,57,60,60,59,60,59,54,56
-~ RMT (137M) fine-tune,qa4,100,94,87,83,80,75,64,51,38,26,24,20
-~ ARMT (137M) fine-tune,qa4,100,,,100,100,100,100,100,100,100,100,92
-GPT-2 (137M),qa5,45,19,,,,,,,,,,
-mamba-2.8b-hf,qa5,75,58,43,9,0,,,,,,,
-rwkv-6-world-7b,qa5,79,77,80,61,14,,,,,,,
-v5-Eagle-7B-HF,qa5,75,76,71,57,3,,,,,,,
-Meta-Llama-3-8B-Instruct,qa5,85,78,73,65,73,,,,,,,
-LLaMA-2-7B-32K,qa5,55,74,70,67,59,51,7,,,,,
-longchat-7b-v1.5-32k,qa5,63,62,62,65,66,67,9,,,,,
-LongAlpaca-13B,qa5,63,61,61,61,62,66,12,,,,,
-Llama-2-7B-32K-Instruct,qa5,63,69,69,67,66,63,9,,,,,
-01-ai/Yi-34B,qa5,88,70,59,48,53,55,4,,,,,
-Mistral-7b-Instruct-v0.2,qa5,70,66,66,67,69,67,67,,,,,
-Mixtral-8x7B-Instruct-v0.1,qa5,80,79,80,73,66,67,56,,,,,
-01-ai/Yi-34B-200k,qa5,78,77,77,76,76,75,76,80,,,,
-Mixtral-8x22B-Instruct-v0.1,qa5,87,84,84,79,69,64,63,63,,,,
-activation-beacon-llama2-7b-chat,qa5,65,67,64,63,57,45,29,17,7,,,
-Yarn-Mistral-7b-128k,qa5,58,47,45,47,47,38,23,17,19,,,
-chatglm3-6b-128k,qa5,70,69,64,60,61,61,58,55,26,,,
-activation-beacon-mistral-7b,qa5,68,66,66,74,74,66,67,55,28,,,
-01-ai/Yi-9B-200k,qa5,76,59,50,57,57,45,52,47,40,,,
-Phi-3-mini-128k-instruct,qa5,79,66,76,72,72,73,71,64,23,,,
-ai21labs/Jamba-v0.1,qa5,83,70,64,62,58,64,63,60,50,,,
-c4ai-command-r-v01,qa5,86,82,81,78,75,79,72,70,66,,,
-Phi-3-medium-128k-instruct,qa5,77,85,84,81,82,82,81,78,69,,,
-GPT-4,qa5,96,95,92,90,93,85,82,60,51,,,
-~ Mamba (130M) fine-tune,qa5,98,99,98,99,99,99,98,99,98,,,
-Llama3-ChatQA-1.5-8B + RAG,qa5,80,77,78,77,78,77,78,76,75,75,76,67
-~ RMT (137M) fine-tune,qa5,100,100,99,99,99,94,90,89,86,69,63,44
-~ ARMT (137M) fine-tune,qa5,"99,6",,,"98,1","98,2","98,1",98,"97,9","96,9","95,3","93,4","76,6"

+Model,task,0K,1K,2K,4K,8K,16K,32K,64K,128K,512K,1M,10M
+GPT-2 (137M),avg,27,15,,,,,,,,,,
+mamba-2.8b-hf,avg,70,52,35,9,0,,,,,,,
+rwkv-6-world-7b,avg,56,55,48,35,7,,,,,,,
+v5-Eagle-7B-HF,avg,62,54,48,41,2,,,,,,,
+Meta-Llama-3-8B-Instruct,avg,64,60,58,50,44,,,,,,,
+LLaMA-2-7B-32K,avg,41,53,45,40,39,32,3,,,,,
+longchat-7b-v1.5-32k,avg,46,42,40,41,42,39,5,,,,,
+LongAlpaca-13B,avg,48,47,46,43,40,36,4,,,,,
+Llama-2-7B-32K-Instruct,avg,49,52,49,43,40,35,5,,,,,
+01-ai/Yi-34B,avg,72,52,43,37,38,31,4,,,,,
+Mistral-7b-Instruct-v0.2,avg,60,56,52,49,45,42,37,,,,,
+Mixtral-8x7B-Instruct-v0.1,avg,65,63,60,55,50,46,40,,,,,
+01-ai/Yi-34B-200k,avg,65,59,56,54,52,50,48,48,,,,
+Mixtral-8x22B-Instruct-v0.1,avg,75,73,70,65,58,51,43,35,,,,
+activation-beacon-llama2-7b-chat,avg,55,52,47,43,36,23,16,8,6,,,
+Yarn-Mistral-7b-128k,avg,51,52,43,40,38,30,16,10,9,,,
+chatglm3-6b-128k,avg,56,55,51,48,46,41,36,21,13,,,
+activation-beacon-mistral-7b,avg,59,56,51,48,43,37,36,27,14,,,
+01-ai/Yi-9B-200k,avg,52,55,48,46,45,36,37,29,24,,,
+Phi-3-mini-128k-instruct,avg,64,57,55,51,50,46,42,37,7,,,
+ai21labs/Jamba-v0.1,avg,65,53,50,48,46,45,41,40,34,,,
+c4ai-command-r-v01,avg,64,64,63,61,59,52,51,46,38,,,
+Meta-Llama-3.1-8B-Instruct,avg,67,68,66,66,62,60,56,49,39,,,
+Phi-3-medium-128k-instruct,avg,72,70,67,62,60,57,53,45,30,,,
+GPT-4,avg,87,81,77,74,71,64,53,43,36,,,
+Meta-Llama-3.1-70B-Instruct,avg,85,81,78,74,70,65,59,53,45,,,
+~ Mamba (130M) fine-tune,avg,,,,"98,7","98,5","98,5","98,1",97,"92,5",,,
+Llama3-ChatQA-1.5-8B + RAG,avg,48,48,47,46,45,45,44,42,45,42,39,37
+~ RMT (137M) fine-tune,avg,"99,36","97,4","94,66","92,32","89,9","85,62","77,88","69,86","58,52","46,36","42,84","33,78"
+~ ARMT (137M) fine-tune,avg,"99,32",,,"98,1","98,2","98,1",98,"97,9","96,9","95,3","93,4","76,6"
+GPT-2 (137M),qa1,35,13,,,,,,,,,,
+mamba-2.8b-hf,qa1,65,56,40,7,1,,,,,,,
+rwkv-6-world-7b,qa1,65,62,53,39,5,,,,,,,
+v5-Eagle-7B-HF,qa1,68,58,52,36,3,,,,,,,
+Meta-Llama-3-8B-Instruct,qa1,98,93,90,79,62,,,,,,,
+LLaMA-2-7B-32K,qa1,54,57,33,26,34,32,3,,,,,
+longchat-7b-v1.5-32k,qa1,52,60,56,55,50,42,4,,,,,
+LongAlpaca-13B,qa1,58,55,58,50,37,23,2,,,,,
+Llama-2-7B-32K-Instruct,qa1,65,61,52,41,35,23,3,,,,,
+01-ai/Yi-34B,qa1,99,59,51,34,46,31,4,,,,,
+Mistral-7b-Instruct-v0.2,qa1,92,86,75,64,63,57,45,,,,,
+Mixtral-8x7B-Instruct-v0.1,qa1,99,92,84,77,65,53,49,,,,,
+01-ai/Yi-34B-200k,qa1,85,73,68,66,63,65,62,60,,,,
+Mixtral-8x22B-Instruct-v0.1,qa1,100,99,95,89,79,63,40,38,,,,
+activation-beacon-llama2-7b-chat,qa1,85,81,67,65,48,21,16,6,5,,,
+Yarn-Mistral-7b-128k,qa1,61,71,58,45,51,34,21,8,8,,,
+chatglm3-6b-128k,qa1,82,77,74,72,67,56,47,13,13,,,
+activation-beacon-mistral-7b,qa1,92,86,73,59,47,42,42,27,9,,,
+01-ai/Yi-9B-200k,qa1,33,82,69,59,56,47,44,32,29,,,
+Phi-3-mini-128k-instruct,qa1,97,84,72,69,70,60,53,38,1,,,
+ai21labs/Jamba-v0.1,qa1,90,72,66,63,65,58,50,49,38,,,
+c4ai-command-r-v01,qa1,98,95,94,91,89,68,70,50,30,,,
+Phi-3-medium-128k-instruct,qa1,100,93,87,80,81,72,69,58,21,,,
+GPT-4,qa1,99,100,100,95,93,84,62,58,39,,,
+~ Mamba (130M) fine-tune,qa1,100,100,100,100,100,100,100,100,100,92,,
+Llama3-ChatQA-1.5-8B + RAG,qa1,60,62,60,58,58,60,60,56,64,54,55,50
+~ RMT (137M) fine-tune,qa1,100,100,100,100,100,100,99,96,94,87,84,66
+~ ARMT (137M) fine-tune,qa1,100,,,100,100,100,100,100,100,99,99,89
+GPT-2 (137M),qa2,21,17,,,,,,,,,,
+mamba-2.8b-hf,qa2,68,47,24,8,0,,,,,,,
+rwkv-6-world-7b,qa2,42,26,20,16,3,,,,,,,
+v5-Eagle-7B-HF,qa2,43,27,25,19,2,,,,,,,
+Meta-Llama-3-8B-Instruct,qa2,47,46,49,39,20,,,,,,,
+LLaMA-2-7B-32K,qa2,37,40,37,26,27,21,2,,,,,
+longchat-7b-v1.5-32k,qa2,31,19,14,13,18,14,2,,,,,
+LongAlpaca-13B,qa2,30,30,28,29,26,18,2,,,,,
+Llama-2-7B-32K-Instruct,qa2,41,40,36,27,20,16,3,,,,,
+01-ai/Yi-34B,qa2,67,43,32,30,23,15,4,,,,,
+Mistral-7b-Instruct-v0.2,qa2,46,34,32,22,11,9,7,,,,,
+Mixtral-8x7B-Instruct-v0.1,qa2,58,51,46,35,27,18,14,,,,,
+01-ai/Yi-34B-200k,qa2,62,51,46,42,35,32,27,30,,,,
+Mixtral-8x22B-Instruct-v0.1,qa2,77,65,61,56,48,41,33,11,,,,
+activation-beacon-llama2-7b-chat,qa2,52,37,28,25,23,11,9,4,2,,,
+Yarn-Mistral-7b-128k,qa2,47,48,37,30,34,21,12,5,3,,,
+chatglm3-6b-128k,qa2,51,46,45,39,38,31,24,6,5,,,
+activation-beacon-mistral-7b,qa2,45,35,32,28,22,14,12,10,2,,,
+01-ai/Yi-9B-200k,qa2,67,52,43,39,31,25,22,12,8,,,
+Phi-3-mini-128k-instruct,qa2,57,38,38,36,34,23,22,15,2,,,
+ai21labs/Jamba-v0.1,qa2,57,43,42,39,37,29,26,20,16,,,
+c4ai-command-r-v01,qa2,64,58,56,54,50,39,37,32,16,,,
+Phi-3-medium-128k-instruct,qa2,76,62,58,51,44,41,27,14,11,,,
+GPT-4,qa2,88,79,72,68,65,59,42,25,25,,,
+~ Mamba (130M) fine-tune,qa2,98,98,98,98,98,98,98,95,87,,,
+Llama3-ChatQA-1.5-8B + RAG,qa2,28,25,22,19,14,13,9,7,6,6,2,2
+~ RMT (137M) fine-tune,qa2,100,100,99,98,97,94,82,59,39,25,22,19
+~ ARMT (137M) fine-tune,qa2,100,,,100,100,100,100,100,100,99,99,84
+GPT-2 (137M),qa3,6,8,,,,,,,,,,
+mamba-2.8b-hf,qa3,48,39,21,8,0,,,,,,,
+rwkv-6-world-7b,qa3,40,45,28,24,4,,,,,,,
+v5-Eagle-7B-HF,qa3,43,34,30,40,1,,,,,,,
+Meta-Llama-3-8B-Instruct,qa3,33,28,30,26,11,,,,,,,
+LLaMA-2-7B-32K,qa3,32,38,34,28,27,21,1,,,,,
+longchat-7b-v1.5-32k,qa3,22,16,15,17,21,22,4,,,,,
+LongAlpaca-13B,qa3,25,26,26,25,24,27,2,,,,,
+Llama-2-7B-32K-Instruct,qa3,35,36,34,26,23,20,2,,,,,
+01-ai/Yi-34B,qa3,45,34,24,20,17,12,4,,,,,
+Mistral-7b-Instruct-v0.2,qa3,36,34,31,30,24,18,12,,,,,
+Mixtral-8x7B-Instruct-v0.1,qa3,34,32,31,30,27,29,24,,,,,
+01-ai/Yi-34B-200k,qa3,35,30,27,24,24,22,22,26,,,,
+Mixtral-8x22B-Instruct-v0.1,qa3,53,56,49,39,31,27,26,26,,,,
+activation-beacon-llama2-7b-chat,qa3,33,25,25,21,20,17,13,5,5,,,
+Yarn-Mistral-7b-128k,qa3,31,36,33,32,27,25,9,13,7,,,
+chatglm3-6b-128k,qa3,33,37,31,31,27,25,23,17,9,,,
+activation-beacon-mistral-7b,qa3,36,33,25,21,18,15,15,15,16,,,
+01-ai/Yi-9B-200k,qa3,34,33,29,24,25,21,20,20,8,,,
+Phi-3-mini-128k-instruct,qa3,32,41,31,27,26,24,21,22,4,,,
+ai21labs/Jamba-v0.1,qa3,32,31,29,26,24,22,22,21,26,,,
+c4ai-command-r-v01,qa3,25,28,26,28,26,30,28,33,24,,,
+Phi-3-medium-128k-instruct,qa3,53,51,45,35,30,30,27,25,17,,,
+GPT-4,qa3,56,63,57,56,53,45,31,31,32,,,
+~ Mamba (130M) fine-tune,qa3,97,97,97,97,97,96,95,92,81,,,
+Llama3-ChatQA-1.5-8B + RAG,qa3,17,18,17,17,16,17,15,13,19,17,10,11
+~ RMT (137M) fine-tune,qa3,97,94,88,81,73,66,55,55,36,25,22,21
+~ ARMT (137M) fine-tune,qa3,97,,,92,92,92,91,90,86,80,72,37
+GPT-2 (137M),qa4,29,18,,,,,,,,,,
+mamba-2.8b-hf,qa4,96,59,47,12,0,,,,,,,
+rwkv-6-world-7b,qa4,54,65,57,35,7,,,,,,,
+v5-Eagle-7B-HF,qa4,79,74,63,55,3,,,,,,,
+Meta-Llama-3-8B-Instruct,qa4,58,55,50,43,52,,,,,,,
+LLaMA-2-7B-32K,qa4,26,54,51,51,46,36,3,,,,,
+longchat-7b-v1.5-32k,qa4,60,55,52,57,57,49,4,,,,,
+LongAlpaca-13B,qa4,65,61,58,52,50,44,4,,,,,
+Llama-2-7B-32K-Instruct,qa4,39,52,54,56,55,52,6,,,,,
+01-ai/Yi-34B,qa4,59,56,51,55,52,43,4,,,,,
+Mistral-7b-Instruct-v0.2,qa4,54,58,58,60,60,58,54,,,,,
+Mixtral-8x7B-Instruct-v0.1,qa4,55,60,59,61,63,61,58,,,,,
+01-ai/Yi-34B-200k,qa4,64,65,64,63,61,56,54,44,,,,
+Mixtral-8x22B-Instruct-v0.1,qa4,56,62,59,62,62,60,54,39,,,,
+activation-beacon-llama2-7b-chat,qa4,40,50,52,43,34,22,14,9,10,,,
+Yarn-Mistral-7b-128k,qa4,60,56,43,45,32,31,16,7,8,,,
+chatglm3-6b-128k,qa4,45,48,42,38,35,32,27,13,11,,,
+activation-beacon-mistral-7b,qa4,53,58,60,57,53,50,45,29,15,,,
+01-ai/Yi-9B-200k,qa4,49,47,50,50,54,43,45,36,33,,,
+Phi-3-mini-128k-instruct,qa4,54,56,56,50,49,50,45,47,5,,,
+ai21labs/Jamba-v0.1,qa4,64,50,49,49,48,52,46,49,38,,,
+c4ai-command-r-v01,qa4,46,58,59,54,56,46,46,47,52,,,
+Phi-3-medium-128k-instruct,qa4,54,61,63,64,64,61,59,52,33,,,
+GPT-4,qa4,98,70,63,60,52,47,46,40,32,,,
+~ Mamba (130M) fine-tune,qa4,100,100,100,100,100,100,99,100,98,,,
+Llama3-ChatQA-1.5-8B + RAG,qa4,53,58,56,59,57,60,60,59,60,59,54,56
+~ RMT (137M) fine-tune,qa4,100,94,87,83,80,75,64,51,38,26,24,20
+~ ARMT (137M) fine-tune,qa4,100,,,100,100,100,100,100,100,100,100,92
+GPT-2 (137M),qa5,45,19,,,,,,,,,,
+mamba-2.8b-hf,qa5,75,58,43,9,0,,,,,,,
+rwkv-6-world-7b,qa5,79,77,80,61,14,,,,,,,
+v5-Eagle-7B-HF,qa5,75,76,71,57,3,,,,,,,
+Meta-Llama-3-8B-Instruct,qa5,85,78,73,65,73,,,,,,,
+LLaMA-2-7B-32K,qa5,55,74,70,67,59,51,7,,,,,
+longchat-7b-v1.5-32k,qa5,63,62,62,65,66,67,9,,,,,
+LongAlpaca-13B,qa5,63,61,61,61,62,66,12,,,,,
+Llama-2-7B-32K-Instruct,qa5,63,69,69,67,66,63,9,,,,,
+01-ai/Yi-34B,qa5,88,70,59,48,53,55,4,,,,,
+Mistral-7b-Instruct-v0.2,qa5,70,66,66,67,69,67,67,,,,,
+Mixtral-8x7B-Instruct-v0.1,qa5,80,79,80,73,66,67,56,,,,,
+01-ai/Yi-34B-200k,qa5,78,77,77,76,76,75,76,80,,,,
+Mixtral-8x22B-Instruct-v0.1,qa5,87,84,84,79,69,64,63,63,,,,
+activation-beacon-llama2-7b-chat,qa5,65,67,64,63,57,45,29,17,7,,,
+Yarn-Mistral-7b-128k,qa5,58,47,45,47,47,38,23,17,19,,,
+chatglm3-6b-128k,qa5,70,69,64,60,61,61,58,55,26,,,
+activation-beacon-mistral-7b,qa5,68,66,66,74,74,66,67,55,28,,,
+01-ai/Yi-9B-200k,qa5,76,59,50,57,57,45,52,47,40,,,
+Phi-3-mini-128k-instruct,qa5,79,66,76,72,72,73,71,64,23,,,
+ai21labs/Jamba-v0.1,qa5,83,70,64,62,58,64,63,60,50,,,
+c4ai-command-r-v01,qa5,86,82,81,78,75,79,72,70,66,,,
+Phi-3-medium-128k-instruct,qa5,77,85,84,81,82,82,81,78,69,,,
+GPT-4,qa5,96,95,92,90,93,85,82,60,51,,,
+~ Mamba (130M) fine-tune,qa5,98,99,98,99,99,99,98,99,98,,,
+Llama3-ChatQA-1.5-8B + RAG,qa5,80,77,78,77,78,77,78,76,75,75,76,67
+~ RMT (137M) fine-tune,qa5,100,100,99,99,99,94,90,89,86,69,63,44
+~ ARMT (137M) fine-tune,qa5,"99,6",,,"98,1","98,2","98,1",98,"97,9","96,9","95,3","93,4","76,6"
+llama3.1:8b-instruct-fp16,qa1,86,,,70,67,,,,,,,,
+llama3.1:70b-instruct-fp16,qa1,99,,,79,69,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa1,100,,,90,89,,,,,,,,
+llama3.1:8b-instruct-fp16,qa2,47,,,35,37,,,,,,,,
+llama3.1:70b-instruct-fp16,qa2,71,,,43,39,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa2,77,,,53,53,,,,,,,,
+llama3.1:8b-instruct-fp16,qa3,27,,,28,28,,,,,,,,
+llama3.1:70b-instruct-fp16,qa3,39,,,24,24,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa3,34,,,40,30,,,,,,,,
+llama3.1:8b-instruct-fp16,qa4,69,,,56,54,,,,,,,,
+llama3.1:70b-instruct-fp16,qa4,91,,,64,66,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa4,95,,,64,57,,,,,,,,
+llama3.1:8b-instruct-fp16,qa5,81,,,73,71,,,,,,,,
+llama3.1:70b-instruct-fp16,qa5,86,,,81,78,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa5,88,,,86,89,,,,,,,,
+llama3.1:8b-instruct-fp16,qa6,95,,,78,70,,,,,,,,
+llama3.1:70b-instruct-fp16,qa6,97,,,90,92,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa6,97,,,93,91,,,,,,,,
+llama3.1:8b-instruct-fp16,qa7,46,,,12,12,,,,,,,,
+llama3.1:70b-instruct-fp16,qa7,92,,,47,38,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa7,95,,,59,56,,,,,,,,
+llama3.1:8b-instruct-fp16,qa8,39,,,25,15,,,,,,,,
+llama3.1:70b-instruct-fp16,qa8,91,,,47,36,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa8,94,,,66,64,,,,,,,,
+llama3.1:8b-instruct-fp16,qa9,95,,,86,79,,,,,,,,
+llama3.1:70b-instruct-fp16,qa9,97,,,96,93,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa9,99,,,96,94,,,,,,,,
+llama3.1:8b-instruct-fp16,qa10,77,,,65,63,,,,,,,,
+llama3.1:70b-instruct-fp16,qa10,86,,,84,76,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa10,89,,,89,91,,,,,,,,
+llama3.1:8b-instruct-fp16,qa11,88,,,,,,,,,,
+llama3.1:70b-instruct-fp16,qa11,100,,,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa11,100,,,,,,,,,,
+llama3.1:8b-instruct-fp16,qa12,76,,,,,,,,,,
+llama3.1:70b-instruct-fp16,qa12,100,,,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa12,95,,,,,,,,,,
+llama3.1:8b-instruct-fp16,qa13,67,,,,,,,,,,
+llama3.1:70b-instruct-fp16,qa13,99,,,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa13,100,,,,,,,,,,
+llama3.1:8b-instruct-fp16,qa14,86,,,,,,,,,,
+llama3.1:70b-instruct-fp16,qa14,86,,,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa14,90,,,,,,,,,,
+llama3.1:8b-instruct-fp16,qa15,78,,,,,,,,,,
+llama3.1:70b-instruct-fp16,qa15,95,,,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa15,83,,,,,,,,,,
+llama3.1:8b-instruct-fp16,qa16,63,,,,,,,,,,
+llama3.1:70b-instruct-fp16,qa16,78,,,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa16,92,,,,,,,,,,
+llama3.1:8b-instruct-fp16,qa17,51,,,,,,,,,,
+llama3.1:70b-instruct-fp16,qa17,79,,,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa17,77,,,,,,,,,,
+llama3.1:8b-instruct-fp16,qa18,76,,,,,,,,,,
+llama3.1:70b-instruct-fp16,qa18,97,,,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa18,98,,,,,,,,,,
+llama3.1:8b-instruct-fp16,qa19,11,,,,,,,,,,
+llama3.1:70b-instruct-fp16,qa19,51,,,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa19,34,,,,,,,,,,
+llama3.1:8b-instruct-fp16,qa20,45,,,,,,,,,,
+llama3.1:70b-instruct-fp16,qa20,73,,,,,,,,,,
+qwen2.5:72b-instruct-fp16,qa20,56,,,,,,,,,,
+llama3.1:8b-instruct-fp16,avg,65,,,59,55,,,,,,,,
+llama3.1:70b-instruct-fp16,avg,85,,,74,68,,,,,,,,
+qwen2.5:72b-instruct-fp16,avg,84,,,82,79,,,,,,,,