Repository: open-compass/opencompass
Issue: [Bug] Academic Leaderboard 结果无法对齐，Summary只展示部分
Root cause: Configuration Error

### 先决条件

- [X] 我已经搜索过 [问题](https://github.com/open-compass/opencompass/issues/) 和 [讨论](https://github.com/open-compass/opencompass/discussions) 但未得到预期的帮助。
- [X] 错误在 [最新版本](https://github.com/open-compass/opencompass) 中尚未被修复。

### 问题类型

我正在使用官方支持的任务/模型/数据集进行评估。

### 环境

{'CUDA available': True,
 'CUDA_HOME': '/usr/local/cuda-11.4',
 'GCC': 'gcc (GCC) 8.3.1 20190311 (Red Hat 8.3.1-3)',
 'GPU 0,1,2,3,4,5,6,7': 'Tesla V100-PCIE-32GB',
 'MMEngine': '0.10.1',
 'NVCC': 'Cuda compilation tools, release 11.4, V11.4.152',
 'OpenCV': '4.9.0',
 'PyTorch': '2.5.1+cu124',
 'PyTorch compiling details': 'PyTorch built with:\n'
                              '  - GCC 9.3\n'
                              '  - C++ Version: 201703\n'
                              '  - Intel(R) oneAPI Math Kernel Library Version '
                              '2024.2-Product Build 20240605 for Intel(R) 64 '
                              'architecture applications\n'
                              '  - Intel(R) MKL-DNN v3.5.3 (Git Hash '
                              '66f0cb9eb66affd2da3bf5f8d897376f04aae6af)\n'
                              '  - OpenMP 201511 (a.k.a. OpenMP 4.5)\n'
                              '  - LAPACK is enabled (usually provided by '
                              'MKL)\n'
                              '  - NNPACK is enabled\n'
                              '  - CPU capability usage: AVX512\n'
                              '  - CUDA Runtime 12.4\n'
                              '  - NVCC architecture flags: '
                              '-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86;-gencode;arch=compute_90,code=sm_90\n'
                              '  - CuDNN 90.1\n'
                              '  - Magma 2.6.1\n'
                              '  - Build settings: BLAS_INFO=mkl, '
                              'BUILD_TYPE=Release, CUDA_VERSION=12.4, '
                              'CUDNN_VERSION=9.1.0, '
                              'CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, '
                              'CXX_FLAGS= -D_GLIBCXX_USE_CXX11_ABI=0 '
                              '-fabi-version=11 -fvisibility-inlines-hidden '
                              '-DUSE_PTHREADPOOL -DNDEBUG -DUSE_KINETO '
                              '-DLIBKINETO_NOROCTRACER -DLIBKINETO_NOXPUPTI=ON '
                              '-DUSE_FBGEMM -DUSE_PYTORCH_QNNPACK '
                              '-DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE '
                              '-O2 -fPIC -Wall -Wextra -Werror=return-type '
                              '-Werror=non-virtual-dtor -Werror=bool-operation '
                              '-Wnarrowing -Wno-missing-field-initializers '
                              '-Wno-type-limits -Wno-array-bounds '
                              '-Wno-unknown-pragmas -Wno-unused-parameter '
                              '-Wno-strict-overflow -Wno-strict-aliasing '
                              '-Wno-stringop-overflow -Wsuggest-override '
                              '-Wno-psabi -Wno-error=old-style-cast '
                              '-Wno-missing-braces -fdiagnostics-color=always '
                              '-faligned-new -Wno-unused-but-set-variable '
                              '-Wno-maybe-uninitialized -fno-math-errno '
                              '-fno-trapping-math -Werror=format '
                              '-Wno-stringop-overflow, LAPACK_INFO=mkl, '
                              'PERF_WITH_AVX=1, PERF_WITH_AVX2=1, '
                              'TORCH_VERSION=2.5.1, USE_CUDA=ON, USE_CUDNN=ON, '
                              'USE_CUSPARSELT=1, USE_EXCEPTION_PTR=1, '
                              'USE_GFLAGS=OFF, USE_GLOG=OFF, USE_GLOO=ON, '
                              'USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, '
                              'USE_NCCL=1, USE_NNPACK=ON, USE_OPENMP=ON, '
                              'USE_ROCM=OFF, USE_ROCM_KERNEL_ASSERT=OFF, \n',
 'Python': '3.10.13 (main, Sep 11 2023, 13:44:35) [GCC 11.2.0]',
 'TorchVision': '0.20.1+cu124',
 'lmdeploy': "not installed:No module named 'lmdeploy'",
 'numpy_random_seed': 2147483648,
 'opencompass': '0.3.8+7cdd90d',
 'sys.platform': 'linux',
 'transformers': '4.46.2'}

### 重现问题 - 代码/配置示例

from mmengine.config import read_base
import os.path as osp
from opencompass.partitioners import NaivePartitioner, NumWorkerPartitioner, SizePartitioner
from opencompass.runners import LocalRunner, LocalAPIRunner
from opencompass.tasks import OpenICLInferTask, OpenICLEvalTask


#######################################################################
#                          PART 0  Essential Configs                  #
#######################################################################
with read_base():
    # Datasets Part
    ## Core Set
    # ## Examination
    from opencompass.configs.datasets.mmlu.mmlu_openai_simple_evals_gen_b618ea import mmlu_datasets
    from opencompass.configs.datasets.mmlu_pro.mmlu_pro_0shot_cot_gen_08c1de import mmlu_pro_datasets
    from opencompass.configs.datasets.cmmlu.cmmlu_0shot_cot_gen_305931 import cmmlu_datasets
    # ## Reasoning
    from opencompass.configs.datasets.bbh.bbh_gen_4a31fa import bbh_datasets
    from opencompass.configs.datasets.gpqa.gpqa_openai_simple_evals_gen_5aeece import gpqa_datasets
    # ## Math
    from opencompass.configs.datasets.math.math_0shot_gen_393424 import math_datasets
    # ## Coding
    from opencompass.configs.datasets.humaneval.humaneval_gen_8e312c import humaneval_datasets
    # ## Instruction Following
    from opencompass.configs.datasets.IFEval.IFEval_gen_3321a3 import ifeval_datasets

    # Summarizer
    from opencompass.configs.summarizers.groups.mmlu import mmlu_summary_groups
    from opencompass.configs.summarizers.groups.mmlu_pro import mmlu_pro_summary_groups
    from opencompass.configs.summarizers.groups.cmmlu import cmmlu_summary_groups
    from opencompass.configs.summarizers.groups.bbh import bbh_summary_groups


    # Model List
    # from opencompass.configs.models.qwen.lmdeploy_qwen2_1_5b_instruct import models as lmdeploy_qwen2_1_5b_instruct_model
    # from opencompass.configs.models.hf_internlm.lmdeploy_internlm2_5_7b_chat import models as hf_internlm2_5_7b_chat_model
    # from opencompass.configs.models.openbmb.hf_minicpm_2b_sft_bf16 import models as hf_minicpm_2b_sft_bf16_model
    # from opencompass.configs.models.yi.hf_yi_1_5_6b_chat import models as hf_yi_1_5_6b_chat_model
    # from opencompass.configs.models.gemma.hf_gemma_2b_it import models as hf_gemma_2b_it_model
    # from opencompass.configs.models.yi.hf_yi_1_5_34b_chat import models as hf_yi_1_5_34b_chat_model
    from .models.qwen.qwen2_7b_chat_api import models as qwen2_7b_chat_model
#######################################################################
#                          PART 1  Datasets List                      #
#######################################################################
# datasets list for evaluation
datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])


#######################################################################
#                       PART 2  Datset Summarizer                     #
#######################################################################
# with read_base():

core_summary_groups = [
    {
        'name': 'core_average',
        'subsets': [
            ['mmlu', 'accuracy'],
            ['mmlu_pro', 'accuracy'],
            # ['cmmlu', 'naive_average'],
            ['cmmlu', 'accuracy'],
            ['bbh', 'score'],
            ['math', 'accuracy'],
            ['openai_humaneval', 'humaneval_pass@1'],
            ['GPQA_diamond', 'accuracy'],
            ['IFEval', 'Prompt-level-strict-accuracy'],
        ],
    },
]

summarizer = dict(
    dataset_abbrs=[
        ['core_average', 'naive_average'],
        ['mmlu', 'accuracy'],
        ['mmlu_pro', 'accuracy'],
        ['cmmlu', 'accuracy'],
        ['bbh', 'score'],
        ['math', 'accuracy'],
        ['openai_humaneval', 'humaneval_pass@1'],
        ['GPQA_diamond', 'accuracy'],
        ['IFEval', 'Prompt-level-strict-accuracy'],
        '',

        ['mmlu', 'accuracy'],
        ['mmlu-stem', 'accuracy'],
        ['mmlu-social-science', 'accuracy'],
        ['mmlu-humanities', 'accuracy'],
        ['mmlu-other', 'accuracy'],

        '',
        ['mmlu_pro', 'accuracy'],
        ['mmlu_pro_math','accuracy'],
        ['mmlu_pro_physics', 'accuracy'],
        ['mmlu_pro_chemistry', 'accuracy'],
        ['mmlu_pro_law', 'accuracy'],
        ['mmlu_pro_engineering', 'accuracy'],
        ['mmlu_pro_other', 'accuracy'],
        ['mmlu_pro_economics', 'accuracy'],
        ['mmlu_pro_health', 'accuracy'],
        ['mmlu_pro_psychology', 'accuracy'],
        ['mmlu_pro_business', 'accuracy'],
        ['mmlu_pro_biology', 'accuracy'],
        ['mmlu_pro_philosophy', 'accuracy'],
        ['mmlu_pro_computer_science','accuracy'],
        ['mmlu_pro_history', 'accuracy'],
        '',
        ['cmmlu', 'accuracy'],
        ['cmmlu-stem', 'accuracy'],
        ['cmmlu-social-science', 'accuracy'],
        ['cmmlu-humanities', 'accuracy'],
        ['cmmlu-other', 'accuracy'],
        ['cmmlu-china-specific', 'accuracy'],
        '',
        ['bbh', 'extract_rate'],
        ['math', 'extract_rate'],
        # ['openai_humaneval', 'extract_rate'],
        ['GPQA_diamond', 'extract_rate'],
        # ['IFEval', 'extract_rate'],
        '',
        ['mmlu', 'extract_rate'],
        ['mmlu-stem', 'extract_rate'],
        ['mmlu-social-science', 'extract_rate'],
        ['mmlu-humanities', 'extract_rate'],
        ['mmlu-other', 'extract_rate'],
        '',
        ['mmlu_pro', 'extract_rate'],
        ['mmlu_pro_math', 'extract_rate'],
        ['mmlu_pro_physics', 'extract_rate'],
        ['mmlu_pro_chemistry', 'extract_rate'],
        ['mmlu_pro_law', 'extract_rate'],
        ['mmlu_pro_engineering', 'extract_rate'],
        ['mmlu_pro_other', 'extract_rate'],
        ['mmlu_pro_economics', 'extract_rate'],
        ['mmlu_pro_health', 'extract_rate'],
        ['mmlu_pro_psychology', 'extract_rate'],
        ['mmlu_pro_business', 'extract_rate'],
        ['mmlu_pro_biology', 'extract_rate'],
        ['mmlu_pro_philosophy', 'extract_rate'],
        ['mmlu_pro_computer_science', 'extract_rate'],
        ['mmlu_pro_history', 'extract_rate'],
        '',
        ['cmmlu', 'extract_rate'],
        ['cmmlu-stem', 'extract_rate'],
        ['cmmlu-social-science', 'extract_rate'],
        ['cmmlu-humanities', 'extract_rate'],
        ['cmmlu-other', 'extract_rate'],
        ['cmmlu-china-specific', 'extract_rate'],

    ],
    summary_groups=sum(
        [v for k, v in locals().items() if k.endswith('_summary_groups')], []),
)


#######################################################################
#                        PART 3  Models  List                         #
#######################################################################

models = sum([v for k, v in locals().items() if k.endswith('_model')], [])



#######################################################################
#                 PART 4  Inference/Evaluation Configuaration         #
#######################################################################

# # Local Runner
# infer = dict(
#     partitioner=dict(
#         type=NumWorkerPartitioner,
#         num_worker=8
#     ),
#     runner=dict(
#         type=LocalRunner,
#         max_num_workers=16,
#         retry=0, # Modify if needed
#         task=dict(type=OpenICLInferTask)
#     ),
# )

# Local API Runner
infer = dict(
    partitioner=dict(
        type=NumWorkerPartitioner,
        num_worker=16
    ),
    runner=dict(
        type=LocalAPIRunner,
        max_num_workers=16,
        concurrent_users=1,
        # retry=0, # Modify if needed
        task=dict(type=OpenICLInferTask)
    ),
)

# eval with local runner
eval = dict(
    partitioner=dict(type=NaivePartitioner, n=16),
    runner=dict(
        type=LocalRunner,
        max_num_workers=16,
        task=dict(type=OpenICLEvalTask)),
)


#######################################################################
#                      PART 5  Utils Configuaration                   #
#######################################################################
base_exp_dir = 'outputs/corebench_v1_9/'
work_dir = osp.join(base_exp_dir, 'chat_objective')


### 重现问题 - 命令或脚本

python run.py confgis/eval_academic_leaderboard_2020407_api.py

### 重现问题 - 错误信息

![image](https://github.com/user-attachments/assets/0cb2df08-3b35-4237-a817-a885d73eb409)
![image](https://github.com/user-attachments/assets/0b8f3069-e41e-499b-bf53-ca68c932eabf)


### 其他信息

_No response_

You can use the latest academic leaderboard config in `configs/eval_academic_leaderboard_202412.py`