Repository: open-compass/VLMEvalKit
Issue: 报错
Root cause: Validation Gap

from vlmeval.config import supported_VLM
model = supported_VLM['MiniCPM-V-2_6']()
# 前向单张图片
ret = model.generate(['assets/apple.jpg', 'What is in this image?'])
print(ret)  # 这张图片上有一个带叶子的红苹果
# 前向多张图片
ret = model.generate(['assets/apple.jpg', 'assets/apple.jpg', 'How many apples are there in the provided images? '])
print(ret)  # 提供的图片中有两个苹果



---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In[1], line 1
----> 1 from vlmeval.config import supported_VLM
      2 model = supported_VLM['MiniCPM-V-2_6']()
      3 # 前向单张图片

File [~/autodl-tmp/MiniCPM-V/eval_mm/vlmevalkit/vlmeval/__init__.py:12](https://a445102-afae-6f6d8a60.neimeng.seetacloud.com:6443/jupyter/lab/tree/autodl-tmp/~/autodl-tmp/MiniCPM-V/eval_mm/vlmevalkit/vlmeval/__init__.py#line=11)
     10 from .vlm import *
     11 from .config import *
---> 12 from .tools import cli
     14 load_env()
     16 __version__ = '0.2rc1'

File [~/autodl-tmp/MiniCPM-V/eval_mm/vlmevalkit/vlmeval/tools.py:67](https://a445102-afae-6f6d8a60.neimeng.seetacloud.com:6443/jupyter/lab/tree/autodl-tmp/~/autodl-tmp/MiniCPM-V/eval_mm/vlmevalkit/vlmeval/tools.py#line=66)
     63 dataset_levels['l23'] = dataset_levels['l2'] + dataset_levels['l3']
     64 dataset_levels['l123'] = dataset_levels['l12'] + dataset_levels['l3']
     66 models = {
---> 67     '4.33.0': list(qwen_series) + list(xcomposer_series) + [
     68         'mPLUG-Owl2', 'flamingov2', 'VisualGLM_6b', 'MMAlaya', 'PandaGPT_13B', 'VXVERSE'
     69     ] + list(idefics_series) + list(minigpt4_series) + list(instructblip_series),
     70     '4.37.0': [x for x in llava_series if 'next' not in x] + list(internvl_series) + [
     71         'TransCore_M', 'emu2_chat', 'MiniCPM-V', 'MiniCPM-V-2', 'OmniLMM_12B',
     72         'cogvlm-grounding-generalist', 'cogvlm-chat', 'cogvlm2-llama3-chat-19B',
     73     ] + list(xtuner_series) + list(yivl_series) + list(deepseekvl_series) + list(cambrian_series),
     74     '4.40.0': [
     75         'idefics2_8b', 'Bunny-llama3-8B', 'MiniCPM-Llama3-V-2_5', '360VL-70B', 'Phi-3-Vision',
     76     ] + list(wemm_series),
     77     'latest': ['paligemma-3b-mix-448', 'MiniCPM-V-2_6', 'glm-4v-9b'] + [x for x in llava_series if 'next' in x]
     78     + list(chameleon_series) + list(ovis_series) + list(mantis_series),
     79     'api': list(api_models)
     80 }
     82 # SKIP_MODELS will be skipped in report_missing and run APIs
     83 SKIP_MODELS = [
     84     'MGM_7B', 'GPT4V_HIGH', 'GPT4V', 'flamingov2', 'PandaGPT_13B',
     85     'GeminiProVision', 'Step1V-0701', 'SenseChat-5-Vision',
   (...)
     92     'mPLUG-Owl2', 'MMAlaya', 'OmniLMM_12B', 'emu2_chat', 'VXVERSE'
     93 ] + list(minigpt4_series) + list(instructblip_series) + list(xtuner_series) + list(chameleon_series) + list(vila_series)

NameError: name 'qwen_series' is not defined

请问这是什么原因？


Hi, 
the **qwen series** dict is defined in [config](https://github.com/open-compass/VLMEvalKit/blob/5316c5f1b9ee3cec99a9ebcbf7f9684e9487b83b/vlmeval/config.py#L131C15-L131C16). 
Please refer to this line and check it .