Repository: open-compass/opencompass
Issue: [Bug] LMEvaluator超时导致整个eval失败
Root cause: Validation Gap

### Prerequisite

- [x] I have searched [Issues](https://github.com/open-compass/opencompass/issues/) and [Discussions](https://github.com/open-compass/opencompass/discussions) but cannot get the expected help.
- [x] The bug has not been fixed in the [latest version](https://github.com/open-compass/opencompass).

### Type

I have modified the code (config is not considered code), or I'm working on my own tasks/models/datasets.

### Environment

{'CUDA available': True,
 'GCC': 'gcc (Ubuntu 9.4.0-1ubuntu1~20.04.2) 9.4.0',
 'MMEngine': '0.10.6',
 'MUSA available': False,
 'OpenCV': '4.11.0',
 'PyTorch': '2.5.1',
 'PyTorch compiling details': 'PyTorch built with:\n'
                              '  - GCC 10.2\n'
                              '  - C++ Version: 201703\n'
                              '  - Intel(R) MKL-DNN v3.5.3 (Git Hash '
                              '66f0cb9eb66affd2da3bf5f8d897376f04aae6af)\n'
                              '  - OpenMP 201511 (a.k.a. OpenMP 4.5)\n'
                              '  - LAPACK is enabled (usually provided by '
                              'MKL)\n'
                              '  - NNPACK is enabled\n'
                              '  - CPU capability usage: NO AVX\n'
                              '  - Build settings: BLAS_INFO=open, '
                              'BUILD_TYPE=Release, '
                              'CXX_COMPILER=/opt/rh/devtoolset-10/root/usr/bin/c++, '
                              'CXX_FLAGS=-ffunction-sections -fdata-sections '
                              '-D_GLIBCXX_USE_CXX11_ABI=0 -fabi-version=11 '
                              '-fvisibility-inlines-hidden -DUSE_PTHREADPOOL '
                              '-DNDEBUG -DUSE_KINETO -DLIBKINETO_NOCUPTI '
                              '-DLIBKINETO_NOROCTRACER -DLIBKINETO_NOXPUPTI=ON '
                              '-DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK '
                              '-DSYMBOLICATE_MOBILE_DEBUG_HANDLE -O2 -fPIC '
                              '-Wall -Wextra -Werror=return-type '
                              '-Werror=non-virtual-dtor -Werror=bool-operation '
                              '-Wnarrowing -Wno-missing-field-initializers '
                              '-Wno-type-limits -Wno-array-bounds '
                              '-Wno-unknown-pragmas -Wno-unused-parameter '
                              '-Wno-strict-overflow -Wno-strict-aliasing '
                              '-Wno-stringop-overflow -Wsuggest-override '
                              '-Wno-psabi -Wno-error=old-style-cast '
                              '-Wno-missing-braces -fdiagnostics-color=always '
                              '-faligned-new -Wno-unused-but-set-variable '
                              '-Wno-maybe-uninitialized -fno-math-errno '
                              '-fno-trapping-math -Werror=format '
                              '-Wno-stringop-overflow, LAPACK_INFO=open, '
                              'TORCH_VERSION=2.5.1, USE_CUDA=OFF, '
                              'USE_CUDNN=OFF, USE_CUSPARSELT=OFF, '
                              'USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, '
                              'USE_GLOG=OFF, USE_GLOO=ON, USE_MKLDNN=ON, '
                              'USE_MPI=OFF, USE_NCCL=OFF, USE_NNPACK=ON, '
                              'USE_OPENMP=ON, USE_ROCM=OFF, '
                              'USE_ROCM_KERNEL_ASSERT=OFF, \n',
 'Python': '3.10.2 (main, Jan  8 2025, 06:13:36) [GCC 9.4.0]',
 'TorchVision': '0.20.1',
 'lmdeploy': "not installed:No module named 'lmdeploy'",
 'numpy_random_seed': 2147483648,
 'opencompass': '0.3.9+',
 'sys.platform': 'linux',
 'transformers': '4.51.3'}

### Reproduces the problem - code/configuration sample

```
from mmengine.config import read_base

with read_base():
     from opencompass.configs.datasets.subjective.alignbench.alignbench_judgeby_critiquellm_new import alignbench_datasets
from opencompass.models import VLLMwithChatTemplate, OpenAISDK
from opencompass.partitioners import NumWorkerPartitioner
from opencompass.partitioners.sub_naive import SubjectiveNaivePartitioner
from opencompass.runners import LocalRunner
from opencompass.tasks import OpenICLInferTask
from opencompass.tasks.subjective_eval import SubjectiveEvalTask
from opencompass.summarizers import DefaultSubjectiveSummarizer


datasets = sum(
    (v for k, v in locals().items() if k.endswith('_datasets')),
    [],
)

api_meta_template = dict(
    round=[
        dict(role='HUMAN', api_role='HUMAN'),
        dict(role='BOT', api_role='BOT', generate=True),
    ],
    reserved_roles=[dict(role='SYSTEM', api_role='SYSTEM')],
)

# -------------Inference Stage ----------------------------------------
# For subjective evaluation, we often set do sample for models
models = [
    dict(
        type=VLLMwithChatTemplate,
        abbr='qwen2.5-7b-instruct-vllm',
        path='/Qwen-Series/custom_Qwen2.5-7B-Instruct',
        model_kwargs=dict(tensor_parallel_size=1),
        max_out_len=1024,
        batch_size=8,
        # generation_kwargs=dict(do_sample=True, top_p=0.8, top_k=20, temperature=0.7),
        generation_kwargs=dict(temperature=0),
        run_cfg=dict(num_gpus=1),
    )
]


infer = dict(
    partitioner=dict(type=NumWorkerPartitioner, num_worker=8, num_split=None),
    runner=dict(type=LocalRunner, max_num_workers=16, task=dict(type=OpenICLInferTask)),
)
# -------------Evalation Stage ----------------------------------------

## ------------- JudgeLLM Configuration
judge_models = [dict(
    abbr='vllm-qwen2-5-72b-instruct',
    type=OpenAISDK,
    path='Qwen2_5_72B',
    key='EMPTY',
    retry=10,
    tokenizer_path='/Qwen-Series/Qwen2.5-72B-Instruct',
    openai_api_base=['http://0.0.0.0:8080/v1'],
    meta_template=api_meta_template,
    query_per_second=50,
    batch_size=32,
    temperature=0,
    verbose=False,
    max_out_len=16384,
    max_seq_len=32768,
)]

## ------------- Evaluation Configuration
eval = dict(
    partitioner=dict(type=SubjectiveNaivePartitioner, models=models, judge_models=judge_models,),
    runner=dict(type=LocalRunner, max_num_workers=16, task=dict(type=SubjectiveEvalTask)),
)

summarizer = dict(type=DefaultSubjectiveSummarizer)
work_dir = 'outputs/score_subjective/'

```

### Reproduces the problem - command or script

```python run.py configs/eval_subjective.py```
由于我使用了自己训练过的qwen25-7b，导致prediction某些回复出现大量重复数字例如(100000000000000000000000000000000000000000000000000000000000000000000)，我发现类似这种会让72b接口回复失败并且直接超时，导致整个eval程序失败
```
Traceback (most recent call last):
  File "/opencompass-main/opencompass/tasks/subjective_eval.py", line 450, in <module>
    inferencer.run()
  File "/opencompass-main/opencompass/tasks/subjective_eval.py", line 93, in run
    self._score(model_cfg, dataset_cfg, eval_cfg, output_column,
  File "/opencompass-main/opencompass/tasks/subjective_eval.py", line 382, in _score
    result = icl_evaluator.score(predictions=model_preds,
  File "/opencompass-main/opencompass/openicl/icl_evaluator/lm_evaluator.py", line 319, in score
    self.inferencer.inference(retriever=retriever,
  File "/opencompass-main/opencompass/openicl/icl_inferencer/icl_gen_inferencer.py", line 153, in inference
    results = self.model.generate_from_template(
  File "/opencompass-main/opencompass/models/base.py", line 201, in generate_from_template
    return self.generate(inputs, max_out_len=max_out_len, **kwargs)
  File "/opencompass-main/opencompass/models/openai_api.py", line 176, in generate
    results = list(
  File "/usr/local/python3.10.2/lib/python3.10/site-packages/tqdm/std.py", line 1181, in __iter__
    for obj in iterable:
  File "/usr/local/python3.10.2/lib/python3.10/concurrent/futures/_base.py", line 609, in result_iterator
    yield fs.pop().result()
  File "/usr/local/python3.10.2/lib/python3.10/concurrent/futures/_base.py", line 446, in result
    return self.__get_result()
  File "/usr/local/python3.10.2/lib/python3.10/concurrent/futures/_base.py", line 391, in __get_result
    raise self._exception
  File "/usr/local/python3.10.2/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opencompass-main/opencompass/models/openai_api.py", line 655, in _generate
    raise RuntimeError('Calling OpenAI API failed after retrying for '
RuntimeError: Calling OpenAI API failed after retrying for 10 times. Check the logs for details.
```

### Reproduces the problem - error message

我的想法是在推理处添加后处理对这一batch的prediction做判断如果出现上述predict则跳过或者在最终结果处(results目录下的json)写入固定的失败结果。但是我不知道具体应该在哪个位置的代码进行修改
或者
请问还有什么好的办法能够解决这个问题吗？

### Other information

_No response_