Repository: open-compass/opencompass
Issue: [Bug] 评测lawbench数据集时偶现异常
Root cause: Resource Mishandling

### 先决条件

- [X] 我已经搜索过 [问题](https://github.com/open-compass/opencompass/issues/) 和 [讨论](https://github.com/open-compass/opencompass/discussions) 但未得到预期的帮助。
- [X] 错误在 [最新版本](https://github.com/open-compass/opencompass) 中尚未被修复。

### 问题类型

我正在使用官方支持的任务/模型/数据集进行评估。

### 环境

{'CUDA available': False,compass.utils;import pprint;pprint.pprint(dict(opencompass.utils.collect_env()))"
 'GCC': 'Apple clang version 15.0.0 (clang-1500.3.9.4)',
 'MMEngine': '0.10.3',
 'MUSA available': False,
 'OpenCV': '4.9.0',
 'PyTorch': '2.2.1',
 'PyTorch compiling details': 'PyTorch built with:\n'
                              '  - GCC 4.2\n'
                              '  - C++ Version: 201703\n'
                              '  - clang 13.1.6\n'
                              '  - Intel(R) oneAPI Math Kernel Library Version '
                              '2022.2-Product Build 20220801 for Intel(R) 64 '
                              'architecture applications\n'
                              '  - Intel(R) MKL-DNN v3.3.2 (Git Hash '
                              '2dc95a2ad0841e29db8b22fbccaf3e5da7992b01)\n'
                              '  - OpenMP 201811\n'
                              '  - LAPACK is enabled (usually provided by '
                              'MKL)\n'
                              '  - NNPACK is enabled\n'
                              '  - CPU capability usage: AVX2\n'
                              '  - Build settings: BLAS_INFO=mkl, '
                              'BUILD_TYPE=Release, '
                              'CXX_COMPILER=/Applications/Xcode_13.3.1.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/c++, '
                              'CXX_FLAGS= -fvisibility-inlines-hidden '
                              '-DUSE_PTHREADPOOL -DNDEBUG -DUSE_KINETO '
                              '-DLIBKINETO_NOCUPTI -DLIBKINETO_NOROCTRACER '
                              '-DUSE_FBGEMM -DUSE_QNNPACK '
                              '-DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK '
                              '-DUSE_PYTORCH_METAL_EXPORT '
                              '-DSYMBOLICATE_MOBILE_DEBUG_HANDLE '
                              '-DUSE_COREML_DELEGATE -O2 -fPIC -Wall -Wextra '
                              '-Werror=return-type -Werror=non-virtual-dtor '
                              '-Werror=braced-scalar-init '
                              '-Werror=range-loop-construct '
                              '-Werror=bool-operation -Wnarrowing '
                              '-Wno-missing-field-initializers '
                              '-Wno-type-limits -Wno-array-bounds '
                              '-Wno-unknown-pragmas -Wno-unused-parameter '
                              '-Wno-unused-function -Wno-unused-result '
                              '-Wno-strict-overflow -Wno-strict-aliasing '
                              '-Wvla-extension -Wsuggest-override '
                              '-Wnewline-eof -Winconsistent-missing-override '
                              '-Winconsistent-missing-destructor-override '
                              '-Wno-pass-failed -Wno-error=pedantic '
                              '-Wno-error=old-style-cast '
                              '-Wno-error=inconsistent-missing-override '
                              '-Wno-error=inconsistent-missing-destructor-override '
                              '-Wconstant-conversion '
                              '-Wno-invalid-partial-specialization '
                              '-Wno-missing-braces -Qunused-arguments '
                              '-fcolor-diagnostics -faligned-new '
                              '-Wno-unused-but-set-variable -fno-math-errno '
                              '-fno-trapping-math -Werror=format -DUSE_MPS '
                              '-Wno-unused-private-field -Wno-missing-braces, '
                              'LAPACK_INFO=mkl, PERF_WITH_AVX=1, '
                              'PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, '
                              'TORCH_VERSION=2.2.1, USE_CUDA=OFF, '
                              'USE_CUDNN=OFF, USE_EXCEPTION_PTR=1, '
                              'USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, '
                              'USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=OFF, '
                              'USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF, '
                              'USE_ROCM_KERNEL_ASSERT=OFF, \n',
 'Python': '3.10.14 (main, Mar 21 2024, 11:24:58) [Clang 14.0.6 ]',
 'TorchVision': '0.17.1',
 'numpy_random_seed': 2147483648,
 'opencompass': '0.2.2+da643a9',
 'sys.platform': 'darwin'}

### 重现问题 - 代码/配置示例

from mmengine.config import read_base
from opencompass.models import KServeAPI, OpenAI
from opencompass.partitioners import NaivePartitioner
from opencompass.runners.hanhai_infer import HanhaiInferRunner
from opencompass.runners.hanhai_eval import HanhaiEvalRunner
from opencompass.tasks import OpenICLInferTask
from opencompass.tasks import OpenICLEvalTask
from opencompass.summarizers import DefaultSummarizer, HanhaiSummarizer


with read_base():
    from .datasets.lawbench.lawbench_zero_shot_gen_002588 import lawbench_datasets as lawbench_zero_shot_datasets
    from .datasets.lawbench.lawbench_one_shot_gen_002588 import lawbench_datasets as lawbench_one_shot_datasets

datasets = [
    *lawbench_zero_shot_datasets,
    *lawbench_one_shot_datasets
]

api_meta_template = dict(
    round=[
            dict(role='HUMAN', api_role='HUMAN'),
            dict(role='BOT', api_role='BOT', generate=True),
    ],
)

models = [
    dict(
        type=KServeAPI,
        abbr='qzhou-law-0315',
        path='qzhou-law-0315',
        header={'Custom-Host': 'qzhou-law-0315.kubeflow-user-example-com.svc.cluster.local'},
        url='http://platform.ai.ksyun.com/v2/models/qzhou-law-0315/infer',
        model_type='chat',
        meta_template=api_meta_template,
        query_per_second=1,
        max_out_len=1024,
        max_seq_len=2048,
        batch_size=1
    ),
]

infer = dict(
    partitioner=dict(type=NaivePartitioner),
    runner=dict(
        type=HanhaiInferRunner,
        max_num_workers=14,
        concurrent_users=2,
        task=dict(type=OpenICLInferTask)
    ),
)

eval = dict(
    partitioner=dict(type=NaivePartitioner),
    runner=dict(
        type=HanhaiEvalRunner,
        max_num_workers=14,
        max_workers_per_gpu=1,
        task=dict(type=OpenICLEvalTask)
    ),
)


### 重现问题 - 命令或脚本

python3 run.py ./configs/eval_hanhai_api_58.py  --debug

### 重现问题 - 错误信息

Traceback (most recent call last):
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/datasets/lawbench/utils/parallel_to_m2.py", line 221, in <module>
    main(args)
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/datasets/lawbench/utils/parallel_to_m2.py", line 138, in main
    timeout_indices = firsttime_process(args)
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/datasets/lawbench/utils/parallel_to_m2.py", line 88, in firsttime_process
    lines = open(args.file, "r", encoding="utf-8").read().strip().split("\n")  # format: id src tgt1 tgt2...
FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmp_pred_501.para'
Traceback (most recent call last):
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/datasets/lawbench/utils/parallel_to_m2.py", line 221, in <module>
    main(args)
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/datasets/lawbench/utils/parallel_to_m2.py", line 138, in main
    timeout_indices = firsttime_process(args)
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/datasets/lawbench/utils/parallel_to_m2.py", line 88, in firsttime_process
    lines = open(args.file, "r", encoding="utf-8").read().strip().split("\n")  # format: id src tgt1 tgt2...
FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmp_gold_501.para'
Traceback (most recent call last):
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/datasets/lawbench/utils/compare_m2_for_evaluation.py", line 433, in <module>
    main()
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/datasets/lawbench/utils/compare_m2_for_evaluation.py", line 8, in main
    hyp_m2 = open(args.hyp).read().strip().split("\n\n")[args.start:args.end] if args.start is not None and args.end is not None else open(args.hyp).read().strip().split("\n\n")
FileNotFoundError: [Errno 2] No such file or directory: '/tmp/tmp_pred_501.para.m2'
Traceback (most recent call last):
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/tasks/openicl_eval.py", line 363, in <module>
    inferencer.run()
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/tasks/openicl_eval.py", line 109, in run
    self._score()
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/tasks/openicl_eval.py", line 218, in _score
    result = icl_evaluator.score(**preds)
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/datasets/lawbench/lawbench.py", line 69, in score
    scores = funct_dict[self.index](data_dict)
  File "/Users/zhangliang/evaluation/executor/opencompass/opencompass/datasets/lawbench/evaluation_functions/wsjd.py", line 42, in compute_wsjd
    output = subprocess.check_output(f"python3 compare_m2_for_evaluation.py -hyp /tmp/tmp_pred_{uid}.para.m2 -ref /tmp/tmp_gold_{uid}.para.m2", shell = True)
  File "/Users/zhangliang/anaconda3/envs/evaluation/lib/python3.10/subprocess.py", line 421, in check_output
    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
  File "/Users/zhangliang/anaconda3/envs/evaluation/lib/python3.10/subprocess.py", line 526, in run
    raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command 'python3 compare_m2_for_evaluation.py -hyp /tmp/tmp_pred_501.para.m2 -ref /tmp/tmp_gold_501.para.m2' returned non-zero exit status 1.


### 其他信息

当Lawbench数据集的0-short和1-short两个任务都在运行，他们在计算wsjd时生成的文件名相同，其中一个任务把生成的临时文件删除了，导致第二个任务报错

0.3.1的版本还是有这个问题