Repository: EleutherAI/lm-evaluation-harness
Issue: Endless humaneval test
Root cause: Resource Mishandling

When testing Humaneval, I often find myself in a seemingly endless wait after the generation is complete. What could be the possible reasons for this?

scripts：
```bash
export HF_ALLOW_CODE_EVAL=1

lm_eval --model sglang \
    --model_args pretrained=Qwen/Qwen2.5-3b,dp_size=4,dtype=auto,mem_fraction_static=0.8\
    --tasks humaneval\
    --batch_size 32\
    --confirm_run_unsafe_code
```

log:
```bash
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/spawn.py", line 122, in spawn_main
    exitcode = _main(fd, parent_sentinel)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/spawn.py", line 131, in _main
    prepare(preparation_data)
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/spawn.py", line 246, in prepare
    _fixup_main_from_path(data['init_main_from_path'])
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/spawn.py", line 297, in _fixup_main_from_path
    main_content = runpy.run_path(main_path,
                   ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<frozen runpy>", line 287, in run_path
  File "<frozen runpy>", line 98, in _run_module_code
  File "<frozen runpy>", line 88, in _run_code
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/env/rjh/bin/lm_eval", line 5, in <module>
Traceback (most recent call last):
    from lm_eval.__main__ import cli_evaluate
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/lm-evaluation-harness/lm_eval/__init__.py", line 4, in <module>
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/scheduler.py", line 1757, in run_scheduler_process
    scheduler.event_loop_overlap()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/scheduler.py", line 536, in event_loop_overlap
    self.check_memory()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/scheduler.py", line 914, in check_memory
    + self.tree_cache.evictable_size()
      ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/mem_cache/radix_cache.py", line 304, in evictable_size
    def evictable_size(self):
    
KeyboardInterrupt
    from .evaluator import evaluate, simple_evaluate
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/lm-evaluation-harness/lm_eval/evaluator.py", line 15, in <module>
    import lm_eval.models
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/lm-evaluation-harness/lm_eval/models/__init__.py", line 1, in <module>
    from . import (
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/lm-evaluation-harness/lm_eval/models/vllm_causallms.py", line 28, in <module>
    from vllm import LLM, SamplingParams
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/__init__.py", line 11, in <module>
    from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/engine/arg_utils.py", line 13, in <module>
Traceback (most recent call last):
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/scheduler.py", line 1757, in run_scheduler_process
    scheduler.event_loop_overlap()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/scheduler.py", line 536, in event_loop_overlap
    self.check_memory()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/scheduler.py", line 913, in check_memory
    self.token_to_kv_pool_allocator.available_size()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/mem_cache/memory_pool.py", line 142, in available_size
    return len(self.free_slots)
           ^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/torch/_tensor.py", line 1088, in __len__
    if torch._C._get_tracing_state():
       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
KeyboardInterrupt
Traceback (most recent call last):
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/detokenizer_manager.py", line 263, in run_detokenizer_process
    manager.event_loop()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/detokenizer_manager.py", line 107, in event_loop
    recv_obj = self.recv_from_scheduler.recv_pyobj()
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/zmq/sugar/socket.py", line 972, in recv_pyobj
    msg = self.recv(flags)
          ^^^^^^^^^^^^^^^^
  File "_zmq.py", line 1156, in zmq.backend.cython._zmq.Socket.recv
  File "_zmq.py", line 1191, in zmq.backend.cython._zmq.Socket.recv
  File "_zmq.py", line 1278, in zmq.backend.cython._zmq._recv_copy
  File "_zmq.py", line 160, in zmq.backend.cython._zmq._check_rc
KeyboardInterrupt
Traceback (most recent call last):
    from vllm.config import (CacheConfig, CompilationConfig, ConfigFormat,
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/config.py", line 24, in <module>
Traceback (most recent call last):
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/scheduler.py", line 1757, in run_scheduler_process
    scheduler.event_loop_overlap()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/scheduler.py", line 507, in event_loop_overlap
    recv_reqs = self.recv_requests()
                ^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/scheduler.py", line 548, in recv_requests
    recv_req = self.recv_from_tokenizer.recv_pyobj(zmq.NOBLOCK)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
    self.run()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/zmq/sugar/socket.py", line 972, in recv_pyobj
    msg = self.recv(flags)
          ^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "_zmq.py", line 1156, in zmq.backend.cython._zmq.Socket.recv
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/scheduler.py", line 1757, in run_scheduler_process
    scheduler.event_loop_overlap()
  File "_zmq.py", line 1191, in zmq.backend.cython._zmq.Socket.recv
  File "_zmq.py", line 1278, in zmq.backend.cython._zmq._recv_copy
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "_zmq.py", line 171, in zmq.backend.cython._zmq._check_rc
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/scheduler.py", line 507, in event_loop_overlap
    recv_reqs = self.recv_requests()
                ^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/zmq/error.py", line 106, in __init__
    super().__init__(EAGAIN)
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/sglang/srt/managers/scheduler.py", line 548, in recv_requests
    recv_req = self.recv_from_tokenizer.recv_pyobj(zmq.NOBLOCK)
               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/zmq/error.py", line 46, in __init__
    self.strerror = strerror(errno)
                    ^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/zmq/sugar/socket.py", line 972, in recv_pyobj
    msg = self.recv(flags)
          ^^^^^^^^^^^^^^^^
  File "_zmq.py", line 1156, in zmq.backend.cython._zmq.Socket.recv
  File "_zmq.py", line 1191, in zmq.backend.cython._zmq.Socket.recv
KeyboardInterrupt
  File "_zmq.py", line 1278, in zmq.backend.cython._zmq._recv_copy
  File "_zmq.py", line 171, in zmq.backend.cython._zmq._check_rc
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/zmq/error.py", line 106, in __init__
    super().__init__(EAGAIN)
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/zmq/error.py", line 46, in __init__
    self.strerror = strerror(errno)
                    ^^^^^^^^^^^^^^^
KeyboardInterrupt
    from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/model_executor/__init__.py", line 3, in <module>
    from vllm.model_executor.parameter import (BasevLLMParameter,
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/model_executor/parameter.py", line 9, in <module>
    from vllm.distributed import get_tensor_model_parallel_rank
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/distributed/__init__.py", line 3, in <module>
    from .communication_op import *
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/distributed/communication_op.py", line 8, in <module>
    from .parallel_state import get_tp_group
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/distributed/parallel_state.py", line 40, in <module>
    import vllm.distributed.kv_transfer.kv_transfer_agent as kv_transfer
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/distributed/kv_transfer/kv_transfer_agent.py", line 16, in <module>
    from vllm.distributed.kv_transfer.kv_connector.factory import (
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/distributed/kv_transfer/kv_connector/factory.py", line 6, in <module>
    from .base import KVConnectorBase
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/distributed/kv_transfer/kv_connector/base.py", line 15, in <module>
    from vllm.sequence import IntermediateTensors
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/sequence.py", line 17, in <module>
    from vllm.inputs import SingletonInputs, SingletonInputsAdapter
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/inputs/__init__.py", line 9, in <module>
    from .registry import (DummyData, InputContext, InputProcessingContext,
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/inputs/registry.py", line 15, in <module>
    from vllm.transformers_utils.tokenizer import (AnyTokenizer,
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/transformers_utils/tokenizer.py", line 20, in <module>
    from vllm.transformers_utils.tokenizers import MistralTokenizer
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/transformers_utils/tokenizers/__init__.py", line 3, in <module>
    from .mistral import (MistralTokenizer, maybe_serialize_tool_calls,
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/vllm/transformers_utils/tokenizers/mistral.py", line 14, in <module>
    from vllm.utils import is_list_of
  File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
  File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
  File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
  File "<frozen importlib._bootstrap_external>", line 995, in exec_module
  File "<frozen importlib._bootstrap_external>", line 1128, in get_code
  File "<frozen importlib._bootstrap_external>", line 757, in _compile_bytecode
KeyboardInterrupt
^CProcess Process-1:
Traceback (most recent call last):
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/.cache/modules/evaluate_modules/metrics/evaluate-metric--code_eval/78d307ea938083398db7d9815f03ed661e9c15f60d77880ce007a8a02648f176/code_eval.py", line 179, in _compute
[rank0]: Traceback (most recent call last):
[rank0]:   File "<string>", line 1, in <module>
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/spawn.py", line 122, in spawn_main
[rank0]:     exitcode = _main(fd, parent_sentinel)
[rank0]:                ^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/spawn.py", line 135, in _main
[rank0]:     return self._bootstrap(parent_sentinel)
[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 332, in _bootstrap
[rank0]:     threading._shutdown()
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/threading.py", line 1624, in _shutdown
[rank0]:     lock.acquire()
[rank0]: KeyboardInterrupt
[rank0]: Traceback (most recent call last):
[rank0]:   File "<string>", line 1, in <module>
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/spawn.py", line 122, in spawn_main
[rank0]:     exitcode = _main(fd, parent_sentinel)
[rank0]:                ^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/spawn.py", line 135, in _main
[rank0]:     return self._bootstrap(parent_sentinel)
[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 332, in _bootstrap
[rank0]:     threading._shutdown()
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/threading.py", line 1624, in _shutdown
[rank0]:     lock.acquire()
[rank0]: KeyboardInterrupt
[rank0]: Traceback (most recent call last):
[rank0]:   File "<string>", line 1, in <module>
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/spawn.py", line 122, in spawn_main
[rank0]:     exitcode = _main(fd, parent_sentinel)
[rank0]:                ^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/spawn.py", line 135, in _main
[rank0]:     return self._bootstrap(parent_sentinel)
[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 332, in _bootstrap
[rank0]:     threading._shutdown()
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/threading.py", line 1624, in _shutdown
[rank0]:     lock.acquire()
[rank0]: KeyboardInterrupt
    for future in as_completed(futures):
                  ^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/concurrent/futures/_base.py", line 243, in as_completed
[rank0]: Traceback (most recent call last):
[rank0]:   File "<string>", line 1, in <module>
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/spawn.py", line 122, in spawn_main
[rank0]:     exitcode = _main(fd, parent_sentinel)
[rank0]:                ^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/spawn.py", line 135, in _main
[rank0]:     return self._bootstrap(parent_sentinel)
[rank0]:            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/process.py", line 332, in _bootstrap
[rank0]:     threading._shutdown()
[rank0]:   File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/threading.py", line 1624, in _shutdown
[rank0]:     lock.acquire()
[rank0]: KeyboardInterrupt
    waiter.event.wait(wait_timeout)
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/threading.py", line 655, in wait
    signaled = self._cond.wait(timeout)
               ^^^^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/threading.py", line 355, in wait
    waiter.acquire()
KeyboardInterrupt

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/env/rjh/bin/lm_eval", line 8, in <module>
    sys.exit(cli_evaluate())
             ^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/lm-evaluation-harness/lm_eval/__main__.py", line 389, in cli_evaluate
    results = evaluator.simple_evaluate(
              ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/lm-evaluation-harness/lm_eval/utils.py", line 423, in _wrapper
    return fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/lm-evaluation-harness/lm_eval/evaluator.py", line 316, in simple_evaluate
    results = evaluate(
              ^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/lm-evaluation-harness/lm_eval/utils.py", line 423, in _wrapper
    return fn(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/lm-evaluation-harness/lm_eval/evaluator.py", line 570, in evaluate
    metrics = task.process_results(
              ^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/lm-evaluation-harness/lm_eval/api/task.py", line 1662, in process_results
    result_score = self._metric_fn_list[metric](
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/lm-evaluation-harness/lm_eval/tasks/humaneval/utils.py", line 18, in pass_at_k
    res = compute_.compute(
          ^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/site-packages/evaluate/module.py", line 467, in compute
    output = self._compute(**inputs, **compute_kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/.cache/modules/evaluate_modules/metrics/evaluate-metric--code_eval/78d307ea938083398db7d9815f03ed661e9c15f60d77880ce007a8a02648f176/code_eval.py", line 164, in _compute
    with ThreadPoolExecutor(max_workers=num_workers) as executor:
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/concurrent/futures/_base.py", line 647, in __exit__
    self.shutdown(wait=True)
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/concurrent/futures/thread.py", line 239, in shutdown
    t.join()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/threading.py", line 1149, in join
    self._wait_for_tstate_lock()
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/threading.py", line 1169, in _wait_for_tstate_lock
    if lock.acquire(block, timeout):
       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
KeyboardInterrupt
/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/resource_tracker.py:123: UserWarning: resource_tracker: process died unexpectedly, relaunching.  Some resources might leak.
  warnings.warn('resource_tracker: process died unexpectedly, '
Traceback (most recent call last):
  File "/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/ruanjunhao04/env/rjh/lib/python3.12/multiprocessing/resource_tracker.py", line 240, in main
    cache[rtype].remove(name)
KeyError: '/loky-81423-ez1tax2b'
```

Hi! It [uses](https://github.com/EleutherAI/lm-evaluation-harness/blob/1afb190ce95340453d60aaccce9c3e5ad7888805/lm_eval/tasks/humaneval/utils.py#L18) the `code_eval` metric from [evaluate](https://huggingface.co/spaces/evaluate-metric/code_eval). You could try adding a lower `timeout` kwarg, but 3 seconds seems reasonable to me. Could also try `num_workers` if you have more cpus.

Could also run a test script to check if it's working properly in your environment. Something like:
```python
from evaluate import load
code_eval = load("code_eval")
test_cases = [["assert add(2,3)==5"]]* 100
candidates = [[["def add(a,b): return a*b", "def add(a, b): return a+b"]]]* 100
for t,c in zip (test_cases, candidates)
   pass_at_k, results = code_eval.compute(references=t, predictions=c, k=[1],timeout=3)
```


Thank you for your help. It seems the issue is with `code_eval = load("code_eval")`. I'm unable to load it in offline environment. Do you have any suggestions?


> Thank you for your help. It seems the issue is with `code_eval = load("code_eval")`. I'm unable to load it in offline environment. Do you have any suggestions?

you could copy from the source code ([here](https://github.com/huggingface/evaluate/blob/5aa3982a9a8c86e506860e381d428a64b0cce73b/metrics/code_eval/code_eval.py#L155-L213) and [here](https://github.com/huggingface/evaluate/blob/main/metrics/code_eval/execute.py)). Most of the code seems to be scaffolding: basically it's [concatenating](https://github.com/huggingface/evaluate/blob/5aa3982a9a8c86e506860e381d428a64b0cce73b/metrics/code_eval/code_eval.py#L172) the prediction string (+ newline) with the reference string and [calling](https://github.com/huggingface/evaluate/blob/5aa3982a9a8c86e506860e381d428a64b0cce73b/metrics/code_eval/execute.py#L76) `exec` on it to see if it runs correctly

I modified `lm-evaluation-harness/lm_eval/tasks/humaneval/utils.py` (the following example I can run without problem)

```python

try:
    compute_ = hf_evaluate.load("/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/evaluate/metrics/code_eval")
    test_cases = ["assert add(2, 3)==5"]
    candidates = [["def add(a,b): return a*b"]]
    results = compute_.compute(references=test_cases, predictions=candidates, k=[1])
except Exception as e:
    raise e


def pass_at_k(references: list[str], predictions: list[list[str]], k: list[int] = None):
    global compute_
    assert k is not None
    if isinstance(k, int):
        k = [k]
    res = compute_.compute(
        references=references,
        predictions=predictions,
        k=k,
        num_workers=64, timeout=3.0
    )
    return res[0]
```

but still
```bash
Running generate_until requests:   1%|▌                                                                                   | 1/164 [00:14<38:25, 14.14s/it]2025-03-25 15:34:02,392 - INFO - flashinfer.jit: Loading JIT ops: cascade
2025-03-25 15:34:02,409 - INFO - flashinfer.jit: Loading JIT ops: cascade
2025-03-25 15:34:02,411 - INFO - flashinfer.jit: Loading JIT ops: cascade
2025-03-25 15:34:02,412 - INFO - flashinfer.jit: Loading JIT ops: cascade
2025-03-25 15:34:02,799 - INFO - flashinfer.jit: Finished loading JIT ops: cascade
2025-03-25 15:34:02,895 - INFO - flashinfer.jit: Finished loading JIT ops: cascade
2025-03-25 15:34:02,991 - INFO - flashinfer.jit: Finished loading JIT ops: cascade
2025-03-25 15:34:03,096 - INFO - flashinfer.jit: Finished loading JIT ops: cascade
Running generate_until requests: 100%|██████████████████████████████████████████████████████████████████████████████████| 164/164 [00:36<00:00,  4.52it/s]

```

: ( damn seems here exist some other problem

you can also use the `--predict_only` flag to skip the metric calculation and then run it locally using the saved samples file (run it on `filtered_resps`)

> you can also use the `--predict_only` flag to skip the metric calculation and then run it locally using the saved samples file (run it on `filtered_resps`)

After --predict_only, I successfully saved the response. But I sam till curious about how to run it on filtered_resps

run this [pass_at_k](https://github.com/EleutherAI/lm-evaluation-harness/blob/1afb190ce95340453d60aaccce9c3e5ad7888805/lm_eval/tasks/humaneval/utils.py#L13) function, with the references=[`target`] and predictions=`filtered_resps` from the samples jsonl file. `k = [1]`

I've provided a simple modification to the utils for reference for those who encounter the same situation. You need to follow @baberabb's instructions first to generate a JSONL file containing the sampling results, then rewrite the path in the main function. One thing to note is that I'm not sure if code_eval has changed, but in any case, the `res` I return here is a tuple. Therefore, it cannot be accessed directly using string indexing.
```python
import evaluate as hf_evaluate
import json
import os
os.environ["HF_ALLOW_CODE_EVAL"] = "1"
try:
    compute_ = hf_evaluate.load("/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/evaluate/metrics/code_eval")
    test_cases = ["assert add(2, 3)==5"]
    candidates = [["def add(a,b): return a*b"]]
    results = compute_.compute(references=test_cases, predictions=candidates, k=[1])
except Exception as e:
    raise e


def pass_at_k(references: list[str], predictions: list[list[str]], k: list[int] = None):
    global compute_
    assert k is not None
    if isinstance(k, int):
        k = [k]
    res = compute_.compute(
        references=references,
        predictions=predictions,
        k=k,
        num_workers=64, timeout=3.0
    )

    return res[0][f'pass@{k[0]}']


def build_predictions(resps: list[list[str]], docs: list[dict]) -> list[list[str]]:
    return [[doc["prompt"] + r for r in resp] for resp, doc in zip(resps, docs)]


def build_predictions_instruct(
    resps: list[list[str]], docs: list[dict]
) -> list[list[str]]:
    return [
        [
            doc["prompt"] + (r if r.rfind("```") == -1 else r[: r.rfind("```")])
            for r in resp
        ]
        for resp, doc in zip(resps, docs)
    ]

if __name__=='__main__':

    file_dir='/mnt/dolphinfs/hdd_pool/docker/user/hadoop-aipnlp/INS/ruanjunhao04/ruanjunhao/ndp/tests/fuck/__mnt__dolphinfs__hdd_pool__docker__user__hadoop-aipnlp__INS__ruanjunhao04__ruanjunhao__ndp__trained_model__qwen2.5-3b-ntp-318__checkpoint-14676/samples_humaneval_2025-03-25T20-02-13.169208.jsonl'
    references =[]
    predictions =[]
    with open(file_dir, 'r') as f:
        for line in f:
            data = json.loads(line)
            target = data.get('target')
            response = data.get('filtered_resps')[0]

            references.append(target)
            predictions.append(response)

    # Assuming 'response' directly contains the list of generated code snippets
    filtered_resps = predictions

    # Run pass_at_k with k=[1]
    k_values = [1]

    results = pass_at_k(references=references, predictions=filtered_resps, k=1)
    print(f"Pass@k for k={k_values}: {results}")

```

( Although I'm still puzzled as to why separating the execution works, while directly using lm eval for end-to-end evaluation consistently causes it to get stuck.