Repository: open-compass/opencompass

Issue:
[Bug] How can I evaluate Llama2-7b-chat-hf in turbomind style?
### Prerequisite

- [X] I have searched [Issues](https://github.com/InternLM/opencompass/issues/) and [Discussions](https://github.com/InternLM/opencompass/discussions) but cannot get the expected help.
- [X] The bug has not been fixed in the [latest version](https://github.com/InternLM/opencompass).

### Type

I have modified the code (config is not considered code), or I'm working on my own tasks/models/datasets.

### Environment

```shell
/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/torchvision/io/image.py:13: UserWarning: Failed to load image Python extension: '/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/torchvision/image.so: undefined symbol: _ZN3c104cuda20CUDACachingAllocator9allocatorE'If you don't plan on using image functionality from `torchvision.io`, you can ignore this warning. Otherwise, there might be something wrong with your environment. Did you have `libjpeg` or `libpng` installed before building `torchvision` from source?
  warn(
{'CUDA available': True,
 'CUDA_HOME': '/data/cuda',
 'GCC': 'gcc (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0',
 'GPU 0,1,2,3,4,5,6,7': 'NVIDIA A100-SXM4-40GB',
 'MMEngine': '0.8.4',
 'NVCC': 'Cuda compilation tools, release 11.0, V11.0.221',
 'OpenCV': '4.8.0',
 'PyTorch': '1.13.1+cu117',
 'PyTorch compiling details': 'PyTorch built with:\n'
                              '  - GCC 9.3\n'
                              '  - C++ Version: 201402\n'
                              '  - Intel(R) Math Kernel Library Version '
                              '2020.0.0 Product Build 20191122 for Intel(R) 64 '
                              'architecture applications\n'
                              '  - Intel(R) MKL-DNN v2.6.0 (Git Hash '
                              '52b5f107dd9cf10910aaa19cb47f3abf9b349815)\n'
                              '  - OpenMP 201511 (a.k.a. OpenMP 4.5)\n'
                              '  - LAPACK is enabled (usually provided by '
                              'MKL)\n'
                              '  - NNPACK is enabled\n'
                              '  - CPU capability usage: AVX2\n'
                              '  - CUDA Runtime 11.7\n'
                              '  - NVCC architecture flags: '
                              '-gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86\n'
                              '  - CuDNN 8.5\n'
                              '  - Magma 2.6.1\n'
                              '  - Build settings: BLAS_INFO=mkl, '
                              'BUILD_TYPE=Release, CUDA_VERSION=11.7, '
                              'CUDNN_VERSION=8.5.0, '
                              'CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, '
                              'CXX_FLAGS= -fabi-version=11 -Wno-deprecated '
                              '-fvisibility-inlines-hidden -DUSE_PTHREADPOOL '
                              '-fopenmp -DNDEBUG -DUSE_KINETO -DUSE_FBGEMM '
                              '-DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK '
                              '-DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE '
                              '-DEDGE_PROFILER_USE_KINETO -O2 -fPIC '
                              '-Wno-narrowing -Wall -Wextra '
                              '-Werror=return-type -Werror=non-virtual-dtor '
                              '-Wno-missing-field-initializers '
                              '-Wno-type-limits -Wno-array-bounds '
                              '-Wno-unknown-pragmas -Wunused-local-typedefs '
                              '-Wno-unused-parameter -Wno-unused-function '
                              '-Wno-unused-result -Wno-strict-overflow '
                              '-Wno-strict-aliasing '
                              '-Wno-error=deprecated-declarations '
                              '-Wno-stringop-overflow -Wno-psabi '
                              '-Wno-error=pedantic -Wno-error=redundant-decls '
                              '-Wno-error=old-style-cast '
                              '-fdiagnostics-color=always -faligned-new '
                              '-Wno-unused-but-set-variable '
                              '-Wno-maybe-uninitialized -fno-math-errno '
                              '-fno-trapping-math -Werror=format '
                              '-Werror=cast-function-type '
                              '-Wno-stringop-overflow, LAPACK_INFO=mkl, '
                              'PERF_WITH_AVX=1, PERF_WITH_AVX2=1, '
                              'PERF_WITH_AVX512=1, TORCH_VERSION=1.13.1, '
                              'USE_CUDA=ON, USE_CUDNN=ON, USE_EXCEPTION_PTR=1, '
                              'USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, '
                              'USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=ON, '
                              'USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF, \n',
 'Python': '3.10.12 (main, Jul  5 2023, 18:54:27) [GCC 11.2.0]',
 'TorchVision': '0.15.2',
 'numpy_random_seed': 2147483648,
 'opencompass': '0.1.2+b2d602f',
 'sys.platform': 'linux'}
```

### Reproduces the problem - code/configuration sample

```python
# eval_llama2_chat_7b_turbomind.py

'''
the model "/data/home/xxx/examples/model_deployed/llama2-7b" use the below command to generate:
CUDA_VISIBLE_DEVICES=6 python3 -m lmdeploy.serve.turbomind.deploy llama2 /data/home/xxx/model/llama2/7b/origin/chat-hf --dst-path /data/home/xxx/examples/model_deployed/llama2-7b
'''

from mmengine.config import read_base
from opencompass.models.turbomind import TurboMindModel

with read_base():
    # choose a list of datasets
    from .datasets.SuperGLUE_CB.SuperGLUE_CB_gen import CB_datasets
    # and output the results in a choosen format
    from .summarizers.medium import summarizer

datasets = [*CB_datasets]


_meta_template = dict(
    round=[
        dict(role="HUMAN", api_role="HUMAN"),
        dict(role="BOT", api_role="BOT", generate=True),
    ],
)
models = [
    dict(
        type=TurboMindModel,
        abbr='llama2-7b-chat-tb',
        path="llama2-7b-chat",
        model_path='/data/home/xxx/examples/model_deployed/llama2-7b',
        max_out_len=100,
        max_seq_len=2048,
        batch_size=16,
        meta_template=_meta_template,
        run_cfg=dict(num_gpus=2, num_procs=1),
    )
]
```

### Reproduces the problem - command or script

```shell
CUDA_VISIBLE_DEVICES=0,1 python run.py configs/eval_llama2_chat_7b_turbomind.py -w outputs/turbomind
```

### Reproduces the problem - error message

```python
# /infer/llama2-7b-chat-tb/CB.out
09/05 11:14:03 - OpenCompass - INFO - Task [llama2-7b-chat-tb/CB]
/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/mmengine/utils/manager.py:113: UserWarning: <class 'mmengine.logging.logger.MMLogger'> instance named of OpenCompass has been created, the method `get_instance` should not accept any other arguments
  warnings.warn(
[TM][INFO] Barrier(1)
[TM][INFO] Barrier(1)
[WARNING] gemm_config.in is not found; using default GEMM algo
[TM][INFO] NCCL group_id = 0
[TM][INFO] [LlamaCacheManager] max_entry_count = 48
[TM][INFO] [LlamaCacheManager] chunk_size = 1
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 1
[TM][INFO] [LlamaCacheManager][allocate] free = 1
[TM][INFO] [internalThreadEntry] 0
/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/fuzzywuzzy/fuzz.py:11: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning
  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')
09/05 11:14:16 - OpenCompass - INFO - Start inferencing [llama2-7b-chat-tb/CB]
[2023-09-05 11:14:16,930] [opencompass.openicl.icl_inferencer.icl_gen_inferencer] [INFO] Starting inference process...

  0%|          | 0/4 [00:00<?, ?it/s][TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 53882
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 4104
[TM][INFO] [init] max_input_length = 96
[TM][INFO] [init] max_context_len = 96
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        53882            0         96           96             96               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 95, max_input_len = 95, max_context_len = 95
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 19.28 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 96
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        53882           96            196         0
[TM][INFO] ------------------------- step = 100 -------------------------
[TM][INFO] ------------------------- step = 110 -------------------------
[TM][INFO] ------------------------- step = 120 -------------------------
[TM][INFO] ------------------------- step = 130 -------------------------
[TM][INFO] ------------------------- step = 140 -------------------------
[TM][INFO] ------------------------- step = 150 -------------------------
[TM][INFO] ------------------------- step = 160 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 53882
[TM][INFO] [LlamaCacheManager][update] 53882
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 26574
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 2
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 4104
[TM][INFO] [init] max_input_length = 96
[TM][INFO] [init] max_context_len = 96
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        26574            0         96           96             96               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 95, max_input_len = 95, max_context_len = 95
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 15.31 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 96
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        26574           96            196         0
[TM][INFO] ------------------------- step = 100 -------------------------
[TM][INFO] ------------------------- step = 110 -------------------------
[TM][INFO] ------------------------- step = 120 -------------------------
[TM][INFO] ------------------------- step = 130 -------------------------
[TM][INFO] ------------------------- step = 140 -------------------------
[TM][INFO] ------------------------- step = 150 -------------------------
[TM][INFO] ------------------------- step = 160 -------------------------
[TM][INFO] ------------------------- step = 170 -------------------------
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 1
[TM][INFO] [LlamaCacheManager][create] 91626
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 3
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 2
[TM][INFO] [init] session_len = 4104
[TM][INFO] [init] max_input_length = 103
[TM][INFO] [init] max_context_len = 103
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    1        91626            0        103          103            103               0          0
[TM][INFO] [decodeContext] base = 1, count = 1
[TM][INFO] [decodeContext] offset = 1, batch_size = 1, token_num = 102, max_input_len = 102, max_context_len = 102
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 15.47 ms
[TM][INFO] [initGen] batch_size = 2
[TM][INFO] [initGen] max_context_len = 180
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        26574          180            196         0
[TM][INFO] [initGen]    1        91626          103            280         0
[TM][INFO] ------------------------- step = 180 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 26574
[TM][INFO] [LlamaCacheManager][update] 26574
[TM][INFO] [synchronize] batch_size = 1
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 109
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        91626          109            203         0
[TM][INFO] ------------------------- step = 110 -------------------------
[TM][INFO] ------------------------- step = 120 -------------------------
[TM][INFO] ------------------------- step = 130 -------------------------
[TM][INFO] ------------------------- step = 140 -------------------------
[TM][INFO] ------------------------- step = 150 -------------------------
[TM][INFO] ------------------------- step = 160 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 91626
[TM][INFO] [LlamaCacheManager][update] 91626
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 41550
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 4
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 4104
[TM][INFO] [init] max_input_length = 122
[TM][INFO] [init] max_context_len = 122
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        41550            0        122          122            122               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 121, max_input_len = 121, max_context_len = 121
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 16.28 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 122
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        41550          122            222         0
[TM][INFO] ------------------------- step = 130 -------------------------
[TM][INFO] ------------------------- step = 140 -------------------------
[TM][INFO] ------------------------- step = 150 -------------------------
[TM][INFO] ------------------------- step = 160 -------------------------
[TM][INFO] ------------------------- step = 170 -------------------------
[TM][INFO] ------------------------- step = 180 -------------------------
[TM][INFO] ------------------------- step = 190 -------------------------
[TM][INFO] ------------------------- step = 200 -------------------------
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 1
[TM][INFO] [LlamaCacheManager][create] 47329
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 5
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 2
[TM][INFO] [init] session_len = 4104
[TM][INFO] [init] max_input_length = 122
[TM][INFO] [init] max_context_len = 122
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    1        47329            0        122          122            122               0          0
[TM][INFO] [decodeContext] base = 1, count = 1
[TM][INFO] [decodeContext] offset = 1, batch_size = 1, token_num = 121, max_input_len = 121, max_context_len = 121
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 15.68 ms
[TM][INFO] [initGen] batch_size = 2
[TM][INFO] [initGen] max_context_len = 205
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        41550          205            222         0
[TM][INFO] [initGen]    1        47329          122            305         0
[TM][INFO] [finishRequest] slot = 0, id = 41550
[TM][INFO] [LlamaCacheManager][update] 41550
[TM][INFO] [synchronize] batch_size = 1
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 126
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        47329          126            222         0
[TM][INFO] ------------------------- step = 130 -------------------------
[TM][INFO] ------------------------- step = 140 -------------------------
[TM][INFO] ------------------------- step = 150 -------------------------
[TM][INFO] ------------------------- step = 160 -------------------------
[TM][INFO] ------------------------- step = 170 -------------------------
[TM][INFO] ------------------------- step = 180 -------------------------
[TM][INFO] ------------------------- step = 190 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 47329
[TM][INFO] [LlamaCacheManager][update] 47329
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 70334
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 6
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 4104
[TM][INFO] [init] max_input_length = 106
[TM][INFO] [init] max_context_len = 106
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        70334            0        106          106            106               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 105, max_input_len = 105, max_context_len = 105
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 15.54 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 106
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        70334          106            206         0
[TM][INFO] ------------------------- step = 110 -------------------------
[TM][INFO] ------------------------- step = 120 -------------------------
[TM][INFO] ------------------------- step = 130 -------------------------
[TM][INFO] ------------------------- step = 140 -------------------------
[TM][INFO] ------------------------- step = 150 -------------------------
[TM][INFO] ------------------------- step = 160 -------------------------
[TM][INFO] ------------------------- step = 170 -------------------------
[TM][INFO] ------------------------- step = 180 -------------------------
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 1
[TM][INFO] [LlamaCacheManager][create] 24312
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 7
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -11) local_rank: 0 (pid: 692914) of binary: /data/home/xxx/.conda/envs/opencompass/bin/python
Traceback (most recent call last):
  File "/data/home/xxx/.conda/envs/opencompass/bin/torchrun", line 8, in <module>
    sys.exit(main())
  File "/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper
    return f(*args, **kwargs)
  File "/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main
    run(args)
  File "/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run
    elastic_launch(
  File "/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__
    return launch_agent(self._config, self._entrypoint, list(args))
  File "/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent
    raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
============================================================
/data/home/xxx/examples/opencompass/opencompass/tasks/openicl_infer.py FAILED
------------------------------------------------------------
Failures:
  <NO_OTHER_FAILURES>
------------------------------------------------------------
Root Cause (first observed failure):
[0]:
  time      : 2023-09-05_11:14:26
  host      : node3
  rank      : 0 (local_rank: 0)
  exitcode  : -11 (pid: 692914)
  error_file: <N/A>
  traceback : Signal 11 (SIGSEGV) received by PID 692914
============================================================

# /eval/llama2-7b-chat-tb/CB.out
/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/fuzzywuzzy/fuzz.py:11: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning
  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')
09/05 11:14:30 - OpenCompass - ERROR - /data/home/xxx/examples/opencompass/opencompass/tasks/openicl_eval.py - _score - 163 - Task [llama2-7b-chat-tb/CB]: No predictions found.
/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/mmengine/utils/manager.py:113: UserWarning: <class 'mmengine.logging.logger.MMLogger'> instance named of OpenCompass has been created, the method `get_instance` should not accept any other arguments
  warnings.warn(
09/05 11:14:30 - OpenCompass - INFO - time elapsed: 1.33s
```

### Other information

what's the reason of it? I just copy the official demo and change the path of turbomind model and _meta_template.

And the official demo which use internLM-7B is CUDA of memory in one A100-40g.It's weird. Maybe the info is helpful for you.I paste it below:

```python
# the comand I run: CUDA_VISIBLE_DEVICES=1 python run.py configs/eval_internlm_chat_7b_turbomind.py -w outputs/turbomind

09/05 11:39:10 - OpenCompass - INFO - Task [internlm-chat-7b-tb/CB]
/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/mmengine/utils/manager.py:113: UserWarning: <class 'mmengine.logging.logger.MMLogger'> instance named of OpenCompass has been created, the method `get_instance` should not accept any other arguments
  warnings.warn(
WARNING: Can not find tokenizer.json. It may take long time to initialize the tokenizer.
[TM][INFO] Barrier(1)
[TM][INFO] Barrier(1)
[WARNING] gemm_config.in is not found; using default GEMM algo
[TM][INFO] NCCL group_id = 0
[TM][INFO] [LlamaCacheManager] max_entry_count = 48
[TM][INFO] [LlamaCacheManager] chunk_size = 1
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 1
[TM][INFO] [LlamaCacheManager][allocate] free = 1
[TM][INFO] [internalThreadEntry] 0
/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/fuzzywuzzy/fuzz.py:11: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning
  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')
09/05 11:39:25 - OpenCompass - INFO - Start inferencing [internlm-chat-7b-tb/CB]
[2023-09-05 11:39:25,906] [opencompass.openicl.icl_inferencer.icl_gen_inferencer] [INFO] Starting inference process...

  0%|          | 0/4 [00:00<?, ?it/s][TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 14063
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 69
[TM][INFO] [init] max_context_len = 69
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        14063            0         69           69             69               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 68, max_input_len = 68, max_context_len = 68
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 16.78 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 69
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        14063           69            169         0
[TM][INFO] ------------------------- step = 70 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 14063
[TM][INFO] [LlamaCacheManager][update] 14063
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 86789
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 2
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 83
[TM][INFO] [init] max_context_len = 83
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        86789            0         83           83             83               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 82, max_input_len = 82, max_context_len = 82
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 18.15 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 83
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        86789           83            183         0
[TM][INFO] [finishRequest] slot = 0, id = 86789
[TM][INFO] [LlamaCacheManager][update] 86789
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 25732
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 3
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 118
[TM][INFO] [init] max_context_len = 118
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        25732            0        118          118            118               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 117, max_input_len = 117, max_context_len = 117
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 18.49 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 118
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        25732          118            218         0
[TM][INFO] ------------------------- step = 120 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 25732
[TM][INFO] [LlamaCacheManager][update] 25732
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 52964
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 4
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 165
[TM][INFO] [init] max_context_len = 165
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        52964            0        165          165            165               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 164, max_input_len = 164, max_context_len = 164
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 23.38 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 165
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        52964          165            265         0
[TM][INFO] [finishRequest] slot = 0, id = 52964
[TM][INFO] [LlamaCacheManager][update] 52964
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 5241
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 5
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 167
[TM][INFO] [init] max_context_len = 167
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0         5241            0        167          167            167               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 166, max_input_len = 166, max_context_len = 166
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 23.44 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 167
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0         5241          167            267         0
[TM][INFO] ------------------------- step = 170 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 5241
[TM][INFO] [LlamaCacheManager][update] 5241
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 78719
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 6
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 186
[TM][INFO] [init] max_context_len = 186
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        78719            0        186          186            186               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 185, max_input_len = 185, max_context_len = 185
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 22.33 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 186
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        78719          186            286         0
[TM][INFO] [finishRequest] slot = 0, id = 78719
[TM][INFO] [LlamaCacheManager][update] 78719
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 59472
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 7
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 126
[TM][INFO] [init] max_context_len = 126
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        59472            0        126          126            126               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 125, max_input_len = 125, max_context_len = 125
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 17.59 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 126
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        59472          126            226         0
[TM][INFO] [finishRequest] slot = 0, id = 59472
[TM][INFO] [LlamaCacheManager][update] 59472
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 94552
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 8
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 108
[TM][INFO] [init] max_context_len = 108
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        94552            0        108          108            108               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 107, max_input_len = 107, max_context_len = 107
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 17.32 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 108
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        94552          108            208         0
[TM][INFO] ------------------------- step = 110 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 94552
[TM][INFO] [LlamaCacheManager][update] 94552
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 24824
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 9
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 197
[TM][INFO] [init] max_context_len = 197
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        24824            0        197          197            197               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 196, max_input_len = 196, max_context_len = 196
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 24.89 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 197
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        24824          197            297         0
[TM][INFO] [finishRequest] slot = 0, id = 24824
[TM][INFO] [LlamaCacheManager][update] 24824
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 15743
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 10
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 98
[TM][INFO] [init] max_context_len = 98
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        15743            0         98           98             98               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 97, max_input_len = 97, max_context_len = 97
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 18.46 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 98
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        15743           98            198         0
[TM][INFO] ------------------------- step = 100 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 15743
[TM][INFO] [LlamaCacheManager][update] 15743
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 66892
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 11
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 126
[TM][INFO] [init] max_context_len = 126
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        66892            0        126          126            126               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 125, max_input_len = 125, max_context_len = 125
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 18.92 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 126
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        66892          126            226         0
[TM][INFO] [finishRequest] slot = 0, id = 66892
[TM][INFO] [LlamaCacheManager][update] 66892
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 92465
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 12
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 105
[TM][INFO] [init] max_context_len = 105
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        92465            0        105          105            105               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 104, max_input_len = 104, max_context_len = 104
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 18.72 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 105
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        92465          105            205         0
[TM][INFO] [finishRequest] slot = 0, id = 92465
[TM][INFO] [LlamaCacheManager][update] 92465
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 41446
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 13
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 99
[TM][INFO] [init] max_context_len = 99
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        41446            0         99           99             99               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 98, max_input_len = 98, max_context_len = 98
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 18.45 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 99
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        41446           99            199         0
[TM][INFO] ------------------------- step = 100 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 41446
[TM][INFO] [LlamaCacheManager][update] 41446
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 4375
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 14
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 102
[TM][INFO] [init] max_context_len = 102
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0         4375            0        102          102            102               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 101, max_input_len = 101, max_context_len = 101
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 18.54 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 102
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0         4375          102            202         0
[TM][INFO] [finishRequest] slot = 0, id = 4375
[TM][INFO] [LlamaCacheManager][update] 4375
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 755
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 15
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 67
[TM][INFO] [init] max_context_len = 67
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0          755            0         67           67             67               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 66, max_input_len = 66, max_context_len = 66
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 16.12 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 67
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0          755           67            167         0
[TM][INFO] [finishRequest] slot = 0, id = 755
[TM][INFO] [LlamaCacheManager][update] 755
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 85042
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 16
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 116
[TM][INFO] [init] max_context_len = 116
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        85042            0        116          116            116               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 115, max_input_len = 115, max_context_len = 115
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 18.69 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 116
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        85042          116            216         0
[TM][INFO] [finishRequest] slot = 0, id = 85042
[TM][INFO] [LlamaCacheManager][update] 85042
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 85262
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 17
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 69
[TM][INFO] [init] max_context_len = 69
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        85262            0         69           69             69               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 68, max_input_len = 68, max_context_len = 68
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 15.23 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 69
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        85262           69            169         0
[TM][INFO] ------------------------- step = 70 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 85262
[TM][INFO] [LlamaCacheManager][update] 85262
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 71975
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 18
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 83
[TM][INFO] [init] max_context_len = 83
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        71975            0         83           83             83               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 82, max_input_len = 82, max_context_len = 82
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 17.01 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 83
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        71975           83            183         0
[TM][INFO] [finishRequest] slot = 0, id = 71975
[TM][INFO] [LlamaCacheManager][update] 71975
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 90400
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 19
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 118
[TM][INFO] [init] max_context_len = 118
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        90400            0        118          118            118               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 117, max_input_len = 117, max_context_len = 117
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 18.68 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 118
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        90400          118            218         0
[TM][INFO] ------------------------- step = 120 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 90400
[TM][INFO] [LlamaCacheManager][update] 90400
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 16473
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 20
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 165
[TM][INFO] [init] max_context_len = 165
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        16473            0        165          165            165               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 164, max_input_len = 164, max_context_len = 164
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 23.42 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 165
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        16473          165            265         0
[TM][INFO] [finishRequest] slot = 0, id = 16473
[TM][INFO] [LlamaCacheManager][update] 16473
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 29773
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 21
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 167
[TM][INFO] [init] max_context_len = 167
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        29773            0        167          167            167               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 166, max_input_len = 166, max_context_len = 166
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 23.48 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 167
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        29773          167            267         0
[TM][INFO] ------------------------- step = 170 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 29773
[TM][INFO] [LlamaCacheManager][update] 29773
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 79665
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 22
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 186
[TM][INFO] [init] max_context_len = 186
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        79665            0        186          186            186               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 185, max_input_len = 185, max_context_len = 185
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 22.25 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 186
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        79665          186            286         0
[TM][INFO] [finishRequest] slot = 0, id = 79665
[TM][INFO] [LlamaCacheManager][update] 79665
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 23731
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 23
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 126
[TM][INFO] [init] max_context_len = 126
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        23731            0        126          126            126               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 125, max_input_len = 125, max_context_len = 125
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 18.84 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 126
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        23731          126            226         0
[TM][INFO] [finishRequest] slot = 0, id = 23731
[TM][INFO] [LlamaCacheManager][update] 23731
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 13154
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
[TM][INFO] [LlamaCacheManager][allocate] count = 24
[TM][INFO] [LlamaCacheManager][allocate] free = 0
[TM][INFO] [init] infer_request_count = 1
[TM][INFO] [init] batch_size = 1
[TM][INFO] [init] session_len = 2056
[TM][INFO] [init] max_input_length = 108
[TM][INFO] [init] max_context_len = 108
[TM][INFO] [init] slot  sequence_id  history_len  input_len  context_len  tmp_input_len  token_ids.size  cache_len
[TM][INFO] [init]    0        13154            0        108          108            108               0          0
[TM][INFO] [decodeContext] base = 0, count = 1
[TM][INFO] [decodeContext] offset = 0, batch_size = 1, token_num = 107, max_input_len = 107, max_context_len = 107
[TM][INFO] context decoding start
[TM][INFO] context decoding end
[TM][INFO] [decodeContext] 18.60 ms
[TM][INFO] [initGen] batch_size = 1
[TM][INFO] [initGen] max_context_len = 108
[TM][INFO] [initGen] slot  sequence_id  context_len  seq_limit_len  finished
[TM][INFO] [initGen]    0        13154          108            208         0
[TM][INFO] ------------------------- step = 110 -------------------------
[TM][INFO] [finishRequest] slot = 0, id = 13154
[TM][INFO] [LlamaCacheManager][update] 13154
[TM][INFO] [forward] Enqueue requests
[TM][INFO] [synchronize] batch_size = 0
[TM][INFO] [LlamaCacheManager][create] 43068
[TM][INFO] [forward] Wait for requests to complete ...
[TM][INFO] [LlamaCacheManager][allocate]
[TM][INFO] [LlamaCacheManager][allocate] malloc 1
terminate called after throwing an instance of 'std::runtime_error'
  what():  [TM][ERROR] CUDA runtime error: out of memory /lmdeploy/src/turbomind/utils/allocator.h:223 

ERROR:torch.distributed.elastic.multiprocessing.api:failed (exitcode: -6) local_rank: 0 (pid: 713111) of binary: /data/home/xxx/.conda/envs/opencompass/bin/python
Traceback (most recent call last):
  File "/data/home/xxx/.conda/envs/opencompass/bin/torchrun", line 8, in <module>
    sys.exit(main())
  File "/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 346, in wrapper
    return f(*args, **kwargs)
  File "/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/torch/distributed/run.py", line 762, in main
    run(args)
  File "/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/torch/distributed/run.py", line 753, in run
    elastic_launch(
  File "/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 132, in __call__
    return launch_agent(self._config, self._entrypoint, list(args))
  File "/data/home/xxx/.conda/envs/opencompass/lib/python3.10/site-packages/torch/distributed/launcher/api.py", line 246, in launch_agent
    raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError: 
============================================================
/data/home/xxx/examples/opencompass/opencompass/tasks/openicl_infer.py FAILED
------------------------------------------------------------
Failures:
  <NO_OTHER_FAILURES>
------------------------------------------------------------
Root Cause (first observed failure):
[0]:
  time      : 2023-09-05_11:39:54
  host      : node3
  rank      : 0 (local_rank: 0)
  exitcode  : -6 (pid: 713111)
  error_file: <N/A>
  traceback : Signal 6 (SIGABRT) received by PID 713111
============================================================

```


Discussion:
Thanks for the bug report, LMDeploy team is working hard on handling this problem. 
@tonysy you can assign this issue to me.

@amulil The reason for this issue is that turbomind didn't handle the occupied resources when the instances were destroyed.

Resolving patch:
diff --git a/configs/eval_internlm_chat_7b_turbomind.py b/configs/eval_internlm_chat_7b_turbomind.py
--- a/configs/eval_internlm_chat_7b_turbomind.py
+++ b/configs/eval_internlm_chat_7b_turbomind.py
@@ -3,30 +3,30 @@
 
 with read_base():
     # choose a list of datasets
-    from .datasets.SuperGLUE_CB.SuperGLUE_CB_gen import CB_datasets
+    from .datasets.gsm8k.gsm8k_gen_1d7fe4 import gsm8k_datasets
     # and output the results in a choosen format
     from .summarizers.medium import summarizer
 
-datasets = [*CB_datasets]
+datasets = [*gsm8k_datasets]
 
 
-_meta_template = dict(
+meta_template = dict(
     round=[
-        dict(role='HUMAN', begin='
:', end='<eoh>\n'),
+        dict(role='HUMAN', begin='
:', end='\n'),
         dict(role='BOT', begin='
:', end='<eoa>\n', generate=True),
     ],
-)
+    eos_token_id=103028)
 
 models = [
     dict(
         type=TurboMindModel,
-        abbr='internlm-chat-7b-tb',
+        abbr='internlm-chat-7b-turbomind',
         path="internlm-chat-7b",
-        model_path='./workspace',
+        tis_addr='0.0.0.0:33337',
         max_out_len=100,
         max_seq_len=2048,
         batch_size=16,
-        meta_template=_meta_template,
+        meta_template=meta_template,
         run_cfg=dict(num_gpus=1, num_procs=1),
     )
 ]
diff --git a/opencompass/models/turbomind.py b/opencompass/models/turbomind.py
--- a/opencompass/models/turbomind.py
+++ b/opencompass/models/turbomind.py
@@ -1,10 +1,11 @@
-import os.path as osp
-import random
+import logging
+import threading
 from concurrent.futures import ThreadPoolExecutor
 from typing import Dict, List, Optional, Union
 
-from opencompass.models.base import BaseModel
-from opencompass.models.base_api import TokenBucket
+from lmdeploy.serve.turbomind.chatbot import Chatbot
+
+from opencompass.models.base import BaseModel, LMTemplateParser
 from opencompass.utils.logging import get_logger
 from opencompass.utils.prompt import PromptList
 
@@ -43,10 +44,8 @@ class TurboMindModel(BaseModel):
     def __init__(
         self,
         path: str,
-        model_path: str,
+        tis_addr: str = '0.0.0.0:33337',
         max_seq_len: int = 2048,
-        query_per_second: int = 1,
-        retry: int = 2,
         meta_template: Optional[Dict] = None,
     ):
 
@@ -54,27 +53,17 @@ def __init__(
                          max_seq_len=max_seq_len,
                          meta_template=meta_template)
         self.logger = get_logger()
-
-        from lmdeploy import turbomind as tm
-        from lmdeploy.model import MODELS as LMMODELS
-        from lmdeploy.turbomind.tokenizer import Tokenizer as LMTokenizer
-
-        self.retry = retry
-
-        tokenizer_model_path = osp.join(model_path, 'triton_models',
-                                        'tokenizer')
-        self.tokenizer = LMTokenizer(tokenizer_model_path)
-        tm_model = tm.TurboMind(model_path, eos_id=self.tokenizer.eos_token_id)
-        self.model_name = tm_model.model_name
-        self.model = LMMODELS.get(self.model_name)()
-        self.generator = tm_model.create_instance()
-        self.token_bucket = TokenBucket(query_per_second)
+        self.template_parser = LMTemplateParser(meta_template)
+        self.eos_token_id = None
+        if meta_template and 'eos_token_id' in meta_template:
+            self.eos_token_id = meta_template['eos_token_id']
+        self.tis_addr = tis_addr
 
     def generate(
         self,
         inputs: List[str or PromptList],
         max_out_len: int = 512,
-        temperature: float = 0.0,
+        temperature: float = 1.0,
     ) -> List[str]:
         """Generate results given a list of inputs.
 
@@ -91,10 +80,10 @@ def generate(
         Returns:
             List[str]: A list of generated strings.
         """
-        prompts = inputs
+
         with ThreadPoolExecutor() as executor:
             results = list(
-                executor.map(self._generate, prompts,
+                executor.map(self._generate, inputs,
                              [max_out_len] * len(inputs),
                              [temperature] * len(inputs)))
         return results
@@ -106,12 +95,12 @@ def wait(self):
         """
         return self.token_bucket.get_token()
 
-    def _generate(self, input: str or PromptList, max_out_len: int,
+    def _generate(self, prompt: str or PromptList, max_out_len: int,
                   temperature: float) -> str:
         """Generate results given a list of inputs.
 
         Args:
-            inputs (str or PromptList): A string or PromptDict.
+            prompt (str or PromptList): A string or PromptDict.
                 The PromptDict should be organized in OpenCompass'
                 API format.
             max_out_len (int): The maximum length of the output.
@@ -123,39 +112,21 @@ def _generate(self, input: str or PromptList, max_out_len: int,
         Returns:
             str: The generated string.
         """
-        assert isinstance(input, (str, PromptList))
-
         assert type(
-            input
-        ) is str, 'We only support string for TurboMind Python API now'
-
-        intput_token_ids = self.tokenizer.encode(input)
-
-        for _ in range(self.retry):
-            self.wait()
-            session_id = random.randint(1, 100000)
-            nth_round = 0
-            for outputs in self.generator.stream_infer(
-                    session_id=session_id,
-                    input_ids=[intput_token_ids],
-                    stream_output=False,
-                    request_output_len=max_out_len,
-                    sequence_start=(nth_round == 0),
-                    sequence_end=False,
-                    step=0,
-                    stop=False,
-                    top_k=40,
-                    top_p=0.8,
-                    temperature=temperature,
-                    repetition_penalty=1.0,
-                    ignore_eos=False,
-                    random_seed=random.getrandbits(64)
-                    if nth_round == 0 else None):
-                pass
-
-        output_token_ids, _ = outputs[0]
-        # decode output_token_ids
-        response = self.tokenizer.decode(output_token_ids)
-        response = valid_str(response)
-
+            prompt) is str, 'We only support string for TurboMind RPC API'
+        chatbot = Chatbot(self.tis_addr,
+                          temperature=temperature,
+                          capability='completion',
+                          top_k=1,
+                          log_level=logging.ERROR)
+
+        for status, text, n_token in chatbot.stream_infer(
+                session_id=threading.currentThread().ident,
+                prompt=prompt,
+                request_output_len=max_out_len,
+                sequence_start=True,
+                sequence_end=True):
+            continue
+        response = valid_str(text)
+        response = response.replace('<eoa>', '')
         return response


Test patch: