Repository: InternLM/lmdeploy

Issue:
[Bug] Api Server长时间大批量推理后出现降速现象，最终出现CUDA runtime error: out of memory
### Checklist

- [X] 1. I have searched related issues but cannot get the expected help.
- [X] 2. The bug has not been fixed in the latest version.

### Describe the bug

通过最新的docker镜像启动一个api server，客户端使用api进行大批量、长时间的推理调用（总量20万次推理左右）。在经过大约10万次推理以后可观察到推理速度由最初的平均7it/s降至1it/s左右。
此时通过nvidia-smi观察显存占用，发现与调用开始时并无明显变化。
继续推理一段时间后，api server会报CUDA runtime error: out of memory /opt/lmdeploy/src/turbomind/utils/allocator.h:231 错误，然后可观察到服务重启，推理速度恢复正常。
推理速度降至1it/s时的显存状态：
![屏幕截图 2024-02-27 105210](https://github.com/InternLM/lmdeploy/assets/3341682/3126e5be-5b03-4155-87b2-f021a03255c6)


### Reproduction

参考 #636 实现了batch调用chat_completions_v1接口的方法，客户端通过该方法调用大批量的推理任务：
```python
import asyncio
from tqdm.asyncio import tqdm
from typing import Union, List, Dict, Optional
from dataclasses import dataclass, field
from lmdeploy.serve.openai.api_client import APIClient
from loguru import logger

@dataclass
class SingleChatReq:
    messages: Union[str, List[Dict[str, str]]]
    temperature: Optional[float] = 0.7
    top_p: Optional[float] = 1.0
    n: Optional[int] = 1
    max_tokens: Optional[int] = 512
    stop: Optional[Union[str, List[str]]] = None
    # stream: Optional[bool] = False  ## not supported yet
    presence_penalty: Optional[float] = 0.0
    frequency_penalty: Optional[float] = 0.0
    # user: Optional[str] = None  ## useless
    repetition_penalty: Optional[float] = 1.0
    # session_id: Optional[int] = -1  ## not supported yet
    ignore_eos: Optional[bool] = False
    skip_special_tokens: Optional[bool] = True
    extra_kwargs: Optional[dict] = field(default_factory=dict)


class ExtendBatchAPIClient(APIClient):
    def chat_completions_v1_batch(self,
                                  model: str,
                                  chat_requests: List[SingleChatReq],
                                  max_retries: int = 5) -> list:
        assert max_retries >= 0
        batch_size = len(chat_requests)
        batch_arr = chat_requests
        outputs = [""] * batch_size

        async def _try_req(single_req: SingleChatReq):
            try_count = 0
            while try_count <= max_retries:
                try:
                    response = await asyncio.to_thread(
                        lambda: next(
                            self.chat_completions_v1(
                                model=model,
                                messages=single_req.messages,
                                temperature=single_req.temperature,
                                top_p=single_req.top_p,
                                n=single_req.n,
                                max_tokens=single_req.max_tokens,
                                stop=single_req.stop,
                                presence_penalty=single_req.presence_penalty,
                                frequency_penalty=single_req.frequency_penalty,
                                repetition_penalty=single_req.repetition_penalty,
                                ignore_eos=single_req.ignore_eos,
                                skip_special_tokens=single_req.skip_special_tokens,
                                **single_req.extra_kwargs
                            )
                        )
                    )
                    return response
                except Exception as e:
                    try_count += 1
                    logger.warning(f"Request error, retry count: {try_count}. Exception: {e}")
                    if try_count <= max_retries:
                        await asyncio.sleep(1)
            raise RuntimeError("Max retries exceeded")

        async def get_response(_id):
            single_req = batch_arr[_id]
            response = await _try_req(single_req)
            outputs[_id] = response

        async def gather():
            tasks = [asyncio.ensure_future(get_response(_id)) for _id in range(batch_size)]
            try:
                await tqdm.gather(*tasks)
            except Exception as e:
                for t in tasks:
                    t.cancel()
                raise e

        asyncio.get_event_loop().run_until_complete(gather())
        return outputs
```

### Environment

```Shell
sys.platform: linux
Python: 3.8.10 (default, Nov 22 2023, 10:22:35) [GCC 9.4.0]
CUDA available: True
MUSA available: False
numpy_random_seed: 2147483648
GPU 0: NVIDIA GeForce RTX 4090 D
CUDA_HOME: /usr/local/cuda
NVCC: Cuda compilation tools, release 11.8, V11.8.89
GCC: x86_64-linux-gnu-gcc (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0
PyTorch: 2.1.0+cu118
PyTorch compiling details: PyTorch built with:
  - GCC 9.3
  - C++ Version: 201703
  - Intel(R) oneAPI Math Kernel Library Version 2022.2-Product Build 20220804 for Intel(R) 64 architecture applications
  - Intel(R) MKL-DNN v3.1.1 (Git Hash 64f6bcbcbab628e96f33a62c3e975f8535a7bde4)
  - OpenMP 201511 (a.k.a. OpenMP 4.5)
  - LAPACK is enabled (usually provided by MKL)
  - NNPACK is enabled
  - CPU capability usage: AVX2
  - CUDA Runtime 11.8
  - NVCC architecture flags: -gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86;-gencode;arch=compute_37,code=sm_37;-gencode;arch=compute_90,code=sm_90
  - CuDNN 8.7
  - Magma 2.6.1
  - Build settings: BLAS_INFO=mkl, BUILD_TYPE=Release, CUDA_VERSION=11.8, CUDNN_VERSION=8.7.0, CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, CXX_FLAGS= -D_GLIBCXX_USE_CXX11_ABI=0 -fabi-version=11 -fvisibility-inlines-hidden -DUSE_PTHREADPOOL -DNDEBUG -DUSE_KINETO -DLIBKINETO_NOROCTRACER -DUSE_FBGEMM -DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK -DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE -O2 -fPIC -Wall -Wextra -Werror=return-type -Werror=non-virtual-dtor -Werror=bool-operation -Wnarrowing -Wno-missing-field-initializers -Wno-type-limits -Wno-array-bounds -Wno-unknown-pragmas -Wno-unused-parameter -Wno-unused-function -Wno-unused-result -Wno-strict-overflow -Wno-strict-aliasing -Wno-stringop-overflow -Wno-psabi -Wno-error=pedantic -Wno-error=old-style-cast -Wno-invalid-partial-specialization -Wno-unused-private-field -Wno-aligned-allocation-unavailable -Wno-missing-braces -fdiagnostics-color=always -faligned-new -Wno-unused-but-set-variable -Wno-maybe-uninitialized -fno-math-errno -fno-trapping-math -Werror=format -Werror=cast-function-type -Wno-stringop-overflow, LAPACK_INFO=mkl, PERF_WITH_AVX=1, PERF_WITH_AVX2=1, PERF_WITH_AVX512=1, TORCH_DISABLE_GPU_ASSERTS=ON, TORCH_VERSION=2.1.0, USE_CUDA=ON, USE_CUDNN=ON, USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, USE_GLOG=OFF, USE_MKL=ON, USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=1, USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF,

TorchVision: 0.16.0+cu118
LMDeploy: 0.2.4+24ea5dc
transformers: 4.37.1
gradio: 3.50.2
fastapi: 0.109.2
pydantic: 2.6.1
```


### Error traceback

```Shell
服务端报错时的日志：

...
INFO:     192.168.16.15:55098 - "POST /v1/chat/completions HTTP/1.1" 200 OK
INFO:     192.168.16.15:55140 - "POST /v1/chat/completions HTTP/1.1" 200 OK
INFO:     192.168.16.15:55062 - "POST /v1/chat/completions HTTP/1.1" 200 OK
INFO:     192.168.16.15:55152 - "POST /v1/chat/completions HTTP/1.1" 200 OK
terminate called after throwing an instance of 'std::runtime_error'
  what():  [TM][ERROR] CUDA runtime error: out of memory /opt/lmdeploy/src/turbomind/utils/allocator.h:231 


=============================
== Triton Inference Server ==
=============================

NVIDIA Release 22.12 (build 50109463)
Triton Server Version 2.29.0

Copyright (c) 2018-2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.

Various files include modifications (c) NVIDIA CORPORATION & AFFILIATES.  All rights reserved.

This container image and its contents are governed by the NVIDIA Deep Learning Container License.
By pulling and using the container, you accept the terms and conditions of this license:
https://developer.nvidia.com/ngc/nvidia-deep-learning-container-license

2024-02-26 10:24:00,949 - lmdeploy - WARNING - Best matched chat template name: internlm2-chat-7b
2024-02-26 10:24:00,968 - lmdeploy - WARNING - model_source: hf_model
2024-02-26 10:24:02,600 - lmdeploy - WARNING - model_config:

[llama]
model_name = internlm2-chat-7b
tensor_para_size = 1
head_num = 32
kv_head_num = 8
vocab_size = 92544
num_layer = 32
inter_size = 14336
norm_eps = 1e-05
attn_bias = 0
start_id = 1
end_id = 2
session_len = 32768
weight_type = fp16
rotary_embedding = 128
rope_theta = 1000000.0
size_per_head = 128
group_size = 0
max_batch_size = 128
max_context_token_num = 1
step_length = 1
cache_max_entry_count = 0.2
cache_block_seq_len = 128
cache_chunk_size = -1
num_tokens_per_iter = 0
max_prefill_iters = 1
extra_tokens_per_iter = 0
use_context_fmha = 1
quant_policy = 0
max_position_embeddings = 32768
rope_scaling_factor = 0.0
use_logn_attn = 0


2024-02-26 10:24:02,760 - lmdeploy - WARNING - get 259 model params
/opt/py38/lib/python3.8/site-packages/fuzzywuzzy/fuzz.py:11: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning
  warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')
```
```



Discussion:
补充信息：
模型：internlm2-chat-7b
cache_max_entry_count：0.2
temperature：0.1
top_p：0.8
感觉像是 #1179 解决的问题。原本可能有请求的响应没有结束，会一直占用，导致最后能用于响应请求的实例减少。
> 感觉像是 #1179 解决的问题。原本可能有请求的响应没有结束，会一直占用，导致最后能用于响应请求的实例减少。

从现象上来看似乎就是这个问题，在我实际的测试中最后有15个请求block住了，我用更新后的版本测试一下
> 感觉像是 #1179 解决的问题。原本可能有请求的响应没有结束，会一直占用，导致最后能用于响应请求的实例减少。

用main分支重新制作镜像后进行测试，block的问题不再出现了，但依然在17w左右次请求后服务重启。
```
lmdeploy_server_deploy-api_server-1  | INFO:     192.168.16.15:58614 - "POST /v1/chat/completions HTTP/1.1" 200 OK
lmdeploy_server_deploy-api_server-1  | INFO:     192.168.16.15:58576 - "POST /v1/chat/completions HTTP/1.1" 200 OK
lmdeploy_server_deploy-api_server-1  | INFO:     192.168.16.15:58630 - "POST /v1/chat/completions HTTP/1.1" 200 OK
lmdeploy_server_deploy-api_server-1  | INFO:     192.168.16.15:58604 - "POST /v1/chat/completions HTTP/1.1" 200 OK
lmdeploy_server_deploy-api_server-1  | INFO:     192.168.16.15:58648 - "POST /v1/chat/completions HTTP/1.1" 200 OK
lmdeploy_server_deploy-api_server-1  | INFO:     192.168.16.15:58638 - "POST /v1/chat/completions HTTP/1.1" 200 OK
lmdeploy_server_deploy-api_server-1  | INFO:     192.168.16.15:58644 - "POST /v1/chat/completions HTTP/1.1" 200 OK
lmdeploy_server_deploy-api_server-1  | terminate called after throwing an instance of 'std::bad_function_call'
lmdeploy_server_deploy-api_server-1  |   what():  bad_function_call
lmdeploy_server_deploy-api_server-1  | 
lmdeploy_server_deploy-api_server-1  | =============================
lmdeploy_server_deploy-api_server-1  | == Triton Inference Server ==
lmdeploy_server_deploy-api_server-1  | =============================
lmdeploy_server_deploy-api_server-1  | 
lmdeploy_server_deploy-api_server-1  | NVIDIA Release 22.12 (build 50109463)
lmdeploy_server_deploy-api_server-1  | Triton Server Version 2.29.0
lmdeploy_server_deploy-api_server-1  | 
lmdeploy_server_deploy-api_server-1  | Copyright (c) 2018-2022, NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
lmdeploy_server_deploy-api_server-1  | 
lmdeploy_server_deploy-api_server-1  | Various files include modifications (c) NVIDIA CORPORATION & AFFILIATES.  All rights reserved.
lmdeploy_server_deploy-api_server-1  | 
lmdeploy_server_deploy-api_server-1  | This container image and its contents are governed by the NVIDIA Deep Learning Container License.
lmdeploy_server_deploy-api_server-1  | By pulling and using the container, you accept the terms and conditions of this license:
lmdeploy_server_deploy-api_server-1  | https://developer.nvidia.com/ngc/nvidia-deep-learning-container-license
lmdeploy_server_deploy-api_server-1  | 
lmdeploy_server_deploy-api_server-1  | 2024-02-27 19:14:58,300 - lmdeploy - WARNING - Best matched chat template name: internlm2-chat-7b
lmdeploy_server_deploy-api_server-1  | 2024-02-27 19:14:58,320 - lmdeploy - WARNING - model_source: hf_model
lmdeploy_server_deploy-api_server-1  | 2024-02-27 19:14:59,974 - lmdeploy - WARNING - model_config:
lmdeploy_server_deploy-api_server-1  | 
lmdeploy_server_deploy-api_server-1  | [llama]
lmdeploy_server_deploy-api_server-1  | model_name = internlm2-chat-7b
lmdeploy_server_deploy-api_server-1  | tensor_para_size = 1
lmdeploy_server_deploy-api_server-1  | head_num = 32
lmdeploy_server_deploy-api_server-1  | kv_head_num = 8
lmdeploy_server_deploy-api_server-1  | vocab_size = 92544
lmdeploy_server_deploy-api_server-1  | num_layer = 32
lmdeploy_server_deploy-api_server-1  | inter_size = 14336
lmdeploy_server_deploy-api_server-1  | norm_eps = 1e-05
lmdeploy_server_deploy-api_server-1  | attn_bias = 0
lmdeploy_server_deploy-api_server-1  | start_id = 1
lmdeploy_server_deploy-api_server-1  | end_id = 2
lmdeploy_server_deploy-api_server-1  | session_len = 32768
lmdeploy_server_deploy-api_server-1  | weight_type = fp16
lmdeploy_server_deploy-api_server-1  | rotary_embedding = 128
lmdeploy_server_deploy-api_server-1  | rope_theta = 1000000.0
lmdeploy_server_deploy-api_server-1  | size_per_head = 128
lmdeploy_server_deploy-api_server-1  | group_size = 0
lmdeploy_server_deploy-api_server-1  | max_batch_size = 128
lmdeploy_server_deploy-api_server-1  | max_context_token_num = 1
lmdeploy_server_deploy-api_server-1  | step_length = 1
lmdeploy_server_deploy-api_server-1  | cache_max_entry_count = 0.2
lmdeploy_server_deploy-api_server-1  | cache_block_seq_len = 128
lmdeploy_server_deploy-api_server-1  | cache_chunk_size = -1
lmdeploy_server_deploy-api_server-1  | num_tokens_per_iter = 8192
lmdeploy_server_deploy-api_server-1  | max_prefill_iters = 4
lmdeploy_server_deploy-api_server-1  | extra_tokens_per_iter = 0
lmdeploy_server_deploy-api_server-1  | use_context_fmha = 1
lmdeploy_server_deploy-api_server-1  | quant_policy = 0
lmdeploy_server_deploy-api_server-1  | max_position_embeddings = 32768
lmdeploy_server_deploy-api_server-1  | rope_scaling_factor = 0.0
lmdeploy_server_deploy-api_server-1  | use_logn_attn = 0
lmdeploy_server_deploy-api_server-1  | 
lmdeploy_server_deploy-api_server-1  | 
lmdeploy_server_deploy-api_server-1  | 2024-02-27 19:15:00,130 - lmdeploy - WARNING - get 259 model params
lmdeploy_server_deploy-api_server-1  | /opt/py38/lib/python3.8/site-packages/fuzzywuzzy/fuzz.py:11: UserWarning: Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning
lmdeploy_server_deploy-api_server-1  |   warnings.warn('Using slow pure-python SequenceMatcher. Install python-Levenshtein to remove this warning')
```

OK。也就是main分支性能保持不变了，但是会最后程序挂了是吗？可以设置下日志等级不？调成 `--log-level DEBUG`。最好可以分享下具体请求的内容。
因为数据集比较大，不太方便直接放上来，我正在尝试一个简单的测试用例，如果可以复现就把脚本放上来。不过在检查结果的时候发现了一些奇怪的现象：
![image](https://github.com/InternLM/lmdeploy/assets/3341682/68882dba-2e29-4016-9c14-7c2e7c28ed06)
我们的prompt希望模型最终以json格式返回，我过滤出了无法解析的样本，`llm_response`是每个请求response里的内容，`text`, `product_type`, `aspect`是每个请求拼接至prompt里的内容，可以发现这些错误样本的response内容有不少与inputs无关，甚至看起来像是其它request的response。总共样本量在20万左右，这种情况的样本大概出现了250个，发生的机率并不是很高。
然后我拿这个样本的inputs单独推理了一遍，可以得到正确的response：
`{"label": "未提及", "reason": "新闻原文中没有直接提及小麦的出口量，而是讨论了小麦产量和阿根廷政府的改革承诺。"}`
所以看起来应该还是在某个地方存在线程安全的问题？
这些错误样本的内容在其他回答里有出现吗？可以重点关注下这个
确实有，找到个很典型的样例，其中一个正确的样本：
```
{
  'text': '此外，受到Sviblov制裁影响的还包括一家正在开发全球最大锌矿之一的公司Ozernoye。该矿山产量预计可占到全球产量的4%，被市场视为锌供应中的重要补给。但该矿山在上月突遭火灾，其产能预计推迟至2024年才能上线。即便该矿山明年投产，由于美国制裁原因，西方公司将很难与Ozernoye矿业公司进行贸易。而据报道，国际巨头托克（Trafigura）、瑞典Open Mineral等公司此前都对该矿表示出了兴趣，并一直在竞标，希望达成采购锌矿石的长期合同。'
  'product_type': '锌',
  'aspect': '出口量',
  'llm_response': '{"label": "未提及", "reason": "新闻原文中提到了Ozernoye锌矿的产能预计推迟至2024年才能上线，以及美国制裁对西方公司与Ozernoye矿业公司进行贸易的影响，但并未直接提及锌的出口量。"}'
}
```

response错误样本：
```
{
  'text': '此外，在连续几次干旱之后，明年天气好转的前景可能会使农民受益。据专家预测，厄尔尼诺现象明年将给阿根廷带来更多降雨，这将有助于提高明年6月和7月种植的小麦作物产量。',
  'product_type': '小麦',
  'aspect': '库存',
  'llm_response': '该矿山明年投产，由于美国制裁原因，西方公司将很难与Ozernoye矿业公司进行贸易。而据报道，国际巨头托克（Trafigura）、瑞典Open Mineral等公司此前都对该矿表示出了兴趣，并一直在竞标，希望达成采购锌矿石的长期合同。\n目标商品品种：锌\n目标属性：出口量 
 \n 
 assistant\n{"label": "未提及", "reason": "新闻原文中提到了Ozernoye锌矿的产能预计推迟至2024年才能上线，以及美国制裁对西方公司与Ozernoye矿业公司进行贸易的影响，但并未直接提及锌的出口量。"}'
}
```

可以明显看到，错误样本response开头的"**该矿山明年投产，由于美国制裁原因，西方公司将很难与Ozernoye矿业公司进行贸易。而据报道，国际巨头托克（Trafigura）、瑞典Open Mineral等公司此前都对该矿表示出了兴趣，并一直在竞标，希望达成采购锌矿石的长期合同。\n目标商品品种：锌\n目标属性：出口量**"就是正确样本prompt的一部分
也就是说，错误结果的这部分内容，实际上是其他问题的prompt截取出来的。这个感觉不像是 python 线程错乱产生的问题，更像是 engine 将 cache 作为结果输出了。
请问有没试过temperature = 0 的情况下， 相同的问题，但是使用模型单条问题生成的结果和在大批量推理中生成的结果不一样的情况出现，谢谢
虽然还不太熟悉enging的cache机制，但倒是发现有个地方似乎有点隐患。

首先是server接口这：
https://github.com/InternLM/lmdeploy/blob/d15fcf87499e6fcc6705dfb3799d8087f31ae7ef/lmdeploy/serve/openai/api_server.py#L293C2-L294C54

只要客户端请求时不填写session_id，那么在大量并发时势必会出现相同的session_id。
根据AsyncEngine的实现：
https://github.com/InternLM/lmdeploy/blob/d15fcf87499e6fcc6705dfb3799d8087f31ae7ef/lmdeploy/serve/async_engine.py#L283C5-L291C25
在generator耗尽时，其余请求都会阻塞在这：
https://github.com/InternLM/lmdeploy/blob/d15fcf87499e6fcc6705dfb3799d8087f31ae7ef/lmdeploy/serve/async_engine.py#L485C13-L485C68

但此时会有种情况，假设session_id = 1的请求有两个，其中一个请求`R1`正在generate，另外一个请求`R2`在wait。如果一个session_id != 1的请求结束，释放了一个generator，恰好此时R2拿到了这个generator就会结束阻塞，开始进行generate。如果我没理解错的话，这个地方可能会出现问题：
https://github.com/InternLM/lmdeploy/blob/d15fcf87499e6fcc6705dfb3799d8087f31ae7ef/lmdeploy/serve/async_engine.py#L516C2-L516C73
相当于有两个session_id同为1的请求在交替修改这个dict。

同时还会有种可能，如果R1在R2之前结束生成，根据：
https://github.com/InternLM/lmdeploy/blob/d15fcf87499e6fcc6705dfb3799d8087f31ae7ef/lmdeploy/serve/async_engine.py#L280C8-L282C1
R1会尝试归还自己的generator，但此时实际归还的是R2的。

不知道我这样理解有没有问题？
哦，如果客户端都不指定 session_id 调用的话，我内部会随机生成。这里是可能出现两个 instance 在跑一个 session_id。你那边方便试一试在客户端代码里，每个请求都赋予不同的 session_id 吗？

Resolving patch:
diff --git a/lmdeploy/serve/openai/api_server.py b/lmdeploy/serve/openai/api_server.py
--- a/lmdeploy/serve/openai/api_server.py
+++ b/lmdeploy/serve/openai/api_server.py
@@ -1,7 +1,6 @@
 # Copyright (c) OpenMMLab. All rights reserved.
 import asyncio
 import os
-import random
 import time
 from http import HTTPStatus
 from typing import AsyncGenerator, List, Literal, Optional, Union
@@ -31,6 +30,7 @@
 class VariableInterface:
     """A IO interface maintaining variables."""
     async_engine: AsyncEngine = None
+    session_id: int = 0
     api_keys: Optional[List[str]] = None
     qos_engine: QosEngine = None
     request_hosts = []
@@ -143,7 +143,8 @@ async def chat_completions_v1_qos(request: ChatCompletionRequestQos,
     - frequency_penalty (replaced with repetition_penalty)
     """
     if request.session_id == -1:
-        request.session_id = random.randint(1, 10086)
+        VariableInterface.session_id += 1
+        request.session_id = VariableInterface.session_id
     error_check_ret = await check_request(request)
     if error_check_ret is not None:
         return error_check_ret
@@ -291,7 +292,8 @@ async def chat_completions_v1(request: ChatCompletionRequest,
     - frequency_penalty (replaced with repetition_penalty)
     """
     if request.session_id == -1:
-        request.session_id = random.randint(1, 10086)
+        VariableInterface.session_id += 1
+        request.session_id = VariableInterface.session_id
     error_check_ret = await check_request(request)
     if error_check_ret is not None:
         return error_check_ret
@@ -449,7 +451,8 @@ async def completions_v1_qos(request: CompletionRequestQos,
     - frequency_penalty (replaced with repetition_penalty)
     """
     if request.session_id == -1:
-        request.session_id = random.randint(1, 10086)
+        VariableInterface.session_id += 1
+        request.session_id = VariableInterface.session_id
     error_check_ret = await check_request(request)
     if error_check_ret is not None:
         return error_check_ret
@@ -600,7 +603,8 @@ async def completions_v1(request: CompletionRequest,
     - frequency_penalty (replaced with repetition_penalty)
     """
     if request.session_id == -1:
-        request.session_id = random.randint(1, 10086)
+        VariableInterface.session_id += 1
+        request.session_id = VariableInterface.session_id
     error_check_ret = await check_request(request)
     if error_check_ret is not None:
         return error_check_ret
@@ -797,7 +801,8 @@ async def chat_interactive_v1_qos(request: GenerateRequestQos,
     - user_id (str): for qos; if not specified, will set to "default"
     """
     if request.session_id == -1:
-        request.session_id = random.randint(10087, 23333)
+        VariableInterface.session_id += 1
+        request.session_id = VariableInterface.session_id
 
     if VariableInterface.qos_engine is None:
         return create_error_response(
@@ -872,7 +877,8 @@ async def chat_interactive_v1(request: GenerateRequest,
         VariableInterface.async_engine.stop_session(request.session_id)
         return {'text': '', 'tokens': 0, 'finish_reason': None}
     if request.session_id == -1:
-        request.session_id = random.randint(10087, 23333)
+        VariableInterface.session_id += 1
+        request.session_id = VariableInterface.session_id
 
     async_engine = VariableInterface.async_engine
     sequence_start = async_engine.id2step.get(str(request.session_id), 0) == 0


Test patch: