update
Browse files- config.json +6 -5
- configuration_lmdeploy.py +3 -1
- modeling_lmdeploy.py +9 -6
config.json
CHANGED
@@ -1,11 +1,11 @@
|
|
1 |
{
|
2 |
"architectures": [
|
3 |
-
"
|
4 |
],
|
5 |
"auto_map": {
|
6 |
-
"AutoConfig": "configuration_lmdeploy.
|
7 |
-
"AutoModel": "modeling_lmdeploy.
|
8 |
-
"AutoModelForCausalLM": "modeling_lmdeploy.
|
9 |
},
|
10 |
"turbomind": {
|
11 |
"model_name": "internlm-chat-20b",
|
@@ -35,5 +35,6 @@
|
|
35 |
"max_position_embeddings": 2048,
|
36 |
"use_dynamic_ntk": 0,
|
37 |
"use_logn_attn": 0
|
38 |
-
}
|
|
|
39 |
}
|
|
|
1 |
{
|
2 |
"architectures": [
|
3 |
+
"LMDeployForCausalLM"
|
4 |
],
|
5 |
"auto_map": {
|
6 |
+
"AutoConfig": "configuration_lmdeploy.LMDeployConfig",
|
7 |
+
"AutoModel": "modeling_lmdeploy.LMDeployForCausalLM",
|
8 |
+
"AutoModelForCausalLM": "modeling_lmdeploy.LMDeployForCausalLM"
|
9 |
},
|
10 |
"turbomind": {
|
11 |
"model_name": "internlm-chat-20b",
|
|
|
35 |
"max_position_embeddings": 2048,
|
36 |
"use_dynamic_ntk": 0,
|
37 |
"use_logn_attn": 0
|
38 |
+
},
|
39 |
+
"lmdeploy_version": "0.0.14"
|
40 |
}
|
configuration_lmdeploy.py
CHANGED
@@ -7,7 +7,8 @@ from lmdeploy.turbomind.deploy.target_model.base import TurbomindModelConfig
|
|
7 |
from lmdeploy.version import __version__ as lm_version
|
8 |
|
9 |
|
10 |
-
class
|
|
|
11 |
|
12 |
def __init__(self, turbomind: dict = None, **kwargs):
|
13 |
default_tm_cfg = copy.deepcopy(
|
@@ -33,3 +34,4 @@ class LmdeployConfig(PretrainedConfig):
|
|
33 |
return config, kwargs
|
34 |
else:
|
35 |
return config
|
|
|
|
7 |
from lmdeploy.version import __version__ as lm_version
|
8 |
|
9 |
|
10 |
+
class LMDeployConfig(PretrainedConfig):
|
11 |
+
"""Lmdeploy config."""
|
12 |
|
13 |
def __init__(self, turbomind: dict = None, **kwargs):
|
14 |
default_tm_cfg = copy.deepcopy(
|
|
|
34 |
return config, kwargs
|
35 |
else:
|
36 |
return config
|
37 |
+
|
modeling_lmdeploy.py
CHANGED
@@ -7,14 +7,15 @@ from itertools import count
|
|
7 |
from queue import Queue
|
8 |
from typing import List, Optional, Tuple, Union
|
9 |
|
|
|
10 |
from transformers import PretrainedConfig
|
11 |
from transformers.modeling_utils import PreTrainedModel
|
12 |
from transformers.utils import logging
|
13 |
|
14 |
from lmdeploy.turbomind import TurboMind
|
15 |
-
from lmdeploy.turbomind.utils import
|
16 |
|
17 |
-
from .configuration_lmdeploy import
|
18 |
|
19 |
logger = logging.get_logger(__name__)
|
20 |
|
@@ -55,11 +56,11 @@ class Session:
|
|
55 |
return self._error
|
56 |
|
57 |
|
58 |
-
class
|
59 |
-
config_class =
|
60 |
|
61 |
def __init__(self,
|
62 |
-
config:
|
63 |
*inputs,
|
64 |
model_path: str = None,
|
65 |
**kwargs):
|
@@ -90,7 +91,7 @@ class LmdeployForCausalLM(PreTrainedModel):
|
|
90 |
if os.path.isdir(pretrained_model_name_or_path):
|
91 |
local_folder = pretrained_model_name_or_path
|
92 |
else:
|
93 |
-
local_folder =
|
94 |
pretrained_model_name_or_path,
|
95 |
revision=revision,
|
96 |
cache_dir=cache_dir,
|
@@ -137,6 +138,7 @@ class LmdeployForCausalLM(PreTrainedModel):
|
|
137 |
sequence_end=False,
|
138 |
stop=True):
|
139 |
pass
|
|
|
140 |
finally:
|
141 |
self.que.put(generator)
|
142 |
|
@@ -222,3 +224,4 @@ class LmdeployForCausalLM(PreTrainedModel):
|
|
222 |
session._step = _step + response_size
|
223 |
|
224 |
yield response, session
|
|
|
|
7 |
from queue import Queue
|
8 |
from typing import List, Optional, Tuple, Union
|
9 |
|
10 |
+
from huggingface_hub import snapshot_download
|
11 |
from transformers import PretrainedConfig
|
12 |
from transformers.modeling_utils import PreTrainedModel
|
13 |
from transformers.utils import logging
|
14 |
|
15 |
from lmdeploy.turbomind import TurboMind
|
16 |
+
from lmdeploy.turbomind.utils import get_gen_param
|
17 |
|
18 |
+
from .configuration_lmdeploy import LMDeployConfig
|
19 |
|
20 |
logger = logging.get_logger(__name__)
|
21 |
|
|
|
56 |
return self._error
|
57 |
|
58 |
|
59 |
+
class LMDeployForCausalLM(PreTrainedModel):
|
60 |
+
config_class = LMDeployConfig
|
61 |
|
62 |
def __init__(self,
|
63 |
+
config: LMDeployConfig,
|
64 |
*inputs,
|
65 |
model_path: str = None,
|
66 |
**kwargs):
|
|
|
91 |
if os.path.isdir(pretrained_model_name_or_path):
|
92 |
local_folder = pretrained_model_name_or_path
|
93 |
else:
|
94 |
+
local_folder = snapshot_download(
|
95 |
pretrained_model_name_or_path,
|
96 |
revision=revision,
|
97 |
cache_dir=cache_dir,
|
|
|
138 |
sequence_end=False,
|
139 |
stop=True):
|
140 |
pass
|
141 |
+
session._error = 1
|
142 |
finally:
|
143 |
self.que.put(generator)
|
144 |
|
|
|
224 |
session._step = _step + response_size
|
225 |
|
226 |
yield response, session
|
227 |
+
|