Upload BaichuanForCausalLM
Browse files- config.json +31 -0
- configuration_baichuan.py +69 -0
- generation_config.json +14 -0
- generation_utils.py +83 -0
- modeling_baichuan.py +785 -0
- pytorch_model-00001-of-00002.bin +3 -0
- pytorch_model-00002-of-00002.bin +3 -0
- pytorch_model.bin.index.json +234 -0
- quantizer.py +210 -0
config.json
ADDED
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"_from_model_config": true,
|
3 |
+
"_name_or_path": "../../../Baichuan2-7B-Chat",
|
4 |
+
"architectures": [
|
5 |
+
"BaichuanForCausalLM"
|
6 |
+
],
|
7 |
+
"auto_map": {
|
8 |
+
"AutoConfig": "configuration_baichuan.BaichuanConfig",
|
9 |
+
"AutoModelForCausalLM": "modeling_baichuan.BaichuanForCausalLM"
|
10 |
+
},
|
11 |
+
"bos_token_id": 1,
|
12 |
+
"eos_token_id": 2,
|
13 |
+
"hidden_act": "silu",
|
14 |
+
"hidden_size": 4096,
|
15 |
+
"initializer_range": 0.02,
|
16 |
+
"intermediate_size": 11008,
|
17 |
+
"max_position_embeddings": 4096,
|
18 |
+
"model_max_length": 4096,
|
19 |
+
"model_type": "baichuan",
|
20 |
+
"num_attention_heads": 32,
|
21 |
+
"num_hidden_layers": 32,
|
22 |
+
"pad_token_id": 0,
|
23 |
+
"rms_norm_eps": 1e-06,
|
24 |
+
"tie_word_embeddings": false,
|
25 |
+
"tokenizer_class": "BaichuanTokenizer",
|
26 |
+
"torch_dtype": "bfloat16",
|
27 |
+
"transformers_version": "4.32.0",
|
28 |
+
"use_cache": true,
|
29 |
+
"vocab_size": 125696,
|
30 |
+
"z_loss_weight": 0
|
31 |
+
}
|
configuration_baichuan.py
ADDED
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Copyright 2023 Baichuan Inc. All Rights Reserved.
|
2 |
+
|
3 |
+
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
|
4 |
+
#
|
5 |
+
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
6 |
+
# and OPT implementations in this library. It has been modified from its
|
7 |
+
# original forms to accommodate minor architectural differences compared
|
8 |
+
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
9 |
+
#
|
10 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
11 |
+
# you may not use this file except in compliance with the License.
|
12 |
+
# You may obtain a copy of the License at
|
13 |
+
#
|
14 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
15 |
+
#
|
16 |
+
# Unless required by applicable law or agreed to in writing, software
|
17 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
18 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
19 |
+
# See the License for the specific language governing permissions and
|
20 |
+
# limitations under the License.
|
21 |
+
|
22 |
+
from transformers.configuration_utils import PretrainedConfig
|
23 |
+
from transformers.utils import logging
|
24 |
+
|
25 |
+
|
26 |
+
logger = logging.get_logger(__name__)
|
27 |
+
|
28 |
+
|
29 |
+
class BaichuanConfig(PretrainedConfig):
|
30 |
+
model_type = "baichuan"
|
31 |
+
keys_to_ignore_at_inference = ["past_key_values"]
|
32 |
+
|
33 |
+
def __init__(
|
34 |
+
self,
|
35 |
+
vocab_size=125696,
|
36 |
+
hidden_size=4096,
|
37 |
+
intermediate_size=11008,
|
38 |
+
num_hidden_layers=32,
|
39 |
+
num_attention_heads=32,
|
40 |
+
hidden_act="silu",
|
41 |
+
max_position_embeddings=4096,
|
42 |
+
initializer_range=0.02,
|
43 |
+
rms_norm_eps=1e-6,
|
44 |
+
use_cache=True,
|
45 |
+
pad_token_id=0,
|
46 |
+
bos_token_id=1,
|
47 |
+
eos_token_id=2,
|
48 |
+
tie_word_embeddings=False,
|
49 |
+
z_loss_weight=0,
|
50 |
+
**kwargs,
|
51 |
+
):
|
52 |
+
self.vocab_size = vocab_size
|
53 |
+
self.max_position_embeddings = max_position_embeddings
|
54 |
+
self.hidden_size = hidden_size
|
55 |
+
self.intermediate_size = intermediate_size
|
56 |
+
self.num_hidden_layers = num_hidden_layers
|
57 |
+
self.num_attention_heads = num_attention_heads
|
58 |
+
self.hidden_act = hidden_act
|
59 |
+
self.initializer_range = initializer_range
|
60 |
+
self.rms_norm_eps = rms_norm_eps
|
61 |
+
self.use_cache = use_cache
|
62 |
+
self.z_loss_weight = z_loss_weight
|
63 |
+
super().__init__(
|
64 |
+
pad_token_id=pad_token_id,
|
65 |
+
bos_token_id=bos_token_id,
|
66 |
+
eos_token_id=eos_token_id,
|
67 |
+
tie_word_embeddings=tie_word_embeddings,
|
68 |
+
**kwargs,
|
69 |
+
)
|
generation_config.json
ADDED
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"assistant_token_id": 196,
|
3 |
+
"bos_token_id": 1,
|
4 |
+
"do_sample": true,
|
5 |
+
"eos_token_id": 2,
|
6 |
+
"max_new_tokens": 2048,
|
7 |
+
"pad_token_id": 0,
|
8 |
+
"repetition_penalty": 1.05,
|
9 |
+
"temperature": 0.3,
|
10 |
+
"top_k": 5,
|
11 |
+
"top_p": 0.85,
|
12 |
+
"transformers_version": "4.32.0",
|
13 |
+
"user_token_id": 195
|
14 |
+
}
|
generation_utils.py
ADDED
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from typing import List
|
2 |
+
from queue import Queue
|
3 |
+
|
4 |
+
import torch
|
5 |
+
|
6 |
+
|
7 |
+
def build_chat_input(model, tokenizer, messages: List[dict], max_new_tokens: int=0):
|
8 |
+
def _parse_messages(messages, split_role="user"):
|
9 |
+
system, rounds = "", []
|
10 |
+
round = []
|
11 |
+
for i, message in enumerate(messages):
|
12 |
+
if message["role"] == "system":
|
13 |
+
assert i == 0
|
14 |
+
system = message["content"]
|
15 |
+
continue
|
16 |
+
if message["role"] == split_role and round:
|
17 |
+
rounds.append(round)
|
18 |
+
round = []
|
19 |
+
round.append(message)
|
20 |
+
if round:
|
21 |
+
rounds.append(round)
|
22 |
+
return system, rounds
|
23 |
+
|
24 |
+
max_new_tokens = max_new_tokens or model.generation_config.max_new_tokens
|
25 |
+
max_input_tokens = model.config.model_max_length - max_new_tokens
|
26 |
+
system, rounds = _parse_messages(messages, split_role="user")
|
27 |
+
system_tokens = tokenizer.encode(system)
|
28 |
+
max_history_tokens = max_input_tokens - len(system_tokens)
|
29 |
+
|
30 |
+
history_tokens = []
|
31 |
+
for round in rounds[::-1]:
|
32 |
+
round_tokens = []
|
33 |
+
for message in round:
|
34 |
+
if message["role"] == "user":
|
35 |
+
round_tokens.append(model.generation_config.user_token_id)
|
36 |
+
else:
|
37 |
+
round_tokens.append(model.generation_config.assistant_token_id)
|
38 |
+
round_tokens.extend(tokenizer.encode(message["content"]))
|
39 |
+
if len(history_tokens) == 0 or len(history_tokens) + len(round_tokens) <= max_history_tokens:
|
40 |
+
history_tokens = round_tokens + history_tokens # concat left
|
41 |
+
if len(history_tokens) < max_history_tokens:
|
42 |
+
continue
|
43 |
+
break
|
44 |
+
|
45 |
+
input_tokens = system_tokens + history_tokens
|
46 |
+
if messages[-1]["role"] != "assistant":
|
47 |
+
input_tokens.append(model.generation_config.assistant_token_id)
|
48 |
+
input_tokens = input_tokens[-max_input_tokens:] # truncate left
|
49 |
+
return torch.LongTensor([input_tokens]).to(model.device)
|
50 |
+
|
51 |
+
|
52 |
+
class TextIterStreamer:
|
53 |
+
def __init__(self, tokenizer, skip_prompt=False, skip_special_tokens=False):
|
54 |
+
self.tokenizer = tokenizer
|
55 |
+
self.skip_prompt = skip_prompt
|
56 |
+
self.skip_special_tokens = skip_special_tokens
|
57 |
+
self.tokens = []
|
58 |
+
self.text_queue = Queue()
|
59 |
+
self.next_tokens_are_prompt = True
|
60 |
+
|
61 |
+
def put(self, value):
|
62 |
+
if self.skip_prompt and self.next_tokens_are_prompt:
|
63 |
+
self.next_tokens_are_prompt = False
|
64 |
+
else:
|
65 |
+
if len(value.shape) > 1:
|
66 |
+
value = value[0]
|
67 |
+
self.tokens.extend(value.tolist())
|
68 |
+
self.text_queue.put(
|
69 |
+
self.tokenizer.decode(self.tokens, skip_special_tokens=self.skip_special_tokens))
|
70 |
+
|
71 |
+
def end(self):
|
72 |
+
self.text_queue.put(None)
|
73 |
+
|
74 |
+
def __iter__(self):
|
75 |
+
return self
|
76 |
+
|
77 |
+
def __next__(self):
|
78 |
+
value = self.text_queue.get()
|
79 |
+
if value is None:
|
80 |
+
raise StopIteration()
|
81 |
+
else:
|
82 |
+
return value
|
83 |
+
|
modeling_baichuan.py
ADDED
@@ -0,0 +1,785 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# Copyright 2023 Baichuan Inc. All Rights Reserved.
|
2 |
+
|
3 |
+
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
|
4 |
+
#
|
5 |
+
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
6 |
+
# and OPT implementations in this library. It has been modified from its
|
7 |
+
# original forms to accommodate minor architectural differences compared
|
8 |
+
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
9 |
+
#
|
10 |
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
11 |
+
# you may not use this file except in compliance with the License.
|
12 |
+
# You may obtain a copy of the License at
|
13 |
+
#
|
14 |
+
# http://www.apache.org/licenses/LICENSE-2.0
|
15 |
+
#
|
16 |
+
# Unless required by applicable law or agreed to in writing, software
|
17 |
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
18 |
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
19 |
+
# See the License for the specific language governing permissions and
|
20 |
+
# limitations under the License.
|
21 |
+
|
22 |
+
|
23 |
+
from .configuration_baichuan import BaichuanConfig
|
24 |
+
from .generation_utils import build_chat_input, TextIterStreamer
|
25 |
+
|
26 |
+
import math
|
27 |
+
from typing import List, Optional, Tuple, Union
|
28 |
+
from threading import Thread
|
29 |
+
|
30 |
+
import torch
|
31 |
+
import torch.utils.checkpoint
|
32 |
+
from torch import nn
|
33 |
+
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss
|
34 |
+
from torch.nn import functional as F
|
35 |
+
from transformers import PreTrainedModel, PretrainedConfig
|
36 |
+
from transformers.activations import ACT2FN
|
37 |
+
from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
|
38 |
+
from transformers.generation.utils import GenerationConfig
|
39 |
+
from transformers.utils import logging, ContextManagers
|
40 |
+
|
41 |
+
import os
|
42 |
+
from contextlib import contextmanager
|
43 |
+
logger = logging.get_logger(__name__)
|
44 |
+
|
45 |
+
try:
|
46 |
+
from xformers import ops as xops
|
47 |
+
except ImportError:
|
48 |
+
xops = None
|
49 |
+
logger.warning(
|
50 |
+
"Xformers is not installed correctly. If you want to use memory_efficient_attention to accelerate training use the following command to install Xformers\npip install xformers."
|
51 |
+
)
|
52 |
+
|
53 |
+
|
54 |
+
# Copied from transformers.models.bart.modeling_bart._make_causal_mask
|
55 |
+
def _make_causal_mask(
|
56 |
+
input_ids_shape: torch.Size, dtype: torch.dtype, device: torch.device, past_key_values_length: int = 0
|
57 |
+
):
|
58 |
+
"""
|
59 |
+
Make causal mask used for bi-directional self-attention.
|
60 |
+
"""
|
61 |
+
bsz, tgt_len = input_ids_shape
|
62 |
+
mask = torch.full((tgt_len, tgt_len), torch.tensor(torch.finfo(dtype).min, device=device), device=device)
|
63 |
+
mask_cond = torch.arange(mask.size(-1), device=device)
|
64 |
+
mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0)
|
65 |
+
mask = mask.to(dtype)
|
66 |
+
|
67 |
+
if past_key_values_length > 0:
|
68 |
+
mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype, device=device), mask], dim=-1)
|
69 |
+
return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length)
|
70 |
+
|
71 |
+
def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None):
|
72 |
+
"""
|
73 |
+
Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`.
|
74 |
+
"""
|
75 |
+
if len(mask.size()) == 3:
|
76 |
+
bsz, src_len, _ = mask.size()
|
77 |
+
tgt_len = tgt_len if tgt_len is not None else src_len
|
78 |
+
expanded_mask = mask[:,None,:,:].expand(bsz, 1, tgt_len, src_len).to(dtype)
|
79 |
+
else:
|
80 |
+
bsz, src_len = mask.size()
|
81 |
+
tgt_len = tgt_len if tgt_len is not None else src_len
|
82 |
+
expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype)
|
83 |
+
|
84 |
+
inverted_mask = 1.0 - expanded_mask
|
85 |
+
|
86 |
+
return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min)
|
87 |
+
|
88 |
+
|
89 |
+
class RMSNorm(nn.Module):
|
90 |
+
def __init__(self, hidden_size, eps=1e-6):
|
91 |
+
"""
|
92 |
+
RMSNorm is equivalent to T5LayerNorm
|
93 |
+
"""
|
94 |
+
super().__init__()
|
95 |
+
self.weight = nn.Parameter(torch.ones(hidden_size))
|
96 |
+
self.variance_epsilon = eps
|
97 |
+
|
98 |
+
def forward(self, hidden_states):
|
99 |
+
variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
|
100 |
+
hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
|
101 |
+
|
102 |
+
# convert into half-precision if necessary
|
103 |
+
if self.weight.dtype in [torch.float16, torch.bfloat16]:
|
104 |
+
hidden_states = hidden_states.to(self.weight.dtype)
|
105 |
+
|
106 |
+
return self.weight * hidden_states
|
107 |
+
|
108 |
+
|
109 |
+
class RotaryEmbedding(torch.nn.Module):
|
110 |
+
def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
|
111 |
+
super().__init__()
|
112 |
+
self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim))
|
113 |
+
self.max_seq_len_cached = max_position_embeddings
|
114 |
+
t = torch.arange(self.max_seq_len_cached, device=self.inv_freq.device, dtype=torch.float32)
|
115 |
+
freqs = torch.outer(t, self.inv_freq)
|
116 |
+
emb = torch.cat((freqs, freqs), dim=-1)
|
117 |
+
self.cos_cached = emb.cos()[None, None, :, :].to(torch.float32)
|
118 |
+
self.sin_cached = emb.sin()[None, None, :, :].to(torch.float32)
|
119 |
+
def forward(self, x, seq_len=None):
|
120 |
+
# x: [bs, num_attention_heads, seq_len, head_size]
|
121 |
+
# This `if` block is unlikely to be run after we build sin/cos in `__init__`. Keep the logic here just in case.
|
122 |
+
if seq_len > self.max_seq_len_cached:
|
123 |
+
self.max_seq_len_cached = seq_len
|
124 |
+
t = torch.arange(self.max_seq_len_cached, device=self.inv_freq.device, dtype=torch.float32)
|
125 |
+
freqs = torch.outer(t, self.inv_freq)
|
126 |
+
emb = torch.cat((freqs, freqs), dim=-1)
|
127 |
+
self.cos_cached = emb.cos()[None, None, :, :].to(torch.float32).to(x.device)
|
128 |
+
self.sin_cached = emb.sin()[None, None, :, :].to(torch.float32).to(x.device)
|
129 |
+
elif self.cos_cached.device != x.device:
|
130 |
+
self.cos_cached = self.cos_cached.to(x.device)
|
131 |
+
self.sin_cached = self.sin_cached.to(x.device)
|
132 |
+
return (
|
133 |
+
self.cos_cached[:, :, :seq_len, ...],
|
134 |
+
self.sin_cached[:, :, :seq_len, ...],
|
135 |
+
)
|
136 |
+
|
137 |
+
|
138 |
+
def rotate_half(x):
|
139 |
+
"""Rotates half the hidden dims of the input."""
|
140 |
+
x1 = x[..., : x.shape[-1] // 2]
|
141 |
+
x2 = x[..., x.shape[-1] // 2:]
|
142 |
+
return torch.cat((-x2, x1), dim=-1)
|
143 |
+
|
144 |
+
|
145 |
+
def apply_rotary_pos_emb(q, k, cos_, sin_, position_ids):
|
146 |
+
cos = cos_.squeeze(1).squeeze(0) # [seq_len, dim]
|
147 |
+
sin = sin_.squeeze(1).squeeze(0) # [seq_len, dim]
|
148 |
+
cos = cos[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim]
|
149 |
+
sin = sin[position_ids].unsqueeze(1) # [bs, 1, seq_len, dim]
|
150 |
+
q_embed = (q.float() * cos) + (rotate_half(q.float()) * sin)
|
151 |
+
k_embed = (k.float() * cos) + (rotate_half(k.float()) * sin)
|
152 |
+
return q_embed.to(q.dtype), k_embed.to(k.dtype)
|
153 |
+
|
154 |
+
|
155 |
+
class MLP(nn.Module):
|
156 |
+
def __init__(
|
157 |
+
self,
|
158 |
+
hidden_size: int,
|
159 |
+
intermediate_size: int,
|
160 |
+
hidden_act: str,
|
161 |
+
):
|
162 |
+
super().__init__()
|
163 |
+
self.gate_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
164 |
+
self.down_proj = nn.Linear(intermediate_size, hidden_size, bias=False)
|
165 |
+
self.up_proj = nn.Linear(hidden_size, intermediate_size, bias=False)
|
166 |
+
self.act_fn = ACT2FN[hidden_act]
|
167 |
+
|
168 |
+
def forward(self, x):
|
169 |
+
return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
|
170 |
+
|
171 |
+
|
172 |
+
class Attention(nn.Module):
|
173 |
+
"""Multi-headed attention from 'Attention Is All You Need' paper"""
|
174 |
+
def __init__(self, config: BaichuanConfig):
|
175 |
+
super().__init__()
|
176 |
+
self.config = config
|
177 |
+
self.hidden_size = config.hidden_size
|
178 |
+
self.num_heads = config.num_attention_heads
|
179 |
+
self.head_dim = self.hidden_size // self.num_heads
|
180 |
+
self.max_position_embeddings = config.max_position_embeddings
|
181 |
+
|
182 |
+
if (self.head_dim * self.num_heads) != self.hidden_size:
|
183 |
+
raise ValueError(
|
184 |
+
f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
|
185 |
+
f" and `num_heads`: {self.num_heads})."
|
186 |
+
)
|
187 |
+
self.W_pack = nn.Linear(self.hidden_size, 3 * self.hidden_size, bias=False)
|
188 |
+
self.o_proj = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=False)
|
189 |
+
self.rotary_emb = RotaryEmbedding(self.head_dim, max_position_embeddings=self.max_position_embeddings)
|
190 |
+
|
191 |
+
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
|
192 |
+
return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous()
|
193 |
+
|
194 |
+
def forward(
|
195 |
+
self,
|
196 |
+
hidden_states: torch.Tensor,
|
197 |
+
attention_mask: Optional[torch.Tensor] = None,
|
198 |
+
position_ids: Optional[torch.LongTensor] = None,
|
199 |
+
past_key_value: Optional[Tuple[torch.Tensor]] = None,
|
200 |
+
output_attentions: bool = False,
|
201 |
+
use_cache: bool = False,
|
202 |
+
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
|
203 |
+
bsz, q_len, _ = hidden_states.size()
|
204 |
+
|
205 |
+
proj = self.W_pack(hidden_states)
|
206 |
+
proj = proj.unflatten(-1, (3, self.hidden_size)).unsqueeze(0).transpose(0, -2).squeeze(-2)
|
207 |
+
query_states = proj[0].view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
208 |
+
key_states = proj[1].view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
209 |
+
value_states = proj[2].view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
|
210 |
+
|
211 |
+
kv_seq_len = key_states.shape[-2]
|
212 |
+
if past_key_value is not None:
|
213 |
+
kv_seq_len += past_key_value[0].shape[-2]
|
214 |
+
cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
|
215 |
+
query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin, position_ids)
|
216 |
+
# [bsz, nh, t, hd]
|
217 |
+
|
218 |
+
if past_key_value is not None:
|
219 |
+
# reuse k, v, self_attention
|
220 |
+
key_states = torch.cat([past_key_value[0], key_states], dim=2)
|
221 |
+
value_states = torch.cat([past_key_value[1], value_states], dim=2)
|
222 |
+
|
223 |
+
past_key_value = (key_states, value_states) if use_cache else None
|
224 |
+
if xops is not None and self.training:
|
225 |
+
attn_weights = None
|
226 |
+
query_states = query_states.transpose(1, 2)
|
227 |
+
key_states = key_states.transpose(1, 2)
|
228 |
+
value_states = value_states.transpose(1, 2)
|
229 |
+
attn_output = xops.memory_efficient_attention(
|
230 |
+
query_states, key_states, value_states, attn_bias=xops.LowerTriangularMask()
|
231 |
+
)
|
232 |
+
else:
|
233 |
+
with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=True):
|
234 |
+
attn_output = F.scaled_dot_product_attention(query_states, key_states, value_states, attn_mask = attention_mask)
|
235 |
+
attn_output = attn_output.transpose(1, 2)
|
236 |
+
attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
|
237 |
+
attn_output = self.o_proj(attn_output)
|
238 |
+
|
239 |
+
if not output_attentions:
|
240 |
+
attn_weights = None
|
241 |
+
|
242 |
+
return attn_output, attn_weights, past_key_value
|
243 |
+
|
244 |
+
|
245 |
+
class DecoderLayer(nn.Module):
|
246 |
+
def __init__(self, config: BaichuanConfig):
|
247 |
+
super().__init__()
|
248 |
+
self.hidden_size = config.hidden_size
|
249 |
+
self.self_attn = Attention(config=config)
|
250 |
+
self.mlp = MLP(
|
251 |
+
hidden_size=self.hidden_size,
|
252 |
+
intermediate_size=config.intermediate_size,
|
253 |
+
hidden_act=config.hidden_act,
|
254 |
+
)
|
255 |
+
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
256 |
+
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
257 |
+
|
258 |
+
def forward(
|
259 |
+
self,
|
260 |
+
hidden_states: torch.Tensor,
|
261 |
+
attention_mask: Optional[torch.Tensor] = None,
|
262 |
+
position_ids: Optional[torch.LongTensor] = None,
|
263 |
+
past_key_value: Optional[Tuple[torch.Tensor]] = None,
|
264 |
+
output_attentions: Optional[bool] = False,
|
265 |
+
use_cache: Optional[bool] = False,
|
266 |
+
) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
|
267 |
+
|
268 |
+
residual = hidden_states
|
269 |
+
|
270 |
+
hidden_states = self.input_layernorm(hidden_states)
|
271 |
+
|
272 |
+
# Self Attention
|
273 |
+
hidden_states, self_attn_weights, present_key_value = self.self_attn(
|
274 |
+
hidden_states=hidden_states,
|
275 |
+
attention_mask=attention_mask,
|
276 |
+
position_ids=position_ids,
|
277 |
+
past_key_value=past_key_value,
|
278 |
+
output_attentions=output_attentions,
|
279 |
+
use_cache=use_cache,
|
280 |
+
)
|
281 |
+
hidden_states = residual + hidden_states
|
282 |
+
|
283 |
+
# Fully Connected
|
284 |
+
residual = hidden_states
|
285 |
+
hidden_states = self.post_attention_layernorm(hidden_states)
|
286 |
+
hidden_states = self.mlp(hidden_states)
|
287 |
+
hidden_states = residual + hidden_states
|
288 |
+
|
289 |
+
outputs = (hidden_states,)
|
290 |
+
|
291 |
+
if output_attentions:
|
292 |
+
outputs += (self_attn_weights,)
|
293 |
+
|
294 |
+
if use_cache:
|
295 |
+
outputs += (present_key_value,)
|
296 |
+
|
297 |
+
return outputs
|
298 |
+
|
299 |
+
|
300 |
+
class BaichuanPreTrainedModel(PreTrainedModel):
|
301 |
+
config_class = BaichuanConfig
|
302 |
+
base_model_prefix = "model"
|
303 |
+
supports_gradient_checkpointing = True
|
304 |
+
_no_split_modules = ["DecoderLayer"]
|
305 |
+
_keys_to_ignore_on_load_unexpected = [r"decoder\.version"]
|
306 |
+
|
307 |
+
def _init_weights(self, module):
|
308 |
+
std = self.config.initializer_range
|
309 |
+
if isinstance(module, nn.Linear):
|
310 |
+
module.weight.data.normal_(mean=0.0, std=std)
|
311 |
+
if module.bias is not None:
|
312 |
+
module.bias.data.zero_()
|
313 |
+
elif isinstance(module, nn.Embedding):
|
314 |
+
module.weight.data.normal_(mean=0.0, std=std)
|
315 |
+
if module.padding_idx is not None:
|
316 |
+
module.weight.data[module.padding_idx].zero_()
|
317 |
+
|
318 |
+
def _set_gradient_checkpointing(self, module, value=False):
|
319 |
+
if isinstance(module, BaichuanModel):
|
320 |
+
module.gradient_checkpointing = value
|
321 |
+
|
322 |
+
|
323 |
+
class BaichuanModel(BaichuanPreTrainedModel):
|
324 |
+
def __init__(self, config: BaichuanConfig):
|
325 |
+
super().__init__(config)
|
326 |
+
self.padding_idx = config.pad_token_id
|
327 |
+
self.vocab_size = config.vocab_size
|
328 |
+
|
329 |
+
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
|
330 |
+
self.layers = nn.ModuleList([DecoderLayer(config) for _ in range(config.num_hidden_layers)])
|
331 |
+
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
332 |
+
|
333 |
+
self.gradient_checkpointing = False
|
334 |
+
# Initialize weights and apply final processing
|
335 |
+
self.post_init()
|
336 |
+
|
337 |
+
def get_input_embeddings(self):
|
338 |
+
return self.embed_tokens
|
339 |
+
|
340 |
+
def set_input_embeddings(self, value):
|
341 |
+
self.embed_tokens = value
|
342 |
+
|
343 |
+
# Copied from transformers.models.bart.modeling_bart.BartDecoder._prepare_decoder_attention_mask
|
344 |
+
def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length):
|
345 |
+
# create causal mask
|
346 |
+
# [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
|
347 |
+
combined_attention_mask = None
|
348 |
+
if input_shape[-1] > 1:
|
349 |
+
combined_attention_mask = _make_causal_mask(
|
350 |
+
input_shape,
|
351 |
+
inputs_embeds.dtype,
|
352 |
+
device=inputs_embeds.device,
|
353 |
+
past_key_values_length=past_key_values_length,
|
354 |
+
)
|
355 |
+
|
356 |
+
if attention_mask is not None:
|
357 |
+
# [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len]
|
358 |
+
expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]).to(
|
359 |
+
inputs_embeds.device
|
360 |
+
)
|
361 |
+
combined_attention_mask = (
|
362 |
+
expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask
|
363 |
+
)
|
364 |
+
|
365 |
+
return combined_attention_mask
|
366 |
+
|
367 |
+
def forward(
|
368 |
+
self,
|
369 |
+
input_ids: torch.LongTensor = None,
|
370 |
+
attention_mask: Optional[torch.Tensor] = None,
|
371 |
+
position_ids: Optional[torch.LongTensor] = None,
|
372 |
+
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
373 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
374 |
+
use_cache: Optional[bool] = None,
|
375 |
+
output_attentions: Optional[bool] = None,
|
376 |
+
output_hidden_states: Optional[bool] = None,
|
377 |
+
return_dict: Optional[bool] = None,
|
378 |
+
) -> Union[Tuple, BaseModelOutputWithPast]:
|
379 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
380 |
+
output_hidden_states = (
|
381 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
382 |
+
)
|
383 |
+
use_cache = use_cache if use_cache is not None else self.config.use_cache
|
384 |
+
|
385 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
386 |
+
|
387 |
+
# retrieve input_ids and inputs_embeds
|
388 |
+
if input_ids is not None and inputs_embeds is not None:
|
389 |
+
raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time")
|
390 |
+
elif input_ids is not None:
|
391 |
+
batch_size, seq_length = input_ids.shape
|
392 |
+
elif inputs_embeds is not None:
|
393 |
+
batch_size, seq_length, _ = inputs_embeds.shape
|
394 |
+
else:
|
395 |
+
raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds")
|
396 |
+
|
397 |
+
seq_length_with_past = seq_length
|
398 |
+
past_key_values_length = 0
|
399 |
+
|
400 |
+
if past_key_values is not None:
|
401 |
+
past_key_values_length = past_key_values[0][0].shape[2]
|
402 |
+
seq_length_with_past = seq_length_with_past + past_key_values_length
|
403 |
+
|
404 |
+
if position_ids is None:
|
405 |
+
device = input_ids.device if input_ids is not None else inputs_embeds.device
|
406 |
+
position_ids = torch.arange(
|
407 |
+
past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device
|
408 |
+
)
|
409 |
+
position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
|
410 |
+
else:
|
411 |
+
position_ids = position_ids.view(-1, seq_length).long()
|
412 |
+
|
413 |
+
if inputs_embeds is None:
|
414 |
+
inputs_embeds = self.embed_tokens(input_ids)
|
415 |
+
# embed positions
|
416 |
+
if attention_mask is None:
|
417 |
+
attention_mask = torch.ones(
|
418 |
+
(batch_size, seq_length_with_past), dtype=torch.bool, device=inputs_embeds.device
|
419 |
+
)
|
420 |
+
attention_mask = self._prepare_decoder_attention_mask(
|
421 |
+
attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length
|
422 |
+
)
|
423 |
+
|
424 |
+
hidden_states = inputs_embeds
|
425 |
+
|
426 |
+
if self.gradient_checkpointing and self.training:
|
427 |
+
if use_cache:
|
428 |
+
logger.warning_once(
|
429 |
+
"`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
|
430 |
+
)
|
431 |
+
use_cache = False
|
432 |
+
|
433 |
+
# decoder layers
|
434 |
+
all_hidden_states = () if output_hidden_states else None
|
435 |
+
all_self_attns = () if output_attentions else None
|
436 |
+
next_decoder_cache = () if use_cache else None
|
437 |
+
|
438 |
+
for idx, decoder_layer in enumerate(self.layers):
|
439 |
+
if output_hidden_states:
|
440 |
+
all_hidden_states += (hidden_states,)
|
441 |
+
|
442 |
+
past_key_value = past_key_values[idx] if past_key_values is not None else None
|
443 |
+
|
444 |
+
if self.gradient_checkpointing and self.training:
|
445 |
+
|
446 |
+
def create_custom_forward(module):
|
447 |
+
def custom_forward(*inputs):
|
448 |
+
# None for past_key_value
|
449 |
+
return module(*inputs, output_attentions, None)
|
450 |
+
|
451 |
+
return custom_forward
|
452 |
+
|
453 |
+
layer_outputs = torch.utils.checkpoint.checkpoint(
|
454 |
+
create_custom_forward(decoder_layer),
|
455 |
+
hidden_states,
|
456 |
+
attention_mask,
|
457 |
+
position_ids,
|
458 |
+
None,
|
459 |
+
)
|
460 |
+
else:
|
461 |
+
layer_outputs = decoder_layer(
|
462 |
+
hidden_states,
|
463 |
+
attention_mask=attention_mask,
|
464 |
+
position_ids=position_ids,
|
465 |
+
past_key_value=past_key_value,
|
466 |
+
output_attentions=output_attentions,
|
467 |
+
use_cache=use_cache,
|
468 |
+
)
|
469 |
+
|
470 |
+
hidden_states = layer_outputs[0]
|
471 |
+
|
472 |
+
if use_cache:
|
473 |
+
next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)
|
474 |
+
|
475 |
+
if output_attentions:
|
476 |
+
all_self_attns += (layer_outputs[1],)
|
477 |
+
|
478 |
+
hidden_states = self.norm(hidden_states)
|
479 |
+
|
480 |
+
# add hidden states from the last decoder layer
|
481 |
+
if output_hidden_states:
|
482 |
+
all_hidden_states += (hidden_states,)
|
483 |
+
|
484 |
+
next_cache = next_decoder_cache if use_cache else None
|
485 |
+
if not return_dict:
|
486 |
+
return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
|
487 |
+
return BaseModelOutputWithPast(
|
488 |
+
last_hidden_state=hidden_states,
|
489 |
+
past_key_values=next_cache,
|
490 |
+
hidden_states=all_hidden_states,
|
491 |
+
attentions=all_self_attns,
|
492 |
+
)
|
493 |
+
|
494 |
+
|
495 |
+
class NormHead(nn.Module):
|
496 |
+
def __init__(self, hidden_size, vocab_size, bias=False):
|
497 |
+
super().__init__()
|
498 |
+
self.weight = nn.Parameter(torch.empty((vocab_size, hidden_size)))
|
499 |
+
nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))
|
500 |
+
self.first_flag = True
|
501 |
+
|
502 |
+
def forward(self, hidden_states):
|
503 |
+
if self.training:
|
504 |
+
norm_weight = nn.functional.normalize(self.weight)
|
505 |
+
self.first_flag = True
|
506 |
+
elif self.first_flag:
|
507 |
+
self.first_flag = False
|
508 |
+
self.weight.data = nn.functional.normalize(self.weight)
|
509 |
+
norm_weight = self.weight
|
510 |
+
else:
|
511 |
+
norm_weight = self.weight
|
512 |
+
return nn.functional.linear(hidden_states, norm_weight)
|
513 |
+
|
514 |
+
_init_weights = True
|
515 |
+
@contextmanager
|
516 |
+
def no_init_weights(_enable=True):
|
517 |
+
global _init_weights
|
518 |
+
old_init_weights = _init_weights
|
519 |
+
if _enable:
|
520 |
+
_init_weights = False
|
521 |
+
try:
|
522 |
+
yield
|
523 |
+
finally:
|
524 |
+
_init_weights = old_init_weights
|
525 |
+
|
526 |
+
class BaichuanForCausalLM(BaichuanPreTrainedModel):
|
527 |
+
def __init__(self, config, *model_args, **model_kwargs):
|
528 |
+
super().__init__(config, *model_args, **model_kwargs)
|
529 |
+
self.model = BaichuanModel(config)
|
530 |
+
|
531 |
+
self.lm_head = NormHead(config.hidden_size, config.vocab_size, bias=False)
|
532 |
+
if hasattr(config, "quantization_config") and isinstance(config.quantization_config, dict) and config.quantization_config.get('load_in_4bit', False):
|
533 |
+
try:
|
534 |
+
from .quantizer import quantize_offline, init_model_weight_int4
|
535 |
+
except ImportError:
|
536 |
+
raise ImportError(f"Needs QLinear to run quantize.")
|
537 |
+
quantize_offline(self, 4)
|
538 |
+
# Initialize weights and apply final processing
|
539 |
+
self.post_init()
|
540 |
+
|
541 |
+
def get_input_embeddings(self):
|
542 |
+
return self.model.embed_tokens
|
543 |
+
|
544 |
+
def set_input_embeddings(self, value):
|
545 |
+
self.model.embed_tokens = value
|
546 |
+
|
547 |
+
def get_output_embeddings(self):
|
548 |
+
return self.lm_head
|
549 |
+
|
550 |
+
def set_output_embeddings(self, new_embeddings):
|
551 |
+
self.lm_head = new_embeddings
|
552 |
+
|
553 |
+
def set_decoder(self, decoder):
|
554 |
+
self.model = decoder
|
555 |
+
|
556 |
+
def get_decoder(self):
|
557 |
+
return self.model
|
558 |
+
|
559 |
+
@classmethod
|
560 |
+
def from_pretrained(
|
561 |
+
cls,
|
562 |
+
pretrained_model_name_or_path: Optional[Union[str, os.PathLike]],
|
563 |
+
*model_args,
|
564 |
+
config: Optional[Union[PretrainedConfig, str, os.PathLike]] = None,
|
565 |
+
cache_dir: Optional[Union[str, os.PathLike]] = None,
|
566 |
+
ignore_mismatched_sizes: bool = False,
|
567 |
+
force_download: bool = False,
|
568 |
+
local_files_only: bool = False,
|
569 |
+
token: Optional[Union[str, bool]] = None,
|
570 |
+
revision: str = "main",
|
571 |
+
use_safetensors: bool = None,
|
572 |
+
**kwargs,
|
573 |
+
):
|
574 |
+
# Load config if we don't provide a configuration
|
575 |
+
if not isinstance(config, PretrainedConfig):
|
576 |
+
config_path = config if config is not None else pretrained_model_name_or_path
|
577 |
+
config, model_kwargs = cls.config_class.from_pretrained(
|
578 |
+
config_path,
|
579 |
+
cache_dir=cache_dir,
|
580 |
+
return_unused_kwargs=True,
|
581 |
+
force_download=force_download,
|
582 |
+
resume_download=False,
|
583 |
+
proxies=None,
|
584 |
+
local_files_only=local_files_only,
|
585 |
+
token=token,
|
586 |
+
revision=revision,
|
587 |
+
subfolder="",
|
588 |
+
_from_auto=False,
|
589 |
+
_from_pipeline=None,
|
590 |
+
**kwargs,
|
591 |
+
)
|
592 |
+
else:
|
593 |
+
model_kwargs = kwargs
|
594 |
+
|
595 |
+
if hasattr(config, "quantization_config") and config.quantization_config['load_in_4bit']:
|
596 |
+
try:
|
597 |
+
from .quantizer import init_model_weight_int4
|
598 |
+
from accelerate import init_empty_weights, dispatch_model, infer_auto_device_map
|
599 |
+
from accelerate.utils import CustomDtype
|
600 |
+
from accelerate.utils import get_balanced_memory
|
601 |
+
except ImportError:
|
602 |
+
raise ImportError(f"Needs import model weight init func to run quantize.")
|
603 |
+
# Instantiate model.
|
604 |
+
init_contexts = [no_init_weights(_enable=True)]
|
605 |
+
init_contexts.append(init_empty_weights())
|
606 |
+
with ContextManagers(init_contexts):
|
607 |
+
model = cls(config)
|
608 |
+
|
609 |
+
model_file = os.path.join(pretrained_model_name_or_path, 'pytorch_model.bin')
|
610 |
+
state_dict = torch.load(model_file, map_location="cpu")
|
611 |
+
model.is_quantized = True
|
612 |
+
|
613 |
+
device_map = kwargs.pop("device_map", None)
|
614 |
+
torch_dtype = kwargs.pop("torch_dtype", None)
|
615 |
+
|
616 |
+
if device_map is not None:
|
617 |
+
kwargs = {"no_split_module_classes": model._no_split_modules}
|
618 |
+
target_dtype = CustomDtype.INT4
|
619 |
+
max_memory = get_balanced_memory(
|
620 |
+
model,
|
621 |
+
dtype=target_dtype,
|
622 |
+
low_zero=(device_map == "balanced_low_0"),
|
623 |
+
max_memory=None,
|
624 |
+
**kwargs,
|
625 |
+
)
|
626 |
+
kwargs["max_memory"] = max_memory
|
627 |
+
device_map = infer_auto_device_map(model, dtype=target_dtype, **kwargs)
|
628 |
+
|
629 |
+
model = init_model_weight_int4(config, model, state_dict)
|
630 |
+
|
631 |
+
# Set model in evaluation mode to deactivate DropOut modules by default
|
632 |
+
model.eval()
|
633 |
+
# If it is a model with generation capabilities, attempt to load the generation config
|
634 |
+
if model.can_generate():
|
635 |
+
try:
|
636 |
+
model.generation_config = GenerationConfig.from_pretrained(
|
637 |
+
pretrained_model_name_or_path,
|
638 |
+
cache_dir=cache_dir,
|
639 |
+
force_download=force_download,
|
640 |
+
resume_download=False,
|
641 |
+
proxies=None,
|
642 |
+
local_files_only=local_files_only,
|
643 |
+
token=token,
|
644 |
+
revision=revision,
|
645 |
+
subfolder="",
|
646 |
+
_from_auto=False,
|
647 |
+
_from_pipeline=None,
|
648 |
+
**kwargs,
|
649 |
+
)
|
650 |
+
except (OSError, TypeError):
|
651 |
+
logger.info(
|
652 |
+
"Generation config file not found, using a generation config created from the model config."
|
653 |
+
)
|
654 |
+
pass
|
655 |
+
|
656 |
+
if device_map is not None:
|
657 |
+
dispatch_model(model, device_map=device_map)
|
658 |
+
|
659 |
+
return model
|
660 |
+
return super(BaichuanForCausalLM, cls).from_pretrained(pretrained_model_name_or_path, *model_args,
|
661 |
+
config=config, cache_dir=cache_dir, ignore_mismatched_sizes=ignore_mismatched_sizes,
|
662 |
+
force_download=force_download, local_files_only=local_files_only, token=token, revision=revision,
|
663 |
+
use_safetensors=use_safetensors, **kwargs)
|
664 |
+
|
665 |
+
def forward(
|
666 |
+
self,
|
667 |
+
input_ids: torch.LongTensor = None,
|
668 |
+
attention_mask: Optional[torch.Tensor] = None,
|
669 |
+
position_ids: Optional[torch.LongTensor] = None,
|
670 |
+
past_key_values: Optional[List[torch.FloatTensor]] = None,
|
671 |
+
inputs_embeds: Optional[torch.FloatTensor] = None,
|
672 |
+
labels: Optional[torch.LongTensor] = None,
|
673 |
+
use_cache: Optional[bool] = None,
|
674 |
+
output_attentions: Optional[bool] = None,
|
675 |
+
output_hidden_states: Optional[bool] = None,
|
676 |
+
return_dict: Optional[bool] = None,
|
677 |
+
) -> Union[Tuple, CausalLMOutputWithPast]:
|
678 |
+
|
679 |
+
output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
|
680 |
+
output_hidden_states = (
|
681 |
+
output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
|
682 |
+
)
|
683 |
+
return_dict = return_dict if return_dict is not None else self.config.use_return_dict
|
684 |
+
|
685 |
+
# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
|
686 |
+
outputs = self.model(
|
687 |
+
input_ids=input_ids,
|
688 |
+
attention_mask=attention_mask,
|
689 |
+
position_ids=position_ids,
|
690 |
+
past_key_values=past_key_values,
|
691 |
+
inputs_embeds=inputs_embeds,
|
692 |
+
use_cache=use_cache,
|
693 |
+
output_attentions=output_attentions,
|
694 |
+
output_hidden_states=output_hidden_states,
|
695 |
+
return_dict=return_dict,
|
696 |
+
)
|
697 |
+
|
698 |
+
hidden_states = outputs[0]
|
699 |
+
logits = self.lm_head(hidden_states)
|
700 |
+
loss = None
|
701 |
+
if labels is not None:
|
702 |
+
# Shift so that tokens < n predict n
|
703 |
+
shift_logits = logits[..., :-1, :].contiguous()
|
704 |
+
shift_labels = labels[..., 1:].contiguous()
|
705 |
+
# Flatten the tokens
|
706 |
+
loss_fct = CrossEntropyLoss()
|
707 |
+
shift_logits = shift_logits.view(-1, self.config.vocab_size)
|
708 |
+
shift_labels = shift_labels.view(-1)
|
709 |
+
softmax_normalizer = shift_logits.max(-1).values ** 2
|
710 |
+
z_loss = self.config.z_loss_weight * softmax_normalizer.mean()
|
711 |
+
# Enable model parallelism
|
712 |
+
shift_labels = shift_labels.to(shift_logits.device)
|
713 |
+
loss = loss_fct(shift_logits, shift_labels) + z_loss
|
714 |
+
|
715 |
+
if not return_dict:
|
716 |
+
output = (logits,) + outputs[1:]
|
717 |
+
return (loss,) + output if loss is not None else output
|
718 |
+
|
719 |
+
return CausalLMOutputWithPast(
|
720 |
+
loss=loss,
|
721 |
+
logits=logits,
|
722 |
+
past_key_values=outputs.past_key_values,
|
723 |
+
hidden_states=outputs.hidden_states,
|
724 |
+
attentions=outputs.attentions,
|
725 |
+
)
|
726 |
+
|
727 |
+
def prepare_inputs_for_generation(
|
728 |
+
self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
|
729 |
+
):
|
730 |
+
if past_key_values:
|
731 |
+
input_ids = input_ids[:, -1:]
|
732 |
+
|
733 |
+
position_ids = kwargs.get("position_ids", None)
|
734 |
+
if attention_mask is not None and position_ids is None:
|
735 |
+
# create position_ids on the fly for batch generation
|
736 |
+
position_ids = attention_mask.long().cumsum(-1) - 1
|
737 |
+
position_ids.masked_fill_(attention_mask == 0, 1)
|
738 |
+
if past_key_values:
|
739 |
+
position_ids = position_ids[:, -1].unsqueeze(-1)
|
740 |
+
|
741 |
+
# if `inputs_embeds` are passed, we only want to use them in the 1st generation step
|
742 |
+
if inputs_embeds is not None and past_key_values is None:
|
743 |
+
model_inputs = {"inputs_embeds": inputs_embeds}
|
744 |
+
else:
|
745 |
+
model_inputs = {"input_ids": input_ids}
|
746 |
+
|
747 |
+
model_inputs.update(
|
748 |
+
{
|
749 |
+
"position_ids": position_ids,
|
750 |
+
"past_key_values": past_key_values,
|
751 |
+
"use_cache": kwargs.get("use_cache"),
|
752 |
+
"attention_mask": attention_mask,
|
753 |
+
}
|
754 |
+
)
|
755 |
+
return model_inputs
|
756 |
+
|
757 |
+
@staticmethod
|
758 |
+
def _reorder_cache(past_key_values, beam_idx):
|
759 |
+
reordered_past = ()
|
760 |
+
for layer_past in past_key_values:
|
761 |
+
reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),)
|
762 |
+
return reordered_past
|
763 |
+
|
764 |
+
def quantize(self, bits: int):
|
765 |
+
try:
|
766 |
+
from .quantizer import quantize_online
|
767 |
+
except ImportError:
|
768 |
+
raise ImportError(f"Needs QLinear to run quantize.")
|
769 |
+
return quantize_online(self, bits)
|
770 |
+
|
771 |
+
def chat(self, tokenizer, messages: List[dict], stream=False,
|
772 |
+
generation_config: Optional[GenerationConfig]=None):
|
773 |
+
generation_config = generation_config or self.generation_config
|
774 |
+
input_ids = build_chat_input(self, tokenizer, messages, generation_config.max_new_tokens)
|
775 |
+
if stream:
|
776 |
+
streamer = TextIterStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
|
777 |
+
Thread(target=self.generate, kwargs=dict(
|
778 |
+
inputs=input_ids, streamer=streamer,
|
779 |
+
generation_config=generation_config,
|
780 |
+
)).start()
|
781 |
+
return streamer
|
782 |
+
else:
|
783 |
+
outputs = self.generate(input_ids, generation_config=generation_config)
|
784 |
+
response = tokenizer.decode(outputs[0][len(input_ids[0]):], skip_special_tokens=True)
|
785 |
+
return response
|
pytorch_model-00001-of-00002.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:362b0abf256a768a0a377038df252511fbfb9a313f29c9b966d5c8571c026787
|
3 |
+
size 9934622796
|
pytorch_model-00002-of-00002.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:d266af6663c057ca585342954d9709db3d3d9a9601f7ba92c70ef0453b9cee3a
|
3 |
+
size 5077401163
|
pytorch_model.bin.index.json
ADDED
@@ -0,0 +1,234 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
{
|
2 |
+
"metadata": {
|
3 |
+
"total_size": 15011946496
|
4 |
+
},
|
5 |
+
"weight_map": {
|
6 |
+
"lm_head.weight": "pytorch_model-00002-of-00002.bin",
|
7 |
+
"model.embed_tokens.weight": "pytorch_model-00001-of-00002.bin",
|
8 |
+
"model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
9 |
+
"model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
10 |
+
"model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
11 |
+
"model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
12 |
+
"model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
13 |
+
"model.layers.0.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
14 |
+
"model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
15 |
+
"model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
16 |
+
"model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
17 |
+
"model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
18 |
+
"model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
19 |
+
"model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
20 |
+
"model.layers.1.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
21 |
+
"model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
22 |
+
"model.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
23 |
+
"model.layers.10.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
24 |
+
"model.layers.10.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
25 |
+
"model.layers.10.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
26 |
+
"model.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
27 |
+
"model.layers.10.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
28 |
+
"model.layers.10.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
29 |
+
"model.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
30 |
+
"model.layers.11.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
31 |
+
"model.layers.11.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
32 |
+
"model.layers.11.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
33 |
+
"model.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
34 |
+
"model.layers.11.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
35 |
+
"model.layers.11.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
36 |
+
"model.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
37 |
+
"model.layers.12.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
38 |
+
"model.layers.12.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
39 |
+
"model.layers.12.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
40 |
+
"model.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
41 |
+
"model.layers.12.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
42 |
+
"model.layers.12.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
43 |
+
"model.layers.13.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
44 |
+
"model.layers.13.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
45 |
+
"model.layers.13.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
46 |
+
"model.layers.13.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
47 |
+
"model.layers.13.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
48 |
+
"model.layers.13.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
49 |
+
"model.layers.13.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
50 |
+
"model.layers.14.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
51 |
+
"model.layers.14.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
52 |
+
"model.layers.14.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
53 |
+
"model.layers.14.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
54 |
+
"model.layers.14.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
55 |
+
"model.layers.14.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
56 |
+
"model.layers.14.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
57 |
+
"model.layers.15.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
58 |
+
"model.layers.15.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
59 |
+
"model.layers.15.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
60 |
+
"model.layers.15.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
61 |
+
"model.layers.15.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
62 |
+
"model.layers.15.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
63 |
+
"model.layers.15.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
64 |
+
"model.layers.16.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
65 |
+
"model.layers.16.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
66 |
+
"model.layers.16.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
67 |
+
"model.layers.16.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
68 |
+
"model.layers.16.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
69 |
+
"model.layers.16.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
70 |
+
"model.layers.16.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
71 |
+
"model.layers.17.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
72 |
+
"model.layers.17.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
73 |
+
"model.layers.17.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
74 |
+
"model.layers.17.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
75 |
+
"model.layers.17.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
76 |
+
"model.layers.17.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
77 |
+
"model.layers.17.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
78 |
+
"model.layers.18.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
79 |
+
"model.layers.18.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
80 |
+
"model.layers.18.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
81 |
+
"model.layers.18.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
82 |
+
"model.layers.18.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
83 |
+
"model.layers.18.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
84 |
+
"model.layers.18.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
85 |
+
"model.layers.19.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
86 |
+
"model.layers.19.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
87 |
+
"model.layers.19.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
88 |
+
"model.layers.19.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
89 |
+
"model.layers.19.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
90 |
+
"model.layers.19.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
91 |
+
"model.layers.19.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
92 |
+
"model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
93 |
+
"model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
94 |
+
"model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
95 |
+
"model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
96 |
+
"model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
97 |
+
"model.layers.2.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
98 |
+
"model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
99 |
+
"model.layers.20.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
100 |
+
"model.layers.20.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
101 |
+
"model.layers.20.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
102 |
+
"model.layers.20.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
103 |
+
"model.layers.20.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
104 |
+
"model.layers.20.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
105 |
+
"model.layers.20.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
106 |
+
"model.layers.21.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
107 |
+
"model.layers.21.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
108 |
+
"model.layers.21.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
109 |
+
"model.layers.21.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
110 |
+
"model.layers.21.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
111 |
+
"model.layers.21.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
112 |
+
"model.layers.21.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
113 |
+
"model.layers.22.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
114 |
+
"model.layers.22.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
115 |
+
"model.layers.22.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
116 |
+
"model.layers.22.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
117 |
+
"model.layers.22.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
118 |
+
"model.layers.22.self_attn.W_pack.weight": "pytorch_model-00002-of-00002.bin",
|
119 |
+
"model.layers.22.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
120 |
+
"model.layers.23.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
121 |
+
"model.layers.23.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
122 |
+
"model.layers.23.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
123 |
+
"model.layers.23.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
124 |
+
"model.layers.23.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
125 |
+
"model.layers.23.self_attn.W_pack.weight": "pytorch_model-00002-of-00002.bin",
|
126 |
+
"model.layers.23.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
127 |
+
"model.layers.24.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
128 |
+
"model.layers.24.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
129 |
+
"model.layers.24.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
130 |
+
"model.layers.24.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
131 |
+
"model.layers.24.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
132 |
+
"model.layers.24.self_attn.W_pack.weight": "pytorch_model-00002-of-00002.bin",
|
133 |
+
"model.layers.24.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
134 |
+
"model.layers.25.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
135 |
+
"model.layers.25.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
136 |
+
"model.layers.25.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
137 |
+
"model.layers.25.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
138 |
+
"model.layers.25.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
139 |
+
"model.layers.25.self_attn.W_pack.weight": "pytorch_model-00002-of-00002.bin",
|
140 |
+
"model.layers.25.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
141 |
+
"model.layers.26.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
142 |
+
"model.layers.26.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
143 |
+
"model.layers.26.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
144 |
+
"model.layers.26.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
145 |
+
"model.layers.26.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
146 |
+
"model.layers.26.self_attn.W_pack.weight": "pytorch_model-00002-of-00002.bin",
|
147 |
+
"model.layers.26.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
148 |
+
"model.layers.27.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
149 |
+
"model.layers.27.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
150 |
+
"model.layers.27.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
151 |
+
"model.layers.27.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
152 |
+
"model.layers.27.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
153 |
+
"model.layers.27.self_attn.W_pack.weight": "pytorch_model-00002-of-00002.bin",
|
154 |
+
"model.layers.27.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
155 |
+
"model.layers.28.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
156 |
+
"model.layers.28.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
157 |
+
"model.layers.28.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
158 |
+
"model.layers.28.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
159 |
+
"model.layers.28.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
160 |
+
"model.layers.28.self_attn.W_pack.weight": "pytorch_model-00002-of-00002.bin",
|
161 |
+
"model.layers.28.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
162 |
+
"model.layers.29.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
163 |
+
"model.layers.29.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
164 |
+
"model.layers.29.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
165 |
+
"model.layers.29.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
166 |
+
"model.layers.29.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
167 |
+
"model.layers.29.self_attn.W_pack.weight": "pytorch_model-00002-of-00002.bin",
|
168 |
+
"model.layers.29.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
169 |
+
"model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
170 |
+
"model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
171 |
+
"model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
172 |
+
"model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
173 |
+
"model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
174 |
+
"model.layers.3.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
175 |
+
"model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
176 |
+
"model.layers.30.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
177 |
+
"model.layers.30.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
178 |
+
"model.layers.30.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
179 |
+
"model.layers.30.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
180 |
+
"model.layers.30.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
181 |
+
"model.layers.30.self_attn.W_pack.weight": "pytorch_model-00002-of-00002.bin",
|
182 |
+
"model.layers.30.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
183 |
+
"model.layers.31.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
184 |
+
"model.layers.31.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
|
185 |
+
"model.layers.31.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
|
186 |
+
"model.layers.31.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
|
187 |
+
"model.layers.31.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
|
188 |
+
"model.layers.31.self_attn.W_pack.weight": "pytorch_model-00002-of-00002.bin",
|
189 |
+
"model.layers.31.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
|
190 |
+
"model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
191 |
+
"model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
192 |
+
"model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
193 |
+
"model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
194 |
+
"model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
195 |
+
"model.layers.4.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
196 |
+
"model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
197 |
+
"model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
198 |
+
"model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
199 |
+
"model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
200 |
+
"model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
201 |
+
"model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
202 |
+
"model.layers.5.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
203 |
+
"model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
204 |
+
"model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
205 |
+
"model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
206 |
+
"model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
207 |
+
"model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
208 |
+
"model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
209 |
+
"model.layers.6.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
210 |
+
"model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
211 |
+
"model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
212 |
+
"model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
213 |
+
"model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
214 |
+
"model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
215 |
+
"model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
216 |
+
"model.layers.7.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
217 |
+
"model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
218 |
+
"model.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
219 |
+
"model.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
220 |
+
"model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
221 |
+
"model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
222 |
+
"model.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
223 |
+
"model.layers.8.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
224 |
+
"model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
225 |
+
"model.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
226 |
+
"model.layers.9.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
|
227 |
+
"model.layers.9.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
|
228 |
+
"model.layers.9.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
|
229 |
+
"model.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
|
230 |
+
"model.layers.9.self_attn.W_pack.weight": "pytorch_model-00001-of-00002.bin",
|
231 |
+
"model.layers.9.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
|
232 |
+
"model.norm.weight": "pytorch_model-00002-of-00002.bin"
|
233 |
+
}
|
234 |
+
}
|
quantizer.py
ADDED
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import bitsandbytes as bnb
|
2 |
+
from bitsandbytes.nn.modules import Params4bit, Int8Params
|
3 |
+
import torch
|
4 |
+
|
5 |
+
def Params4bitCuda(self, device):
|
6 |
+
self.data = self.data.cuda(device)
|
7 |
+
self.quant_state[0] = self.quant_state[0].cuda(device)
|
8 |
+
self.quant_state[4][0] = self.quant_state[4][0].cuda(device)
|
9 |
+
self.quant_state[4][1][0] = self.quant_state[4][1][0].cuda(device)
|
10 |
+
self.quant_state[4][1][1] = self.quant_state[4][1][1].cuda(device)
|
11 |
+
|
12 |
+
self.quant_state[6] = self.quant_state[6].cuda(device)
|
13 |
+
return self
|
14 |
+
|
15 |
+
class Linear4bitOnline(torch.nn.Module):
|
16 |
+
def __init__(self, weight, bias, quant_type):
|
17 |
+
super().__init__()
|
18 |
+
self.weight = Params4bit(
|
19 |
+
weight.data, requires_grad=False, compress_statistics=True, quant_type=quant_type
|
20 |
+
)
|
21 |
+
self.compute_dtype = None
|
22 |
+
#self.weight.cuda(weight.device)
|
23 |
+
self.bias = bias
|
24 |
+
|
25 |
+
def forward(self, x: torch.Tensor):
|
26 |
+
# weights are cast automatically as Int8Params, but the bias has to be cast manually
|
27 |
+
if self.bias is not None and self.bias.dtype != x.dtype:
|
28 |
+
self.bias.data = self.bias.data.to(x.dtype)
|
29 |
+
|
30 |
+
if getattr(self.weight, "quant_state", None) is None:
|
31 |
+
print(
|
32 |
+
"FP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first."
|
33 |
+
)
|
34 |
+
inp_dtype = x.dtype
|
35 |
+
if self.compute_dtype is not None:
|
36 |
+
x = x.to(self.compute_dtype)
|
37 |
+
|
38 |
+
bias = None if self.bias is None else self.bias.to(self.compute_dtype)
|
39 |
+
out = bnb.matmul_4bit(
|
40 |
+
x, self.weight.t(), bias=bias, quant_state=self.weight.quant_state
|
41 |
+
)
|
42 |
+
|
43 |
+
out = out.to(inp_dtype)
|
44 |
+
|
45 |
+
return out
|
46 |
+
|
47 |
+
class Linear8bitLtOnline(torch.nn.Module):
|
48 |
+
def __init__(
|
49 |
+
self,
|
50 |
+
weight,
|
51 |
+
bias,
|
52 |
+
has_fp16_weights=True,
|
53 |
+
memory_efficient_backward=False,
|
54 |
+
threshold=0.0,
|
55 |
+
index=None,
|
56 |
+
):
|
57 |
+
super().__init__()
|
58 |
+
assert (
|
59 |
+
not memory_efficient_backward
|
60 |
+
), "memory_efficient_backward is no longer required and the argument is deprecated in 0.37.0 and will be removed in 0.39.0"
|
61 |
+
self.state = bnb.MatmulLtState()
|
62 |
+
self.index = index
|
63 |
+
|
64 |
+
# Necessary for stacked layers
|
65 |
+
self.state.threshold = threshold
|
66 |
+
self.state.has_fp16_weights = has_fp16_weights
|
67 |
+
self.state.memory_efficient_backward = memory_efficient_backward
|
68 |
+
if threshold > 0.0 and not has_fp16_weights:
|
69 |
+
self.state.use_pool = True
|
70 |
+
|
71 |
+
self.weight = Int8Params(
|
72 |
+
weight.data,
|
73 |
+
has_fp16_weights=has_fp16_weights,
|
74 |
+
requires_grad=has_fp16_weights,
|
75 |
+
)
|
76 |
+
self.bias = bias
|
77 |
+
|
78 |
+
def init_8bit_state(self):
|
79 |
+
self.state.CB = self.weight.CB
|
80 |
+
self.state.SCB = self.weight.SCB
|
81 |
+
self.weight.CB = None
|
82 |
+
self.weight.SCB = None
|
83 |
+
|
84 |
+
def forward(self, x: torch.Tensor):
|
85 |
+
self.state.is_training = self.training
|
86 |
+
if self.weight.CB is not None:
|
87 |
+
self.init_8bit_state()
|
88 |
+
|
89 |
+
# weights are cast automatically as Int8Params, but the bias has to be cast manually
|
90 |
+
if self.bias is not None and self.bias.dtype != x.dtype:
|
91 |
+
self.bias.data = self.bias.data.to(x.dtype)
|
92 |
+
|
93 |
+
out = bnb.matmul(x, self.weight, bias=self.bias, state=self.state)
|
94 |
+
|
95 |
+
if not self.state.has_fp16_weights:
|
96 |
+
if self.state.CB is not None and self.state.CxB is not None:
|
97 |
+
# we converted 8-bit row major to turing/ampere format in the first inference pass
|
98 |
+
# we no longer need the row-major weight
|
99 |
+
del self.state.CB
|
100 |
+
self.weight.data = self.state.CxB
|
101 |
+
return out
|
102 |
+
|
103 |
+
def quantize_offline(model, bits: int):
|
104 |
+
assert (bits == 4), f'bits: {bits} is not supported'
|
105 |
+
|
106 |
+
for i, layer in enumerate(model.model.layers):
|
107 |
+
layer.self_attn.W_pack = bnb.nn.Linear4bit(
|
108 |
+
layer.self_attn.W_pack.weight.shape[1],
|
109 |
+
layer.self_attn.W_pack.weight.shape[0],
|
110 |
+
False,
|
111 |
+
torch.float16,
|
112 |
+
compress_statistics=True,
|
113 |
+
quant_type="nf4",
|
114 |
+
)
|
115 |
+
layer.self_attn.o_proj = bnb.nn.Linear4bit(
|
116 |
+
layer.self_attn.o_proj.weight.shape[1],
|
117 |
+
layer.self_attn.o_proj.weight.shape[0],
|
118 |
+
False,
|
119 |
+
torch.float16,
|
120 |
+
compress_statistics=True,
|
121 |
+
quant_type="nf4",
|
122 |
+
)
|
123 |
+
|
124 |
+
layer.mlp.gate_proj = bnb.nn.Linear4bit(
|
125 |
+
layer.mlp.gate_proj.weight.shape[1],
|
126 |
+
layer.mlp.gate_proj.weight.shape[0],
|
127 |
+
False,
|
128 |
+
torch.float16,
|
129 |
+
compress_statistics=True,
|
130 |
+
quant_type="nf4",
|
131 |
+
)
|
132 |
+
layer.mlp.down_proj = bnb.nn.Linear4bit(
|
133 |
+
layer.mlp.down_proj.weight.shape[1],
|
134 |
+
layer.mlp.down_proj.weight.shape[0],
|
135 |
+
False,
|
136 |
+
torch.float16,
|
137 |
+
compress_statistics=True,
|
138 |
+
quant_type="nf4",
|
139 |
+
)
|
140 |
+
layer.mlp.up_proj = bnb.nn.Linear4bit(
|
141 |
+
layer.mlp.up_proj.weight.shape[1],
|
142 |
+
layer.mlp.up_proj.weight.shape[0],
|
143 |
+
False,
|
144 |
+
torch.float16,
|
145 |
+
compress_statistics=True,
|
146 |
+
quant_type="nf4",
|
147 |
+
)
|
148 |
+
return model
|
149 |
+
|
150 |
+
def quantize_online(model, bits: int):
|
151 |
+
def quant(weight, bias=None):
|
152 |
+
if bits == 8:
|
153 |
+
linear = Linear8bitLtOnline(
|
154 |
+
weight,
|
155 |
+
bias,
|
156 |
+
has_fp16_weights=False,
|
157 |
+
threshold=6.0,
|
158 |
+
)
|
159 |
+
if bias is not None:
|
160 |
+
linear.bias = torch.nn.Parameter(bias)
|
161 |
+
elif bits == 4:
|
162 |
+
linear = Linear4bitOnline(
|
163 |
+
weight,
|
164 |
+
bias,
|
165 |
+
quant_type="nf4", #fp4/nf4
|
166 |
+
)
|
167 |
+
else:
|
168 |
+
raise ValueError("quantize only support 4/8 bit")
|
169 |
+
return linear
|
170 |
+
|
171 |
+
for i, layer in enumerate(model.model.layers):
|
172 |
+
layer.self_attn.W_pack = quant(layer.self_attn.W_pack.weight)
|
173 |
+
layer.self_attn.o_proj = quant(layer.self_attn.o_proj.weight)
|
174 |
+
layer.mlp.gate_proj = quant(layer.mlp.gate_proj.weight)
|
175 |
+
layer.mlp.down_proj = quant(layer.mlp.down_proj.weight)
|
176 |
+
layer.mlp.up_proj = quant(layer.mlp.up_proj.weight)
|
177 |
+
return model
|
178 |
+
|
179 |
+
def init_model_weight_int4(config, model, state_dict):
|
180 |
+
#replace Params4bit.cuda with Params4bitCuda
|
181 |
+
Params4bit.cuda = Params4bitCuda
|
182 |
+
|
183 |
+
for i in range(config.num_hidden_layers):
|
184 |
+
weight_data = state_dict[f'model.layers.{i}.self_attn.W_pack.weight.data']
|
185 |
+
weight_quant_state = state_dict[f'model.layers.{i}.self_attn.W_pack.weight.quant_state']
|
186 |
+
model.model.layers[i].self_attn.W_pack.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
|
187 |
+
|
188 |
+
weight_data = state_dict[f'model.layers.{i}.self_attn.o_proj.weight.data']
|
189 |
+
weight_quant_state = state_dict[f'model.layers.{i}.self_attn.o_proj.weight.quant_state']
|
190 |
+
model.model.layers[i].self_attn.o_proj.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
|
191 |
+
|
192 |
+
weight_data = state_dict[f'model.layers.{i}.mlp.gate_proj.weight.data']
|
193 |
+
weight_quant_state = state_dict[f'model.layers.{i}.mlp.gate_proj.weight.quant_state']
|
194 |
+
model.model.layers[i].mlp.gate_proj.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
|
195 |
+
|
196 |
+
weight_data = state_dict[f'model.layers.{i}.mlp.up_proj.weight.data']
|
197 |
+
weight_quant_state = state_dict[f'model.layers.{i}.mlp.up_proj.weight.quant_state']
|
198 |
+
model.model.layers[i].mlp.up_proj.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
|
199 |
+
|
200 |
+
weight_data = state_dict[f'model.layers.{i}.mlp.down_proj.weight.data']
|
201 |
+
weight_quant_state = state_dict[f'model.layers.{i}.mlp.down_proj.weight.quant_state']
|
202 |
+
model.model.layers[i].mlp.down_proj.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
|
203 |
+
|
204 |
+
model.model.layers[i].input_layernorm.weight = state_dict[f'model.layers.{i}.input_layernorm.weight']
|
205 |
+
model.model.layers[i].post_attention_layernorm.weight = state_dict[f'model.layers.{i}.post_attention_layernorm.weight']
|
206 |
+
|
207 |
+
model.model.embed_tokens.weight = state_dict['model.embed_tokens.weight']
|
208 |
+
model.model.norm.weight = state_dict['model.norm.weight']
|
209 |
+
model.lm_head.weight = state_dict['lm_head.weight']
|
210 |
+
return model
|