sakuraumi commited on
Commit
09699fb
1 Parent(s): 2a74bc7

Upload 8 files

Browse files
sakura_13b_model_v0.5_4bits_autogptq_40k/configuration_baichuan.py CHANGED
@@ -9,7 +9,7 @@ class BaichuanConfig(PretrainedConfig):
9
 
10
  def __init__(
11
  self,
12
- vocab_size=64000,
13
  hidden_size=5120,
14
  intermediate_size=13696,
15
  num_hidden_layers=40,
 
9
 
10
  def __init__(
11
  self,
12
+ vocab_size=125696,
13
  hidden_size=5120,
14
  intermediate_size=13696,
15
  num_hidden_layers=40,
sakura_13b_model_v0.5_4bits_autogptq_40k/generation_utils.py ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from typing import List
2
+ from queue import Queue
3
+
4
+ import torch
5
+
6
+
7
+ def build_chat_input(model, tokenizer, messages: List[dict], max_new_tokens: int=0):
8
+ def _parse_messages(messages, split_role="user"):
9
+ system, rounds = "", []
10
+ round = []
11
+ for i, message in enumerate(messages):
12
+ if message["role"] == "system":
13
+ assert i == 0
14
+ system = message["content"]
15
+ continue
16
+ if message["role"] == split_role and round:
17
+ rounds.append(round)
18
+ round = []
19
+ round.append(message)
20
+ if round:
21
+ rounds.append(round)
22
+ return system, rounds
23
+
24
+ max_new_tokens = max_new_tokens or model.generation_config.max_new_tokens
25
+ max_input_tokens = model.config.model_max_length - max_new_tokens
26
+ system, rounds = _parse_messages(messages, split_role="user")
27
+ system_tokens = tokenizer.encode(system)
28
+ max_history_tokens = max_input_tokens - len(system_tokens)
29
+
30
+ history_tokens = []
31
+ for round in rounds[::-1]:
32
+ round_tokens = []
33
+ for message in round:
34
+ if message["role"] == "user":
35
+ round_tokens.append(model.generation_config.user_token_id)
36
+ else:
37
+ round_tokens.append(model.generation_config.assistant_token_id)
38
+ round_tokens.extend(tokenizer.encode(message["content"]))
39
+ if len(history_tokens) == 0 or len(history_tokens) + len(round_tokens) <= max_history_tokens:
40
+ history_tokens = round_tokens + history_tokens # concat left
41
+ if len(history_tokens) < max_history_tokens:
42
+ continue
43
+ break
44
+
45
+ input_tokens = system_tokens + history_tokens
46
+ if messages[-1]["role"] != "assistant":
47
+ input_tokens.append(model.generation_config.assistant_token_id)
48
+ input_tokens = input_tokens[-max_input_tokens:] # truncate left
49
+ return torch.LongTensor([input_tokens]).to(model.device)
50
+
51
+
52
+ class TextIterStreamer:
53
+ def __init__(self, tokenizer, skip_prompt=False, skip_special_tokens=False):
54
+ self.tokenizer = tokenizer
55
+ self.skip_prompt = skip_prompt
56
+ self.skip_special_tokens = skip_special_tokens
57
+ self.tokens = []
58
+ self.text_queue = Queue()
59
+ self.next_tokens_are_prompt = True
60
+
61
+ def put(self, value):
62
+ if self.skip_prompt and self.next_tokens_are_prompt:
63
+ self.next_tokens_are_prompt = False
64
+ else:
65
+ if len(value.shape) > 1:
66
+ value = value[0]
67
+ self.tokens.extend(value.tolist())
68
+ self.text_queue.put(
69
+ self.tokenizer.decode(self.tokens, skip_special_tokens=self.skip_special_tokens))
70
+
71
+ def end(self):
72
+ self.text_queue.put(None)
73
+
74
+ def __iter__(self):
75
+ return self
76
+
77
+ def __next__(self):
78
+ value = self.text_queue.get()
79
+ if value is None:
80
+ raise StopIteration()
81
+ else:
82
+ return value
83
+
sakura_13b_model_v0.5_4bits_autogptq_40k/modeling_baichuan.py ADDED
@@ -0,0 +1,827 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2023, Baichuan Intelligent Technology. All rights reserved.
2
+
3
+ from .configuration_baichuan import BaichuanConfig
4
+ from .generation_utils import build_chat_input, TextIterStreamer
5
+
6
+ import math
7
+ from threading import Thread
8
+ from typing import List, Optional, Tuple, Union
9
+
10
+ import torch
11
+ from torch import nn
12
+ from torch.nn import CrossEntropyLoss
13
+ from torch.nn import functional as F
14
+ from transformers import PreTrainedModel, PretrainedConfig
15
+ from transformers.activations import ACT2FN
16
+ from transformers.generation.utils import GenerationConfig
17
+ from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
18
+ from transformers.utils import logging, ContextManagers
19
+
20
+ import os
21
+ from contextlib import contextmanager
22
+ from accelerate import init_empty_weights
23
+
24
+ logger = logging.get_logger(__name__)
25
+
26
+ try:
27
+ from xformers import ops as xops
28
+ except ImportError:
29
+ xops = None
30
+ logger.warning(
31
+ "Xformers is not installed correctly. If you want to use memory_efficient_attention to accelerate training use the following command to install Xformers\npip install xformers."
32
+ )
33
+
34
+
35
+ def _get_interleave(n):
36
+ def _get_interleave_power_of_2(n):
37
+ start = 2 ** (-(2 ** -(math.log2(n) - 3)))
38
+ ratio = start
39
+ return [start * ratio**i for i in range(n)]
40
+
41
+ if math.log2(n).is_integer():
42
+ return _get_interleave_power_of_2(n)
43
+ else:
44
+ closest_power_of_2 = 2 ** math.floor(math.log2(n))
45
+ return (
46
+ _get_interleave_power_of_2(closest_power_of_2)
47
+ + _get_interleave(2 * closest_power_of_2)[0::2][: n - closest_power_of_2]
48
+ )
49
+
50
+
51
+ def _fill_with_neg_inf(t):
52
+ """FP16-compatible function that fills a tensor with -inf."""
53
+ return t.float().fill_(float("-inf")).type_as(t)
54
+
55
+
56
+ def _buffered_future_mask(tensor, maxpos, alibi, attn_heads):
57
+ _future_mask = torch.triu(_fill_with_neg_inf(torch.zeros([maxpos, maxpos])), 1)
58
+ _future_mask = _future_mask.unsqueeze(0) + alibi
59
+ new_future_mask = _future_mask.to(tensor)
60
+ return new_future_mask[: tensor.shape[0] * attn_heads, :maxpos, :maxpos]
61
+
62
+
63
+ def _gen_alibi_mask(tensor, n_head, max_pos):
64
+ slopes = torch.Tensor(_get_interleave(n_head))
65
+ position_point = torch.arange(max_pos) - max_pos + 1
66
+ position_point = position_point.unsqueeze(0).unsqueeze(0).expand(n_head, -1, -1)
67
+ diag = torch.diag(position_point[0])
68
+ position_point = position_point - diag.unsqueeze(0).unsqueeze(0).transpose(-1, -2)
69
+ alibi = slopes.unsqueeze(1).unsqueeze(1) * position_point
70
+ alibi = alibi.view(n_head, 1, max_pos)
71
+ alibi_mask = torch.triu(_fill_with_neg_inf(torch.zeros([max_pos, max_pos])), 1)
72
+ alibi_mask = alibi_mask.unsqueeze(0) + alibi
73
+ return alibi_mask
74
+
75
+
76
+ class RMSNorm(torch.nn.Module):
77
+ def __init__(self, hidden_size, epsilon=1e-6):
78
+ super().__init__()
79
+ self.weight = torch.nn.Parameter(torch.empty(hidden_size))
80
+ self.epsilon = epsilon
81
+
82
+ def forward(self, hidden_states):
83
+ variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
84
+ hidden_states = hidden_states * torch.rsqrt(variance + self.epsilon)
85
+
86
+ # convert into half-precision
87
+ if self.weight.dtype in [torch.float16, torch.bfloat16]:
88
+ hidden_states = hidden_states.to(self.weight.dtype)
89
+
90
+ return self.weight * hidden_states
91
+
92
+
93
+ class MLP(torch.nn.Module):
94
+ def __init__(
95
+ self,
96
+ hidden_size: int,
97
+ intermediate_size: int,
98
+ hidden_act: str,
99
+ ):
100
+ super().__init__()
101
+ self.gate_proj = torch.nn.Linear(hidden_size, intermediate_size, bias=False)
102
+ self.down_proj = torch.nn.Linear(intermediate_size, hidden_size, bias=False)
103
+ self.up_proj = torch.nn.Linear(hidden_size, intermediate_size, bias=False)
104
+ self.act_fn = ACT2FN[hidden_act]
105
+
106
+ def forward(self, x):
107
+ return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
108
+
109
+
110
+ class BaichuanAttention(torch.nn.Module):
111
+ def __init__(self, config: BaichuanConfig):
112
+ super().__init__()
113
+ self.config = config
114
+ self.hidden_size = config.hidden_size
115
+ self.num_heads = config.num_attention_heads
116
+ self.head_dim = self.hidden_size // self.num_heads
117
+ self.max_position_embeddings = config.model_max_length
118
+
119
+ if (self.head_dim * self.num_heads) != self.hidden_size:
120
+ raise ValueError(
121
+ f"hidden_size {self.hidden_size} is not divisible by num_heads {self.num_heads}"
122
+ )
123
+ self.W_pack = torch.nn.Linear(
124
+ self.hidden_size, 3 * self.hidden_size, bias=False
125
+ )
126
+ self.o_proj = torch.nn.Linear(
127
+ self.num_heads * self.head_dim, self.hidden_size, bias=False
128
+ )
129
+
130
+ def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
131
+ return (
132
+ tensor.view(bsz, seq_len, self.num_heads, self.head_dim)
133
+ .transpose(1, 2)
134
+ .contiguous()
135
+ )
136
+
137
+ def forward(
138
+ self,
139
+ hidden_states: torch.Tensor,
140
+ attention_mask: Optional[torch.Tensor] = None,
141
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
142
+ output_attentions: bool = False,
143
+ use_cache: bool = False,
144
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
145
+ bsz, q_len, _ = hidden_states.size()
146
+
147
+ proj = self.W_pack(hidden_states)
148
+ proj = (
149
+ proj.unflatten(-1, (3, self.hidden_size))
150
+ .unsqueeze(0)
151
+ .transpose(0, -2)
152
+ .squeeze(-2)
153
+ )
154
+ query_states = (
155
+ proj[0].view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
156
+ )
157
+ key_states = (
158
+ proj[1].view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
159
+ )
160
+ value_states = (
161
+ proj[2].view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
162
+ )
163
+
164
+ kv_seq_len = key_states.shape[-2]
165
+ if past_key_value is not None:
166
+ kv_seq_len += past_key_value[0].shape[-2]
167
+
168
+ if past_key_value is not None:
169
+ # reuse k, v, self_attention
170
+ key_states = torch.cat([past_key_value[0], key_states], dim=2)
171
+ value_states = torch.cat([past_key_value[1], value_states], dim=2)
172
+
173
+ past_key_value = (key_states, value_states) if use_cache else None
174
+ if xops is not None and self.training:
175
+ attn_weights = None
176
+ # query_states = query_states.transpose(1, 2)
177
+ # key_states = key_states.transpose(1, 2)
178
+ # value_states = value_states.transpose(1, 2)
179
+ # attn_output = xops.memory_efficient_attention(
180
+ # query_states, key_states, value_states, attn_bias=attention_mask
181
+ # )
182
+ with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=True, enable_mem_efficient=True):
183
+ attn_output = F.scaled_dot_product_attention(query_states, key_states, value_states, attn_mask = attention_mask)
184
+ attn_output = attn_output.transpose(1, 2)
185
+ else:
186
+ attn_weights = torch.matmul(
187
+ query_states, key_states.transpose(2, 3)
188
+ ) / math.sqrt(self.head_dim)
189
+
190
+ if attention_mask is not None:
191
+ if q_len == 1: # inference with cache
192
+ if len(attention_mask.size()) == 4:
193
+ attention_mask = attention_mask[:, :, -1:, :]
194
+ else:
195
+ attention_mask = attention_mask[:, -1:, :]
196
+ attn_weights = attn_weights + attention_mask
197
+ attn_weights = torch.max(
198
+ attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)
199
+ )
200
+
201
+ attn_weights = torch.nn.functional.softmax(attn_weights, dim=-1)
202
+ attn_output = torch.matmul(attn_weights, value_states)
203
+
204
+ attn_output = attn_output.transpose(1, 2)
205
+ attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
206
+ attn_output = self.o_proj(attn_output)
207
+
208
+ if not output_attentions:
209
+ attn_weights = None
210
+
211
+ return attn_output, attn_weights, past_key_value
212
+
213
+
214
+ class BaichuanLayer(torch.nn.Module):
215
+ def __init__(self, config: BaichuanConfig):
216
+ super().__init__()
217
+ self.hidden_size = config.hidden_size
218
+ self.self_attn = BaichuanAttention(config=config)
219
+ self.mlp = MLP(
220
+ hidden_size=self.hidden_size,
221
+ intermediate_size=config.intermediate_size,
222
+ hidden_act=config.hidden_act,
223
+ )
224
+ self.input_layernorm = RMSNorm(config.hidden_size, epsilon=config.rms_norm_eps)
225
+ self.post_attention_layernorm = RMSNorm(
226
+ config.hidden_size, epsilon=config.rms_norm_eps
227
+ )
228
+
229
+ def forward(
230
+ self,
231
+ hidden_states: torch.Tensor,
232
+ attention_mask: Optional[torch.Tensor] = None,
233
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
234
+ output_attentions: Optional[bool] = False,
235
+ use_cache: Optional[bool] = False,
236
+ ) -> Tuple[
237
+ torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]
238
+ ]:
239
+ residual = hidden_states
240
+
241
+ hidden_states = self.input_layernorm(hidden_states)
242
+
243
+ # Self Attention
244
+ hidden_states, self_attn_weights, present_key_value = self.self_attn(
245
+ hidden_states=hidden_states,
246
+ attention_mask=attention_mask,
247
+ past_key_value=past_key_value,
248
+ output_attentions=output_attentions,
249
+ use_cache=use_cache,
250
+ )
251
+ hidden_states = residual + hidden_states
252
+
253
+ # Fully Connected
254
+ residual = hidden_states
255
+ hidden_states = self.post_attention_layernorm(hidden_states)
256
+ hidden_states = self.mlp(hidden_states)
257
+ hidden_states = residual + hidden_states
258
+
259
+ outputs = (hidden_states,)
260
+
261
+ if use_cache:
262
+ outputs += (present_key_value,)
263
+
264
+ return outputs
265
+
266
+
267
+ class BaichuanPreTrainedModel(PreTrainedModel):
268
+ config_class = BaichuanConfig
269
+ base_model_prefix = "model"
270
+ supports_gradient_checkpointing = True
271
+ _no_split_modules = ["BaichuanLayer"]
272
+ _keys_to_ignore_on_load_unexpected = [r"decoder\.version"]
273
+
274
+ def _init_weights(self, module):
275
+ std = self.config.initializer_range
276
+ if isinstance(module, torch.nn.Linear):
277
+ module.weight.data.normal_(mean=0.0, std=std)
278
+ if module.bias is not None:
279
+ module.bias.data.zero_()
280
+ elif isinstance(module, torch.nn.Embedding):
281
+ module.weight.data.normal_(mean=0.0, std=std)
282
+ if module.padding_idx is not None:
283
+ module.weight.data[module.padding_idx].zero_()
284
+
285
+ def _set_gradient_checkpointing(self, module, value=False):
286
+ if isinstance(module, BaichuanModel):
287
+ module.gradient_checkpointing = value
288
+
289
+
290
+ class BaichuanModel(BaichuanPreTrainedModel):
291
+ def __init__(self, config: BaichuanConfig):
292
+ super().__init__(config)
293
+ self.padding_idx = config.pad_token_id
294
+ self.vocab_size = config.vocab_size
295
+ self.n_head = config.num_attention_heads
296
+ self.embed_tokens = torch.nn.Embedding(
297
+ config.vocab_size, config.hidden_size, self.padding_idx
298
+ )
299
+ self.layers = torch.nn.ModuleList(
300
+ [BaichuanLayer(config) for _ in range(config.num_hidden_layers)]
301
+ )
302
+ self.norm = RMSNorm(config.hidden_size, epsilon=config.rms_norm_eps)
303
+
304
+ self.gradient_checkpointing = config.gradient_checkpointing
305
+ self.post_init()
306
+ self.max_cache_pos = config.model_max_length
307
+ self.first_run = True
308
+ self.alibi_mask = None
309
+
310
+ def get_input_embeddings(self):
311
+ return self.embed_tokens
312
+
313
+ def set_input_embeddings(self, value):
314
+ self.embed_tokens = value
315
+
316
+ def get_alibi_mask(self, tensor, seq_length_with_past):
317
+ if self.training:
318
+ slopes = torch.Tensor(_get_interleave(self.n_head))
319
+ position_point = (
320
+ torch.arange(seq_length_with_past) - seq_length_with_past + 1
321
+ )
322
+ position_point = (
323
+ position_point.unsqueeze(0)
324
+ .unsqueeze(0)
325
+ .expand(self.n_head, seq_length_with_past, -1)
326
+ )
327
+ diag = torch.diag(position_point[0])
328
+ position_point = position_point - diag.unsqueeze(0).unsqueeze(0).transpose(
329
+ -1, -2
330
+ )
331
+ alibi = slopes.unsqueeze(1).unsqueeze(1) * position_point
332
+ mask = _buffered_future_mask(
333
+ tensor, seq_length_with_past, alibi, self.n_head
334
+ )
335
+ else:
336
+ if self.first_run:
337
+ self.first_run = False
338
+ self.register_buffer(
339
+ "future_mask",
340
+ _gen_alibi_mask(tensor, self.n_head, self.max_cache_pos).to(
341
+ tensor
342
+ ),
343
+ persistent=False,
344
+ )
345
+ if seq_length_with_past > self.max_cache_pos:
346
+ self.max_cache_pos = seq_length_with_past
347
+ self.register_buffer(
348
+ "future_mask",
349
+ _gen_alibi_mask(tensor, self.n_head, self.max_cache_pos).to(
350
+ tensor
351
+ ),
352
+ persistent=False,
353
+ )
354
+ mask = self.future_mask[
355
+ : self.n_head, :seq_length_with_past, :seq_length_with_past
356
+ ]
357
+ return mask
358
+
359
+ def forward(
360
+ self,
361
+ input_ids: torch.LongTensor = None,
362
+ attention_mask: Optional[torch.Tensor] = None,
363
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
364
+ inputs_embeds: Optional[torch.FloatTensor] = None,
365
+ use_cache: Optional[bool] = False,
366
+ output_attentions: Optional[bool] = False,
367
+ output_hidden_states: Optional[bool] = False,
368
+ return_dict: Optional[bool] = True,
369
+ ) -> Union[Tuple, BaseModelOutputWithPast]:
370
+ if input_ids is not None and inputs_embeds is not None:
371
+ raise ValueError(
372
+ "You cannot provide both input_ids and inputs_embeds simultaneously"
373
+ )
374
+ elif input_ids is not None:
375
+ batch_size, seq_length = input_ids.shape
376
+ elif inputs_embeds is not None:
377
+ batch_size, seq_length, _ = inputs_embeds.shape
378
+ else:
379
+ raise ValueError("You need to provide input_ids or inputs_embeds")
380
+
381
+ return_dict = (
382
+ return_dict if return_dict is not None else self.config.use_return_dict
383
+ )
384
+
385
+ seq_length_with_past = seq_length
386
+
387
+ if past_key_values is not None:
388
+ past_key_values_length = past_key_values[0][0].shape[2]
389
+ seq_length_with_past = seq_length_with_past + past_key_values_length
390
+
391
+ if inputs_embeds is None:
392
+ inputs_embeds = self.embed_tokens(input_ids)
393
+
394
+ if self.training:
395
+ if (
396
+ self.alibi_mask is None
397
+ or self.alibi_mask.shape[-1] != seq_length_with_past
398
+ ):
399
+ self.alibi_mask = self.get_alibi_mask(
400
+ inputs_embeds, seq_length_with_past
401
+ )
402
+ alibi_mask = self.alibi_mask
403
+ else:
404
+ alibi_mask = self.get_alibi_mask(inputs_embeds, seq_length_with_past)
405
+
406
+ if attention_mask is not None:
407
+ if len(attention_mask.shape) == 2:
408
+ expanded_mask = attention_mask.to(alibi_mask.dtype)
409
+ expanded_mask = torch.tril(
410
+ torch.gt(expanded_mask[:, :, None] * expanded_mask[:, None, :], 0)
411
+ ) * torch.eq(expanded_mask[:, :, None] - expanded_mask[:, None, :], 0)
412
+ else:
413
+ expanded_mask = attention_mask
414
+ bsz = inputs_embeds.size(0)
415
+ src_len, tgt_len = alibi_mask.size()[-2:]
416
+ expanded_mask = (
417
+ expanded_mask.unsqueeze(1)
418
+ .expand(bsz, 1, src_len, tgt_len)
419
+ .to(alibi_mask.dtype)
420
+ )
421
+ inverted_mask = 1.0 - expanded_mask
422
+ inverted_mask = inverted_mask.masked_fill(
423
+ inverted_mask.to(torch.bool), torch.finfo(alibi_mask.dtype).min
424
+ )
425
+ attention_mask = inverted_mask + alibi_mask.unsqueeze(0)
426
+ else:
427
+ attention_mask = alibi_mask
428
+
429
+ hidden_states = inputs_embeds
430
+
431
+ if self.gradient_checkpointing and self.training:
432
+ if use_cache:
433
+ logger.warning_once(
434
+ "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
435
+ )
436
+ use_cache = False
437
+
438
+ # decoder layers
439
+ all_hidden_states = () if output_hidden_states else None
440
+ all_self_attns = () if output_attentions else None
441
+ next_decoder_cache = () if use_cache else None
442
+
443
+ for idx, decoder_layer in enumerate(self.layers):
444
+ if output_hidden_states:
445
+ all_hidden_states += (hidden_states,)
446
+
447
+ past_key_value = (
448
+ past_key_values[idx] if past_key_values is not None else None
449
+ )
450
+
451
+ if self.gradient_checkpointing and self.training:
452
+
453
+ def create_custom_forward(module):
454
+ def custom_forward(*inputs):
455
+ # None for past_key_value
456
+ return module(*inputs, output_attentions, None)
457
+
458
+ return custom_forward
459
+
460
+ layer_outputs = torch.utils.checkpoint.checkpoint(
461
+ create_custom_forward(decoder_layer),
462
+ hidden_states,
463
+ attention_mask,
464
+ None,
465
+ )
466
+ else:
467
+ layer_outputs = decoder_layer(
468
+ hidden_states,
469
+ attention_mask=attention_mask,
470
+ past_key_value=past_key_value,
471
+ output_attentions=output_attentions,
472
+ use_cache=use_cache,
473
+ )
474
+
475
+ hidden_states = layer_outputs[0]
476
+
477
+ if use_cache:
478
+ next_decoder_cache += (layer_outputs[2 if output_attentions else 1],)
479
+
480
+ if output_attentions:
481
+ all_self_attns += (layer_outputs[1],)
482
+
483
+ hidden_states = self.norm(hidden_states)
484
+
485
+ # add hidden states from the last decoder layer
486
+ if output_hidden_states:
487
+ all_hidden_states += (hidden_states,)
488
+
489
+ next_cache = next_decoder_cache if use_cache else None
490
+ if not return_dict:
491
+ return tuple(
492
+ v
493
+ for v in [hidden_states, next_cache, all_hidden_states, all_self_attns]
494
+ if v is not None
495
+ )
496
+ return BaseModelOutputWithPast(
497
+ last_hidden_state=hidden_states,
498
+ past_key_values=next_cache,
499
+ hidden_states=all_hidden_states,
500
+ attentions=all_self_attns,
501
+ )
502
+
503
+
504
+ class NormHead(nn.Module):
505
+ def __init__(self, hidden_size, vocab_size, bias=False):
506
+ super().__init__()
507
+ self.weight = nn.Parameter(torch.empty((vocab_size, hidden_size)))
508
+ nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))
509
+ self.first_flag = True
510
+
511
+ def forward(self, hidden_states):
512
+ if self.training:
513
+ norm_weight = nn.functional.normalize(self.weight)
514
+ self.first_flag = True
515
+ elif self.first_flag:
516
+ self.first_flag = False
517
+ self.weight.data = nn.functional.normalize(self.weight)
518
+ norm_weight = self.weight
519
+ else:
520
+ norm_weight = self.weight
521
+ return nn.functional.linear(hidden_states, norm_weight)
522
+
523
+ _init_weights = True
524
+ @contextmanager
525
+ def no_init_weights(_enable=True):
526
+ global _init_weights
527
+ old_init_weights = _init_weights
528
+ if _enable:
529
+ _init_weights = False
530
+ try:
531
+ yield
532
+ finally:
533
+ _init_weights = old_init_weights
534
+
535
+
536
+ class BaichuanForCausalLM(BaichuanPreTrainedModel):
537
+ def __init__(self, config, *model_args, **model_kwargs):
538
+ super().__init__(config, *model_args, **model_kwargs)
539
+ self.model = BaichuanModel(config)
540
+ self.lm_head = NormHead(config.hidden_size, config.vocab_size, bias=False)
541
+ #if hasattr(config, "quantization_config") and config.quantization_config['load_in_4bit']:
542
+ if hasattr(config, "quantization_config") and isinstance(config.quantization_config, dict) and config.quantization_config.get('load_in_4bit', False):
543
+ try:
544
+ from .quantizer import quantize_offline, init_model_weight_int4
545
+ except ImportError:
546
+ raise ImportError(f"Needs quantize_offline to run quantize.")
547
+ quantize_offline(self, 4)
548
+ # Initialize weights and apply final processing
549
+ self.post_init()
550
+
551
+ def get_input_embeddings(self):
552
+ return self.model.embed_tokens
553
+
554
+ def set_input_embeddings(self, value):
555
+ self.model.embed_tokens = value
556
+
557
+ def get_output_embeddings(self):
558
+ return self.lm_head
559
+
560
+ def set_output_embeddings(self, new_embeddings):
561
+ self.lm_head = new_embeddings
562
+
563
+ def set_decoder(self, decoder):
564
+ self.model = decoder
565
+
566
+ def get_decoder(self):
567
+ return self.model
568
+
569
+ @classmethod
570
+ def from_pretrained(
571
+ cls,
572
+ pretrained_model_name_or_path: Optional[Union[str, os.PathLike]],
573
+ *model_args,
574
+ config: Optional[Union[PretrainedConfig, str, os.PathLike]] = None,
575
+ cache_dir: Optional[Union[str, os.PathLike]] = None,
576
+ ignore_mismatched_sizes: bool = False,
577
+ force_download: bool = False,
578
+ local_files_only: bool = False,
579
+ token: Optional[Union[str, bool]] = None,
580
+ revision: str = "main",
581
+ use_safetensors: bool = None,
582
+ **kwargs,
583
+ ):
584
+
585
+ # Load config if we don't provide a configuration
586
+ if not isinstance(config, PretrainedConfig):
587
+ config_path = config if config is not None else pretrained_model_name_or_path
588
+ config, model_kwargs = cls.config_class.from_pretrained(
589
+ config_path,
590
+ cache_dir=cache_dir,
591
+ return_unused_kwargs=True,
592
+ force_download=force_download,
593
+ resume_download=False,
594
+ proxies=None,
595
+ local_files_only=local_files_only,
596
+ token=token,
597
+ revision=revision,
598
+ subfolder="",
599
+ _from_auto=False,
600
+ _from_pipeline=None,
601
+ **kwargs,
602
+ )
603
+ else:
604
+ model_kwargs = kwargs
605
+
606
+ if hasattr(config, "quantization_config") and config.quantization_config['load_in_4bit']:
607
+ try:
608
+ from .quantizer import init_model_weight_int4
609
+ from accelerate import init_empty_weights, dispatch_model, infer_auto_device_map
610
+ from accelerate.utils import CustomDtype
611
+ from accelerate.utils import get_balanced_memory
612
+ except ImportError:
613
+ raise ImportError(f"Needs import model weight init func to run quantize.")
614
+ # Instantiate model.
615
+ init_contexts = [no_init_weights(_enable=True)]
616
+ init_contexts.append(init_empty_weights())
617
+ with ContextManagers(init_contexts):
618
+ model = cls(config)
619
+
620
+ model_file = os.path.join(pretrained_model_name_or_path, 'pytorch_model.bin')
621
+ state_dict = torch.load(model_file, map_location="cpu")
622
+ model.is_quantized = True
623
+
624
+ device_map = kwargs.pop("device_map", None)
625
+ torch_dtype = kwargs.pop("torch_dtype", None)
626
+ if device_map is not None:
627
+ kwargs = {"no_split_module_classes": model._no_split_modules}
628
+ target_dtype = CustomDtype.INT4
629
+ max_memory = get_balanced_memory(
630
+ model,
631
+ dtype=target_dtype,
632
+ low_zero=(device_map == "balanced_low_0"),
633
+ max_memory=None,
634
+ **kwargs,
635
+ )
636
+ kwargs["max_memory"] = max_memory
637
+ device_map = infer_auto_device_map(model, dtype=target_dtype, **kwargs)
638
+ model = init_model_weight_int4(config, model, state_dict)
639
+
640
+ # Set model in evaluation mode to deactivate DropOut modules by default
641
+ model.eval()
642
+ # If it is a model with generation capabilities, attempt to load the generation config
643
+ if model.can_generate():
644
+ try:
645
+ model.generation_config = GenerationConfig.from_pretrained(
646
+ pretrained_model_name_or_path,
647
+ cache_dir=cache_dir,
648
+ force_download=force_download,
649
+ resume_download=False,
650
+ proxies=None,
651
+ local_files_only=local_files_only,
652
+ token=token,
653
+ revision=revision,
654
+ subfolder="",
655
+ _from_auto=False,
656
+ _from_pipeline=None,
657
+ **kwargs,
658
+ )
659
+ except (OSError, TypeError):
660
+ logger.info(
661
+ "Generation config file not found, using a generation config created from the model config."
662
+ )
663
+ pass
664
+
665
+ if device_map is not None:
666
+ dispatch_model(model, device_map=device_map)
667
+
668
+ return model
669
+
670
+ return super(BaichuanForCausalLM, cls).from_pretrained(pretrained_model_name_or_path, *model_args,
671
+ config=config, cache_dir=cache_dir, ignore_mismatched_sizes=ignore_mismatched_sizes,
672
+ force_download=force_download, local_files_only=local_files_only, token=token, revision=revision,
673
+ use_safetensors=use_safetensors, **kwargs)
674
+
675
+ def forward(
676
+ self,
677
+ input_ids: torch.LongTensor = None,
678
+ attention_mask: Optional[torch.Tensor] = None,
679
+ past_key_values: Optional[List[torch.FloatTensor]] = None,
680
+ inputs_embeds: Optional[torch.FloatTensor] = None,
681
+ labels: Optional[torch.LongTensor] = None,
682
+ use_cache: Optional[bool] = None,
683
+ output_attentions: Optional[bool] = False,
684
+ output_hidden_states: Optional[bool] = False,
685
+ return_dict: Optional[bool] = True,
686
+ **kwargs,
687
+ ) -> Union[Tuple, CausalLMOutputWithPast]:
688
+ return_dict = (
689
+ return_dict if return_dict is not None else self.config.use_return_dict
690
+ )
691
+
692
+ # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
693
+ outputs = self.model(
694
+ input_ids=input_ids,
695
+ attention_mask=attention_mask,
696
+ past_key_values=past_key_values,
697
+ inputs_embeds=inputs_embeds,
698
+ use_cache=use_cache,
699
+ output_attentions=output_attentions,
700
+ output_hidden_states=output_hidden_states,
701
+ return_dict=return_dict,
702
+ )
703
+
704
+ hidden_states = outputs[0]
705
+ logits = self.lm_head(hidden_states)
706
+ loss = None
707
+ if labels is not None:
708
+ # Shift so that tokens < n predict n
709
+ shift_logits = logits[..., :-1, :].contiguous()
710
+ shift_labels = labels[..., 1:].contiguous()
711
+ # Flatten the tokens
712
+ loss_fct = CrossEntropyLoss()
713
+ shift_logits = shift_logits.view(-1, self.config.vocab_size)
714
+ shift_labels = shift_labels.view(-1)
715
+ softmax_normalizer = shift_logits.max(-1).values ** 2
716
+ z_loss = self.config.z_loss_weight * softmax_normalizer.mean()
717
+ # Enable model parallelism
718
+ shift_labels = shift_labels.to(shift_logits.device)
719
+ loss = loss_fct(shift_logits, shift_labels) + z_loss
720
+
721
+ if not return_dict:
722
+ output = (logits,) + outputs[1:]
723
+ return (loss,) + output if loss is not None else output
724
+
725
+ return CausalLMOutputWithPast(
726
+ loss=loss,
727
+ logits=logits,
728
+ past_key_values=outputs.past_key_values,
729
+ hidden_states=outputs.hidden_states,
730
+ attentions=outputs.attentions,
731
+ )
732
+
733
+ def quantize(self, bits: int):
734
+ try:
735
+ from .quantizer import quantize_online
736
+ except ImportError:
737
+ raise ImportError(f"Needs QLinear to run quantize.")
738
+ return quantize_online(self, bits)
739
+
740
+ def prepare_inputs_for_generation(
741
+ self,
742
+ input_ids: torch.LongTensor,
743
+ past_key_values: Optional[torch.Tensor] = None,
744
+ attention_mask: Optional[torch.Tensor] = None,
745
+ inputs_embeds: Optional[torch.Tensor] = None,
746
+ **kwargs,
747
+ ):
748
+ if past_key_values:
749
+ input_ids = input_ids[:, -1:]
750
+
751
+ # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
752
+ if inputs_embeds is not None and past_key_values is None:
753
+ model_inputs = {"inputs_embeds": inputs_embeds}
754
+ else:
755
+ model_inputs = {"input_ids": input_ids}
756
+
757
+ model_inputs.update(
758
+ {
759
+ "past_key_values": past_key_values,
760
+ "use_cache": kwargs.get("use_cache"),
761
+ "attention_mask": attention_mask,
762
+ }
763
+ )
764
+ return model_inputs
765
+
766
+ @staticmethod
767
+ def _reorder_cache(past_key_values, beam_idx):
768
+ return tuple(
769
+ tuple(past_state.index_select(0, beam_idx) for past_state in layer_past)
770
+ for layer_past in past_key_values
771
+ )
772
+
773
+ def _build_chat_input(
774
+ self, tokenizer, messages: List[dict], max_new_tokens: int = 0
775
+ ):
776
+ max_new_tokens = max_new_tokens or self.generation_config.max_new_tokens
777
+ max_input_tokens = self.config.model_max_length - max_new_tokens
778
+ max_input_tokens = max(self.config.model_max_length // 2, max_input_tokens)
779
+ total_input, round_input = [], []
780
+ for i, message in enumerate(messages[::-1]):
781
+ content_tokens = tokenizer.encode(message["content"])
782
+ if message["role"] == "user":
783
+ round_input = (
784
+ [self.generation_config.user_token_id]
785
+ + content_tokens
786
+ + round_input
787
+ )
788
+ if (
789
+ total_input
790
+ and len(total_input) + len(round_input) > max_input_tokens
791
+ ):
792
+ break
793
+ else:
794
+ total_input = round_input + total_input
795
+ if len(total_input) >= max_input_tokens:
796
+ break
797
+ else:
798
+ round_input = []
799
+ elif message["role"] == "assistant":
800
+ round_input = (
801
+ [self.generation_config.assistant_token_id]
802
+ + content_tokens
803
+ + [self.generation_config.eos_token_id]
804
+ + round_input
805
+ )
806
+ else:
807
+ raise ValueError(f"message role not supported yet: {message['role']}")
808
+ total_input = total_input[-max_input_tokens:] # truncate left
809
+ total_input.append(self.generation_config.assistant_token_id)
810
+ total_input = torch.LongTensor([total_input]).to(self.device)
811
+ return total_input
812
+
813
+ def chat(self, tokenizer, messages: List[dict], stream=False,
814
+ generation_config: Optional[GenerationConfig]=None):
815
+ generation_config = generation_config or self.generation_config
816
+ input_ids = build_chat_input(self, tokenizer, messages, generation_config.max_new_tokens)
817
+ if stream:
818
+ streamer = TextIterStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
819
+ Thread(target=self.generate, kwargs=dict(
820
+ inputs=input_ids, streamer=streamer,
821
+ generation_config=generation_config,
822
+ )).start()
823
+ return streamer
824
+ else:
825
+ outputs = self.generate(input_ids, generation_config=generation_config)
826
+ response = tokenizer.decode(outputs[0][len(input_ids[0]):], skip_special_tokens=True)
827
+ return response
sakura_13b_model_v0.5_4bits_autogptq_40k/quantizer.py ADDED
@@ -0,0 +1,211 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import bitsandbytes as bnb
2
+ from accelerate import init_empty_weights
3
+ from bitsandbytes.nn.modules import Params4bit, Int8Params
4
+ import torch
5
+
6
+ def Params4bitCuda(self, device):
7
+ self.data = self.data.cuda(device)
8
+ self.quant_state[0] = self.quant_state[0].cuda(device)
9
+ self.quant_state[4][0] = self.quant_state[4][0].cuda(device)
10
+ self.quant_state[4][1][0] = self.quant_state[4][1][0].cuda(device)
11
+ self.quant_state[4][1][1] = self.quant_state[4][1][1].cuda(device)
12
+
13
+ self.quant_state[6] = self.quant_state[6].cuda(device)
14
+ return self
15
+
16
+ class Linear4bitOnline(torch.nn.Module):
17
+ def __init__(self, weight, bias, quant_type):
18
+ super().__init__()
19
+ self.weight = Params4bit(
20
+ weight.data, requires_grad=False, compress_statistics=True, quant_type=quant_type
21
+ )
22
+ self.compute_dtype = None
23
+ #self.weight.cuda(weight.device)
24
+ self.bias = bias
25
+
26
+ def forward(self, x: torch.Tensor):
27
+ # weights are cast automatically as Int8Params, but the bias has to be cast manually
28
+ if self.bias is not None and self.bias.dtype != x.dtype:
29
+ self.bias.data = self.bias.data.to(x.dtype)
30
+
31
+ if getattr(self.weight, "quant_state", None) is None:
32
+ print(
33
+ "FP4 quantization state not initialized. Please call .cuda() or .to(device) on the LinearFP4 layer first."
34
+ )
35
+ inp_dtype = x.dtype
36
+ if self.compute_dtype is not None:
37
+ x = x.to(self.compute_dtype)
38
+
39
+ bias = None if self.bias is None else self.bias.to(self.compute_dtype)
40
+ out = bnb.matmul_4bit(
41
+ x, self.weight.t(), bias=bias, quant_state=self.weight.quant_state
42
+ )
43
+
44
+ out = out.to(inp_dtype)
45
+
46
+ return out
47
+
48
+ class Linear8bitLtOnline(torch.nn.Module):
49
+ def __init__(
50
+ self,
51
+ weight,
52
+ bias,
53
+ has_fp16_weights=True,
54
+ memory_efficient_backward=False,
55
+ threshold=0.0,
56
+ index=None,
57
+ ):
58
+ super().__init__()
59
+ assert (
60
+ not memory_efficient_backward
61
+ ), "memory_efficient_backward is no longer required and the argument is deprecated in 0.37.0 and will be removed in 0.39.0"
62
+ self.state = bnb.MatmulLtState()
63
+ self.index = index
64
+
65
+ # Necessary for stacked layers
66
+ self.state.threshold = threshold
67
+ self.state.has_fp16_weights = has_fp16_weights
68
+ self.state.memory_efficient_backward = memory_efficient_backward
69
+ if threshold > 0.0 and not has_fp16_weights:
70
+ self.state.use_pool = True
71
+
72
+ self.weight = Int8Params(
73
+ weight.data,
74
+ has_fp16_weights=has_fp16_weights,
75
+ requires_grad=has_fp16_weights,
76
+ )
77
+ self.bias = bias
78
+
79
+ def init_8bit_state(self):
80
+ self.state.CB = self.weight.CB
81
+ self.state.SCB = self.weight.SCB
82
+ self.weight.CB = None
83
+ self.weight.SCB = None
84
+
85
+ def forward(self, x: torch.Tensor):
86
+ self.state.is_training = self.training
87
+ if self.weight.CB is not None:
88
+ self.init_8bit_state()
89
+
90
+ # weights are cast automatically as Int8Params, but the bias has to be cast manually
91
+ if self.bias is not None and self.bias.dtype != x.dtype:
92
+ self.bias.data = self.bias.data.to(x.dtype)
93
+
94
+ out = bnb.matmul(x, self.weight, bias=self.bias, state=self.state)
95
+
96
+ if not self.state.has_fp16_weights:
97
+ if self.state.CB is not None and self.state.CxB is not None:
98
+ # we converted 8-bit row major to turing/ampere format in the first inference pass
99
+ # we no longer need the row-major weight
100
+ del self.state.CB
101
+ self.weight.data = self.state.CxB
102
+ return out
103
+
104
+ def quantize_offline(model, bits: int):
105
+ assert (bits == 4), f'bits: {bits} is not supported'
106
+
107
+ for i, layer in enumerate(model.model.layers):
108
+ layer.self_attn.W_pack = bnb.nn.Linear4bit(
109
+ layer.self_attn.W_pack.weight.shape[1],
110
+ layer.self_attn.W_pack.weight.shape[0],
111
+ False,
112
+ torch.float16,
113
+ compress_statistics=True,
114
+ quant_type="nf4",
115
+ )
116
+ layer.self_attn.o_proj = bnb.nn.Linear4bit(
117
+ layer.self_attn.o_proj.weight.shape[1],
118
+ layer.self_attn.o_proj.weight.shape[0],
119
+ False,
120
+ torch.float16,
121
+ compress_statistics=True,
122
+ quant_type="nf4",
123
+ )
124
+
125
+ layer.mlp.gate_proj = bnb.nn.Linear4bit(
126
+ layer.mlp.gate_proj.weight.shape[1],
127
+ layer.mlp.gate_proj.weight.shape[0],
128
+ False,
129
+ torch.float16,
130
+ compress_statistics=True,
131
+ quant_type="nf4",
132
+ )
133
+ layer.mlp.down_proj = bnb.nn.Linear4bit(
134
+ layer.mlp.down_proj.weight.shape[1],
135
+ layer.mlp.down_proj.weight.shape[0],
136
+ False,
137
+ torch.float16,
138
+ compress_statistics=True,
139
+ quant_type="nf4",
140
+ )
141
+ layer.mlp.up_proj = bnb.nn.Linear4bit(
142
+ layer.mlp.up_proj.weight.shape[1],
143
+ layer.mlp.up_proj.weight.shape[0],
144
+ False,
145
+ torch.float16,
146
+ compress_statistics=True,
147
+ quant_type="nf4",
148
+ )
149
+ return model
150
+
151
+ def quantize_online(model, bits: int):
152
+ def quant(weight, bias=None):
153
+ if bits == 8:
154
+ linear = Linear8bitLtOnline(
155
+ weight,
156
+ bias,
157
+ has_fp16_weights=False,
158
+ threshold=6.0,
159
+ )
160
+ if bias is not None:
161
+ linear.bias = torch.nn.Parameter(bias)
162
+ elif bits == 4:
163
+ linear = Linear4bitOnline(
164
+ weight,
165
+ bias,
166
+ quant_type="nf4", #fp4/nf4
167
+ )
168
+ else:
169
+ raise ValueError("quantize only support 4/8 bit")
170
+ return linear
171
+
172
+ for i, layer in enumerate(model.model.layers):
173
+ layer.self_attn.W_pack = quant(layer.self_attn.W_pack.weight)
174
+ layer.self_attn.o_proj = quant(layer.self_attn.o_proj.weight)
175
+ layer.mlp.gate_proj = quant(layer.mlp.gate_proj.weight)
176
+ layer.mlp.down_proj = quant(layer.mlp.down_proj.weight)
177
+ layer.mlp.up_proj = quant(layer.mlp.up_proj.weight)
178
+ return model
179
+
180
+ def init_model_weight_int4(config, model, state_dict):
181
+ #replace Params4bit.cuda with Params4bitCuda
182
+ Params4bit.cuda = Params4bitCuda
183
+
184
+ for i in range(config.num_hidden_layers):
185
+ weight_data = state_dict[f'model.layers.{i}.self_attn.W_pack.weight.data']
186
+ weight_quant_state = state_dict[f'model.layers.{i}.self_attn.W_pack.weight.quant_state']
187
+ model.model.layers[i].self_attn.W_pack.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
188
+
189
+ weight_data = state_dict[f'model.layers.{i}.self_attn.o_proj.weight.data']
190
+ weight_quant_state = state_dict[f'model.layers.{i}.self_attn.o_proj.weight.quant_state']
191
+ model.model.layers[i].self_attn.o_proj.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
192
+
193
+ weight_data = state_dict[f'model.layers.{i}.mlp.gate_proj.weight.data']
194
+ weight_quant_state = state_dict[f'model.layers.{i}.mlp.gate_proj.weight.quant_state']
195
+ model.model.layers[i].mlp.gate_proj.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
196
+
197
+ weight_data = state_dict[f'model.layers.{i}.mlp.up_proj.weight.data']
198
+ weight_quant_state = state_dict[f'model.layers.{i}.mlp.up_proj.weight.quant_state']
199
+ model.model.layers[i].mlp.up_proj.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
200
+
201
+ weight_data = state_dict[f'model.layers.{i}.mlp.down_proj.weight.data']
202
+ weight_quant_state = state_dict[f'model.layers.{i}.mlp.down_proj.weight.quant_state']
203
+ model.model.layers[i].mlp.down_proj.weight = Params4bit(weight_data, requires_grad=False, quant_state=weight_quant_state)
204
+
205
+ model.model.layers[i].input_layernorm.weight = state_dict[f'model.layers.{i}.input_layernorm.weight']
206
+ model.model.layers[i].post_attention_layernorm.weight = state_dict[f'model.layers.{i}.post_attention_layernorm.weight']
207
+
208
+ model.model.embed_tokens.weight = state_dict['model.embed_tokens.weight']
209
+ model.model.norm.weight = state_dict['model.norm.weight']
210
+ model.lm_head.weight = state_dict['lm_head.weight']
211
+ return model
sakura_13b_model_v0.5_4bits_autogptq_40k/special_tokens_map.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": true
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": true
15
+ },
16
+ "unk_token": {
17
+ "content": "<unk>",
18
+ "lstrip": false,
19
+ "normalized": true,
20
+ "rstrip": false,
21
+ "single_word": true
22
+ },
23
+ "pad_token": {
24
+ "content": "<unk>",
25
+ "lstrip": false,
26
+ "normalized": true,
27
+ "rstrip": false,
28
+ "single_word": true
29
+ }
30
+ }
sakura_13b_model_v0.5_4bits_autogptq_40k/tokenization_baichuan.py ADDED
@@ -0,0 +1,258 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (c) 2023, Baichuan Intelligent Technology. All rights reserved.
2
+
3
+ import os
4
+ from shutil import copyfile
5
+ from typing import Any, Dict, List, Optional, Tuple
6
+
7
+ import sentencepiece as spm
8
+ from transformers.tokenization_utils import AddedToken, PreTrainedTokenizer
9
+ from transformers.utils import logging
10
+
11
+
12
+ logger = logging.get_logger(__name__)
13
+
14
+ VOCAB_FILES_NAMES = {"vocab_file": "tokenizer.model"}
15
+
16
+ PRETRAINED_VOCAB_FILES_MAP = {
17
+ "vocab_file": {},
18
+ "tokenizer_file": {},
19
+ }
20
+ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {}
21
+
22
+
23
+ class BaichuanTokenizer(PreTrainedTokenizer):
24
+ """
25
+ Construct a Baichuan tokenizer. Based on byte-level Byte-Pair-Encoding.
26
+
27
+ Args:
28
+ vocab_file (`str`):
29
+ Path to the vocabulary file.
30
+ """
31
+
32
+ vocab_files_names = VOCAB_FILES_NAMES
33
+ pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
34
+ max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
35
+ model_input_names = ["input_ids", "attention_mask"]
36
+
37
+ def __init__(
38
+ self,
39
+ vocab_file,
40
+ unk_token="<unk>",
41
+ bos_token="<s>",
42
+ eos_token="</s>",
43
+ pad_token=None,
44
+ sp_model_kwargs: Optional[Dict[str, Any]] = None,
45
+ add_bos_token=True,
46
+ add_eos_token=False,
47
+ clean_up_tokenization_spaces=False,
48
+ **kwargs,
49
+ ):
50
+ self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
51
+ bos_token = (
52
+ AddedToken(bos_token, lstrip=False, rstrip=False)
53
+ if isinstance(bos_token, str)
54
+ else bos_token
55
+ )
56
+ eos_token = (
57
+ AddedToken(eos_token, lstrip=False, rstrip=False)
58
+ if isinstance(eos_token, str)
59
+ else eos_token
60
+ )
61
+ unk_token = (
62
+ AddedToken(unk_token, lstrip=False, rstrip=False)
63
+ if isinstance(unk_token, str)
64
+ else unk_token
65
+ )
66
+ pad_token = (
67
+ AddedToken(pad_token, lstrip=False, rstrip=False)
68
+ if isinstance(pad_token, str)
69
+ else pad_token
70
+ )
71
+ super().__init__(
72
+ bos_token=bos_token,
73
+ eos_token=eos_token,
74
+ unk_token=unk_token,
75
+ pad_token=pad_token,
76
+ add_bos_token=add_bos_token,
77
+ add_eos_token=add_eos_token,
78
+ sp_model_kwargs=self.sp_model_kwargs,
79
+ clean_up_tokenization_spaces=clean_up_tokenization_spaces,
80
+ **kwargs,
81
+ )
82
+ self.vocab_file = vocab_file
83
+ self.add_bos_token = add_bos_token
84
+ self.add_eos_token = add_eos_token
85
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
86
+ self.sp_model.Load(vocab_file)
87
+
88
+ def __getstate__(self):
89
+ state = self.__dict__.copy()
90
+ state["sp_model"] = None
91
+ return state
92
+
93
+ def __setstate__(self, d):
94
+ self.__dict__ = d
95
+ self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
96
+ self.sp_model.Load(self.vocab_file)
97
+
98
+ @property
99
+ def vocab_size(self):
100
+ """Returns vocab size"""
101
+ return self.sp_model.get_piece_size()
102
+
103
+ def get_vocab(self):
104
+ """Returns vocab as a dict"""
105
+ vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
106
+ vocab.update(self.added_tokens_encoder)
107
+ return vocab
108
+
109
+ def _tokenize(self, text):
110
+ """Returns a tokenized string."""
111
+ return self.sp_model.encode(text, out_type=str)
112
+
113
+ def _convert_token_to_id(self, token):
114
+ """Converts a token (str) in an id using the vocab."""
115
+ return self.sp_model.piece_to_id(token)
116
+
117
+ def _convert_id_to_token(self, index):
118
+ """Converts an index (integer) in a token (str) using the vocab."""
119
+ token = self.sp_model.IdToPiece(index)
120
+ return token
121
+
122
+ def convert_tokens_to_string(self, tokens):
123
+ """Converts a sequence of tokens (string) in a single string."""
124
+ current_sub_tokens = []
125
+ out_string = ""
126
+ prev_is_special = False
127
+ for i, token in enumerate(tokens):
128
+ # make sure that special tokens are not decoded using sentencepiece model
129
+ if token in self.all_special_tokens:
130
+ if not prev_is_special and i != 0:
131
+ out_string += " "
132
+ out_string += self.sp_model.decode(current_sub_tokens) + token
133
+ prev_is_special = True
134
+ current_sub_tokens = []
135
+ else:
136
+ current_sub_tokens.append(token)
137
+ prev_is_special = False
138
+ out_string += self.sp_model.decode(current_sub_tokens)
139
+ return out_string
140
+
141
+ def save_vocabulary(
142
+ self, save_directory, filename_prefix: Optional[str] = None
143
+ ) -> Tuple[str]:
144
+ """
145
+ Save the vocabulary and special tokens file to a directory.
146
+
147
+ Args:
148
+ save_directory (`str`):
149
+ The directory in which to save the vocabulary.
150
+
151
+ Returns:
152
+ `Tuple(str)`: Paths to the files saved.
153
+ """
154
+ if not os.path.isdir(save_directory):
155
+ logger.error(f"Vocabulary path ({save_directory}) should be a directory")
156
+ return
157
+ out_vocab_file = os.path.join(
158
+ save_directory,
159
+ (filename_prefix + "-" if filename_prefix else "")
160
+ + VOCAB_FILES_NAMES["vocab_file"],
161
+ )
162
+
163
+ if os.path.abspath(self.vocab_file) != os.path.abspath(
164
+ out_vocab_file
165
+ ) and os.path.isfile(self.vocab_file):
166
+ copyfile(self.vocab_file, out_vocab_file)
167
+ elif not os.path.isfile(self.vocab_file):
168
+ with open(out_vocab_file, "wb") as fi:
169
+ content_spiece_model = self.sp_model.serialized_model_proto()
170
+ fi.write(content_spiece_model)
171
+
172
+ return (out_vocab_file,)
173
+
174
+ def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
175
+ bos_token_id = [self.bos_token_id] if self.add_bos_token else []
176
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
177
+
178
+ output = bos_token_id + token_ids_0 + eos_token_id
179
+
180
+ if token_ids_1 is not None:
181
+ output = output + bos_token_id + token_ids_1 + eos_token_id
182
+
183
+ return output
184
+
185
+ def get_special_tokens_mask(
186
+ self,
187
+ token_ids_0: List[int],
188
+ token_ids_1: Optional[List[int]] = None,
189
+ already_has_special_tokens: bool = False,
190
+ ) -> List[int]:
191
+ """
192
+ Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
193
+ special tokens using the tokenizer `prepare_for_model` method.
194
+
195
+ Args:
196
+ token_ids_0 (`List[int]`):
197
+ List of IDs.
198
+ token_ids_1 (`List[int]`, *optional*):
199
+ Optional second list of IDs for sequence pairs.
200
+ already_has_special_tokens (`bool`, *optional*, defaults to `False`):
201
+ Whether or not the token list is already formatted with special tokens for the model.
202
+
203
+ Returns:
204
+ `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
205
+ """
206
+ if already_has_special_tokens:
207
+ return super().get_special_tokens_mask(
208
+ token_ids_0=token_ids_0,
209
+ token_ids_1=token_ids_1,
210
+ already_has_special_tokens=True,
211
+ )
212
+
213
+ bos_token_id = [1] if self.add_bos_token else []
214
+ eos_token_id = [1] if self.add_eos_token else []
215
+
216
+ if token_ids_1 is None:
217
+ return bos_token_id + ([0] * len(token_ids_0)) + eos_token_id
218
+ return (
219
+ bos_token_id
220
+ + ([0] * len(token_ids_0))
221
+ + eos_token_id
222
+ + bos_token_id
223
+ + ([0] * len(token_ids_1))
224
+ + eos_token_id
225
+ )
226
+
227
+ def create_token_type_ids_from_sequences(
228
+ self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
229
+ ) -> List[int]:
230
+ """
231
+ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT
232
+ sequence pair mask has the following format:
233
+
234
+ ```
235
+ 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
236
+ | first sequence | second sequence |
237
+ ```
238
+
239
+ if token_ids_1 is None, only returns the first portion of the mask (0s).
240
+
241
+ Args:
242
+ token_ids_0 (`List[int]`):
243
+ List of ids.
244
+ token_ids_1 (`List[int]`, *optional*):
245
+ Optional second list of IDs for sequence pairs.
246
+
247
+ Returns:
248
+ `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
249
+ """
250
+ bos_token_id = [self.bos_token_id] if self.add_bos_token else []
251
+ eos_token_id = [self.eos_token_id] if self.add_eos_token else []
252
+
253
+ output = [0] * len(bos_token_id + token_ids_0 + eos_token_id)
254
+
255
+ if token_ids_1 is not None:
256
+ output += [1] * len(bos_token_id + token_ids_1 + eos_token_id)
257
+
258
+ return output
sakura_13b_model_v0.5_4bits_autogptq_40k/tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79452955be6b419a65984273a9f08af86042e1c2a75ee3ba989cbf620a133cc2
3
+ size 2001107
sakura_13b_model_v0.5_4bits_autogptq_40k/tokenizer_config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": false,
3
+ "add_eos_token": false,
4
+ "auto_map": {
5
+ "AutoTokenizer": [
6
+ "tokenization_baichuan.BaichuanTokenizer",
7
+ null
8
+ ]
9
+ },
10
+ "bos_token": {
11
+ "__type": "AddedToken",
12
+ "content": "<s>",
13
+ "lstrip": false,
14
+ "normalized": true,
15
+ "rstrip": false,
16
+ "single_word": true
17
+ },
18
+ "clean_up_tokenization_spaces": false,
19
+ "eos_token": {
20
+ "__type": "AddedToken",
21
+ "content": "</s>",
22
+ "lstrip": false,
23
+ "normalized": true,
24
+ "rstrip": false,
25
+ "single_word": true
26
+ },
27
+ "model_max_length": 4096,
28
+ "pad_token": {
29
+ "__type": "AddedToken",
30
+ "content": "<unk>",
31
+ "lstrip": false,
32
+ "normalized": true,
33
+ "rstrip": false,
34
+ "single_word": true
35
+ },
36
+ "sp_model_kwargs": {},
37
+ "tokenizer_class": "BaichuanTokenizer",
38
+ "unk_token": {
39
+ "__type": "AddedToken",
40
+ "content": "<unk>",
41
+ "lstrip": false,
42
+ "normalized": true,
43
+ "rstrip": false,
44
+ "single_word": true
45
+ }
46
+ }