HachiML commited on
Commit
78fd17a
1 Parent(s): 32303b5

End of training

Browse files
Files changed (4) hide show
  1. README.md +54 -0
  2. generation_config.json +6 -0
  3. model.safetensors +1 -1
  4. modeling_bit_llama.py +169 -0
README.md ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ tags:
3
+ - generated_from_trainer
4
+ model-index:
5
+ - name: myBit-Llama2-jp-127M-test-26
6
+ results: []
7
+ ---
8
+
9
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
10
+ should probably proofread and complete it, then remove this comment. -->
11
+
12
+ # myBit-Llama2-jp-127M-test-26
13
+
14
+ This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
15
+ It achieves the following results on the evaluation set:
16
+ - eval_loss: 3.1801
17
+ - eval_runtime: 317.4695
18
+ - eval_samples_per_second: 674.229
19
+ - eval_steps_per_second: 7.024
20
+ - epoch: 0.29
21
+ - step: 12000
22
+
23
+ ## Model description
24
+
25
+ More information needed
26
+
27
+ ## Intended uses & limitations
28
+
29
+ More information needed
30
+
31
+ ## Training and evaluation data
32
+
33
+ More information needed
34
+
35
+ ## Training procedure
36
+
37
+ ### Training hyperparameters
38
+
39
+ The following hyperparameters were used during training:
40
+ - learning_rate: 0.0024
41
+ - train_batch_size: 96
42
+ - eval_batch_size: 96
43
+ - seed: 42
44
+ - optimizer: Adam with betas=(0.9,0.95) and epsilon=1e-08
45
+ - lr_scheduler_type: linear
46
+ - lr_scheduler_warmup_steps: 5000
47
+ - num_epochs: 1
48
+
49
+ ### Framework versions
50
+
51
+ - Transformers 4.38.2
52
+ - Pytorch 2.2.1+cu121
53
+ - Datasets 2.18.0
54
+ - Tokenizers 0.15.2
generation_config.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "transformers_version": "4.38.2"
6
+ }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:afd7ba0601fb2a6d05245cdef4582b4809a247b0d1f7d97dd6e9981057f8d7a3
3
  size 510960712
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1c4722baff370de49071554aef57d4053cf7ae8524287a6444f53998b12cfcc7
3
  size 510960712
modeling_bit_llama.py ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import warnings
2
+ from typing import Optional, Tuple
3
+ from transformers.models.llama.modeling_llama import (
4
+ LlamaConfig,
5
+ LlamaModel,
6
+ LlamaForCausalLM,
7
+ LlamaAttention,
8
+ LlamaFlashAttention2,
9
+ LlamaSdpaAttention,
10
+ LlamaMLP,
11
+ LlamaDecoderLayer,
12
+ )
13
+ from mybitnet.bitnet import BitLinear, BitLinear158b
14
+ import torch
15
+ from torch import nn
16
+
17
+ class BitLlamaConfig(LlamaConfig):
18
+ model_type = "bit_llama"
19
+
20
+ def __init__(self, bitnet_type="1.58b", bits=8, **kwargs):
21
+ super().__init__(**kwargs)
22
+ self.bitnet_type = bitnet_type
23
+ if self.bitnet_type not in ["1.58b", "1b"]:
24
+ raise ValueError("bitnet_type must be either '1.58b' or '1b'.")
25
+ self.bits = bits
26
+
27
+
28
+ class BitLlamaMLP(LlamaMLP):
29
+ def __init__(self, config):
30
+ super().__init__(config)
31
+ if config.bitnet_type=="1b":
32
+ self.gate_proj = BitLinear(self.hidden_size, self.intermediate_size, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=False)
33
+ self.up_proj = BitLinear(self.hidden_size, self.intermediate_size, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
34
+ self.down_proj = BitLinear(self.intermediate_size, self.hidden_size, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
35
+ elif config.bitnet_type=="1.58b":
36
+ self.gate_proj = BitLinear158b(self.hidden_size, self.intermediate_size, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
37
+ self.up_proj = BitLinear158b(self.hidden_size, self.intermediate_size, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
38
+ self.down_proj = BitLinear158b(self.intermediate_size, self.hidden_size, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
39
+ else:
40
+ raise ValueError("bitnet_type must be either '1.58b' or '1b'.")
41
+
42
+ class BitLlamaAttention(LlamaAttention):
43
+ def __init__(self, config: BitLlamaConfig, layer_idx: Optional[int] = None):
44
+ super().__init__(config)
45
+ if config.bitnet_type=="1b":
46
+ self.q_proj = BitLinear(self.hidden_size, self.num_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
47
+ self.k_proj = BitLinear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
48
+ self.v_proj = BitLinear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
49
+ self.o_proj = BitLinear(self.hidden_size, self.hidden_size, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
50
+ elif config.bitnet_type=="1.58b":
51
+ self.q_proj = BitLinear158b(self.hidden_size, self.num_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
52
+ self.k_proj = BitLinear158b(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
53
+ self.v_proj = BitLinear158b(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
54
+ self.o_proj = BitLinear158b(self.hidden_size, self.hidden_size, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
55
+ else:
56
+ raise ValueError("bitnet_type must be either '1.58b' or '1b'.")
57
+
58
+ class BitLlamaFlashAttention2(LlamaFlashAttention2):
59
+ def __init__(self, config: BitLlamaConfig, layer_idx: Optional[int] = None):
60
+ super().__init__(config, layer_idx)
61
+ if config.bitnet_type=="1b":
62
+ self.q_proj = BitLinear(self.hidden_size, self.num_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
63
+ self.k_proj = BitLinear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
64
+ self.v_proj = BitLinear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
65
+ self.o_proj = BitLinear(self.hidden_size, self.hidden_size, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
66
+ elif config.bitnet_type=="1.58b":
67
+ self.q_proj = BitLinear158b(self.hidden_size, self.num_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
68
+ self.k_proj = BitLinear158b(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
69
+ self.v_proj = BitLinear158b(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
70
+ self.o_proj = BitLinear158b(self.hidden_size, self.hidden_size, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
71
+ else:
72
+ raise ValueError("bitnet_type must be either '1.58b' or '1b'.")
73
+
74
+ class BitLlamaSdpaAttention(LlamaSdpaAttention):
75
+ def __init__(self, config: BitLlamaConfig, layer_idx: Optional[int] = None):
76
+ super().__init__(config, layer_idx)
77
+ if config.bitnet_type=="1b":
78
+ self.q_proj = BitLinear(self.hidden_size, self.num_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
79
+ self.k_proj = BitLinear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
80
+ self.v_proj = BitLinear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
81
+ self.o_proj = BitLinear(self.hidden_size, self.hidden_size, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits, flg_before_linear=True)
82
+ elif config.bitnet_type=="1.58b":
83
+ self.q_proj = BitLinear158b(self.hidden_size, self.num_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
84
+ self.k_proj = BitLinear158b(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
85
+ self.v_proj = BitLinear158b(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
86
+ self.o_proj = BitLinear158b(self.hidden_size, self.hidden_size, bias=False, rms_norm_eps=config.rms_norm_eps, bits=config.bits)
87
+ else:
88
+ raise ValueError("bitnet_type must be either '1.58b' or '1b'.")
89
+
90
+ BITLLAMA_ATTENTION_CLASSES = {
91
+ "eager": BitLlamaAttention,
92
+ "flash_attention_2": BitLlamaFlashAttention2,
93
+ "sdpa": BitLlamaSdpaAttention,
94
+ }
95
+
96
+ class BitLlamaDecoderLayer(LlamaDecoderLayer):
97
+ def __init__(self, config: BitLlamaConfig, layer_idx: int):
98
+ super().__init__(config, layer_idx)
99
+ self.self_attn = BITLLAMA_ATTENTION_CLASSES[config._attn_implementation](config=config, layer_idx=layer_idx)
100
+ self.mlp = BitLlamaMLP(config)
101
+ del self.input_layernorm
102
+ del self.post_attention_layernorm
103
+
104
+ def forward(
105
+ self,
106
+ hidden_states: torch.Tensor,
107
+ attention_mask: Optional[torch.Tensor] = None,
108
+ position_ids: Optional[torch.LongTensor] = None,
109
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
110
+ output_attentions: Optional[bool] = False,
111
+ use_cache: Optional[bool] = False,
112
+ cache_position: Optional[torch.LongTensor] = None,
113
+ **kwargs,
114
+ ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
115
+ """
116
+ refers: https://github.com/huggingface/transformers/blob/c5f0288bc7d76f65996586f79f69fba8867a0e67/src/transformers/models/llama/modeling_llama.py#L693
117
+ """
118
+ if "padding_mask" in kwargs:
119
+ warnings.warn(
120
+ "Passing `padding_mask` is deprecated and will be removed in v4.37. Please make sure use `attention_mask` instead.`"
121
+ )
122
+
123
+ residual = hidden_states
124
+
125
+ # Self Attention
126
+ hidden_states, self_attn_weights, present_key_value = self.self_attn(
127
+ hidden_states=hidden_states,
128
+ attention_mask=attention_mask,
129
+ position_ids=position_ids,
130
+ past_key_value=past_key_value,
131
+ output_attentions=output_attentions,
132
+ use_cache=use_cache,
133
+ cache_position=cache_position,
134
+ **kwargs,
135
+ )
136
+ hidden_states = residual + hidden_states
137
+
138
+ # Fully Connected
139
+ residual = hidden_states
140
+ hidden_states = self.mlp(hidden_states)
141
+ hidden_states = residual + hidden_states
142
+
143
+ outputs = (hidden_states,)
144
+
145
+ if output_attentions:
146
+ outputs += (self_attn_weights,)
147
+
148
+ if use_cache:
149
+ outputs += (present_key_value,)
150
+
151
+ return outputs
152
+
153
+ class BitLlamaModel(LlamaModel):
154
+ config_class = BitLlamaConfig
155
+
156
+ def __init__(self, config: BitLlamaConfig):
157
+ super().__init__(config)
158
+ self.layers = nn.ModuleList(
159
+ [BitLlamaDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
160
+ )
161
+
162
+ class BitLlamaForCausalLM(LlamaForCausalLM):
163
+ config_class = BitLlamaConfig
164
+
165
+ def __init__(self, config: BitLlamaConfig):
166
+ super().__init__(config)
167
+ self.model = BitLlamaModel(config)
168
+ self.lm_head = BitLinear(config.hidden_size, config.vocab_size, bias=False, bits=config.bits, flg_before_linear=True)
169
+