|
--- |
|
datasets: |
|
- abideen/Cosmopedia-100k-pretrain |
|
tags: |
|
- Mistral |
|
- 1bit |
|
- bitnet |
|
- abideen |
|
- M7 |
|
- Liminerity |
|
--- |
|
"""this is my second attempt at converting a model float16 quantized model to 1.5bit. i used my model liminerity/M7-7b for the base model and |
|
trained on: abideen/cosmopedia-100k-pretain dataset and used his google colab project to make this""" |
|
|
|
#EXAMPLE INFERENCE CODE FROM ABIDEEN'S COLAB PROJECT |
|
``` |
|
from transformers import AutoModelForCausalLM, AutoTokenizer |
|
from transformers.models.llama.modeling_llama import * |
|
# Load a pretrained BitNet model |
|
model = "liminerity/Bitnet-M7-70M" |
|
tokenizer = AutoTokenizer.from_pretrained(model) |
|
model = AutoModelForCausalLM.from_pretrained(model) |
|
|
|
|
|
def activation_quant(x): |
|
scale = 127.0 / x.abs().max(dim=-1, keepdim=True).values.clamp_(min=1e-5) |
|
y = (x * scale).round().clamp_(-128, 127) |
|
y = y / scale |
|
return y |
|
def weight_quant(w): |
|
scale = 1.0 / w.abs().mean().clamp_(min=1e-5) |
|
u = (w * scale).round().clamp_(-1, 1) |
|
u = u / scale |
|
return u |
|
|
|
class BitLinear(nn.Linear): |
|
def forward(self, x): |
|
w = self.weight # a weight tensor with shape [d, k] |
|
x = x.to(w.device) |
|
RMSNorm = LlamaRMSNorm(x.shape[-1]).to(w.device) |
|
x_norm = RMSNorm(x) |
|
# A trick for implementing Straight−Through−Estimator (STE) using detach() |
|
x_quant = x_norm + (activation_quant(x_norm) - x_norm).detach() |
|
w_quant = w + (weight_quant(w) - w).detach() |
|
y = F.linear(x_quant, w_quant) |
|
return y |
|
|
|
def convert_to_bitnet(model, copy_weights): |
|
for name, module in model.named_modules(): |
|
# Replace linear layers with BitNet |
|
if isinstance(module, LlamaSdpaAttention) or isinstance(module, LlamaMLP): |
|
for child_name, child_module in module.named_children(): |
|
if isinstance(child_module, nn.Linear): |
|
bitlinear = BitLinear(child_module.in_features, child_module.out_features, child_module.bias is not None).to(device="cuda:0") |
|
if copy_weights: |
|
bitlinear.weight = child_module.weight |
|
if child_module.bias is not None: |
|
bitlinear.bias = child_module.bias |
|
setattr(module, child_name, bitlinear) |
|
# Remove redundant input_layernorms |
|
elif isinstance(module, LlamaDecoderLayer): |
|
for child_name, child_module in module.named_children(): |
|
if isinstance(child_module, LlamaRMSNorm) and child_name == "input_layernorm": |
|
setattr(module, child_name, nn.Identity().to(device="cuda:0")) |
|
|
|
|
|
convert_to_bitnet(model, copy_weights=True) |
|
model.to(device="cuda:0") |
|
|
|
prompt = "What is Machine Learning?" |
|
inputs = tokenizer(prompt, return_tensors="pt").to(model.device) |
|
generate_ids = model.generate(inputs.input_ids, max_length=50) |
|
tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] |
|
``` |
