I can't modify the C++ code it's not mine I can only edit the export python script. I want it to split the model in two files consolidated.00.pth consolidated.01.pth with the good layer size.

Here is how the model is loaded:

this is the llama_model_function:

static bool llama_model_load(
const std::string & fname,
llama_context & lctx,
int n_ctx,
int n_parts,
ggml_type memory_type,
bool vocab_only,
llama_progress_callback progress_callback,
void progress_callback_user_data) {
fprintf(stderr, “%s: loading model from ‘%s’ - please wait …\n”, func, fname.c_str());

lctx.t_start_us = ggml_time_us();

auto & model = lctx.model;
auto & vocab = lctx.vocab;

auto fin = std::ifstream(fname, std::ios::binary);
if (!fin) {
fprintf(stderr, “%s: failed to open ‘%s’\n”, func, fname.c_str());
return false;
}

std::vector<char> f_buf(10241024);
fin.rdbuf()->pubsetbuf(f_buf.data(), f_buf.size());

fin.seekg(0, fin.end);
const size_t file_size = fin.tellg();
fin.seekg(0);

// verify magic
{
uint32_t magic;
fin.read((char *) &magic, sizeof(magic));
if (magic == LLAMA_FILE_MAGIC_UNVERSIONED) {
fprintf(stderr, “%s: invalid model file ‘%s’ (too old, regenerate your model files or convert them with convert-unversioned-ggml-to-ggml.py!)\n”,
func, fname.c_str());
return false;
}
if (magic != LLAMA_FILE_MAGIC) {
return report_bad_magic(fname.c_str(), magic, LLAMA_FILE_MAGIC);
}

uint32_t format_version;
fin.read((char *) &format_version, sizeof(format_version));

if (format_version != LLAMA_FILE_VERSION) {
fprintf(stderr, “%s: invalid model file ‘%s’ (unsupported format version %” PRIu32 “, expected %d)\n”,
func, fname.c_str(), format_version, LLAMA_FILE_VERSION);
return false;
}
}

int n_ff = 0;

// load hparams
{
auto & hparams = model.hparams;

fin.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
//fin.read((char *) &hparams.n_ctx, sizeof(hparams.n_ctx));
fin.read((char *) &hparams.n_embd, sizeof(hparams.n_embd));
fin.read((char ) &hparams.n_mult, sizeof(hparams.n_mult));
fin.read((char ) &hparams.n_head, sizeof(hparams.n_head));
fin.read((char ) &hparams.n_layer, sizeof(hparams.n_layer));
fin.read((char ) &hparams.n_rot, sizeof(hparams.n_rot));
fin.read((char ) &hparams.f16, sizeof(hparams.f16));

hparams.n_ctx = n_ctx;

n_ff = ((2(4hparams.n_embd)/3 + hparams.n_mult - 1)/hparams.n_mult)hparams.n_mult;

if (n_parts < 1) {
n_parts = LLAMA_N_PARTS.at(hparams.n_embd);
}

// temp warning to tell the user to use “–n_parts”
if (hparams.f16 == 4 && n_parts != 1) {
fprintf(stderr, “%s: GPTQ model detected - are you sure n_parts should be %d? we normally expect it to be 1\n”, func, n_parts);
fprintf(stderr, “%s: use ‘–n_parts 1’ if necessary\n”, func);
}

if (hparams.n_layer == 32) {
model.type = e_model::MODEL_7B;
}

if (hparams.n_layer == 40) {
model.type = e_model::MODEL_13B;
}

if (hparams.n_layer == 60) {
model.type = e_model::MODEL_30B;
}

if (hparams.n_layer == 80) {
model.type = e_model::MODEL_65B;
}

fprintf(stderr, “%s: n_vocab = %d\n”, func, hparams.n_vocab);
fprintf(stderr, “%s: n_ctx = %d\n”, func, hparams.n_ctx);
fprintf(stderr, “%s: n_embd = %d\n”, func, hparams.n_embd);
fprintf(stderr, “%s: n_mult = %d\n”, func, hparams.n_mult);
fprintf(stderr, “%s: n_head = %d\n”, func, hparams.n_head);
fprintf(stderr, “%s: n_layer = %d\n”, func, hparams.n_layer);
fprintf(stderr, “%s: n_rot = %d\n”, func, hparams.n_rot);
fprintf(stderr, “%s: f16 = %d\n”, func, hparams.f16);
fprintf(stderr, “%s: n_ff = %d\n”, func, n_ff);
fprintf(stderr, “%s: n_parts = %d\n”, func, n_parts);
fprintf(stderr, “%s: type = %d\n”, func, model.type);
}

// load vocab
{
std::string word;
vocab.id_to_token.resize(model.hparams.n_vocab);
std::vector<char> tmp(64);

for (int i = 0; i < model.hparams.n_vocab; i++) {
uint32_t len;
fin.read((char ) &len, sizeof(len));

word.resize(len);
if (len > 0) {
tmp.resize(len);
fin.read(tmp.data(), len);
word.assign(tmp.data(), len);
} else {
word.clear();
}

float score;
fin.read((char ) &score, sizeof(score));

vocab.token_to_id[word] = i;

auto &tok_score = vocab.id_to_token[i];
tok_score.tok = word;
tok_score.score = score;
}
}

if (vocab_only) {
return true;
}

// for the big tensors, we have the option to store the data in 16-bit floats or quantized
// in order to save memory and also to speed up the computation
// wtype is for per-layer weights, while vtype is for other weights
ggml_type wtype, vtype;
switch (model.hparams.f16) {
case 0: wtype = vtype = GGML_TYPE_F32; break;
case 1: wtype = vtype = GGML_TYPE_F16; break;
case 2: wtype = vtype = GGML_TYPE_Q4_0; break;
case 3: wtype = vtype = GGML_TYPE_Q4_1; break;
case 4: wtype = GGML_TYPE_Q4_1; vtype = GGML_TYPE_F16; break;
default:
{
fprintf(stderr, “%s: invalid model file ‘%s’ (bad f16 value %d)\n”,
func, fname.c_str(), model.hparams.f16);
return false;
}
}

// map model into memory
char mm_addr = NULL;
model.mm_addr = mmap_file(fname.c_str(), &model.mm_length);
if (model.mm_addr == NULL) {
fprintf(stderr, “%s: failed to mmap ‘%s’\n”, func, fname.c_str());
return false;
}
mm_addr = (char )model.mm_addr;
fprintf(stderr, “%s: ggml map size = %6.2f MB\n”, func, model.mm_length/(1024.01024.0));

auto & ctx = model.ctx;

size_t ctx_size = 0;
{
const auto &hparams = model.hparams;
const int n_layer = hparams.n_layer;
ctx_size += (5 + 10n_layer)256; // object overhead
fprintf(stderr, “%s: ggml ctx size = %6.2f KB\n”, func, ctx_size/1024.0);
}

// print memory requirements
{
const size_t scale = memory_type == GGML_TYPE_F32 ? 2 : 1;

// this is the total memory required to run the inference
const size_t mem_required =
ctx_size +
model.mm_length +
MEM_REQ_SCRATCH0.at(model.type) +
MEM_REQ_SCRATCH1.at(model.type) +
MEM_REQ_EVAL.at (model.type);

// this is the memory required by one llama_state
const size_t mem_required_state =
scaleMEM_REQ_KV_SELF.at(model.type);

fprintf(stderr, “%s: mem required = %7.2f MB (+ %7.2f MB per state)\n”, func,
mem_required / 1024.0 / 1024.0, mem_required_state / 1024.0 / 1024.0);
}

// create the ggml context
{
lctx.model.buf.resize(ctx_size);

struct ggml_init_params params = {
/.mem_size =/ lctx.model.buf.size(),
/.mem_buffer =/ lctx.model.buf.data(),
/.no_alloc =/ true,
};

model.ctx = ggml_init(params);
if (!model.ctx) {
fprintf(stderr, “%s: ggml_init() failed\n”, func);
return false;
}
}

// prepare memory for the weights
{
const auto & hparams = model.hparams;

const int n_embd = hparams.n_embd;
const int n_layer = hparams.n_layer;
const int n_vocab = hparams.n_vocab;

model.layers.resize(n_layer);

model.tok_embeddings = ggml_new_tensor_2d(ctx, vtype, n_embd, n_vocab);

model.norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);
model.output = ggml_new_tensor_2d(ctx, vtype, n_embd, n_vocab);

// map by name
model.tensors[“tok_embeddings.weight”] = model.tok_embeddings;

model.tensors[“norm.weight”] = model.norm;
model.tensors[“output.weight”] = model.output;

for (int i = 0; i < n_layer; ++i) {
auto & layer = model.layers[i];

layer.attention_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);

layer.wq = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.wk = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.wv = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);
layer.wo = ggml_new_tensor_2d(ctx, wtype, n_embd, n_embd);

layer.ffn_norm = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_embd);

layer.w1 = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff);
layer.w2 = ggml_new_tensor_2d(ctx, wtype, n_ff, n_embd);
layer.w3 = ggml_new_tensor_2d(ctx, wtype, n_embd, n_ff);

// map by name
model.tensors[“layers.” + std::to_string(i) + “.attention_norm.weight”] = layer.attention_norm;

model.tensors[“layers.” + std::to_string(i) + “.attention.wq.weight”] = layer.wq;
model.tensors[“layers.” + std::to_string(i) + “.attention.wk.weight”] = layer.wk;
model.tensors[“layers.” + std::to_string(i) + “.attention.wv.weight”] = layer.wv;
model.tensors[“layers.” + std::to_string(i) + “.attention.wo.weight”] = layer.wo;

model.tensors[“layers.” + std::to_string(i) + “.ffn_norm.weight”] = layer.ffn_norm;

model.tensors[“layers.” + std::to_string(i) + “.feed_forward.w1.weight”] = layer.w1;
model.tensors[“layers.” + std::to_string(i) + “.feed_forward.w2.weight”] = layer.w2;
model.tensors[“layers.” + std::to_string(i) + “.feed_forward.w3.weight”] = layer.w3;
}
}

std::vector<uint8_t> tmp;

if (progress_callback) {
progress_callback(0.0, progress_callback_user_data);
}

fprintf(stderr, “%s: loading tensors from ‘%s’\n”, func, fname.c_str());

// load weights
{
size_t total_size = 0;
model.n_loaded = 0;

while (true) {
int32_t n_dims;
int32_t length;
int32_t ftype;

fin.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));
fin.read(reinterpret_cast<char *>(&length), sizeof(length));
fin.read(reinterpret_cast<char *>(&ftype), sizeof(ftype));

if (fin.eof()) {
break;
}

int32_t nelements = 1;
int32_t ne[2] = { 1, 1 };
for (int i = 0; i < n_dims; ++i) {
fin.read(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));
nelements *= ne[i];
}

std::string name(length, 0);
fin.read(&name[0], length);

if (model.tensors.find(name.data()) == model.tensors.end()) {
fprintf(stderr, “%s: unknown tensor ‘%s’ in model file\n”, func, name.data());
return false;
}

auto tensor = model.tensors[name.data()];

if (ggml_nelements(tensor) != nelements) {
fprintf(stderr, “%s: tensor ‘%s’ has wrong size in model file\n”, func, name.data());
return false;
}
if (tensor->ne[0] != ne[0] || tensor->ne[1] != ne[1]) {
fprintf(stderr, “%s: tensor ‘%s’ has wrong shape in model file: got [%” PRId64 “, %” PRId64 “], expected [%d, %d]\n”,
func, name.data(), tensor->ne[0], tensor->ne[1], ne[0], ne[1]);
return false;
}
if (0) {
static const char * ftype_str[] = { “f32”, “f16”, “q4_0”, “q4_1”, };
fprintf(stderr, “%24s - [%5d, %5d], type = %6s\n”, name.data(), ne[0], ne[1], ftype_str[ftype]);
}

switch (ftype) {
case 0: // f32
case 1: // f16
break;
case 2: // q4_0
case 3: // q4_1
assert(ne[0] % 64 == 0);
break;
default:
fprintf(stderr, “%s: unknown ftype %d in model file\n”, func, ftype);
return false;
};

// load the tensor data into memory without copying or reading it
size_t offset = fin.tellg();
size_t tensor_data_size = ggml_nbytes(tensor);
offset = (offset + 31) & -32;
tensor->data = mm_addr + offset;
fin.seekg(offset + tensor_data_size);
total_size += tensor_data_size;
model.n_loaded++;

// progress
if (progress_callback) {
double current_progress = size_t(fin.tellg()) / double(file_size);
progress_callback(current_progress, progress_callback_user_data);
}
}

fin.close();

fprintf(stderr, “%s: model size = %8.2f MB / num tensors = %d\n”, func, total_size/1024.0/1024.0, model.n_loaded);
if (model.n_loaded == 0) {
fprintf(stderr, “%s: WARN no tensors loaded from model file - assuming empty model for testing\n”, func);
} else if (model.n_loaded != (int) model.tensors.size()) {
fprintf(stderr, “%s: ERROR not all tensors loaded from model file - expected %zu, got %d\n”, func, model.tensors.size(), model.n_loaded);
return false;
}
}

// loading time will be recalculate after the first eval, so
// we take page faults deferred by mmap() into consideration
lctx.t_load_us = ggml_time_us() - lctx.t_start_us;

if (progress_callback) {
progress_callback(1.0, progress_callback_user_data);
}

return true;
}

here is how the model is exported :

#! /usr/bin/env python
# coding=utf-8

“”“
Modified from: https://github.com/tloen/alpaca-lora
”“”

import json
import os

import fire
import torch
from peft import PeftModel
from transformers import LlamaForCausalLM, LlamaTokenizer

CHECKPOINT_PARAMS = {
“7b”: {“dim”: 4096, “multiple_of”: 256, “n_heads”: 32, “n_layers”: 32, “norm_eps”: 1e-06, “vocab_size”: -1},
“13b”: {“dim”: 5120, “multiple_of”: 256, “n_heads”: 40, “n_layers”: 40, “norm_eps”: 1e-06, “vocab_size”: -1},
“30b”: {“dim”: 6656, “multiple_of”: 256, “n_heads”: 52, “n_layers”: 60, “norm_eps”: 1e-06, “vocab_size”: -1},
“65b”: {“dim”: 8192, “multiple_of”: 256, “n_heads”: 64, “n_layers”: 80, “norm_eps”: 1e-06, “vocab_size”: -1},
}


def main(base_model_name_or_path: str, lora_model_name_or_path: str, output_dir: str, checkpoint_size: str = “7b”):

# Retrieve the model parameters
params = CHECKPOINT_PARAMS.get(checkpoint_size)
if params is None:
raise ValueError(
f"Cannot find the right model parameters for {checkpoint_size}. Please choose between {list(CHECKPOINT_PARAMS.keys())}.“
)

# tokenizer = LlamaTokenizer.from_pretrained(base_model_name_or_path)

base_model = LlamaForCausalLM.from_pretrained(
base_model_name_or_path,
load_in_8bit=False,
torch_dtype=torch.float16,
device_map={”“: “cpu”},
)

lora_model = PeftModel.from_pretrained(
base_model,
lora_model_name_or_path,
device_map={”“: “cpu”},
torch_dtype=torch.float16,
)

# merge weights
for layer in lora_model.base_model.model.model.layers:
if hasattr(layer.self_attn.q_proj, “merge_weights”):
layer.self_attn.q_proj.merge_weights = True
if hasattr(layer.self_attn.v_proj, “merge_weights”):
layer.self_attn.v_proj.merge_weights = True
if hasattr(layer.self_attn.k_proj, “merge_weights”):
layer.self_attn.k_proj.merge_weights = True
if hasattr(layer.self_attn.o_proj, “merge_weights”):
layer.self_attn.o_proj.merge_weights = True
if hasattr(layer.mlp.gate_proj, “merge_weights”):
layer.mlp.gate_proj.merge_weights = True
if hasattr(layer.mlp.down_proj, “merge_weights”):
layer.mlp.down_proj.merge_weights = True
if hasattr(layer.mlp.up_proj, “merge_weights”):
layer.mlp.up_proj.merge_weights = True

lora_model.train(False)

lora_model_sd = lora_model.state_dict()

# params = {
# “dim”: 4096,
# “multiple_of”: 256,
# “n_heads”: 32,
# “n_layers”: 32,
# “norm_eps”: 1e-06,
# “vocab_size”: -1,
# }
n_layers = params[“n_layers”]
n_heads = params[“n_heads”]
dim = params[“dim”]
dims_per_head = dim // n_heads
base = 10000.0
inv_freq = 1.0 / (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head))

def permute(w):
return w.view(n_heads, dim // n_heads // 2, 2, dim).transpose(1, 2).reshape(dim, dim)

def unpermute(w):
return w.view(n_heads, 2, dim // n_heads // 2, dim).transpose(1, 2).reshape(dim, dim)

def translate_state_dict_key(k):
k = k.replace(“base_model.model.”, “”)
if k == “model.embed_tokens.weight”:
return “tok_embeddings.weight”
elif k == “model.norm.weight”:
return “norm.weight”
elif k == “lm_head.weight”:
return “output.weight”
elif k.startswith(“model.layers.”):
layer = k.split(”.“)[2]
if k.endswith(”.self_attn.q_proj.weight"):
return f"layers.{layer}.attention.wq.weight"
elif k.endswith(“.self_attn.k_proj.weight”):
return f"layers.{layer}.attention.wk.weight"
elif k.endswith(“.self_attn.v_proj.weight”):
return f"layers.{layer}.attention.wv.weight"
elif k.endswith(“.self_attn.o_proj.weight”):
return f"layers.{layer}.attention.wo.weight"
elif k.endswith(“.mlp.gate_proj.weight”):
return f"layers.{layer}.feed_forward.w1.weight"
elif k.endswith(“.mlp.down_proj.weight”):
return f"layers.{layer}.feed_forward.w2.weight"
elif k.endswith(“.mlp.up_proj.weight”):
return f"layers.{layer}.feed_forward.w3.weight"
elif k.endswith(“.input_layernorm.weight”):
return f"layers.{layer}.attention_norm.weight"
elif k.endswith(“.post_attention_layernorm.weight”):
return f"layers.{layer}.ffn_norm.weight"
elif k.endswith(“rotary_emb.inv_freq”) or “lora” in k:
return None
else:
print(layer, k)
raise NotImplementedError
else:
print(k)
raise NotImplementedError



new_state_dict = {}
for k, v in lora_model_sd.items():
new_k = translate_state_dict_key(k)
if new_k is not None:
if “wq” in new_k or “wk” in new_k:
new_state_dict[new_k] = unpermute(v)
else:
new_state_dict[new_k] = v

os.makedirs(output_dir, exist_ok=True)

  # Split the tensors based on layer index
    n_layers_actual = len([k for k in new_state_dict.keys() if ".attention.wq.weight" in k])
    part1_keys = [k for k in new_state_dict.keys() if not k.startswith("layers.") or int(k.split(".")[1]) < (n_layers_actual // 2)]
    part2_keys = [k for k in new_state_dict.keys() if k not in part1_keys]

    state_dict_part1 = {k: new_state_dict[k] for k in part1_keys}
    state_dict_part2 = {k: new_state_dict[k] for k in part2_keys}

    torch.save(state_dict_part1, output_dir + "/consolidated.00.pth")
    torch.save(state_dict_part2, output_dir + "/consolidated.01.pth")
    with open(output_dir + "/params.json", "w") as f:
        json.dump(params, f)

if name == “main”:
fire.Fire(main)

Here is the problem I have when i run the inference:

./main -m ./models/13B/ggml-model-f16.bin -n 5000 --repeat_penalty 1.0 --color -i -r “User:” -f prompts/chat-with-bob.txt -t 32
main: seed = 1681035697
llama_model_load: loading model from ‘./models/13B/ggml-model-f16.bin’ - please wait …
llama_model_load: n_vocab = 32000
llama_model_load: n_ctx = 512
llama_model_load: n_embd = 5120
llama_model_load: n_mult = 256
llama_model_load: n_head = 40
llama_model_load: n_layer = 40
llama_model_load: n_rot = 128
llama_model_load: f16 = 1
llama_model_load: n_ff = 13824
llama_model_load: n_parts = 2
llama_model_load: type = 2
llama_model_load: ggml map size = 25138.72 MB
llama_model_load: ggml ctx size = 101.25 KB
llama_model_load: mem required = 27186.82 MB (+ 1608.00 MB per state)
llama_model_load: loading tensors from ‘./models/13B/ggml-model-f16.bin’
llama_model_load: tensor ‘layers.20.attention.wq.weight’ has wrong size in model file
llama_init_from_file: failed to load model
main: error: failed to load model ‘./models/13B/ggml-model-f16.bin’
It seems like there is an issue with loading tensors in the llama_model_load function in the C++ code. Specifically, the error message says the tensor ‘layers.20.attention.wq.weight’ has a wrong size.

To further investigate the issue, you can compare the size of the tensor in the model file with the size of the tensor in the ‘model’ object created in the C++ code. Here’s how you can do that:

1. Modify the llama_model_load function to print the actual and expected sizes for the tensor causing the issue. Update the following code block:


if (ggml_nelements(tensor) != nelements) {
    fprintf(stderr, “%s: tensor ‘%s’ has wrong size in model file\n”, func, name.data());
    return false;
}



to


if (ggml_nelements(tensor) != nelements) {
    fprintf(stderr, “%s: tensor ‘%s’ has wrong size in model file - expected %d, got %zu\n”,
            func, name.data(), nelements, ggml_nelements(tensor));
    return false;
}



2. Re-compile your C++ code and re-run the inference.

The modified code should print the actual and expected tensor size during inference, which could shed more light on the issue.

Additionally, ensure that the model is being exported correctly in the Python script. To verify the exported model and check the dimensions of the problematic tensor, you can load the consolidated.00.pth and consolidated.01.pth files in a separate Python script and check the shapes of all tensors.

Here’s sample Python code to check tensor shapes:


import torch

part1 = torch.load(“output_dir/consolidated.00.pth”)
part2 = torch.load(“output_dir/consolidated.01.pth”)

for k, v in part1.items():
    print(k, v.shape)

for k, v in part2.items():
    print(k, v.shape)



Replace “output_dir” with the correct output directory used when exporting your model. This should give you more insight into the dimensions of the tensors in the exported model, specifically the problematic ‘layers.20.attention.wq.weight’. Compare these dimensions with the dimensions reported by the modified C++ code during inference. This will help you understand if there’s an issue with the model export, the C++ code or a mismatch between the two.
I cant change the c++ code I just need to modify the python script to split the model in two files consolidated.00.pth consolidated.01.pth but as you can see the layer 20 has wrong model size. modify the python script so the layer sizes are correct
In that case, you might need to modify the Python script to ensure that the problematic tensor ‘layers.20.attention.wq.weight’ is correctly split between part1 and part2. As suggested in your code, you are expecting tensors from the first half in part1 and tensors from the second half in part2.

I noticed that you have hardcoded the number of layers in the following line:


part1_keys = [k for k in new_state_dict.keys() if not k.startswith(“layers.”) or int(k.split(“.”)[1]) < n_layers // 2]



However, this may cause issues if there is a mismatch between the model’s number of layers and the n_layers variable. To get the correct number of layers from the model, update the line as follows:


n_layers_actual = len([k for k in new_state_dict.keys() if “.attention.wq.weight” in k])
part1_keys = [k for k in new_state_dict.keys() if not k.startswith(“layers.”) or int(k.split(“.”)[1]) < (n_layers_actual // 2)]



With this change, part1 and part2 should be split correctly no matter how many layers the model has. Re-run the export script and check if the tensors are correctly split between the two files.

If you still encounter issues, please share the output of the Python script I provided earlier to check the tensor shapes in the exported files.
the modified export script file:

#! /usr/bin/env python
# coding=utf-8

“”“
Modified from: https://github.com/tloen/alpaca-lora
”“”

import json
import os

import fire
import torch
from peft import PeftModel
from transformers import LlamaForCausalLM, LlamaTokenizer

CHECKPOINT_PARAMS = {
“7b”: {“dim”: 4096, “multiple_of”: 256, “n_heads”: 32, “n_layers”: 32, “norm_eps”: 1e-06, “vocab_size”: -1},
“13b”: {“dim”: 5120, “multiple_of”: 256, “n_heads”: 40, “n_layers”: 40, “norm_eps”: 1e-06, “vocab_size”: -1},
“30b”: {“dim”: 6656, “multiple_of”: 256, “n_heads”: 52, “n_layers”: 60, “norm_eps”: 1e-06, “vocab_size”: -1},
“65b”: {“dim”: 8192, “multiple_of”: 256, “n_heads”: 64, “n_layers”: 80, “norm_eps”: 1e-06, “vocab_size”: -1},
}


def main(base_model_name_or_path: str, lora_model_name_or_path: str, output_dir: str, checkpoint_size: str = “7b”):

# Retrieve the model parameters
params = CHECKPOINT_PARAMS.get(checkpoint_size)
if params is None:
raise ValueError(
f"Cannot find the right model parameters for {checkpoint_size}. Please choose between {list(CHECKPOINT_PARAMS.keys())}.“
)

# tokenizer = LlamaTokenizer.from_pretrained(base_model_name_or_path)

base_model = LlamaForCausalLM.from_pretrained(
base_model_name_or_path,
load_in_8bit=False,
torch_dtype=torch.float16,
device_map={”“: “cpu”},
)

lora_model = PeftModel.from_pretrained(
base_model,
lora_model_name_or_path,
device_map={”“: “cpu”},
torch_dtype=torch.float16,
)

# merge weights
for layer in lora_model.base_model.model.model.layers:
if hasattr(layer.self_attn.q_proj, “merge_weights”):
layer.self_attn.q_proj.merge_weights = True
if hasattr(layer.self_attn.v_proj, “merge_weights”):
layer.self_attn.v_proj.merge_weights = True
if hasattr(layer.self_attn.k_proj, “merge_weights”):
layer.self_attn.k_proj.merge_weights = True
if hasattr(layer.self_attn.o_proj, “merge_weights”):
layer.self_attn.o_proj.merge_weights = True
if hasattr(layer.mlp.gate_proj, “merge_weights”):
layer.mlp.gate_proj.merge_weights = True
if hasattr(layer.mlp.down_proj, “merge_weights”):
layer.mlp.down_proj.merge_weights = True
if hasattr(layer.mlp.up_proj, “merge_weights”):
layer.mlp.up_proj.merge_weights = True

lora_model.train(False)

lora_model_sd = lora_model.state_dict()

# params = {
# “dim”: 4096,
# “multiple_of”: 256,
# “n_heads”: 32,
# “n_layers”: 32,
# “norm_eps”: 1e-06,
# “vocab_size”: -1,
# }
n_layers = params[“n_layers”]
n_heads = params[“n_heads”]
dim = params[“dim”]
dims_per_head = dim // n_heads
base = 10000.0
inv_freq = 1.0 / (base ** (torch.arange(0, dims_per_head, 2).float() / dims_per_head))

def permute(w):
return w.view(n_heads, dim // n_heads // 2, 2, dim).transpose(1, 2).reshape(dim, dim)

def unpermute(w):
return w.view(n_heads, 2, dim // n_heads // 2, dim).transpose(1, 2).reshape(dim, dim)

def translate_state_dict_key(k):
k = k.replace(“base_model.model.”, “”)
if k == “model.embed_tokens.weight”:
return “tok_embeddings.weight”
elif k == “model.norm.weight”:
return “norm.weight”
elif k == “lm_head.weight”:
return “output.weight”
elif k.startswith(“model.layers.”):
layer = k.split(”.“)[2]
if k.endswith(”.self_attn.q_proj.weight"):
return f"layers.{layer}.attention.wq.weight"
elif k.endswith(“.self_attn.k_proj.weight”):
return f"layers.{layer}.attention.wk.weight"
elif k.endswith(“.self_attn.v_proj.weight”):
return f"layers.{layer}.attention.wv.weight"
elif k.endswith(“.self_attn.o_proj.weight”):
return f"layers.{layer}.attention.wo.weight"
elif k.endswith(“.mlp.gate_proj.weight”):
return f"layers.{layer}.feed_forward.w1.weight"
elif k.endswith(“.mlp.down_proj.weight”):
return f"layers.{layer}.feed_forward.w2.weight"
elif k.endswith(“.mlp.up_proj.weight”):
return f"layers.{layer}.feed_forward.w3.weight"
elif k.endswith(“.input_layernorm.weight”):
return f"layers.{layer}.attention_norm.weight"
elif k.endswith(“.post_attention_layernorm.weight”):
return f"layers.{layer}.ffn_norm.weight"
elif k.endswith(“rotary_emb.inv_freq”) or “lora” in k:
return None
else:
print(layer, k)
raise NotImplementedError
else:
print(k)
raise NotImplementedError



new_state_dict = {}
for k, v in lora_model_sd.items():
new_k = translate_state_dict_key(k)
if new_k is not None:
if “wq” in new_k or “wk” in new_k:
new_state_dict[new_k] = unpermute(v)
else:
new_state_dict[new_k] = v

os.makedirs(output_dir, exist_ok=True)
# Split the tensors based on layer index
n_layers_actual = len([k for k in new_state_dict.keys() if “.attention.wq.weight” in k])
part1_keys = [k for k in new_state_dict.keys() if not k.startswith(“layers.”) or int(k.split(“.”)[1]) < (n_layers_actual // 2)]
part2_keys = [k for k in new_state_dict.keys() if k not in part1_keys]

state_dict_part1 = {k: new_state_dict[k] for k in part1_keys}
state_dict_part2 = {k: new_state_dict[k] for k in part2_keys}

torch.save(state_dict_part1, output_dir + “/consolidated.00.pth”)
torch.save(state_dict_part2, output_dir + “/consolidated.01.pth”)
with open(output_dir + “/params.json”, “w”) as f:
json.dump(params, f)

if name == “main”:
fire.Fire(main)

the error is the same:

./main -m ./models/13B/ggml-model-f16.bin -n 5000 --repeat_penalty 1.0 --color -i -r “User:” -f prompts/chat-with-bob.txt -t 32
main: seed = 1681037044
llama_model_load: loading model from ‘./models/13B/ggml-model-f16.bin’ - please wait …
llama_model_load: n_vocab = 32000
llama_model_load: n_ctx = 512
llama_model_load: n_embd = 5120
llama_model_load: n_mult = 256
llama_model_load: n_head = 40
llama_model_load: n_layer = 40
llama_model_load: n_rot = 128
llama_model_load: f16 = 1
llama_model_load: n_ff = 13824
llama_model_load: n_parts = 2
llama_model_load: type = 2
llama_model_load: ggml map size = 25138.72 MB
llama_model_load: ggml ctx size = 101.25 KB
llama_model_load: mem required = 27186.82 MB (+ 1608.00 MB per state)
llama_model_load: loading tensors from ‘./models/13B/ggml-model-f16.bin’
llama_model_load: tensor ‘layers.20.attention.wq.weight’ has wrong size in model file
llama_init_from_file: failed to load model
main: error: failed to load model ‘./models/13B/ggml-model-f16.bin’