Upload 20 files

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+audio_encoder.rknn filter=lfs diff=lfs merge=lfs -text
+librkllmrt.so filter=lfs diff=lfs merge=lfs -text
+qwen.rkllm filter=lfs diff=lfs merge=lfs -text

audio_encoder.rknn

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:397ddd3fccf5cca827b765b68ee2623997e7d4b0e36ad1997b366e6ed28eb4d9
+size 1363407727

audio_encoder_convert_rknn.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+import os
+from rknn.api import RKNN
+from sys import exit
+import argparse
+import cv2
+import numpy as np
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+seq_lengths = [3000]
+batch_sizes = [1]
+mel_size = 128
+
+def convert_encoder():
+    rknn = RKNN(verbose=True)
+
+    ONNX_MODEL=f"audio_encoder.onnx"
+    RKNN_MODEL=ONNX_MODEL.replace(".onnx",".rknn")
+    DATASET="dataset.txt"
+    QUANTIZE=False
+    input_shapes = [[[batch_size, mel_size, seq_length], [batch_size, seq_length]] for batch_size in batch_sizes for seq_length in seq_lengths]
+    print(input_shapes)
+
+    # pre-process config
+    print('--> Config model')
+    rknn.config(quantized_algorithm='normal', quantized_method='channel', target_platform='rk3588', optimization_level=3, dynamic_input=input_shapes) # mean_values=[0.5, 0.5, 0.5], std_values=[0.5, 0.5, 0.5],
+    print('done')
+
+    # Load ONNX model
+    print("--> Loading model")
+    ret = rknn.load_onnx(
+        model=ONNX_MODEL,
+    )
+
+    if ret != 0:
+        print('Load model failed!')
+        exit(ret)
+    print('done')
+
+    # Build model
+    print('--> Building model')
+    ret = rknn.build(do_quantization=QUANTIZE, dataset=DATASET, rknn_batch_size=None)
+    if ret != 0:
+        print('Build model failed!')
+        exit(ret)
+    print('done')
+
+    # export
+    print('--> Export RKNN model')
+    ret = rknn.export_rknn(RKNN_MODEL)
+    if ret != 0:
+        print('Export RKNN model failed!')
+        exit(ret)
+    print('done')
+    # rknn.init_runtime(target='rk3588')
+    # # image embedding
+    # img_path = "test.jpg"
+
+    # normalize_mean = [0.5, 0.5, 0.5]
+    # normalize_std = [0.5, 0.5, 0.5]
+
+    # img = cv2.imread(img_path)
+    # img = cv2.resize(img, (448, 448))
+    # # img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+    # img = img.astype(np.float32)
+    # # img = (img - normalize_mean) / normalize_std
+    # img = img[np.newaxis, :, :, :]
+    # img = img.transpose(0, 3, 1, 2)
+    # np.save("img.npy", img)
+    # rknn.accuracy_analysis(inputs=["img.npy"], target='rk3588')
+# usage: python convert_rknn.py encoder|all
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("model", type=str, help="model to convert", choices=["encoder", "all"], nargs='?')
+    args = parser.parse_args()
+    if args.model is None:
+        args.model = "all"
+    if args.model == "encoder":
+        convert_encoder()
+    elif args.model == "all":
+        convert_encoder()
+    else:
+        print(f"Unknown model: {args.model}")
+        exit(1)

audio_encoder_export_onnx.py

@@ -0,0 +1,88 @@
+import torch
+import torch.nn as nn
+
+from transformers import Qwen2AudioForConditionalGeneration
+
+class Qwen2AudioEncoderWrapper(nn.Module):
+    """包装Qwen2Audio的编码器和映射层用于ONNX导出"""
+    
+    def __init__(self, model):
+        super().__init__()
+        self.audio_tower = model.audio_tower
+        self.projector = model.multi_modal_projector
+        
+    def forward(self, input_features, feature_attention_mask):
+        # 计算音频特征长度
+        audio_feat_lengths = feature_attention_mask.sum(-1)
+        batch_size, _, max_mel_seq_len = input_features.shape
+        
+        # 计算序列长度
+        max_seq_len = (max_mel_seq_len - 2) // 2 + 1
+        seq_range = torch.arange(0, max_seq_len, device=input_features.device).unsqueeze(0)
+        seq_range = seq_range.expand(batch_size, max_seq_len)
+        
+        # 创建attention mask
+        lengths_expand = audio_feat_lengths.unsqueeze(1).expand(batch_size, max_seq_len) 
+        padding_mask = seq_range >= lengths_expand
+        audio_attention_mask = padding_mask.view(batch_size, 1, 1, max_seq_len)
+        audio_attention_mask = audio_attention_mask.expand(batch_size, 1, max_seq_len, max_seq_len)
+        audio_attention_mask = audio_attention_mask.float()
+        audio_attention_mask = audio_attention_mask.masked_fill(audio_attention_mask.bool(), float("-inf"))
+
+        # 获取音频特征
+        audio_outputs = self.audio_tower(input_features, attention_mask=audio_attention_mask)
+        audio_features = audio_outputs.last_hidden_state
+        
+        # 投影到文本空间
+        projected_features = self.projector(audio_features)
+        
+        return projected_features
+
+def export_qwen2audio_encoder(model, save_path, input_shape=(1, 80, 3000)):
+    """
+    导出Qwen2Audio编码器到ONNX格式
+    
+    Args:
+        model: Qwen2AudioForConditionalGeneration模型
+        save_path: 保存ONNX模型的路径
+        input_shape: 输入音频特征的形状 (batch_size, n_mels, seq_len)
+    """
+    
+    wrapper = Qwen2AudioEncoderWrapper(model)
+    wrapper.eval()
+    
+    # 准备样例输入
+    batch_size, n_mels, seq_len = input_shape
+    dummy_input = torch.randn(input_shape)
+    dummy_mask = torch.ones((batch_size, seq_len))
+    
+    # 设置动态轴
+    dynamic_axes = {
+        'input_features': {0: 'batch_size', 2: 'sequence_length'},
+        'feature_attention_mask': {0: 'batch_size', 1: 'sequence_length'},
+        'output': {0: 'batch_size', 1: 'sequence_length'}
+    }
+    
+    # 导出ONNX
+    torch.onnx.export(
+        wrapper,
+        (dummy_input, dummy_mask),
+        save_path,
+        input_names=['input_features', 'feature_attention_mask'],
+        output_names=['output'],
+        dynamic_axes=dynamic_axes,
+        opset_version=17,
+        do_constant_folding=True
+    )
+
+if __name__ == "__main__":
+    # 加载模型
+    model = Qwen2AudioForConditionalGeneration.from_pretrained("../Qwen2-Audio-7B-Instruct/")
+    model.eval()
+
+    # 导出ONNX
+    export_qwen2audio_encoder(
+        model,
+        "audio_encoder.onnx",
+        input_shape=(1, 128, 3000)  # batch_size=1, n_mels=128, seq_len=3000
+    )

config.json

@@ -0,0 +1,20 @@
+{
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 8192,
+  "model_type": "qwen2",
+  "rope_theta": 10000,
+  "rms_norm_eps": 1e-5,
+  "sliding_window": 32768,
+  "torch_dtype": "bfloat16",
+  "use_mrope": false,
+  "vocab_size": 156032,
+
+  "num_hidden_layers": 32,
+  "hidden_size": 4096,
+  "num_attention_heads": 32
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "chat_format": "chatml",
+  "eos_token_id": [151643,151645],
+  "pad_token_id": 151643,
+  "do_sample": true,
+  "top_k": 20,
+  "top_p": 0.5,
+  "temperature": 0.7,
+  "repetition_penalty": 1.1,
+  "transformers_version": "4.38.1"
+}

librkllmrt.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac71a21e0fa68df97ab8145a0beae1c561f31d391ea78c12be675b9d34edea85
+size 6226872

model.safetensors.index.json

@@ -0,0 +1,883 @@
+{
+  "metadata": {
+    "total_size": 16794189824
+  },
+  "weight_map": {
+    "audio_tower.conv1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.conv1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.conv2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.conv2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.embed_positions.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.0.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.1.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.10.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.11.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.12.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.13.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.14.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.15.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.16.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.17.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.18.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.19.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.2.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.20.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.21.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.22.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.23.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.24.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.25.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.26.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.27.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.28.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.29.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.3.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.30.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.31.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.4.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.5.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.6.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.7.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.8.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.fc1.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.fc1.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.fc2.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.fc2.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.final_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.final_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.self_attn.out_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.self_attn.out_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.self_attn_layer_norm.bias": "model-renamed-00001-of-00005.safetensors",
+    "audio_tower.layers.9.self_attn_layer_norm.weight": "model-renamed-00001-of-00005.safetensors",
+    "lm_head.weight": "model-renamed-00005-of-00005.safetensors",
+    "model.embed_tokens.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-renamed-00003-of-00005.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-renamed-00001-of-00005.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-renamed-00004-of-00005.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-renamed-00002-of-00005.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-renamed-00002-of-00005.safetensors",
+    "model.norm.weight": "model-renamed-00004-of-00005.safetensors",
+    "multi_modal_projector.linear.bias": "model-renamed-00001-of-00005.safetensors",
+    "multi_modal_projector.linear.weight": "model-renamed-00001-of-00005.safetensors"
+  }
+}

multiprocess_inference.py

@@ -0,0 +1,334 @@
+import faulthandler
+faulthandler.enable()
+import os
+import random
+import time
+import signal
+from multiprocessing import Process, Queue, Event
+import numpy as np
+from rkllm_binding import *
+from rknnlite.api.rknn_lite import RKNNLite
+import threading
+import librosa
+from transformers import WhisperFeatureExtractor
+
+# 音频编码器进程
+def audio_encoder_process(load_ready_queue, embedding_queue, audio_path_queue, start_event):
+    
+    AUDIO_ENCODER_PATH = "audio_encoder.rknn"
+    
+    # 初始化音频编码器
+    audio_encoder = RKNNLite(verbose=False)
+    model_size = os.path.getsize(AUDIO_ENCODER_PATH)
+    print(f"Start loading audio encoder model (size: {model_size / 1024 / 1024:.2f} MB)")
+    start_time = time.time()
+    audio_encoder.load_rknn(AUDIO_ENCODER_PATH)
+    end_time = time.time()
+    print(f"Audio encoder loaded in {end_time - start_time:.2f} seconds")
+    audio_encoder.init_runtime()
+    
+    # 初始化Whisper特征提取器
+    feature_extractor = WhisperFeatureExtractor.from_pretrained(".")
+    
+    # 通知主进程加载完成
+    load_ready_queue.put("audio_ready")
+    
+    # 等待开始信号
+    start_event.wait()
+    
+    def process_audio(audio_path, audio_encoder, feature_extractor):
+        try:
+            print("Start audio inference...")
+            audio, _ = librosa.load(audio_path, sr=feature_extractor.sampling_rate)
+            feature_extractor_output = feature_extractor(
+                audio,
+                sampling_rate=feature_extractor.sampling_rate, 
+                return_attention_mask=True,
+                padding="max_length"
+            )
+            
+            start_time = time.time()
+            audio_embeddings = audio_encoder.inference(inputs=[
+                feature_extractor_output.input_features.astype(np.float32),
+                feature_extractor_output.attention_mask.astype(np.float32)
+            ], data_format="nhwc")[0].astype(np.float32)
+            end_time = time.time()
+            print(f"Audio encoder inference time: {end_time - start_time:.2f} seconds")
+
+            effective_length = feature_extractor_output.attention_mask.sum(-1)[0]
+            effective_length = (effective_length - 1) // 2 + 1
+            output_lengths = (effective_length - 2) // 2 + 1
+            audio_embeddings = audio_embeddings[:, :output_lengths]
+            print(audio_embeddings.shape)
+            return audio_embeddings
+        except Exception as e:
+            print(f"Error processing audio: {e}")
+            return None
+
+    while True:
+        audio_path = audio_path_queue.get()
+        if audio_path == "STOP":
+            break
+        embeddings = process_audio(audio_path, audio_encoder, feature_extractor)
+        if embeddings is not None:
+            embedding_queue.put(embeddings)
+        else:
+            embedding_queue.put("ERROR")
+
+# LLM进程
+def llm_process(load_ready_queue, embedding_queue, prompt_queue, inference_done_queue, start_event):
+
+    
+    MODEL_PATH = "/home/firefly/qwen.rkllm"
+    handle = None
+    import locale
+    
+    # 获取系统语言
+    system_lang = locale.getdefaultlocale()[0]
+    is_chinese = system_lang and system_lang.startswith('zh')
+    # is_chinese = False
+
+    # 添加进度提示信息列表
+    progress_messages_zh = [
+        "🚀 启动量子加速引擎...",
+        "🧠 神经网络正在苏醒...",
+        "🔄 并行宇宙计算进行中...",
+        "🌟 正在注入能量矩阵...",
+        "🔥 CPU已经到达工作温度，全力运转中...",
+        "🎯 特征向量正在跳跃式生长...",
+        "🎭 多头注意力机制开始营业...",
+        "💨 散热风扇已经进入超音速状态...",
+        "📚 语义解析器正在啃食数据...",
+        "🔍 上下文关联分析师正在加班...",
+        "🎨 视觉特征正在调色盘中混合...",
+        "🤝 跨模态对齐正在相亲相爱中...",
+        "⚡ 深度特征提取器已经深入地心...",
+        "🧪 神经网络正在炼丹中...",
+        "🎲 张量计算已经进入量子态...",
+        "📦 模型参数正在装箱搬运...",
+        "⚖️ 权重矩阵正在天平上找平衡...",
+        "🗺 语义向量正在绘制航海图...",
+        "🎭 注意力头们正在开会讨论...",
+        "🏗 残差模块正在搭建天梯...",
+        "🌈 激活函数正在调制彩虹...",
+        "🎮 张量核心正在玩魔方...",
+        "🎪 循环神经网络正在马戏团表演...",
+        "🎨 特征图正在画饼充饥...",
+        "🔮 模型正在占卜未来...",
+        "🎯 优化器正在进行火箭轨道计算...",
+        "🎪 批归一化正在杂技表演...",
+        "🎭 Dropout正在玩捉迷藏...",
+        "🌪 梯度正在形成龙卷风...",
+        "🎢 反向传播正在过山车..."
+    ]
+    
+    progress_messages_en = [
+        "Loading...",
+        "Extracting...",
+        "Image fusion in progress...",
+        "Matrix multiplication...",
+        "Chip heating up...",
+        "Feature vector calculation...",
+        "Attention mechanism processing...",
+        "Fan speed increasing...",
+        "Semantic parsing...",
+        "Context analysis...",
+        "Visual feature encoding...",
+        "Cross-modal alignment...",
+        "Deep feature extraction...",
+        "Neural network inference...",
+        "Tensor operations...",
+        "Loading model parameters...",
+        "Weight matrix calculation...",
+        "Semantic vector mapping...",
+        "Multi-head attention...",
+        "Residual connection..."
+    ]
+    
+    # 根据语言选择提示信息
+    progress_messages = progress_messages_zh if is_chinese else progress_messages_en
+    
+    # 添加进度提示控制事件
+    progress_stop_event = threading.Event()
+    
+    # 进度提示线程函数
+    def show_progress():
+        while not progress_stop_event.is_set():
+            for msg in progress_messages:
+                if progress_stop_event.is_set():
+                    break
+                print(f"{msg}", flush=True)
+                time.sleep(random.uniform(0.1, 0.4))
+    
+    def signal_handler(signal, frame):
+        print("Ctrl-C pressed, exiting...")
+        global handle
+        if handle:
+            abort(handle)
+            destroy(handle)
+        exit(0)
+    
+    signal.signal(signal.SIGINT, signal_handler)
+    os.environ["RKLLM_LOG_LEVEL"] = "1"
+    
+    inference_count = 0
+    inference_start_time = 0
+    def result_callback(result, userdata, state):
+        nonlocal inference_start_time, inference_count
+        if state == LLMCallState.RKLLM_RUN_NORMAL:
+            if inference_count == 0:
+                progress_stop_event.set()  # 停止进度提示
+                first_token_time = time.time()
+                print("🎉 完成！")
+                print(f"\nTime to first token: {first_token_time - inference_start_time:.2f} seconds")
+            inference_count += 1
+            print(result.contents.text.decode(), end="", flush=True)
+        elif state == LLMCallState.RKLLM_RUN_FINISH:
+            print("\n\n(finished)")
+            inference_done_queue.put("DONE")
+        elif state == LLMCallState.RKLLM_RUN_ERROR:
+            print("\nError occurred during LLM call")
+            inference_done_queue.put("ERROR")
+    
+    # 初始化LLM
+    param = create_default_param()
+    param.model_path = MODEL_PATH.encode()
+    param.img_start = "<|audio_bos|>".encode()
+    param.img_end = "<|audio_eos|>".encode()
+    param.img_content = "<|AUDIO|>".encode()
+    param.max_context_len = 768
+    param.max_new_tokens = 256
+    extend_param = RKLLMExtendParam()
+    extend_param.base_domain_id = 1
+    param.extend_param = extend_param
+    
+    model_size = os.path.getsize(MODEL_PATH)
+    print(f"Start loading language model (size: {model_size / 1024 / 1024:.2f} MB)")
+    start_time = time.time()
+    handle = init(param, result_callback)
+    end_time = time.time()
+    print(f"Language model loaded in {end_time - start_time:.2f} seconds")
+    
+    # 通知主进程加载完成
+    load_ready_queue.put("llm_ready")
+    
+    # 创建推理参数
+    infer_param = RKLLMInferParam()
+    infer_param.mode = RKLLMInferMode.RKLLM_INFER_GENERATE.value
+    
+    while True:
+        prompt = prompt_queue.get()
+        print(f"Received prompt: ===={prompt}\n====")
+        if prompt == "STOP":
+            break
+
+        # 重置计数器和事件
+        inference_count = 0
+        progress_stop_event.clear()
+        
+        # 启动进度提示线程
+        progress_thread = threading.Thread(target=show_progress)
+        progress_thread.daemon = True
+        # progress_thread.start()
+            
+        image_embeddings = embedding_queue.get()
+        if isinstance(image_embeddings, str) and image_embeddings == "ERROR":
+            print("Error processing audio")
+            continue
+        print(image_embeddings.shape)
+        rkllm_input = create_rkllm_input(RKLLMInputType.RKLLM_INPUT_MULTIMODAL,
+                                        prompt=prompt,
+                                        image_embed=image_embeddings)
+        print(f"Start LLM inference...")
+        inference_start_time = time.time()
+        run(handle, rkllm_input, infer_param, None)
+    
+    # 清理
+    destroy(handle)
+
+def main():
+    load_ready_queue = Queue()
+    embedding_queue = Queue()
+    audio_path_queue = Queue()
+    prompt_queue = Queue()
+    inference_done_queue = Queue()
+    start_event = Event()
+    
+    audio_process = Process(target=audio_encoder_process,
+                           args=(load_ready_queue, embedding_queue, audio_path_queue, start_event))
+    lm_process = Process(target=llm_process,
+                        args=(load_ready_queue, embedding_queue, prompt_queue, inference_done_queue, start_event))
+    
+    audio_process.start()
+    time.sleep(10)
+    lm_process.start()
+    
+    # 等待模型加载
+    ready_count = 0
+    while ready_count < 2:
+        status = load_ready_queue.get()
+        print(f"Received ready signal: {status}")
+        ready_count += 1
+    
+    print("All models loaded, starting interactive mode...")
+    start_event.set()
+    
+    # 交互循环
+    try:
+        while True:
+            print("""
+Enter your input (3 empty lines to start inference, Ctrl+C to exit, for example: 
+这是什么声音{{glass-breaking.wav}}?
+What kind of sound is in {{./test.mp3}}?
+Describe the audio in {{./test.mp3}}
+这是什么动物的叫声{{./jntm.mp3}}?
+):
+""")
+            user_input = []
+            empty_lines = 0
+            
+            while empty_lines < 3:
+                line = input()
+                if line.strip() == "":
+                    empty_lines += 1
+                else:
+                    empty_lines = 0
+                user_input.append(line)
+            
+            # 解析输入
+            full_input = "\n".join(user_input[:-3])  # 去掉最后3个空行
+            import re
+            img_match = re.search(r'\{\{(.+?)\}\}', full_input)
+            if not img_match:
+                print("No image path found in input")
+                continue
+                
+            img_path = img_match.group(1)
+            # 将音频标记替换为<image>标记, rkllm的<image>是写死的...
+            prompt = f"""<|im_start|>system
+You are a helpful assistant.<|im_end|>
+<|im_start|>user
+Audio 1: <image>
+{full_input.replace(img_match.group(0), '')}<|im_end|>
+<|im_start|>assistant
+"""
+            audio_path_queue.put(img_path)
+            prompt_queue.put(prompt)
+            
+            # 等待推理完成
+            status = inference_done_queue.get()
+            if status == "ERROR":
+                print("Inference failed")
+            
+    except KeyboardInterrupt:
+        print("\nExiting...")
+        audio_path_queue.put("STOP")
+        prompt_queue.put("STOP")
+    
+    audio_process.join()
+    lm_process.join()
+
+if __name__ == "__main__":
+    main()
+
+#这是什么声音{{./test.mp3}}?

preprocessor_config.json

@@ -0,0 +1,14 @@
+{
+  "chunk_length": 30,
+  "feature_extractor_type": "WhisperFeatureExtractor",
+  "feature_size": 128,
+  "hop_length": 160,
+  "n_fft": 400,
+  "n_samples": 480000,
+  "nb_max_frames": 3000,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "processor_class": "Qwen2AudioProcessor",
+  "return_attention_mask": true,
+  "sampling_rate": 16000
+}

qwen.rkllm

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cfd15dce8ce894421f6af38fc82f46c5c53eefd452d8fc9d36391f98179d2f4a
+size 8428376036

rename_tensors.py

@@ -0,0 +1,46 @@
+import json
+import os
+import shutil
+import mmap
+import re
+
+def rename_tensors():
+    # 读取JSON文件
+    with open('model.safetensors.index.json', 'r') as f:
+        data = json.load(f)
+
+    # 获取所有唯一的safetensors文件名
+    safetensor_files = set(data['weight_map'].values())
+
+    # 复制并重命名safetensors文件
+    for file in safetensor_files:
+        new_file = file.replace('model-', 'model-renamed-')
+        shutil.copy(file, new_file)
+
+        # 在新文件的前1MB范围内替换字符串
+        with open(new_file, 'r+b') as f:
+            mm = mmap.mmap(f.fileno(), 1024*1024)  # 映射前1MB
+            content = mm.read()
+            # 使用字节字符串进行替换
+            content = content.replace(b'"language_model.', b'               "')
+            mm.seek(0)
+            mm.write(content)
+            mm.close()
+
+    # 更新JSON数据
+    new_weight_map = {}
+    for key, value in data['weight_map'].items():
+        new_key = re.sub(r'^language_model.', '', key)
+        new_value = value.replace('model-', 'model-renamed-')
+        new_weight_map[new_key] = new_value
+
+    data['weight_map'] = new_weight_map
+
+    # 写入新的JSON文件
+    with open('model-renamed.safetensors.index.json', 'w') as f:
+        json.dump(data, f, indent=2)
+
+    print("处理完成。新的JSON文件已生成：model-renamed.safetensors.index.json")
+
+if __name__ == "__main__":
+    rename_tensors()

rkllm-convert.py

@@ -0,0 +1,41 @@
+from rkllm.api import RKLLM
+from datasets import load_dataset
+from transformers import AutoTokenizer
+from tqdm import tqdm
+import torch
+from torch import nn
+import os
+# os.environ['CUDA_VISIBLE_DEVICES']='1'
+
+modelpath = '.'
+# modelpath = "./path/to/Qwen-1.8B-F16.gguf"
+llm = RKLLM()
+
+# Load model
+# Use 'export CUDA_VISIBLE_DEVICES=2' to specify GPU device
+# options ['cpu', 'cuda']
+ret = llm.load_huggingface(model=modelpath, model_lora = None, device='cpu')
+# ret = llm.load_gguf(model = modelpath)
+if ret != 0:
+    print('Load model failed!')
+    exit(ret)
+
+# Build model
+dataset = "./data_quant.json"
+# Json file format, please note to add prompt in the input，like this:
+# [{"input":"Human: 你好！\nAssistant: ", "target": "你好！我是人工智能助手KK！"},...]
+
+qparams = None
+# qparams = 'gdq.qparams' # Use extra_qparams
+ret = llm.build(do_quantization=True, optimization_level=1, quantized_dtype='w8a8',
+                quantized_algorithm='normal', target_platform='rk3588', num_npu_core=3, extra_qparams=qparams)
+
+if ret != 0:
+    print('Build model failed!')
+    exit(ret)
+
+# Export rkllm model
+ret = llm.export_rkllm("./qwen.rkllm")
+if ret != 0:
+    print('Export model failed!')
+    exit(ret)

rkllm_binding.py

@@ -0,0 +1,226 @@
+import ctypes
+import numpy as np
+from enum import IntEnum
+from typing import Callable, Any
+
+# Load the shared library
+_lib = ctypes.CDLL("./librkllmrt.so")  # Adjust the library name if necessary
+
+# Define enums
+class LLMCallState(IntEnum):
+    RKLLM_RUN_NORMAL = 0
+    RKLLM_RUN_WAITING = 1
+    RKLLM_RUN_FINISH = 2
+    RKLLM_RUN_ERROR = 3
+    RKLLM_RUN_GET_LAST_HIDDEN_LAYER = 4
+
+class RKLLMInputType(IntEnum):
+    RKLLM_INPUT_PROMPT = 0
+    RKLLM_INPUT_TOKEN = 1
+    RKLLM_INPUT_EMBED = 2
+    RKLLM_INPUT_MULTIMODAL = 3
+
+class RKLLMInferMode(IntEnum):
+    RKLLM_INFER_GENERATE = 0
+    RKLLM_INFER_GET_LAST_HIDDEN_LAYER = 1
+
+# Define structures
+class RKLLMExtendParam(ctypes.Structure):
+    _fields_ = [
+        ("base_domain_id", ctypes.c_int32),
+        ("reserved", ctypes.c_uint8 * 112)
+    ]
+
+class RKLLMParam(ctypes.Structure):
+    _fields_ = [
+        ("model_path", ctypes.c_char_p),
+        ("max_context_len", ctypes.c_int32),
+        ("max_new_tokens", ctypes.c_int32),
+        ("top_k", ctypes.c_int32),
+        ("top_p", ctypes.c_float),
+        ("temperature", ctypes.c_float),
+        ("repeat_penalty", ctypes.c_float),
+        ("frequency_penalty", ctypes.c_float),
+        ("presence_penalty", ctypes.c_float),
+        ("mirostat", ctypes.c_int32),
+        ("mirostat_tau", ctypes.c_float),
+        ("mirostat_eta", ctypes.c_float),
+        ("skip_special_token", ctypes.c_bool),
+        ("is_async", ctypes.c_bool),
+        ("img_start", ctypes.c_char_p),
+        ("img_end", ctypes.c_char_p),
+        ("img_content", ctypes.c_char_p),
+        ("extend_param", RKLLMExtendParam)
+    ]
+
+class RKLLMLoraAdapter(ctypes.Structure):
+    _fields_ = [
+        ("lora_adapter_path", ctypes.c_char_p),
+        ("lora_adapter_name", ctypes.c_char_p),
+        ("scale", ctypes.c_float)
+    ]
+
+class RKLLMEmbedInput(ctypes.Structure):
+    _fields_ = [
+        ("embed", ctypes.POINTER(ctypes.c_float)),
+        ("n_tokens", ctypes.c_size_t)
+    ]
+
+class RKLLMTokenInput(ctypes.Structure):
+    _fields_ = [
+        ("input_ids", ctypes.POINTER(ctypes.c_int32)),
+        ("n_tokens", ctypes.c_size_t)
+    ]
+
+class RKLLMMultiModelInput(ctypes.Structure):
+    _fields_ = [
+        ("prompt", ctypes.c_char_p),
+        ("image_embed", ctypes.POINTER(ctypes.c_float)),
+        ("n_image_tokens", ctypes.c_size_t)
+    ]
+
+class RKLLMInput(ctypes.Structure):
+    class _InputUnion(ctypes.Union):
+        _fields_ = [
+            ("prompt_input", ctypes.c_char_p),
+            ("embed_input", RKLLMEmbedInput),
+            ("token_input", RKLLMTokenInput),
+            ("multimodal_input", RKLLMMultiModelInput)
+        ]
+    _fields_ = [
+        ("input_type", ctypes.c_int),
+        ("_input", _InputUnion)
+    ]
+
+class RKLLMLoraParam(ctypes.Structure):
+    _fields_ = [
+        ("lora_adapter_name", ctypes.c_char_p)
+    ]
+
+class RKLLMPromptCacheParam(ctypes.Structure):
+    _fields_ = [
+        ("save_prompt_cache", ctypes.c_int),
+        ("prompt_cache_path", ctypes.c_char_p)
+    ]
+
+class RKLLMInferParam(ctypes.Structure):
+    _fields_ = [
+        ("mode", ctypes.c_int),
+        ("lora_params", ctypes.POINTER(RKLLMLoraParam)),
+        ("prompt_cache_params", ctypes.POINTER(RKLLMPromptCacheParam))
+    ]
+
+class RKLLMResultLastHiddenLayer(ctypes.Structure):
+    _fields_ = [
+        ("hidden_states", ctypes.POINTER(ctypes.c_float)),
+        ("embd_size", ctypes.c_int),
+        ("num_tokens", ctypes.c_int)
+    ]
+
+class RKLLMResult(ctypes.Structure):
+    _fields_ = [
+        ("text", ctypes.c_char_p),
+        ("token_id", ctypes.c_int32),
+        ("last_hidden_layer", RKLLMResultLastHiddenLayer)
+    ]
+
+# Define callback type
+LLMResultCallback = ctypes.CFUNCTYPE(None, ctypes.POINTER(RKLLMResult), ctypes.c_void_p, ctypes.c_int)
+
+# Define function prototypes
+_lib.rkllm_createDefaultParam.restype = RKLLMParam
+_lib.rkllm_init.argtypes = [ctypes.POINTER(ctypes.c_void_p), ctypes.POINTER(RKLLMParam), LLMResultCallback]
+_lib.rkllm_init.restype = ctypes.c_int
+_lib.rkllm_load_lora.argtypes = [ctypes.c_void_p, ctypes.POINTER(RKLLMLoraAdapter)]
+_lib.rkllm_load_lora.restype = ctypes.c_int
+_lib.rkllm_load_prompt_cache.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
+_lib.rkllm_load_prompt_cache.restype = ctypes.c_int
+_lib.rkllm_release_prompt_cache.argtypes = [ctypes.c_void_p]
+_lib.rkllm_release_prompt_cache.restype = ctypes.c_int
+_lib.rkllm_destroy.argtypes = [ctypes.c_void_p]
+_lib.rkllm_destroy.restype = ctypes.c_int
+_lib.rkllm_run.argtypes = [ctypes.c_void_p, ctypes.POINTER(RKLLMInput), ctypes.POINTER(RKLLMInferParam), ctypes.c_void_p]
+_lib.rkllm_run.restype = ctypes.c_int
+_lib.rkllm_run_async.argtypes = [ctypes.c_void_p, ctypes.POINTER(RKLLMInput), ctypes.POINTER(RKLLMInferParam), ctypes.c_void_p]
+_lib.rkllm_run_async.restype = ctypes.c_int
+_lib.rkllm_abort.argtypes = [ctypes.c_void_p]
+_lib.rkllm_abort.restype = ctypes.c_int
+_lib.rkllm_is_running.argtypes = [ctypes.c_void_p]
+_lib.rkllm_is_running.restype = ctypes.c_int
+
+# Python wrapper functions
+def create_default_param() -> RKLLMParam:
+    return _lib.rkllm_createDefaultParam()
+
+def init(param: RKLLMParam, callback: Callable[[RKLLMResult, Any, LLMCallState], None]) -> ctypes.c_void_p:
+    handle = ctypes.c_void_p()
+    c_callback = LLMResultCallback(callback)
+    status = _lib.rkllm_init(ctypes.byref(handle), ctypes.byref(param), c_callback)
+    if status != 0:
+        raise RuntimeError(f"Failed to initialize RKLLM: {status}")
+    return handle
+
+def load_lora(handle: ctypes.c_void_p, lora_adapter: RKLLMLoraAdapter) -> None:
+    status = _lib.rkllm_load_lora(handle, ctypes.byref(lora_adapter))
+    if status != 0:
+        raise RuntimeError(f"Failed to load Lora adapter: {status}")
+
+def load_prompt_cache(handle: ctypes.c_void_p, prompt_cache_path: str) -> None:
+    status = _lib.rkllm_load_prompt_cache(handle, prompt_cache_path.encode())
+    if status != 0:
+        raise RuntimeError(f"Failed to load prompt cache: {status}")
+
+def release_prompt_cache(handle: ctypes.c_void_p) -> None:
+    status = _lib.rkllm_release_prompt_cache(handle)
+    if status != 0:
+        raise RuntimeError(f"Failed to release prompt cache: {status}")
+
+def destroy(handle: ctypes.c_void_p) -> None:
+    status = _lib.rkllm_destroy(handle)
+    if status != 0:
+        raise RuntimeError(f"Failed to destroy RKLLM: {status}")
+
+def run(handle: ctypes.c_void_p, rkllm_input: RKLLMInput, rkllm_infer_params: RKLLMInferParam, userdata: Any) -> None:
+    status = _lib.rkllm_run(handle, ctypes.byref(rkllm_input), ctypes.byref(rkllm_infer_params), ctypes.c_void_p(userdata))
+    if status != 0:
+        raise RuntimeError(f"Failed to run RKLLM: {status}")
+
+def run_async(handle: ctypes.c_void_p, rkllm_input: RKLLMInput, rkllm_infer_params: RKLLMInferParam, userdata: Any) -> None:
+    status = _lib.rkllm_run_async(handle, ctypes.byref(rkllm_input), ctypes.byref(rkllm_infer_params), ctypes.c_void_p(userdata))
+    if status != 0:
+        raise RuntimeError(f"Failed to run RKLLM asynchronously: {status}")
+
+def abort(handle: ctypes.c_void_p) -> None:
+    status = _lib.rkllm_abort(handle)
+    if status != 0:
+        raise RuntimeError(f"Failed to abort RKLLM: {status}")
+
+def is_running(handle: ctypes.c_void_p) -> bool:
+    return _lib.rkllm_is_running(handle) == 0
+
+# Helper function to convert numpy array to C array
+def numpy_to_c_array(arr: np.ndarray, c_type):
+    return arr.ctypes.data_as(ctypes.POINTER(c_type))
+
+# Helper function to create RKLLMInput
+def create_rkllm_input(input_type: RKLLMInputType, **kwargs) -> RKLLMInput:
+    rkllm_input = RKLLMInput()
+    rkllm_input.input_type = input_type.value
+
+    if input_type == RKLLMInputType.RKLLM_INPUT_PROMPT:
+        rkllm_input._input.prompt_input = kwargs['prompt'].encode()
+    elif input_type == RKLLMInputType.RKLLM_INPUT_EMBED:
+        embed = kwargs['embed']
+        rkllm_input._input.embed_input.embed = numpy_to_c_array(embed, ctypes.c_float)
+        rkllm_input._input.embed_input.n_tokens = embed.shape[1]
+    elif input_type == RKLLMInputType.RKLLM_INPUT_TOKEN:
+        tokens = kwargs['tokens']
+        rkllm_input._input.token_input.input_ids = numpy_to_c_array(tokens, ctypes.c_int32)
+        rkllm_input._input.token_input.n_tokens = tokens.shape[1]
+    elif input_type == RKLLMInputType.RKLLM_INPUT_MULTIMODAL:
+        rkllm_input._input.multimodal_input.prompt = kwargs['prompt'].encode()
+        image_embed = kwargs['image_embed']
+        rkllm_input._input.multimodal_input.image_embed = numpy_to_c_array(image_embed, ctypes.c_float)
+        rkllm_input._input.multimodal_input.n_image_tokens = image_embed.shape[1]
+
+    return rkllm_input

run_rknn.py

@@ -0,0 +1,128 @@
+import os
+import time
+import numpy as np
+from rkllm_binding import *
+from rknnlite.api.rknn_lite import RKNNLite
+from transformers import WhisperFeatureExtractor
+import signal
+import cv2
+import librosa
+
+
+MODEL_PATH = "qwen.rkllm"
+AUDIO_ENCODER_PATH = "audio_encoder.rknn"
+handle = None
+img_size = 448
+
+# exit on ctrl-c
+def signal_handler(signal, frame):
+    print("Ctrl-C pressed, exiting...")
+    global handle
+    if handle:
+        abort(handle)
+        destroy(handle)
+    exit(0)
+
+signal.signal(signal.SIGINT, signal_handler)
+
+# export RKLLM_LOG_LEVEL=1
+os.environ["RKLLM_LOG_LEVEL"] = "1"
+
+inference_count = 0
+inference_start_time = 0
+def result_callback(result, userdata, state):
+    global inference_start_time
+    global inference_count
+    if state == LLMCallState.RKLLM_RUN_NORMAL:
+        if inference_count == 0:
+            first_token_time = time.time()
+            print(f"Time to first token: {first_token_time - inference_start_time:.2f} seconds")
+        inference_count += 1
+        print(result.contents.text.decode(), end="", flush=True)
+    elif state == LLMCallState.RKLLM_RUN_FINISH:
+        print("\n\n(finished)")
+    elif state == LLMCallState.RKLLM_RUN_ERROR:
+        print("\nError occurred during LLM call")
+
+feature_extractor = WhisperFeatureExtractor.from_pretrained(".")
+
+# Initialize audio encoder
+audio_encoder = RKNNLite(verbose=True)
+model_size = os.path.getsize(AUDIO_ENCODER_PATH)
+print(f"Start loading audio encoder model (size: {model_size / 1024 / 1024:.2f} MB)")
+start_time = time.time()
+audio_encoder.load_rknn(AUDIO_ENCODER_PATH)
+end_time = time.time()
+print(f"Audio encoder loaded in {end_time - start_time:.2f} seconds (speed: {model_size / (end_time - start_time) / 1024 / 1024:.2f} MB/s)")
+audio_encoder.init_runtime()
+
+# Initialize RKLLM
+param = create_default_param()
+param.model_path = MODEL_PATH.encode()
+param.img_start = "<|audio_bos|>".encode()
+param.img_end = "<|audio_eos|>".encode()
+param.img_content = "<|AUDIO|>".encode()
+param.max_context_len = 1024
+extend_param = RKLLMExtendParam()
+extend_param.base_domain_id = 1  # iommu domain 0 for audio encoder
+param.extend_param = extend_param
+model_size = os.path.getsize(MODEL_PATH)
+print(f"Start loading language model (size: {model_size / 1024 / 1024:.2f} MB)")
+start_time = time.time()
+handle = init(param, result_callback)
+end_time = time.time()
+print(f"Language model loaded in {end_time - start_time:.2f} seconds (speed: {model_size / (end_time - start_time) / 1024 / 1024:.2f} MB/s)")
+
+
+# audio embedding
+audio_path = "glass-breaking.mp3"
+
+
+print("Start inference...")
+audio, _ = librosa.load(audio_path, sr=feature_extractor.sampling_rate)
+feature_extractor_output = feature_extractor(
+    audio, 
+    sampling_rate=feature_extractor.sampling_rate,
+    return_attention_mask=True,
+    padding="max_length"
+)
+
+print(feature_extractor_output.input_features.shape)
+start_time = time.time()
+audio_embeddings = audio_encoder.inference(inputs=[
+    feature_extractor_output.input_features.astype(np.float32),
+    feature_extractor_output.attention_mask.astype(np.float32)
+], data_format="nhwc")[0].astype(np.float32)
+end_time = time.time()
+print(f"Audio encoder inference time: {end_time - start_time:.2f} seconds")
+print(audio_embeddings.flags)
+print(audio_embeddings.shape)
+
+# Create input. RKLLM is stupid enough to hardcode the <image> tag for embedding.
+prompt = """<|im_start|>system
+You are a helpful assistant.<|im_end|>
+<|im_start|>user
+Audio 1: <image>
+这是什么声音? <|im_end|>
+<|im_start|>assistant
+"""
+
+# # # 2.56->3.25>2.41->10.2
+# # image_embeddings = np.load("image_embeddings_pth_orig.npy")
+# # image_embeddings = np.ascontiguousarray(image_embeddings, dtype=np.float32)
+# # print(f"Loaded embeddings shape: {image_embeddings.shape}")
+
+# # rkllm_input = create_rkllm_input(RKLLMInputType.RKLLM_INPUT_EMBED, embed=image_embeddings)
+
+rkllm_input = create_rkllm_input(RKLLMInputType.RKLLM_INPUT_MULTIMODAL, prompt=prompt, image_embed=audio_embeddings)
+
+# Create inference parameters
+infer_param = RKLLMInferParam()
+infer_param.mode = RKLLMInferMode.RKLLM_INFER_GENERATE.value
+
+# Run RKLLM
+inference_start_time = time.time()
+run(handle, rkllm_input, infer_param, None)
+
+# Clean up
+destroy(handle)