Repository: ggerganov/llama.cpp
File: convert_hf_to_gguf.py (Python)

Code before:

    def prepare_tensors(self):
        super().prepare_tensors()

        if self._experts is not None:
            # flatten `list[dict[str, Tensor]]` into `list[str]`
            experts = [k for d in self._experts for k in d.keys()]
            if len(experts) > 0:
                raise ValueError(f"Unprocessed experts: {experts}")


@ModelBase.register("ModernBertModel", "ModernBertForMaskedLM", "ModernBertForSequenceClassification")
class ModernBertModel(BertModel):
    model_arch = gguf.MODEL_ARCH.MODERN_BERT

    def set_vocab(self):
        self.gguf_writer.add_add_bos_token(True)
        self.gguf_writer.add_add_eos_token(True)
        self.gguf_writer.add_add_sep_token(True)
        self._set_vocab_gpt2()

    def set_gguf_parameters(self):
        super().set_gguf_parameters()
        self.gguf_writer.add_dense_every_n_layers(self.hparams["global_attn_every_n_layers"])
        self.gguf_writer.add_sliding_window(self.hparams["local_attention"])
        self.gguf_writer.add_rope_freq_base(self.hparams["global_rope_theta"])
        self.gguf_writer.add_rope_freq_base_swa(self.hparams["local_rope_theta"])
        self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE)
        self.gguf_writer.add_vocab_size(self.hparams["vocab_size"])

    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
        # rename custom "head" layers to standard bert "cls.predictions" names for compatibility
        if name == "head.norm.weight":
            name = "cls.predictions.transform.LayerNorm.weight"
        elif name == "head.norm.bias":
            name = "cls.predictions.transform.LayerNorm.bias"
        elif name == "head.dense.weight":
            name = "cls.predictions.transform.dense.weight"
        elif name == "head.dense.bias":
            name = "cls.predictions.transform.dense.bias"

        # These layers act as MLM head, so we don't need them
        if name.startswith("decoder."):
            return []

        if name.startswith("model."):
            name = name[6:]

        return super().modify_tensors(data_torch, name, bid)



Review comment (suggestion):
It looks like something may have changed in `transformers` v5. When I try to run `convert_hf_to_gguf.py` it's erroring out with this as a missing key. It looks like that is no longer a standard key in the config object: https://github.com/huggingface/transformers/blob/main/src/transformers/models/modernbert/configuration_modernbert.py#L38

Code after:
        self.gguf_writer.add_vision_use_gelu(True)
        # python notation, e.g. for vision_feature_layer == -1, we pick last layer -> vision_feature_layers_to_drop = 0
        vision_feature_layers_to_drop = -(self.global_config.get("vision_feature_layer", -1) + 1)
        self.gguf_writer.add_vision_block_count(self.find_vparam(self.n_block_keys) - vision_feature_layers_to_drop)

    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
        del bid  # unused
        is_vision_tensor = "vision_tower" in name or "multi_modal_projector" in name

        if is_vision_tensor:
            # remove "model." prefix
            name = name.replace("model.vision_tower.", "vision_tower.")
            name = name.replace("model.multi_modal_projector.", "multi_modal_projector.")

            if "patch_embedding.weight" in name:
                data_torch = data_torch.view(data_torch.shape[0], 16, 16, 3).permute(0, 3, 1, 2)

            return [(self.map_tensor_name(name), data_torch)]

        return [] # skip other tensors


@ModelBase.register("Lfm2AudioForConditionalGeneration")
class LFM2AudioModel(MmprojModel):
    has_vision_encoder = False
    has_audio_encoder = True
    model_name = "Lfm2AudioEncoder"

    _batch_norm_tensors: list[dict[str, Tensor]] | None = None

    def get_audio_config(self) -> dict[str, Any] | None:
        return self.global_config.get("encoder")

    def set_gguf_parameters(self):
        assert self.hparams_audio is not None
        self.hparams_audio["hidden_size"] = self.hparams_audio["d_model"]
        self.hparams_audio["intermediate_size"] = self.hparams_audio["d_model"]
        self.hparams_audio["num_attention_heads"] = self.hparams_audio["n_heads"]
        super().set_gguf_parameters()
        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.LFM2A)
        self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["feat_in"])
        self.gguf_writer.add_audio_attention_layernorm_eps(1e-5)

    def tensor_force_quant(self, name, new_name, bid, n_dims):
        if ".conv" in name and ".weight" in name:
            return gguf.GGMLQuantizationType.F32
        return super().tensor_force_quant(name, new_name, bid, n_dims)

    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
        # skip language model tensors
        if name.startswith("lfm."):