Repository: ggerganov/llama.cpp
File: convert_hf_to_gguf.py (Python)

Code before:
            self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])
            self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])
            self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["moe_shared_expert_intermediate_size"])
            self.gguf_writer.add_expert_count(self.hparams["n_routed_experts"])
            self.gguf_writer.add_expert_shared_count(self.hparams["n_shared_experts"])
            self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])
            self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])
            self.gguf_writer.add_expert_group_count(self.hparams["n_group"])

            # number of experts used per token (top-k)
            if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:
                self.gguf_writer.add_expert_used_count(n_experts_used)

    def set_vocab(self):
        super().set_vocab()

        # The tokenizer _does_ add a BOS token (via post_processor type
        # TemplateProcessing) but does not set add_bos_token to true in the
        # config, so we need to explicitly override it here.
        if not self.is_moe:
            self.gguf_writer.add_add_bos_token(True)

    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
        if self.is_moe and bid is not None:
            if name.endswith("mixer.gate.e_score_correction_bias"):
                new_name = name.replace("e_score_correction_bias", "e_score_correction_bias.bias")
                mapped_name = self.map_tensor_name(new_name)
                return [(mapped_name, data_torch)]

            if name.endswith("mixer.dt_bias"):
                new_name = name.replace("dt_bias", "dt.bias")
                mapped_name = self.map_tensor_name(new_name)
                return [(mapped_name, data_torch)]

            if name.endswith("mixer.conv1d.weight"):
                squeezed_data = data_torch.squeeze()
                mapped_name = self.map_tensor_name(name)
                return [(mapped_name, squeezed_data)]

            if name.endswith("mixer.A_log"):
                transformed_data = -torch.exp(data_torch)
                reshaped_data = transformed_data.squeeze().reshape(-1, 1)
                mapped_name = self.map_tensor_name(name)
                return [(mapped_name, reshaped_data)]

            if name.endswith("mixer.D"):
                reshaped_data = data_torch.squeeze().reshape(-1, 1)
                mapped_name = self.map_tensor_name(name)
                return [(mapped_name, reshaped_data)]

            if name.endswith("mixer.norm.weight"):

Review comment (suggestion):
```suggestion
                new_name = name.replace("e_score_correction_bias", "e_score_correction.bias")
```
I would prefer this (it's consistent with the naming for all other models), ~but I guess it's too late now.~

Code after:
            self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])
            self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"])
            self.gguf_writer.add_expert_shared_feed_forward_length(self.hparams["moe_shared_expert_intermediate_size"])
            self.gguf_writer.add_expert_count(self.hparams["n_routed_experts"])
            self.gguf_writer.add_expert_shared_count(self.hparams["n_shared_experts"])
            self.gguf_writer.add_expert_weights_norm(self.hparams["norm_topk_prob"])
            self.gguf_writer.add_expert_weights_scale(self.hparams["routed_scaling_factor"])
            self.gguf_writer.add_expert_group_count(self.hparams["n_group"])

            # number of experts used per token (top-k)
            if (n_experts_used := self.hparams.get("num_experts_per_tok")) is not None:
                self.gguf_writer.add_expert_used_count(n_experts_used)

    def set_vocab(self):
        super().set_vocab()

        # The tokenizer _does_ add a BOS token (via post_processor type
        # TemplateProcessing) but does not set add_bos_token to true in the
        # config, so we need to explicitly override it here.
        if not self.is_moe:
            self.gguf_writer.add_add_bos_token(True)

    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
        if self.is_moe and bid is not None:
            if name.endswith("mixer.gate.e_score_correction_bias"):
                new_name = name.replace("e_score_correction_bias", "e_score_correction.bias")
                mapped_name = self.map_tensor_name(new_name)
                return [(mapped_name, data_torch)]

            if name.endswith("mixer.dt_bias"):
                new_name = name.replace("dt_bias", "dt.bias")
                mapped_name = self.map_tensor_name(new_name)
                return [(mapped_name, data_torch)]

            if name.endswith("mixer.conv1d.weight"):
                squeezed_data = data_torch.squeeze()
                mapped_name = self.map_tensor_name(name)
                return [(mapped_name, squeezed_data)]

            if name.endswith("mixer.A_log"):
                transformed_data = -torch.exp(data_torch)
                reshaped_data = transformed_data.squeeze().reshape(-1, 1)
                mapped_name = self.map_tensor_name(name)
                return [(mapped_name, reshaped_data)]

            if name.endswith("mixer.D"):
                reshaped_data = data_torch.squeeze().reshape(-1, 1)
                mapped_name = self.map_tensor_name(name)
                return [(mapped_name, reshaped_data)]

            if name.endswith("mixer.norm.weight"):