Repository: ggerganov/llama.cpp
File: convert_hf_to_gguf.py (Python)

Code before:
                self.dir_model, trust_remote_code=True
            )
            vocab = getattr(tokenizer, "vocab", tokenizer.get_vocab())
            vocab_size = self.hparams.get("vocab_size", len(vocab))
            assert max(vocab.values()) < vocab_size

            tokpre = self.get_vocab_base_pre(tokenizer)

            reverse_vocab = {id_: encoded_tok for encoded_tok, id_ in vocab.items()}
            added_vocab = tokenizer.get_added_vocab()

            added_tokens_decoder = tokenizer.added_tokens_decoder

            for i in range(vocab_size):
                if i not in reverse_vocab:
                    tokens.append(f"[PAD{i}]")
                    toktypes.append(gguf.TokenType.UNUSED)
                else:
                    token: str = reverse_vocab[i]

                    if token in added_vocab:
                        if not added_tokens_decoder[i].normalized:
                            previous_token = token
                            token = tokenizer.decode(
                                tokenizer.encode(token, add_special_tokens=False)
                            )
                            if previous_token != token:
                                logger.info(
                                    f"{repr(previous_token)} is encoded and decoded back to {repr(token)} using AutoTokenizer"
                                )

                        if added_tokens_decoder[i].special or self.does_token_look_special(
                            token
                        ):
                            toktypes.append(gguf.TokenType.CONTROL)
                        else:
                            toktypes.append(gguf.TokenType.USER_DEFINED)
                    elif re.fullmatch(r"<0x[0-9A-Fa-f]{2}>", token):
                        toktypes.append(gguf.TokenType.BYTE)  # special
                    else:
                        toktypes.append(gguf.TokenType.NORMAL)
                    tokens.append(token)

            self.gguf_writer.add_tokenizer_model("llama")
            self.gguf_writer.add_tokenizer_pre(tokpre)
            self.gguf_writer.add_token_list(tokens)
            self.gguf_writer.add_token_types(toktypes)

            special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
            special_vocab._set_special_token("bos", 1)
            special_vocab.add_to_gguf(self.gguf_writer)

Review comment (suggestion):
```suggestion
                            token = tokenizer.decode(tokenizer.encode(token, add_special_tokens=False))
```

Code after:
        dt_rank      = self.find_hparam(["mamba_dt_rank"], optional=True) or -(d_model // -16)
        rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-6
        n_kv_head = self.hparams["num_key_value_heads"]
        attn_offset = self.hparams["attn_layer_offset"]
        attn_period = self.hparams["attn_layer_period"]
        n_kv_vec = [0 for _ in range(attn_offset)] + [
            n_kv_head if (i - attn_offset) % attn_period == 0 else 0 for i in range(attn_offset, self.block_count)
        ]

        self.gguf_writer.add_block_count(self.block_count)
        self.gguf_writer.add_context_length(self.find_hparam(["max_position_embeddings", "n_ctx"]))
        self.gguf_writer.add_embedding_length(d_model)
        self.gguf_writer.add_feed_forward_length(self.hparams["intermediate_size"])
        self.gguf_writer.add_head_count(self.hparams["num_attention_heads"])
        self.gguf_writer.add_head_count_kv(n_kv_vec)
        self.gguf_writer.add_ssm_conv_kernel(d_conv)
        self.gguf_writer.add_ssm_inner_size(d_inner)
        self.gguf_writer.add_ssm_state_size(d_state)
        self.gguf_writer.add_ssm_time_step_rank(dt_rank)
        self.gguf_writer.add_layer_norm_rms_eps(rms_norm_eps)
        self.gguf_writer.add_expert_count(self.hparams["num_experts"])
        self.gguf_writer.add_expert_used_count(self.hparams["num_experts_per_tok"])
        self.gguf_writer.add_file_type(self.ftype)

    _experts: list[dict[str, Tensor]] | None = None

    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:

        # Mini-Jamba
        name = name.replace(".moe.", ".feed_forward.")
        if bid is not None:
            moe_offset = self.hparams["expert_layer_offset"]
            moe_period = self.hparams["expert_layer_period"]

            if not (bid >= moe_offset and (bid - moe_offset) % moe_period == 0):
                name = name.replace(".experts.0.", ".")

        # process the experts separately
        if ".feed_forward.experts." in name:
            n_experts = self.hparams["num_experts"]

            assert bid is not None

            if self._experts is None:
                self._experts = [{} for _ in range(self.block_count)]

            self._experts[bid][name] = data_torch

            if len(self._experts[bid]) >= n_experts * 3:

                # merge the experts into a single 3d tensor