Repository: ggerganov/llama.cpp
File: convert_hf_to_gguf.py (Python)

Code before:
            'model.generation_aligner.',
            'generation_aligner.',
            'model.generation_head.',
            'generation_head.',
        )
        if name.startswith(skip_generation_prefixes):
            return []

        # Handle aligner tensors
        if name.startswith(('model.aligner.', 'aligner.')):
            return list(self._map_aligner_tensor(data_torch, name))

        # Handle vision tensors
        if name.startswith(('model.vision_model.', 'vision_model.')):
            return [(self.map_tensor_name(name), data_torch)]

        return []


@ModelBase.register("SolarOpenForCausalLM")
class SolarOpenModel(Glm4MoeModel):
    model_arch = gguf.MODEL_ARCH.GLM4_MOE

    def set_vocab(self):
        from transformers import AutoTokenizer
        tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
        special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
        tokens, toktypes, tokpre = self.get_vocab_base()
        self.gguf_writer.add_tokenizer_model("gpt2")
        self.gguf_writer.add_tokenizer_pre(tokpre)
        self.gguf_writer.add_token_list(tokens)
        self.gguf_writer.add_token_types(toktypes)
        special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["
"])
        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["
"])
        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<unk>"])
        special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["
"])
        special_vocab.add_to_gguf(self.gguf_writer)


###### CONVERSION LOGIC ######


# tree of lazy tensors
class LazyTorchTensor(gguf.LazyBase):
    _tensor_type = torch.Tensor
    # to keep the type-checker happy
    dtype: torch.dtype
    shape: torch.Size

    # only used when converting a torch.Tensor to a np.ndarray
    _dtype_map: dict[torch.dtype, type] = {

Review comment (suggestion):
```suggestion
        tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
```
Any reason for keeping this? There's no custom tokenizer, it should not be needed and the warning can be ignored.

Code after:
            'model.generation_aligner.',
            'generation_aligner.',
            'model.generation_head.',
            'generation_head.',
        )
        if name.startswith(skip_generation_prefixes):
            return []

        # Handle aligner tensors
        if name.startswith(('model.aligner.', 'aligner.')):
            return list(self._map_aligner_tensor(data_torch, name))

        # Handle vision tensors
        if name.startswith(('model.vision_model.', 'vision_model.')):
            return [(self.map_tensor_name(name), data_torch)]

        return []


@ModelBase.register("SolarOpenForCausalLM")
class SolarOpenModel(Glm4MoeModel):
    model_arch = gguf.MODEL_ARCH.GLM4_MOE

    def set_vocab(self):
        from transformers import AutoTokenizer
        tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
        special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True)
        tokens, toktypes, tokpre = self.get_vocab_base()
        self.gguf_writer.add_tokenizer_model("gpt2")
        self.gguf_writer.add_tokenizer_pre(tokpre)
        self.gguf_writer.add_token_list(tokens)
        self.gguf_writer.add_token_types(toktypes)
        special_vocab._set_special_token("eos", tokenizer.get_added_vocab()["
"])
        special_vocab._set_special_token("eot", tokenizer.get_added_vocab()["
"])
        special_vocab._set_special_token("unk", tokenizer.get_added_vocab()["<unk>"])
        special_vocab._set_special_token("bos", tokenizer.get_added_vocab()["
"])
        special_vocab.add_to_gguf(self.gguf_writer)


###### CONVERSION LOGIC ######


# tree of lazy tensors
class LazyTorchTensor(gguf.LazyBase):
    _tensor_type = torch.Tensor
    # to keep the type-checker happy
    dtype: torch.dtype
    shape: torch.Size

    # only used when converting a torch.Tensor to a np.ndarray
    _dtype_map: dict[torch.dtype, type] = {