Repository: ggerganov/llama.cpp
File: gguf-py/gguf/vocab.py (Python)

Code before:
class MistralVocab(Vocab):
    tokenizer_model = "mistral"
    name = "mistral"

    added_tokens_dict: dict[str, int] = {}
    added_tokens_list: list[str] = []

    def __init__(self, base_path: Path):
        if not _mistral_common_installed:
            raise ImportError(
                "To use MistralVocab, please install the `mistral-common` package. "
                "You can install it with `pip install mistral-common`."
            )
        assert _filter_valid_tokenizer_files is not None, "mistral_common is not installed"
        assert MistralTokenizer is not None, "mistral_common is not installed"
        assert Tekkenizer is not None, "mistral_common is not installed"

        logger.info(f"Loading Mistral tokenizer from {base_path}")

        # Find the tokenizer files
        all_files = [f.as_posix() for f in base_path.glob("**/*") if f.is_file()]

        if get_one_valid_tokenizer_file is not None:
            tokenizer_file_path = get_one_valid_tokenizer_file(all_files)
            # get_one_valid_tokenizer_file returns a path rather than just the file name e.g: "ministral3b\tekken.json" instead of "tekken.json"
            tokenizer_file = Path(tokenizer_file_path).name
        else:
            valid_tokenizer_files = _filter_valid_tokenizer_files(all_files)

            if len(valid_tokenizer_files) == 0:
                raise ValueError(f"No tokenizer file found in the directory: {base_path}")
            # If there are multiple tokenizer files, we use tekken.json if it exists, otherwise the versioned one.
            if len(valid_tokenizer_files) > 1:
                if "tekken.json" in valid_tokenizer_files:
                    tokenizer_file = "tekken.json"
                else:
                    tokenizer_file = sorted(valid_tokenizer_files)[-1]
                logger.warning(
                    f"Multiple tokenizer files found in {base_path}. Using {tokenizer_file}"
                )
            else:
                tokenizer_file = valid_tokenizer_files[0]

        self.tokenizer = MistralTokenizer.from_file(
            base_path / tokenizer_file
        ).instruct_tokenizer.tokenizer
        self.tokenizer_type = (
            MistralTokenizerType.tekken
            if isinstance(self.tokenizer, Tekkenizer)
            else MistralTokenizerType.spm
        )

Review comment (bug):
This will fail if the tokenizer file is in a subdirectory, the correct approach would be to drop `tokenizer_file` and create `tokenizer_file_path` in the fallback as well, then use that instead of `base_path / tokenizer_file` further down.

Code after:
class MistralVocab(Vocab):
    tokenizer_model = "mistral"
    name = "mistral"

    added_tokens_dict: dict[str, int] = {}
    added_tokens_list: list[str] = []

    def __init__(self, base_path: Path):
        if not _mistral_common_installed:
            raise ImportError(
                "To use MistralVocab, please install the `mistral-common` package. "
                "You can install it with `pip install mistral-common`."
            )
        assert _filter_valid_tokenizer_files is not None, "mistral_common is not installed"
        assert MistralTokenizer is not None, "mistral_common is not installed"
        assert Tekkenizer is not None, "mistral_common is not installed"

        logger.info(f"Loading Mistral tokenizer from {base_path}")

        # Find the tokenizer files
        all_files = [f.as_posix() for f in base_path.glob("**/*") if f.is_file()]

        if get_one_valid_tokenizer_file is not None:
            tokenizer_file_path = get_one_valid_tokenizer_file(all_files)
        else:
            valid_tokenizer_files = _filter_valid_tokenizer_files(all_files)

            if len(valid_tokenizer_files) == 0:
                raise ValueError(f"No tokenizer file found in the directory: {base_path}")
            # If there are multiple tokenizer files, we use tekken.json if it exists, otherwise the versioned one.
            if len(valid_tokenizer_files) > 1:
                if "tekken.json" in valid_tokenizer_files:
                    tokenizer_file = "tekken.json"
                else:
                    tokenizer_file = sorted(valid_tokenizer_files)[-1]
                logger.warning(
                    f"Multiple tokenizer files found in {base_path}. Using {tokenizer_file}"
                )
            else:
                tokenizer_file = valid_tokenizer_files[0]

            tokenizer_file_path = base_path / tokenizer_file

        self.tokenizer = MistralTokenizer.from_file(
            tokenizer_file_path
        ).instruct_tokenizer.tokenizer
        self.tokenizer_type = (
            MistralTokenizerType.tekken
            if isinstance(self.tokenizer, Tekkenizer)
            else MistralTokenizerType.spm
        )