Repository: coqui-ai/TTS
File: TTS/tts/layers/xtts/tokenizer.py (Python)

Code before:
        return self.tokenizer.encode(txt).ids

    def decode(self, seq):
        if isinstance(seq, torch.Tensor):
            seq = seq.cpu().numpy()
        txt = self.tokenizer.decode(seq, skip_special_tokens=False).replace(" ", "")
        txt = txt.replace("[SPACE]", " ")
        txt = txt.replace("[STOP]", "")
        txt = txt.replace("[UNK]", "")
        return txt

    def preprocess_text(self, txt, lang):
        if lang in ["en", "es", "fr", "de", "pt", "it", "pl", "zh", "ar", "cs", "ru", "nl", "tr", "hu"]:
            txt = multilingual_cleaners(txt, lang)
        elif lang == "ja":
            if self.katsu is None:
                import cutlet

                self.katsu = cutlet.Cutlet()
            txt = japanese_cleaners(txt, self.katsu)
        elif lang == "zh-cn" or lang == "zh":
            txt = chinese_transliterate(txt)
        elif lang == "ko":
            txt = korean_cleaners(txt)
        else:
            raise NotImplementedError()
        return txt

    def __len__(self):
        return self.tokenizer.get_vocab_size()

    def get_number_tokens(self):
        return max(self.tokenizer.get_vocab().values()) + 1


def test_expand_numbers_multilingual():
    test_cases = [
        # English
        ("In 12.5 seconds.", "In twelve point five seconds.", "en"),
        ("There were 50 soldiers.", "There were fifty soldiers.", "en"),
        ("This is a 1st test", "This is a first test", "en"),
        ("That will be $20 sir.", "That will be twenty dollars sir.", "en"),
        ("That will be 20€ sir.", "That will be twenty euro sir.", "en"),
        ("That will be 20.15€ sir.", "That will be twenty euro, fifteen cents sir.", "en"),
        ("That's 100,000.5.", "That's one hundred thousand point five.", "en"),
        # French
        ("En 12,5 secondes.", "En douze virgule cinq secondes.", "fr"),
        ("Il y avait 50 soldats.", "Il y avait cinquante soldats.", "fr"),
        ("Ceci est un 1er test", "Ceci est un premier test", "fr"),
        ("Cela vous fera $20 monsieur.", "Cela vous fera vingt dollars monsieur.", "fr"),
        ("Cela vous fera 20€ monsieur.", "Cela vous fera vingt euros monsieur.", "fr"),

Review comment (performance):
Use a set for more performance?
```suggestion
        if lang in {"en", "es", "fr", "de", "pt", "it", "pl", "ar", "cs", "ru", "nl", "tr", "zh-cn"}:
```

Code after:
            "it": 213,
            "pt": 203,
            "pl": 224,
            "zh-cn": 82,
            "ar": 166,
            "cs": 186,
            "ru": 182,
            "nl": 251,
            "tr": 226,
            "ja": 71,
            "hu": 224,
            "ko": 95,
        }

    @cached_property
    def katsu(self):
        import cutlet
        return cutlet.Cutlet()
    
    def check_input_length(self, txt, lang):
        limit = self.char_limits.get(lang, 250)
        if len(txt) > limit:
            print(f"[!] Warning: The text length exceeds the character limit of {limit} for language '{lang}', this might cause truncated audio.")

    def preprocess_text(self, txt, lang):
        if lang in ["en", "es", "fr", "de", "pt", "it", "pl", "ar", "cs", "ru", "nl", "tr", "zh-cn"]:
            txt = multilingual_cleaners(txt, lang)
            if lang == "zh-cn":
                txt = chinese_transliterate(txt)
        elif lang == "ja":                
            txt = japanese_cleaners(txt, self.katsu)
        else:
            raise NotImplementedError()
        return txt

    def encode(self, txt, lang):
        self.check_input_length(txt, lang)
        txt = self.preprocess_text(txt, lang)
        txt = f"[{lang}]{txt}"
        txt = txt.replace(" ", "[SPACE]")
        return self.tokenizer.encode(txt).ids

    def decode(self, seq):
        if isinstance(seq, torch.Tensor):
            seq = seq.cpu().numpy()
        txt = self.tokenizer.decode(seq, skip_special_tokens=False).replace(" ", "")
        txt = txt.replace("[SPACE]", " ")
        txt = txt.replace("[STOP]", "")
        txt = txt.replace("[UNK]", "")
        return txt