xlm-mlm-17-1280 / tokenizer_config.json
lysandre's picture
lysandre HF staff
Create tokenizer_config.json
feb8a6d
{"do_lowercase_and_remove_accent": false, "id2lang": {"0": "ar", "1": "de", "2": "en", "3": "es", "4": "fr", "5": "hi", "6": "it", "7": "ja", "8": "ko", "9": "nl", "10": "pl", "11": "pt", "12": "ru", "13": "sv", "14": "tr", "15": "vi", "16": "zh"}, "lang2id": {"ar": 0, "de": 1, "en": 2, "es": 3, "fr": 4, "hi": 5, "it": 6, "ja": 7, "ko": 8, "nl": 9, "pl": 10, "pt": 11, "ru": 12, "sv": 13, "tr": 14, "vi": 15, "zh": 16}, "model_max_length": 512}