tiny-random-transfo-xl / tokenizer_config.json
LysandreJik's picture
v2 update
c3eb103
{"special": null, "min_freq": 0, "max_size": null, "lower_case": false, "delimiter": null, "never_split": null, "unk_token": "<unk>", "eos_token": "<eos>", "additional_special_tokens": ["<formula>"], "language": "en", "special_tokens_map_file": null, "tokenizer_file": null, "name_or_path": "transfo-xl-wt103", "tokenizer_class": "TransfoXLTokenizer"}