qcpg-sentences / tokenizer_config.json
Elron's picture
add tokenizer
d4deefe
raw
history blame
1.69 kB
{"eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>", "extra_ids": 0, "additional_special_tokens": ["COND_LEXICAL_DIV_0", "COND_LEXICAL_DIV_10", "COND_LEXICAL_DIV_100", "COND_LEXICAL_DIV_15", "COND_LEXICAL_DIV_20", "COND_LEXICAL_DIV_25", "COND_LEXICAL_DIV_30", "COND_LEXICAL_DIV_35", "COND_LEXICAL_DIV_40", "COND_LEXICAL_DIV_45", "COND_LEXICAL_DIV_5", "COND_LEXICAL_DIV_50", "COND_LEXICAL_DIV_55", "COND_LEXICAL_DIV_60", "COND_LEXICAL_DIV_65", "COND_LEXICAL_DIV_70", "COND_LEXICAL_DIV_75", "COND_LEXICAL_DIV_80", "COND_LEXICAL_DIV_85", "COND_LEXICAL_DIV_90", "COND_LEXICAL_DIV_95", "COND_SEMANTIC_SIM_0", "COND_SEMANTIC_SIM_10", "COND_SEMANTIC_SIM_15", "COND_SEMANTIC_SIM_20", "COND_SEMANTIC_SIM_25", "COND_SEMANTIC_SIM_30", "COND_SEMANTIC_SIM_35", "COND_SEMANTIC_SIM_40", "COND_SEMANTIC_SIM_45", "COND_SEMANTIC_SIM_5", "COND_SEMANTIC_SIM_50", "COND_SEMANTIC_SIM_55", "COND_SEMANTIC_SIM_60", "COND_SEMANTIC_SIM_65", "COND_SEMANTIC_SIM_70", "COND_SEMANTIC_SIM_75", "COND_SEMANTIC_SIM_80", "COND_SEMANTIC_SIM_85", "COND_SEMANTIC_SIM_90", "COND_SEMANTIC_SIM_95", "COND_SYNTACTIC_DIV_0", "COND_SYNTACTIC_DIV_10", "COND_SYNTACTIC_DIV_15", "COND_SYNTACTIC_DIV_20", "COND_SYNTACTIC_DIV_25", "COND_SYNTACTIC_DIV_30", "COND_SYNTACTIC_DIV_35", "COND_SYNTACTIC_DIV_40", "COND_SYNTACTIC_DIV_45", "COND_SYNTACTIC_DIV_5", "COND_SYNTACTIC_DIV_50", "COND_SYNTACTIC_DIV_55", "COND_SYNTACTIC_DIV_60", "COND_SYNTACTIC_DIV_65", "COND_SYNTACTIC_DIV_70", "COND_SYNTACTIC_DIV_75", "COND_SYNTACTIC_DIV_80"], "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "/dccstor/tslm-gen/experiments/cond-bleurt/outputs/t5-base-cond-parabk2-bleurt-lr5e-4-v1", "tokenizer_class": "T5Tokenizer"}