add tokenizer

added_tokens.json

@@ -0,0 +1 @@
+{"<|belief|>": 50257, "<|endofaction|>": 50260, "<|system|>": 50266, "<|endofbelief|>": 50258, "<|task|>": 50267, "<|endofchitchat|>": 50270, "<|response|>": 50261, "<|chitchat|>": 50269, "<|user|>": 50265, "<|action|>": 50259, "<|endoftask|>": 50268, "<|context|>": 50263, "<|endofcontext|>": 50264, "<|endofresponse|>": 50262}

special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>", "additional_special_tokens": ["<|belief|>", "<|endofbelief|>", "<|action|>", "<|endofaction|>", "<|response|>", "<|endofresponse|>", "<|context|>", "<|endofcontext|>", "<|user|>", "<|system|>", "<|task|>", "<|endoftask|>", "<|chitchat|>", "<|endofchitchat|>"]}

tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "gpt2-TOD/", "tokenizer_class": "GPT2Tokenizer"}