add tokenizer

tokenizer.json

@@ -19,21 +19,21 @@
   "added_tokens": [
     {
       "id": 50256,
-      "special": true,
       "content": "<|endoftext|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": true
+      "normalized": true,
+      "special": true
     },
     {
       "id": 50257,
-      "special": true,
       "content": "<|pad|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": null,

tokenizer_config.json

@@ -1 +1 @@
-{"unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "model_max_length": 1024, "return_special_tokens_mask": true, "special_tokens_map_file": null, "name_or_path": "microsoft/DialoGPT-medium", "errors": "replace", "tokenizer_class": "GPT2Tokenizer"}
+{"unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "model_max_length": 1024, "return_special_tokens_mask": true, "special_tokens_map_file": null, "name_or_path": "DingleyMaillotUrgell/homer-bot", "errors": "replace", "tokenizer_class": "GPT2Tokenizer"}