deepparag commited on
Commit
247e86d
1 Parent(s): 9f40232

Upload tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +3 -1
tokenizer_config.json CHANGED
@@ -1,4 +1,5 @@
1
  {
 
2
  "add_prefix_space": false,
3
  "bos_token": {
4
  "__type": "AddedToken",
@@ -18,7 +19,8 @@
18
  },
19
  "errors": "replace",
20
  "model_max_length": 1024,
21
- "name_or_path": "./output-small",
 
22
  "special_tokens_map_file": null,
23
  "tokenizer_class": "GPT2Tokenizer",
24
  "unk_token": {
 
1
  {
2
+ "add_bos_token": false,
3
  "add_prefix_space": false,
4
  "bos_token": {
5
  "__type": "AddedToken",
 
19
  },
20
  "errors": "replace",
21
  "model_max_length": 1024,
22
+ "name_or_path": "microsoft/DialoGPT-small",
23
+ "pad_token": null,
24
  "special_tokens_map_file": null,
25
  "tokenizer_class": "GPT2Tokenizer",
26
  "unk_token": {