tom-brady commited on
Commit
0bd5b68
1 Parent(s): d7eea67

Upload special_tokens_map.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +3 -3
special_tokens_map.json CHANGED
@@ -35,14 +35,14 @@
35
  "<|extra0|>"
36
  ],
37
  "bos_token": {
38
- "content": "<s>",
39
  "lstrip": false,
40
  "normalized": false,
41
  "rstrip": false,
42
  "single_word": false
43
  },
44
  "eos_token": {
45
- "content": "</s>",
46
  "lstrip": false,
47
  "normalized": false,
48
  "rstrip": false,
@@ -56,7 +56,7 @@
56
  "single_word": false
57
  },
58
  "unk_token": {
59
- "content": "<unk>",
60
  "lstrip": false,
61
  "normalized": false,
62
  "rstrip": false,
 
35
  "<|extra0|>"
36
  ],
37
  "bos_token": {
38
+ "content": "<|endoftext|>",
39
  "lstrip": false,
40
  "normalized": false,
41
  "rstrip": false,
42
  "single_word": false
43
  },
44
  "eos_token": {
45
+ "content": "<|endoftext|>",
46
  "lstrip": false,
47
  "normalized": false,
48
  "rstrip": false,
 
56
  "single_word": false
57
  },
58
  "unk_token": {
59
+ "content": "<|endoftext|>",
60
  "lstrip": false,
61
  "normalized": false,
62
  "rstrip": false,