Commit
•
3a4dbce
1
Parent(s):
fb84573
Update tokenizer_config.json
Browse files- tokenizer_config.json +2 -2
tokenizer_config.json
CHANGED
@@ -25,7 +25,7 @@
|
|
25 |
"normalized": false,
|
26 |
"rstrip": true,
|
27 |
"single_word": false,
|
28 |
-
"special":
|
29 |
},
|
30 |
"32000": {
|
31 |
"content": "<|endoftext|>",
|
@@ -121,7 +121,7 @@
|
|
121 |
"normalized": false,
|
122 |
"rstrip": false,
|
123 |
"single_word": false,
|
124 |
-
"special":
|
125 |
},
|
126 |
"32012": {
|
127 |
"content": "<|end_of_text|>",
|
|
|
25 |
"normalized": false,
|
26 |
"rstrip": true,
|
27 |
"single_word": false,
|
28 |
+
"special": true
|
29 |
},
|
30 |
"32000": {
|
31 |
"content": "<|endoftext|>",
|
|
|
121 |
"normalized": false,
|
122 |
"rstrip": false,
|
123 |
"single_word": false,
|
124 |
+
"special": true
|
125 |
},
|
126 |
"32012": {
|
127 |
"content": "<|end_of_text|>",
|