intervitens
commited on
Commit
•
76e349d
1
Parent(s):
d29a005
Update tokenizer_config.json
Browse files- tokenizer_config.json +2 -2
tokenizer_config.json
CHANGED
@@ -64,7 +64,7 @@
|
|
64 |
"normalized": false,
|
65 |
"rstrip": false,
|
66 |
"single_word": false,
|
67 |
-
"special":
|
68 |
},
|
69 |
"8": {
|
70 |
"content": "<|im_end|>",
|
@@ -72,7 +72,7 @@
|
|
72 |
"normalized": false,
|
73 |
"rstrip": false,
|
74 |
"single_word": false,
|
75 |
-
"special":
|
76 |
},
|
77 |
"9": {
|
78 |
"content": "<unused2>",
|
|
|
64 |
"normalized": false,
|
65 |
"rstrip": false,
|
66 |
"single_word": false,
|
67 |
+
"special": true
|
68 |
},
|
69 |
"8": {
|
70 |
"content": "<|im_end|>",
|
|
|
72 |
"normalized": false,
|
73 |
"rstrip": false,
|
74 |
"single_word": false,
|
75 |
+
"special": true
|
76 |
},
|
77 |
"9": {
|
78 |
"content": "<unused2>",
|