Nitral-AI commited on
Commit
47bfa34
·
verified ·
1 Parent(s): fb50b12

Upload 4 files

Browse files
special_tokens_map.json CHANGED
@@ -1,6 +1,6 @@
1
  {
2
  "bos_token": {
3
- "content": "<|im_start|>",
4
  "lstrip": false,
5
  "normalized": false,
6
  "rstrip": false,
 
1
  {
2
  "bos_token": {
3
+ "content": "<s>",
4
  "lstrip": false,
5
  "normalized": false,
6
  "rstrip": false,
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:33e80d8e6833ba3c78edd582b14af07a8fa1ab545ea693e7d5013265b6525f2e
3
  size 17078290
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c45fe029ef085dcd5bdde99c84a04ff6279fcdca9be95f16802c0dbe64fe8363
3
  size 17078290
tokenizer_config.json CHANGED
@@ -116,7 +116,7 @@
116
  "special": true
117
  },
118
  "14": {
119
- "content": "<|im_start|>",
120
  "lstrip": false,
121
  "normalized": false,
122
  "rstrip": false,
 
116
  "special": true
117
  },
118
  "14": {
119
+ "content": "<SPECIAL_14>",
120
  "lstrip": false,
121
  "normalized": false,
122
  "rstrip": false,