Nitral-AI commited on
Commit
fb50b12
·
verified ·
1 Parent(s): a3e46ca

Upload 4 files

Browse files
special_tokens_map.json CHANGED
@@ -1,13 +1,13 @@
1
  {
2
  "bos_token": {
3
- "content": "<s>",
4
  "lstrip": false,
5
  "normalized": false,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
  "eos_token": {
10
- "content": "</s>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
 
1
  {
2
  "bos_token": {
3
+ "content": "<|im_start|>",
4
  "lstrip": false,
5
  "normalized": false,
6
  "rstrip": false,
7
  "single_word": false
8
  },
9
  "eos_token": {
10
+ "content": "<|im_end|>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:b0240ce510f08e6c2041724e9043e33be9d251d1e4a4d94eb68cd47b954b61d2
3
- size 17078292
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:33e80d8e6833ba3c78edd582b14af07a8fa1ab545ea693e7d5013265b6525f2e
3
+ size 17078290
tokenizer_config.json CHANGED
@@ -116,7 +116,7 @@
116
  "special": true
117
  },
118
  "14": {
119
- "content": "<SPECIAL_14>",
120
  "lstrip": false,
121
  "normalized": false,
122
  "rstrip": false,
@@ -124,7 +124,7 @@
124
  "special": true
125
  },
126
  "15": {
127
- "content": "<SPECIAL_15>",
128
  "lstrip": false,
129
  "normalized": false,
130
  "rstrip": false,
 
116
  "special": true
117
  },
118
  "14": {
119
+ "content": "<|im_start|>",
120
  "lstrip": false,
121
  "normalized": false,
122
  "rstrip": false,
 
124
  "special": true
125
  },
126
  "15": {
127
+ "content": "<|im_end|>",
128
  "lstrip": false,
129
  "normalized": false,
130
  "rstrip": false,