bibidentuhanoi commited on
Commit
b31e2f0
1 Parent(s): 9249694

Upload tokenizer

Browse files
special_tokens_map.json CHANGED
@@ -7,14 +7,14 @@
7
  "single_word": false
8
  },
9
  "eos_token": {
10
- "content": "<|im_end|>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
  "pad_token": {
17
- "content": "<|im_end|>",
18
  "lstrip": false,
19
  "normalized": false,
20
  "rstrip": false,
 
7
  "single_word": false
8
  },
9
  "eos_token": {
10
+ "content": "</s>",
11
  "lstrip": false,
12
  "normalized": false,
13
  "rstrip": false,
14
  "single_word": false
15
  },
16
  "pad_token": {
17
+ "content": "<unk>",
18
  "lstrip": false,
19
  "normalized": false,
20
  "rstrip": false,
tokenizer.json CHANGED
@@ -23,21 +23,12 @@
23
  },
24
  {
25
  "id": 2,
26
- "content": "<|im_end|>",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
32
- },
33
- {
34
- "id": 32000,
35
- "content": "<|im_start|>",
36
- "single_word": false,
37
- "lstrip": false,
38
- "rstrip": false,
39
- "normalized": false,
40
- "special": false
41
  }
42
  ],
43
  "normalizer": {
@@ -146,7 +137,7 @@
146
  "vocab": {
147
  "<unk>": 0,
148
  "<s>": 1,
149
- "<|im_end|>": 2,
150
  "<0x00>": 3,
151
  "<0x01>": 4,
152
  "<0x02>": 5,
 
23
  },
24
  {
25
  "id": 2,
26
+ "content": "</s>",
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
30
  "normalized": false,
31
  "special": true
 
 
 
 
 
 
 
 
 
32
  }
33
  ],
34
  "normalizer": {
 
137
  "vocab": {
138
  "<unk>": 0,
139
  "<s>": 1,
140
+ "</s>": 2,
141
  "<0x00>": 3,
142
  "<0x01>": 4,
143
  "<0x02>": 5,
tokenizer_config.json CHANGED
The diff for this file is too large to render. See raw diff