Hugging Face's logo

KingLLM
/
updated_tokenizer

Transformers
Model card Files
xet
 Community
KingLLM commited on
Commit
c91eefe
·
verified ·
1 Parent(s): f953384

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +12 -11
  2. tokenizer.json +2 -2
  3. tokenizer_config.json +19 -11
added_tokens.json CHANGED
@@ -1,22 +1,23 @@
1
  {
2
- "Bioderma": 250013,
3
  "Cerave": 250002,
4
- "Drunk Elephant": 250012,
5
- "EucerinSPF 50": 250018,
6
- "Flipkart": 250019,
7
- "Glycolic Acid": 250015,
8
  "Hydrating Cleanser": 250003,
9
  "Hydro Boost Water Gel": 250006,
10
- "L'Oréal": 250017,
11
  "La Roche-Posay": 250004,
12
- "Myntra": 250021,
13
  "Neutrogena": 250005,
14
  "Niacinamide": 250007,
15
- "Nykka": 250020,
16
  "Retinol": 250008,
17
- "SPF 50": 250010,
18
- "Tatcha": 250011,
19
  "Vitamin C": 250009,
20
  "Walmart": 250016,
21
- "squalane": 250014
 
22
  }
 
1
  {
2
+ "Bioderma": 250014,
3
  "Cerave": 250002,
4
+ "Drunk Elephant": 250013,
5
+ "Eucerin": 250021,
6
+ "EucerinSPF 50": 250011,
7
+ "Flipkart": 250017,
8
  "Hydrating Cleanser": 250003,
9
  "Hydro Boost Water Gel": 250006,
10
+ "L'Oréal": 250010,
11
  "La Roche-Posay": 250004,
12
+ "Myntra": 250019,
13
  "Neutrogena": 250005,
14
  "Niacinamide": 250007,
15
+ "Nykka": 250018,
16
  "Retinol": 250008,
17
+ "SPF 50": 250022,
18
+ "Tatcha": 250012,
19
  "Vitamin C": 250009,
20
  "Walmart": 250016,
21
+ "amazon": 250020,
22
+ "squalane": 250015
23
  }
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:4afb9101502f81996faa15f11d1bf488f147a42dddb54f09d6c78ec9db124902
3
- size 17086920
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6e713901bac6b865d60cb5e0af6742c24d89c104c5edfcfd3fd8ade1787c1d0c
3
+ size 17086930
tokenizer_config.json CHANGED
@@ -113,7 +113,7 @@
113
  "special": false
114
  },
115
  "250010": {
116
- "content": "SPF 50",
117
  "lstrip": false,
118
  "normalized": true,
119
  "rstrip": false,
@@ -121,7 +121,7 @@
121
  "special": false
122
  },
123
  "250011": {
124
- "content": "Tatcha",
125
  "lstrip": false,
126
  "normalized": true,
127
  "rstrip": false,
@@ -129,7 +129,7 @@
129
  "special": false
130
  },
131
  "250012": {
132
- "content": "Drunk Elephant",
133
  "lstrip": false,
134
  "normalized": true,
135
  "rstrip": false,
@@ -137,7 +137,7 @@
137
  "special": false
138
  },
139
  "250013": {
140
- "content": "Bioderma",
141
  "lstrip": false,
142
  "normalized": true,
143
  "rstrip": false,
@@ -145,7 +145,7 @@
145
  "special": false
146
  },
147
  "250014": {
148
- "content": "squalane",
149
  "lstrip": false,
150
  "normalized": true,
151
  "rstrip": false,
@@ -153,7 +153,7 @@
153
  "special": false
154
  },
155
  "250015": {
156
- "content": "Glycolic Acid",
157
  "lstrip": false,
158
  "normalized": true,
159
  "rstrip": false,
@@ -169,7 +169,7 @@
169
  "special": false
170
  },
171
  "250017": {
172
- "content": "L'Oréal",
173
  "lstrip": false,
174
  "normalized": true,
175
  "rstrip": false,
@@ -177,7 +177,7 @@
177
  "special": false
178
  },
179
  "250018": {
180
- "content": "EucerinSPF 50",
181
  "lstrip": false,
182
  "normalized": true,
183
  "rstrip": false,
@@ -185,7 +185,7 @@
185
  "special": false
186
  },
187
  "250019": {
188
- "content": "Flipkart",
189
  "lstrip": false,
190
  "normalized": true,
191
  "rstrip": false,
@@ -193,7 +193,7 @@
193
  "special": false
194
  },
195
  "250020": {
196
- "content": "Nykka",
197
  "lstrip": false,
198
  "normalized": true,
199
  "rstrip": false,
@@ -201,7 +201,15 @@
201
  "special": false
202
  },
203
  "250021": {
204
- "content": "Myntra",
 
 
 
 
 
 
 
 
205
  "lstrip": false,
206
  "normalized": true,
207
  "rstrip": false,
 
113
  "special": false
114
  },
115
  "250010": {
116
+ "content": "L'Oréal",
117
  "lstrip": false,
118
  "normalized": true,
119
  "rstrip": false,
 
121
  "special": false
122
  },
123
  "250011": {
124
+ "content": "EucerinSPF 50",
125
  "lstrip": false,
126
  "normalized": true,
127
  "rstrip": false,
 
129
  "special": false
130
  },
131
  "250012": {
132
+ "content": "Tatcha",
133
  "lstrip": false,
134
  "normalized": true,
135
  "rstrip": false,
 
137
  "special": false
138
  },
139
  "250013": {
140
+ "content": "Drunk Elephant",
141
  "lstrip": false,
142
  "normalized": true,
143
  "rstrip": false,
 
145
  "special": false
146
  },
147
  "250014": {
148
+ "content": "Bioderma",
149
  "lstrip": false,
150
  "normalized": true,
151
  "rstrip": false,
 
153
  "special": false
154
  },
155
  "250015": {
156
+ "content": "squalane",
157
  "lstrip": false,
158
  "normalized": true,
159
  "rstrip": false,
 
169
  "special": false
170
  },
171
  "250017": {
172
+ "content": "Flipkart",
173
  "lstrip": false,
174
  "normalized": true,
175
  "rstrip": false,
 
177
  "special": false
178
  },
179
  "250018": {
180
+ "content": "Nykka",
181
  "lstrip": false,
182
  "normalized": true,
183
  "rstrip": false,
 
185
  "special": false
186
  },
187
  "250019": {
188
+ "content": "Myntra",
189
  "lstrip": false,
190
  "normalized": true,
191
  "rstrip": false,
 
193
  "special": false
194
  },
195
  "250020": {
196
+ "content": "amazon",
197
  "lstrip": false,
198
  "normalized": true,
199
  "rstrip": false,
 
201
  "special": false
202
  },
203
  "250021": {
204
+ "content": "Eucerin",
205
+ "lstrip": false,
206
+ "normalized": true,
207
+ "rstrip": false,
208
+ "single_word": false,
209
+ "special": false
210
+ },
211
+ "250022": {
212
+ "content": "SPF 50",
213
  "lstrip": false,
214
  "normalized": true,
215
  "rstrip": false,