Hugging Face's logo

dvruette
/
oasst-gpt-neox-20b-1000-steps

Text Generation
Transformers
PyTorch
gpt_neox
Inference Endpoints
text-generation-inference
Model card Files Community
1
dvruette commited on
Commit
36c70ed
•
1 Parent(s): 87dbd9f

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +3 -3
  2. tokenizer.json +5 -5
special_tokens_map.json CHANGED
@@ -1,10 +1,10 @@
1
  {
2
  "additional_special_tokens": [
3
- "<|system|>",
4
  "<|prompter|>",
 
 
5
  "<|prefix_end|>",
6
- "<|assistant|>",
7
- "<|prefix_begin|>"
8
  ],
9
  "bos_token": "<|endoftext|>",
10
  "eos_token": "<|endoftext|>",
 
1
  {
2
  "additional_special_tokens": [
 
3
  "<|prompter|>",
4
+ "<|prefix_begin|>",
5
+ "<|system|>",
6
  "<|prefix_end|>",
7
+ "<|assistant|>"
 
8
  ],
9
  "bos_token": "<|endoftext|>",
10
  "eos_token": "<|endoftext|>",
tokenizer.json CHANGED
@@ -230,7 +230,7 @@
230
  },
231
  {
232
  "id": 50277,
233
- "content": "<|system|>",
234
  "single_word": false,
235
  "lstrip": false,
236
  "rstrip": false,
@@ -239,7 +239,7 @@
239
  },
240
  {
241
  "id": 50278,
242
- "content": "<|prompter|>",
243
  "single_word": false,
244
  "lstrip": false,
245
  "rstrip": false,
@@ -248,7 +248,7 @@
248
  },
249
  {
250
  "id": 50279,
251
- "content": "<|prefix_end|>",
252
  "single_word": false,
253
  "lstrip": false,
254
  "rstrip": false,
@@ -257,7 +257,7 @@
257
  },
258
  {
259
  "id": 50280,
260
- "content": "<|assistant|>",
261
  "single_word": false,
262
  "lstrip": false,
263
  "rstrip": false,
@@ -266,7 +266,7 @@
266
  },
267
  {
268
  "id": 50281,
269
- "content": "<|prefix_begin|>",
270
  "single_word": false,
271
  "lstrip": false,
272
  "rstrip": false,
 
230
  },
231
  {
232
  "id": 50277,
233
+ "content": "<|prompter|>",
234
  "single_word": false,
235
  "lstrip": false,
236
  "rstrip": false,
 
239
  },
240
  {
241
  "id": 50278,
242
+ "content": "<|prefix_begin|>",
243
  "single_word": false,
244
  "lstrip": false,
245
  "rstrip": false,
 
248
  },
249
  {
250
  "id": 50279,
251
+ "content": "<|system|>",
252
  "single_word": false,
253
  "lstrip": false,
254
  "rstrip": false,
 
257
  },
258
  {
259
  "id": 50280,
260
+ "content": "<|prefix_end|>",
261
  "single_word": false,
262
  "lstrip": false,
263
  "rstrip": false,
 
266
  },
267
  {
268
  "id": 50281,
269
+ "content": "<|assistant|>",
270
  "single_word": false,
271
  "lstrip": false,
272
  "rstrip": false,