Hugging Face's logo

webpolis
/
zenos-gpt-j-6B-instruct-4bit

Text Generation
Transformers
PyTorch
Safetensors
gptj
Inference Endpoints
4-bit precision
bitsandbytes
Model card Files Community
Nicolas Iglesias commited on
Commit
9fff98c
•
1 Parent(s): 642402e

Upload tokenizer

Browse files
Files changed (3) hide show
  1. added_tokens.json +1 -0
  2. special_tokens_map.json +3 -0
  3. tokenizer_config.json +3 -1
added_tokens.json CHANGED
@@ -1,4 +1,5 @@
1
  {
 
2
  "<|vulgarg|>": 50257,
3
  "[/INST]": 50259,
4
  "[INST]": 50258
 
1
  {
2
+ "<|endoftext|>": 50256,
3
  "<|vulgarg|>": 50257,
4
  "[/INST]": 50259,
5
  "[INST]": 50258
special_tokens_map.json CHANGED
@@ -1,4 +1,7 @@
1
  {
 
 
 
2
  "bos_token": "<|endoftext|>",
3
  "eos_token": "<|endoftext|>",
4
  "unk_token": "<|endoftext|>"
 
1
  {
2
+ "additional_special_tokens": [
3
+ "<|endoftext|>"
4
+ ],
5
  "bos_token": "<|endoftext|>",
6
  "eos_token": "<|endoftext|>",
7
  "unk_token": "<|endoftext|>"
tokenizer_config.json CHANGED
@@ -34,7 +34,9 @@
34
  "special": false
35
  }
36
  },
37
- "additional_special_tokens": [],
 
 
38
  "bos_token": "<|endoftext|>",
39
  "clean_up_tokenization_spaces": true,
40
  "eos_token": "<|endoftext|>",
 
34
  "special": false
35
  }
36
  },
37
+ "additional_special_tokens": [
38
+ "<|endoftext|>"
39
+ ],
40
  "bos_token": "<|endoftext|>",
41
  "clean_up_tokenization_spaces": true,
42
  "eos_token": "<|endoftext|>",