Hugging Face's logo

HuggingFaceM4
/
tiny-random-idefics

Transformers
PyTorch
idefics
pretraining
text-generation-inference
Inference Endpoints
Model card Files Community
5
Leyo commited on
Commit
cb18d77
•
1 Parent(s): 99f3a09

Upload tokenizer

Browse files
Files changed (3) hide show
  1. special_tokens_map.json +2 -14
  2. tokenizer.json +1 -1
  3. tokenizer_config.json +2 -4
special_tokens_map.json CHANGED
@@ -1,19 +1,7 @@
1
  {
2
  "additional_special_tokens": [
3
- {
4
- "content": "<fake_token_around_image>",
5
- "lstrip": false,
6
- "normalized": false,
7
- "rstrip": false,
8
- "single_word": false
9
- },
10
- {
11
- "content": "<image>",
12
- "lstrip": false,
13
- "normalized": false,
14
- "rstrip": false,
15
- "single_word": false
16
- }
17
  ],
18
  "bos_token": {
19
  "content": "<s>",
 
1
  {
2
  "additional_special_tokens": [
3
+ "<image>",
4
+ "<fake_token_around_image>"
 
 
 
 
 
 
 
 
 
 
 
 
5
  ],
6
  "bos_token": {
7
  "content": "<s>",
tokenizer.json CHANGED
@@ -31506,7 +31506,7 @@
31506
  "अ": 31350,
31507
  "╔": 31351,
31508
  "无": 31352,
31509
- "
": 31353,
31510
  "은": 31354,
31511
  "ʷ": 31355,
31512
  "那": 31356,
 
31506
  "अ": 31350,
31507
  "╔": 31351,
31508
  "无": 31352,
31509
+ "
": 31353,
31510
  "은": 31354,
31511
  "ʷ": 31355,
31512
  "那": 31356,
tokenizer_config.json CHANGED
@@ -1,6 +1,4 @@
1
  {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
  "added_tokens_decoder": {
5
  "0": {
6
  "content": "<unk>",
@@ -44,8 +42,8 @@
44
  }
45
  },
46
  "additional_special_tokens": [
47
- "<fake_token_around_image>",
48
- "<image>"
49
  ],
50
  "bos_token": "<s>",
51
  "clean_up_tokenization_spaces": false,
 
1
  {
 
 
2
  "added_tokens_decoder": {
3
  "0": {
4
  "content": "<unk>",
 
42
  }
43
  },
44
  "additional_special_tokens": [
45
+ "<image>",
46
+ "<fake_token_around_image>"
47
  ],
48
  "bos_token": "<s>",
49
  "clean_up_tokenization_spaces": false,