Xenova HF staff multimodalart HF staff commited on
Commit
1a41918
1 Parent(s): 1c6aa5f

add im_start and im_end (#2)

Browse files

- add im_start and im_end (0518759c65f09e0dba2afab61b11e695c0bddf65)


Co-authored-by: Apolinário from multimodal AI art <multimodalart@users.noreply.huggingface.co>

Files changed (2) hide show
  1. tokenizer.json +21 -1
  2. vocab.json +3 -1
tokenizer.json CHANGED
@@ -47,6 +47,24 @@
47
  "rstrip": false,
48
  "normalized": false,
49
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
  }
51
  ],
52
  "normalizer": null,
@@ -100345,7 +100363,9 @@
100345
  "<|fim_prefix|>": 100258,
100346
  "<|fim_middle|>": 100259,
100347
  "<|fim_suffix|>": 100260,
100348
- "<|endofprompt|>": 100276
 
 
100349
  },
100350
  "merges": [
100351
  "Ġ Ġ",
 
47
  "rstrip": false,
48
  "normalized": false,
49
  "special": true
50
+ },
51
+ {
52
+ "id": 100264,
53
+ "content": "<|im_start|>",
54
+ "single_word": false,
55
+ "lstrip": false,
56
+ "rstrip": false,
57
+ "normalized": false,
58
+ "special": true
59
+ },
60
+ {
61
+ "id": 100265,
62
+ "content": "<|im_end|>",
63
+ "single_word": false,
64
+ "lstrip": false,
65
+ "rstrip": false,
66
+ "normalized": false,
67
+ "special": true
68
  }
69
  ],
70
  "normalizer": null,
 
100363
  "<|fim_prefix|>": 100258,
100364
  "<|fim_middle|>": 100259,
100365
  "<|fim_suffix|>": 100260,
100366
+ "<|endofprompt|>": 100276,
100367
+ "<|im_start|>": 100264,
100368
+ "<|im_end|>": 100265
100369
  },
100370
  "merges": [
100371
  "Ġ Ġ",
vocab.json CHANGED
@@ -100259,5 +100259,7 @@
100259
  "<|fim_prefix|>": 100258,
100260
  "<|fim_middle|>": 100259,
100261
  "<|fim_suffix|>": 100260,
100262
- "<|endofprompt|>": 100276
 
 
100263
  }
 
100259
  "<|fim_prefix|>": 100258,
100260
  "<|fim_middle|>": 100259,
100261
  "<|fim_suffix|>": 100260,
100262
+ "<|endofprompt|>": 100276,
100263
+ "<|im_start|>": 100264,
100264
+ "<|im_end|>": 100265
100265
  }