Xenova
/

dbrx-instruct-tokenizer

Transformers.js

Inference Endpoints

Model card Files Files and versions Community

Xenova HF staff commited on Mar 27

Commit

4ec0196

•

1 Parent(s): 39221ee

Upload tokenizer.json

Files changed (1) hide show

tokenizer.json +3 -2

tokenizer.json CHANGED Viewed

@@ -100373,8 +100373,9 @@
       "<|fim_middle|>": 100259,
       "<|fim_suffix|>": 100260,
       "<|endofprompt|>": 100276,
-      "<|im_start|>": 100264,
-      "<|im_end|>": 100265
     },
     "merges": [
       "Ġ Ġ",

       "<|fim_middle|>": 100259,
       "<|fim_suffix|>": 100260,
       "<|endofprompt|>": 100276,
+	  "<|pad|>": 100277,
+      "<|im_start|>": 100278,
+      "<|im_end|>": 100279
     },
     "merges": [
       "Ġ Ġ",