Upload tokenizer.json (#8)

- Upload tokenizer.json (258a8ee5ea0755ff34142ffece44d0ce9f7789c0)


Co-authored-by: Nicolas Patry <Narsil@users.noreply.huggingface.co>

tokenizer.json

@@ -45,7 +45,7 @@
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
-      "normalized": false,
+      "normalized": true,
       "special": true
     },
     {
@@ -71,6 +71,12 @@
           "String": " "
         },
         "content": "▁"
+      },
+      {
+          "type":"Replace",
+          "pattern":{
+              "Regex":"!\\[\\]([^\\]]*)"},
+              "content":"<image>"
       }
     ]
   },
@@ -93415,4 +93421,4 @@
       "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
     ]
   }
-}
+}