Upload tokenizer.json

#10
by Narsil HF staff - opened
Files changed (1) hide show
  1. tokenizer.json +8 -2
tokenizer.json CHANGED
@@ -45,7 +45,7 @@
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
48
- "normalized": false,
49
  "special": true
50
  },
51
  {
@@ -71,6 +71,12 @@
71
  "String": " "
72
  },
73
  "content": "▁"
 
 
 
 
 
 
74
  }
75
  ]
76
  },
@@ -93415,4 +93421,4 @@
93415
  "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
93416
  ]
93417
  }
93418
- }
 
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
48
+ "normalized": true,
49
  "special": true
50
  },
51
  {
 
71
  "String": " "
72
  },
73
  "content": "▁"
74
+ },
75
+ {
76
+ "type":"Replace",
77
+ "pattern":{
78
+ "Regex":"!\\[\\]([^\\]]*)"},
79
+ "content":"<image>"
80
  }
81
  ]
82
  },
 
93421
  "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
93422
  ]
93423
  }
93424
+ }