Leyo commited on
Commit
4b5c078
β€’
1 Parent(s): d93a6e8

revert tokenizer update (#9)

Browse files

- Revert "Upload tokenizer.json (#8)" (ddb9e0429104eb0d8f69110e88c8b036eeaa3f7b)

Files changed (1) hide show
  1. tokenizer.json +2 -8
tokenizer.json CHANGED
@@ -45,7 +45,7 @@
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
48
- "normalized": true,
49
  "special": true
50
  },
51
  {
@@ -71,12 +71,6 @@
71
  "String": " "
72
  },
73
  "content": "▁"
74
- },
75
- {
76
- "type":"Replace",
77
- "pattern":{
78
- "Regex":"!\\[\\]([^\\]]*)"},
79
- "content":"<image>"
80
  }
81
  ]
82
  },
@@ -93421,4 +93415,4 @@
93421
  "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
93422
  ]
93423
  }
93424
- }
 
45
  "single_word": false,
46
  "lstrip": false,
47
  "rstrip": false,
48
+ "normalized": false,
49
  "special": true
50
  },
51
  {
 
71
  "String": " "
72
  },
73
  "content": "▁"
 
 
 
 
 
 
74
  }
75
  ]
76
  },
 
93415
  "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
93416
  ]
93417
  }
93418
+ }