Titovs commited on
Commit
32a11ac
β€’
1 Parent(s): 0aa3937

Upload tokenizer

Browse files
Files changed (2) hide show
  1. special_tokens_map.json +4 -0
  2. tokenizer_config.json +4 -0
special_tokens_map.json CHANGED
@@ -1,5 +1,9 @@
1
  {
2
  "additional_special_tokens": [
 
 
 
 
3
  "▁<PRE>",
4
  "▁<MID>",
5
  "▁<SUF>",
 
1
  {
2
  "additional_special_tokens": [
3
+ "▁<PRE>",
4
+ "▁<MID>",
5
+ "▁<SUF>",
6
+ "▁<EOT>",
7
  "▁<PRE>",
8
  "▁<MID>",
9
  "▁<SUF>",
tokenizer_config.json CHANGED
@@ -60,6 +60,10 @@
60
  }
61
  },
62
  "additional_special_tokens": [
 
 
 
 
63
  "▁<PRE>",
64
  "▁<MID>",
65
  "▁<SUF>",
 
60
  }
61
  },
62
  "additional_special_tokens": [
63
+ "▁<PRE>",
64
+ "▁<MID>",
65
+ "▁<SUF>",
66
+ "▁<EOT>",
67
  "▁<PRE>",
68
  "▁<MID>",
69
  "▁<SUF>",