anuragsingh28 commited on
Commit
f4b6143
1 Parent(s): 8108334

Upload processor

Browse files
Files changed (2) hide show
  1. added_tokens.json +8 -1
  2. tokenizer.json +63 -0
added_tokens.json CHANGED
@@ -1,5 +1,12 @@
1
  {
 
 
 
 
 
2
  "<s_iitcdip>": 57523,
 
3
  "<s_synthdog>": 57524,
4
- "<sep/>": 57522
 
5
  }
 
1
  {
2
+ "</s_answer>": 57530,
3
+ "</s_question>": 57528,
4
+ "<no/>": 57526,
5
+ "<s_answer>": 57529,
6
+ "<s_docvqa>": 57531,
7
  "<s_iitcdip>": 57523,
8
+ "<s_question>": 57527,
9
  "<s_synthdog>": 57524,
10
+ "<sep/>": 57522,
11
+ "<yes/>": 57525
12
  }
tokenizer.json CHANGED
@@ -88,6 +88,69 @@
88
  "rstrip": false,
89
  "normalized": false,
90
  "special": true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
91
  }
92
  ],
93
  "normalizer": {
 
88
  "rstrip": false,
89
  "normalized": false,
90
  "special": true
91
+ },
92
+ {
93
+ "id": 57525,
94
+ "content": "<yes/>",
95
+ "single_word": false,
96
+ "lstrip": false,
97
+ "rstrip": false,
98
+ "normalized": true,
99
+ "special": false
100
+ },
101
+ {
102
+ "id": 57526,
103
+ "content": "<no/>",
104
+ "single_word": false,
105
+ "lstrip": false,
106
+ "rstrip": false,
107
+ "normalized": true,
108
+ "special": false
109
+ },
110
+ {
111
+ "id": 57527,
112
+ "content": "<s_question>",
113
+ "single_word": false,
114
+ "lstrip": false,
115
+ "rstrip": false,
116
+ "normalized": true,
117
+ "special": false
118
+ },
119
+ {
120
+ "id": 57528,
121
+ "content": "</s_question>",
122
+ "single_word": false,
123
+ "lstrip": false,
124
+ "rstrip": false,
125
+ "normalized": true,
126
+ "special": false
127
+ },
128
+ {
129
+ "id": 57529,
130
+ "content": "<s_answer>",
131
+ "single_word": false,
132
+ "lstrip": false,
133
+ "rstrip": false,
134
+ "normalized": true,
135
+ "special": false
136
+ },
137
+ {
138
+ "id": 57530,
139
+ "content": "</s_answer>",
140
+ "single_word": false,
141
+ "lstrip": false,
142
+ "rstrip": false,
143
+ "normalized": true,
144
+ "special": false
145
+ },
146
+ {
147
+ "id": 57531,
148
+ "content": "<s_docvqa>",
149
+ "single_word": false,
150
+ "lstrip": false,
151
+ "rstrip": false,
152
+ "normalized": true,
153
+ "special": false
154
  }
155
  ],
156
  "normalizer": {