sk2003 commited on
Commit
2a57fe3
1 Parent(s): 66c8af2

Upload processor

Browse files
Files changed (1) hide show
  1. tokenizer.json +10 -1
tokenizer.json CHANGED
@@ -1,7 +1,16 @@
1
  {
2
  "version": "1.0",
3
  "truncation": null,
4
- "padding": null,
 
 
 
 
 
 
 
 
 
5
  "added_tokens": [
6
  {
7
  "id": 0,
 
1
  {
2
  "version": "1.0",
3
  "truncation": null,
4
+ "padding": {
5
+ "strategy": {
6
+ "Fixed": 80
7
+ },
8
+ "direction": "Right",
9
+ "pad_to_multiple_of": null,
10
+ "pad_id": 1,
11
+ "pad_type_id": 0,
12
+ "pad_token": "<pad>"
13
+ },
14
  "added_tokens": [
15
  {
16
  "id": 0,