Panchovix commited on
Commit
56b8792
1 Parent(s): 72b9ee1

Upload 4 files

Browse files
Files changed (1) hide show
  1. tokenizer.json +12 -9
tokenizer.json CHANGED
@@ -9,7 +9,7 @@
9
  "single_word": false,
10
  "lstrip": false,
11
  "rstrip": false,
12
- "normalized": false,
13
  "special": true
14
  },
15
  {
@@ -27,7 +27,16 @@
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
30
- "normalized": true,
 
 
 
 
 
 
 
 
 
31
  "special": true
32
  }
33
  ],
@@ -77,16 +86,10 @@
77
  "type_id": 0
78
  }
79
  },
80
- {
81
- "SpecialToken": {
82
- "id": "<s>",
83
- "type_id": 1
84
- }
85
- },
86
  {
87
  "Sequence": {
88
  "id": "B",
89
- "type_id": 1
90
  }
91
  }
92
  ],
 
9
  "single_word": false,
10
  "lstrip": false,
11
  "rstrip": false,
12
+ "normalized": true,
13
  "special": true
14
  },
15
  {
 
27
  "single_word": false,
28
  "lstrip": false,
29
  "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
+ {
34
+ "id": 32000,
35
+ "content": "[PAD]",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
  "special": true
41
  }
42
  ],
 
86
  "type_id": 0
87
  }
88
  },
 
 
 
 
 
 
89
  {
90
  "Sequence": {
91
  "id": "B",
92
+ "type_id": 0
93
  }
94
  }
95
  ],