Hugging Face's logo

mfigurski80
/
narrative-relationship-tokenizer

Model card Files Community
mfigurski80 commited on
Commit
c692c80
•
1 Parent(s): c63950b

Upload tokenizer

Browse files
Files changed (5) hide show
  1. added_tokens.json +6 -0
  2. special_tokens_map.json +2 -2
  3. tokenizer.json +0 -0
  4. tokenizer_config.json +0 -6
  5. vocab.txt +0 -0
added_tokens.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "[/OB]": 30525,
3
+ "[/SU]": 30523,
4
+ "[OB]": 30524,
5
+ "[SU]": 30522
6
+ }
special_tokens_map.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "additional_special_tokens": [
3
  "[SU]",
4
- "[\\SU]",
5
  "[OB]",
6
- "[\\OB]"
7
  ],
8
  "cls_token": "[CLS]",
9
  "mask_token": "[MASK]",
 
1
  {
2
  "additional_special_tokens": [
3
  "[SU]",
4
+ "[/SU]",
5
  "[OB]",
6
+ "[/OB]"
7
  ],
8
  "cls_token": "[CLS]",
9
  "mask_token": "[MASK]",
tokenizer.json CHANGED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -1,10 +1,4 @@
1
  {
2
- "additional_special_tokens": [
3
- "[SU]",
4
- "[\\SU]",
5
- "[OB]",
6
- "[\\OB]"
7
- ],
8
  "cls_token": "[CLS]",
9
  "do_lower_case": true,
10
  "mask_token": "[MASK]",
 
1
  {
 
 
 
 
 
 
2
  "cls_token": "[CLS]",
3
  "do_lower_case": true,
4
  "mask_token": "[MASK]",
vocab.txt CHANGED
The diff for this file is too large to render. See raw diff