Upload tokenizer

README.md

@@ -1,8 +1,8 @@
 ---
 license: apache-2.0
-base_model: facebook/bart-large
 tags:
 - generated_from_trainer
+base_model: facebook/bart-large
 model-index:
 - name: bart-finetuned-lyrlen-128-tokens
   results: []

added_tokens.json

@@ -1,5 +1,4 @@
 {
-  "<P>": 50285,
   "len_1": 50265,
   "len_10": 50274,
   "len_11": 50275,
@@ -13,6 +12,10 @@
   "len_19": 50283,
   "len_2": 50266,
   "len_20": 50284,
+  "len_21": 50285,
+  "len_22": 50286,
+  "len_23": 50287,
+  "len_24": 50288,
   "len_3": 50267,
   "len_4": 50268,
   "len_5": 50269,

special_tokens_map.json

@@ -13,7 +13,13 @@
     "rstrip": false,
     "single_word": false
   },
-  "eos_token": "<P>",
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
   "mask_token": {
     "content": "<mask>",
     "lstrip": true,
@@ -21,7 +27,7 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "<P>",
+  "pad_token": "</s>",
   "sep_token": {
     "content": "</s>",
     "lstrip": false,

tokenizer_config.json

@@ -202,22 +202,46 @@
       "special": false
     },
     "50285": {
-      "content": "<P>",
+      "content": "len_21",
       "lstrip": false,
-      "normalized": false,
+      "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
+    },
+    "50286": {
+      "content": "len_22",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50287": {
+      "content": "len_23",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50288": {
+      "content": "len_24",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
     }
   },
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": true,
   "cls_token": "<s>",
-  "eos_token": "<P>",
+  "eos_token": "</s>",
   "errors": "replace",
   "mask_token": "<mask>",
   "model_max_length": 1024,
-  "pad_token": "<P>",
+  "pad_token": "</s>",
   "sep_token": "</s>",
   "tokenizer_class": "BartTokenizer",
   "unk_token": "<unk>"