HPLT
/

hplt_bert_base_2_0_eus-Latn

Model card Files Files and versions

MariaFjodorowa commited on Feb 27

Commit

4687ac9

·

verified ·

1 Parent(s): 410bad4

Fix tokenizer decoders

Files changed (1) hide show

tokenizer.json +18 -10

tokenizer.json CHANGED Viewed

@@ -1103,16 +1103,11 @@
         "use_regex": true
       },
       {
-        "type": "Metaspace",
-        "replacement": "▁",
-        "prepend_scheme": "always",
-        "split": true
-      },
-      {
-        "type": "Strip",
-        "content": " ",
-        "start": 1,
-        "stop": 0
       },
       {
         "type": "Replace",
@@ -1127,6 +1122,19 @@
           "Regex": "█"
         },
         "content": "\n"
       }
     ]
   },

         "use_regex": true
       },
       {
+        "type": "Replace",
+        "pattern": {
+          "String": "âĸģ"
+        },
+        "content": " "
       },
       {
         "type": "Replace",
           "Regex": "█"
         },
         "content": "\n"
+      },
+      {
+        "type": "Replace",
+        "pattern": {
+          "String": "▁"
+        },
+        "content": " "
+      },
+      {
+        "type": "Strip",
+        "content": " ",
+        "start": 1,
+        "stop": 0
       }
     ]
   },