Xenova
/

xlm-clm-ende-1024

Transformers.js

Model card Files Files and versions Community

Xenova HF staff commited on Sep 7, 2023

Commit

4b5063b

•

1 Parent(s): 8665cc3

Upload tokenizer.json

Files changed (1) hide show

tokenizer.json +11 -1

tokenizer.json CHANGED Viewed

@@ -140,7 +140,17 @@
     "pre_tokenizer": {
         "type": "BertPreTokenizer"
     },
-    "post_processor": null,
     "decoder": {
         "type": "BPEDecoder",
         "suffix": "</w>"

     "pre_tokenizer": {
         "type": "BertPreTokenizer"
     },
+    "post_processor": {
+        "type": "BertProcessing",
+        "sep": [
+            "</s>",
+            2
+        ],
+        "cls": [
+            "<s>",
+            0
+        ]
+    },
     "decoder": {
         "type": "BPEDecoder",
         "suffix": "</w>"