Xenova
/

esm2_t30_150M_UR50D

Transformers.js

Model card Files Files and versions Community

Xenova HF staff commited on Jan 4

Commit

9362a8b

•

1 Parent(s): 4b2cd83

Upload tokenizer.json

Files changed (1) hide show

tokenizer.json +18 -0

tokenizer.json CHANGED Viewed

@@ -328,17 +328,35 @@
       }
     ],
     "pair": [
       {
         "Sequence": {
           "id": "A",
           "type_id": 0
         }
       },
       {
         "Sequence": {
           "id": "B",
           "type_id": 1
         }
       }
     ],
     "special_tokens": {

       }
     ],
     "pair": [
+      {
+        "SpecialToken": {
+          "id": "<cls>",
+          "type_id": 0
+        }
+      },
       {
         "Sequence": {
           "id": "A",
           "type_id": 0
         }
       },
+      {
+        "SpecialToken": {
+          "id": "<eos>",
+          "type_id": 0
+        }
+      },
       {
         "Sequence": {
           "id": "B",
           "type_id": 1
         }
+      },
+      {
+        "SpecialToken": {
+          "id": "<eos>",
+          "type_id": 1
+        }
       }
     ],
     "special_tokens": {