HoangHa
/

init-deberta

Feature Extraction

text-embeddings-inference

Model card Files Files and versions

HoangHa commited on Jun 5, 2024

Commit

283f83d

·

verified ·

1 Parent(s): 902a33c

Update tokenizer.json

Files changed (1) hide show

tokenizer.json +8 -1

tokenizer.json CHANGED Viewed

@@ -77,7 +77,14 @@
     }
   ],
   "normalizer": null,
-  "pre_tokenizer":null,
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [

     }
   ],
   "normalizer": null,
+  "pre_tokenizer":{
+        "type": "Split",
+        "pattern": {
+            "Regex": ".*?]"
+        },
+        "behavior": "Isolated",
+        "invert": false
+    },
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [