ku-nlp
/

deberta-v2-base-japanese

Inference Endpoints

Model card Files Files and versions Community

tealgreen0503 commited on May 21, 2023

Commit

74370b4

•

1 Parent(s): dbaae2b

update: tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +8 -2

tokenizer_config.json CHANGED Viewed

@@ -10,6 +10,12 @@
   "sp_model_kwargs": {},
   "special_tokens_map_file": null,
   "split_by_punct": false,
-  "tokenizer_class": "DebertaV2Tokenizer",
-  "unk_token": "[UNK]"
 }

   "sp_model_kwargs": {},
   "special_tokens_map_file": null,
   "split_by_punct": false,
+  "tokenizer_class": "DebertaV2JumanppTokenizer",
+  "unk_token": "[UNK]",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_deberta_v2_jumanpp.DebertaV2JumanppTokenizer",
+      "tokenization_deberta_v2_jumanpp_fast.DebertaV2JumanppTokenizerFast"
+    ]
+  }
 }