add-custom-tokenizer

README.md

@@ -14,7 +14,7 @@ metrics:
   - accuracy
 mask_token: "[MASK]"
 widget:
-    - text: "京都 大学 で 自然 言語 処理 を [MASK] する 。"
+    - text: "京都大学で自然言語処理を[MASK]する。"
 ---
 
 # Model Card for Japanese DeBERTa V2 base
@@ -29,10 +29,10 @@ You can use this model for masked language modeling as follows:
 
 ```python
 from transformers import AutoTokenizer, AutoModelForMaskedLM
-tokenizer = AutoTokenizer.from_pretrained('ku-nlp/deberta-v2-base-japanese')
+tokenizer = AutoTokenizer.from_pretrained('ku-nlp/deberta-v2-base-japanese', trust_remote_code=True)
 model = AutoModelForMaskedLM.from_pretrained('ku-nlp/deberta-v2-base-japanese')
 
-sentence = '京都 大学 で 自然 言語 処理 を [MASK] する 。'  # input should be segmented into words by Juman++ in advance
+sentence = '京都大学で自然言語処理を[MASK]する。'
 encoding = tokenizer(sentence, return_tensors='pt')
 ...
 ```
@@ -41,7 +41,9 @@ You can also fine-tune this model on downstream tasks.
 
 ## Tokenization
 
-The input text should be segmented into words by [Juman++](https://github.com/ku-nlp/jumanpp) in advance. [Juman++ 2.0.0-rc3](https://github.com/ku-nlp/jumanpp/releases/tag/v2.0.0-rc3) was used for pre-training. Each word is tokenized into subwords by [sentencepiece](https://github.com/google/sentencepiece).
+~~The input text should be segmented into words by [Juman++](https://github.com/ku-nlp/jumanpp) in advance. [Juman++ 2.0.0-rc3](https://github.com/ku-nlp/jumanpp/releases/tag/v2.0.0-rc3) was used for pre-training. Each word is tokenized into subwords by [sentencepiece](https://github.com/google/sentencepiece).~~
+
+UPDATE: The input text is internally segmented by [Juman++](https://github.com/ku-nlp/jumanpp) within `DebertaV2JumanppTokenizer(Fast)`, so there's no need to segment it in advance. To use `DebertaV2JumanppTokenizer(Fast)`, you need to install [Juman++ 2.0.0-rc3](https://github.com/ku-nlp/jumanpp/releases/tag/v2.0.0-rc3) and [rhoknp](https://github.com/ku-nlp/rhoknp).
 
 ## Training data
 

tokenization_deberta_v2_jumanpp.py

@@ -0,0 +1,30 @@
+from transformers import DebertaV2Tokenizer
+
+
+class DebertaV2JumanppTokenizer(DebertaV2Tokenizer):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.juman_tokenizer = JumanppTokenizer()
+
+    def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs) -> tuple[str, dict]:
+        text = self.juman_tokenizer.tokenize(text)
+
+        add_prefix_space = kwargs.pop("add_prefix_space", False)
+        if is_split_into_words or add_prefix_space:
+            text = " " + text
+        return (text, kwargs)
+
+
+class JumanppTokenizer:
+    def __init__(self):
+        try:
+            import rhoknp
+        except ImportError:
+            raise ImportError(
+                "You need to install rhoknp to use JumanppPreTokenizer. "
+                "See https://github.com/ku-nlp/rhoknp for installation."
+            )
+        self.juman = rhoknp.Jumanpp()
+
+    def tokenize(self, text: str) -> str:
+        return " ".join([morpheme.surf for morpheme in self.juman.apply_to_sentence(text).morphemes])

tokenization_deberta_v2_jumanpp_fast.py

@@ -0,0 +1,64 @@
+import copy
+
+from tokenizers import NormalizedString, PreTokenizedString, normalizers, pre_tokenizers
+from transformers import DebertaV2TokenizerFast
+
+
+class DebertaV2JumanppTokenizerFast(DebertaV2TokenizerFast):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.juman_normalizer = normalizers.Sequence(
+            [
+                # cf. https://github.com/ku-nlp/rhoknp/blob/v1.3.0/src/rhoknp/units/sentence.py#L36
+                normalizers.Replace("\r", ""),
+                normalizers.Replace("\n", ""),
+                # cf. https://github.com/ku-nlp/jumanpp/blob/v2.0.0-rc3/src/jumandic/shared/juman_format.cc#L44-L61
+                normalizers.Replace("\t", "\\t"),
+                normalizers.Replace(" ", "　"),
+                normalizers.Replace('"', "”"),
+                normalizers.Replace("<", "＜"),
+                normalizers.Replace(">", "＞"),
+            ]
+        )
+        self.juman_pre_tokenizer = pre_tokenizers.PreTokenizer.custom(JumanppPreTokenizer())
+
+        self.default_normalizer = copy.deepcopy(self.backend_tokenizer.normalizer)
+        self.default_pre_tokenizer = copy.deepcopy(self.backend_tokenizer.pre_tokenizer)
+
+        self.backend_tokenizer.normalizer = normalizers.Sequence(
+            [self.juman_normalizer, self.backend_tokenizer.normalizer]
+        )
+        self.backend_tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
+            [self.juman_pre_tokenizer, self.backend_tokenizer.pre_tokenizer]
+        )
+
+    def save_pretrained(self, *args, **kwargs):
+        self.backend_tokenizer.normalizer = self.default_normalizer
+        self.backend_tokenizer.pre_tokenizer = self.default_pre_tokenizer
+        super().save_pretrained(*args, **kwargs)
+
+        self.backend_tokenizer.normalizer = normalizers.Sequence(
+            [self.juman_normalizer, self.backend_tokenizer.normalizer]
+        )
+        self.backend_tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
+            [self.juman_pre_tokenizer, self.backend_tokenizer.pre_tokenizer]
+        )
+
+
+class JumanppPreTokenizer:
+    def __init__(self):
+        try:
+            import rhoknp
+        except ImportError:
+            raise ImportError(
+                "You need to install rhoknp to use JumanppPreTokenizer. "
+                "See https://github.com/ku-nlp/rhoknp for installation."
+            )
+        self.juman = rhoknp.Jumanpp()
+
+    def pre_tokenize(self, pretok: PreTokenizedString):
+        pretok.split(self.jumanpp_split)
+
+    def jumanpp_split(self, i: int, normalized_string: NormalizedString) -> list[NormalizedString]:
+        offsets = [morpheme.span for morpheme in self.juman.apply_to_sentence(str(normalized_string)).morphemes]
+        return [normalized_string[offset[0]:offset[1]] for offset in offsets]

tokenizer_config.json

@@ -10,6 +10,12 @@
   "sp_model_kwargs": {},
   "special_tokens_map_file": null,
   "split_by_punct": false,
-  "tokenizer_class": "DebertaV2Tokenizer",
-  "unk_token": "[UNK]"
+  "tokenizer_class": "DebertaV2JumanppTokenizer",
+  "unk_token": "[UNK]",
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_deberta_v2_jumanpp.DebertaV2JumanppTokenizer",
+      "tokenization_deberta_v2_jumanpp_fast.DebertaV2JumanppTokenizerFast"
+    ]
+  }
 }