Commit
β’
f39f8de
1
Parent(s):
a711d92
Update README.md
Browse files
README.md
CHANGED
@@ -1,3 +1,86 @@
|
|
1 |
---
|
2 |
license: cc-by-nc-sa-4.0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
3 |
---
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
---
|
2 |
license: cc-by-nc-sa-4.0
|
3 |
+
datasets:
|
4 |
+
- wikipedia
|
5 |
+
- cc100
|
6 |
+
language:
|
7 |
+
- ja
|
8 |
+
library_name: transformers
|
9 |
+
pipeline_tag: fill-mask
|
10 |
---
|
11 |
+
|
12 |
+
BERT-base (Sudachi + WordPiece)
|
13 |
+
===
|
14 |
+
|
15 |
+
## How to load the tokenizer
|
16 |
+
Please download the dictionary file for Sudachi + WordPiece from [our GitHub repository](https://github.com/hitachi-nlp/compare-ja-tokenizer/blob/public/data/dict/sudachi_wordpiece.json).
|
17 |
+
Then you can load the tokenizer by specifying the path of the dictionary file to `dict_path`.
|
18 |
+
|
19 |
+
```python
|
20 |
+
from typing import Optional
|
21 |
+
|
22 |
+
from tokenizers import Tokenizer, NormalizedString, PreTokenizedString
|
23 |
+
from tokenizers.processors import BertProcessing
|
24 |
+
from tokenizers.pre_tokenizers import PreTokenizer
|
25 |
+
from transformers import PreTrainedTokenizerFast
|
26 |
+
|
27 |
+
from sudachipy import tokenizer
|
28 |
+
from sudachipy import dictionary
|
29 |
+
import textspan
|
30 |
+
|
31 |
+
class SudachiPreTokenizer:
|
32 |
+
def __init__(self, mecab_dict_path: Optional[str] = None):
|
33 |
+
self.sudachi = dictionary.Dictionary().create()
|
34 |
+
|
35 |
+
def tokenize(self, sequence: str) -> list[str]:
|
36 |
+
return [token.surface() for token in self.sudachi.tokenize(sequence)]
|
37 |
+
|
38 |
+
def custom_split(self, i: int, normalized_string: NormalizedString) -> list[NormalizedString]:
|
39 |
+
text = str(normalized_string)
|
40 |
+
tokens = self.tokenize(text)
|
41 |
+
tokens_spans = textspan.get_original_spans(tokens, text)
|
42 |
+
return [normalized_string[st:ed] for cahr_spans in tokens_spans for st,ed in cahr_spans]
|
43 |
+
|
44 |
+
def pre_tokenize(self, pretok: PreTokenizedString):
|
45 |
+
pretok.split(self.custom_split)
|
46 |
+
|
47 |
+
# load a pre-tokenizer
|
48 |
+
pre_tokenizer = SudachiPreTokenizer()
|
49 |
+
|
50 |
+
# load a tokenizer
|
51 |
+
dict_path = /path/to/sudachi_wordpiece.json
|
52 |
+
tokenizer = Tokenizer.from_file(dict_path)
|
53 |
+
tokenizer.post_processor = BertProcessing(
|
54 |
+
cls=("[CLS]", tokenizer.token_to_id('[CLS]')),
|
55 |
+
sep=("[SEP]", tokenizer.token_to_id('[SEP]'))
|
56 |
+
)
|
57 |
+
|
58 |
+
# convert to PreTrainedTokenizerFast
|
59 |
+
tokenizer = PreTrainedTokenizerFast(
|
60 |
+
tokenizer_object=tokenizer,
|
61 |
+
unk_token='[UNK]',
|
62 |
+
cls_token='[CLS]',
|
63 |
+
sep_token='[SEP]',
|
64 |
+
pad_token='[PAD]',
|
65 |
+
mask_token='[MASK]'
|
66 |
+
)
|
67 |
+
|
68 |
+
# set a pre-tokenizer
|
69 |
+
tokenizer._tokenizer.pre_tokenizer = PreTokenizer.custom(pre_tokenizer)
|
70 |
+
```
|
71 |
+
|
72 |
+
```python
|
73 |
+
# Test
|
74 |
+
test_str = "γγγ«γ‘γ―γη§γ―ε½’ζ
η΄ θ§£ζε¨γ«γ€γγ¦η η©Άγγγ¦γγΎγγ"
|
75 |
+
tokenizer.convert_ids_to_tokens(tokenizer(test_str).input_ids)
|
76 |
+
# -> ['[CLS]','γ','##γ','##γ«','##γ‘','##γ―','γ','η§','γ―','ε½’ζ
','##η΄ ','解ζ','ε¨','γ«','γ€γ','γ¦','η η©Ά','γ','γ','γ¦','γ','γΎγ','γ','[SEP]']
|
77 |
+
```
|
78 |
+
|
79 |
+
## How to load the model
|
80 |
+
```python
|
81 |
+
from transformers import AutoModelForMaskedLM
|
82 |
+
model = AutoModelForMaskedLM.from_pretrained("hitachi-nlp/bert-base_sudachi-wordpiece")
|
83 |
+
```
|
84 |
+
|
85 |
+
|
86 |
+
**See [our repository](https://github.com/hitachi-nlp/compare-ja-tokenizer) for more details!**
|