ken11
commited on
Commit
·
e1d6e47
1
Parent(s):
635ac3a
update README
Browse files
README.md
CHANGED
@@ -43,6 +43,26 @@ _, result = predict[0, mask_index].topk(5)
|
|
43 |
print(tokenizer.convert_ids_to_tokens(result.tolist()))
|
44 |
```
|
45 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
46 |
## Training Data
|
47 |
学習には
|
48 |
- [日本語Wikipediaの全文](https://ja.wikipedia.org/wiki/Wikipedia:%E3%83%87%E3%83%BC%E3%82%BF%E3%83%99%E3%83%BC%E3%82%B9%E3%83%80%E3%82%A6%E3%83%B3%E3%83%AD%E3%83%BC%E3%83%89)
|
|
|
43 |
print(tokenizer.convert_ids_to_tokens(result.tolist()))
|
44 |
```
|
45 |
|
46 |
+
#### for TensorFlow
|
47 |
+
```py
|
48 |
+
from transformers import (
|
49 |
+
TFAutoModelForMaskedLM, AutoTokenizer
|
50 |
+
)
|
51 |
+
import tensorflow as tf
|
52 |
+
|
53 |
+
|
54 |
+
tokenizer = AutoTokenizer.from_pretrained("ken11/albert-base-japanese-v1-with-japanese-tokenizer")
|
55 |
+
model = TFAutoModelForMaskedLM.from_pretrained("ken11/albert-base-japanese-v1-with-japanese-tokenizer")
|
56 |
+
|
57 |
+
text = "明日は明日の[MASK]が吹く"
|
58 |
+
tokens = tokenizer(text, return_tensors="tf")
|
59 |
+
mask_index = tokens["input_ids"][0].numpy().tolist().index(tokenizer.mask_token_id)
|
60 |
+
predict = model(**tokens)[0]
|
61 |
+
result = tf.math.top_k(predict[0, mask_index], k=5)
|
62 |
+
|
63 |
+
print(tokenizer.convert_ids_to_tokens(result.indices.numpy()))
|
64 |
+
```
|
65 |
+
|
66 |
## Training Data
|
67 |
学習には
|
68 |
- [日本語Wikipediaの全文](https://ja.wikipedia.org/wiki/Wikipedia:%E3%83%87%E3%83%BC%E3%82%BF%E3%83%99%E3%83%BC%E3%82%B9%E3%83%80%E3%82%A6%E3%83%B3%E3%83%AD%E3%83%BC%E3%83%89)
|