Update README.md
Browse files
README.md
CHANGED
@@ -20,8 +20,8 @@ from transformers import AutoTokenizer, AutoModel, DistilBertForMaskedLM
|
|
20 |
import torch
|
21 |
import torch.nn.functional as F
|
22 |
|
23 |
-
tokenizer = AutoTokenizer.from_pretrained('bongsoo/
|
24 |
-
model = DistilBertForMaskedLM.from_pretrained('bongsoo/
|
25 |
|
26 |
text = ['한국의 수도는 [MASK] 이다', '에펠탑은 [MASK]에 있다', '충무공 이순신은 [MASK]에 최고의 장수였다']
|
27 |
tokenized_input = tokenizer(text, max_length=128, truncation=True, padding='max_length', return_tensors='pt')
|
@@ -81,8 +81,8 @@ def mean_pooling(model_output, attention_mask):
|
|
81 |
sentences = ['This is an example sentence', 'Each sentence is converted']
|
82 |
|
83 |
# Load model from HuggingFace Hub
|
84 |
-
tokenizer = AutoTokenizer.from_pretrained('bongsoo/
|
85 |
-
model = AutoModel.from_pretrained('bongsoo/
|
86 |
|
87 |
# Tokenize sentences
|
88 |
encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
|
|
|
20 |
import torch
|
21 |
import torch.nn.functional as F
|
22 |
|
23 |
+
tokenizer = AutoTokenizer.from_pretrained('bongsoo/mdistilbertV2.1', do_lower_case=False)
|
24 |
+
model = DistilBertForMaskedLM.from_pretrained('bongsoo/mdistilbertV2.1')
|
25 |
|
26 |
text = ['한국의 수도는 [MASK] 이다', '에펠탑은 [MASK]에 있다', '충무공 이순신은 [MASK]에 최고의 장수였다']
|
27 |
tokenized_input = tokenizer(text, max_length=128, truncation=True, padding='max_length', return_tensors='pt')
|
|
|
81 |
sentences = ['This is an example sentence', 'Each sentence is converted']
|
82 |
|
83 |
# Load model from HuggingFace Hub
|
84 |
+
tokenizer = AutoTokenizer.from_pretrained('bongsoo/mdistilbertV2.1')
|
85 |
+
model = AutoModel.from_pretrained('bongsoo/mdistilbertV2.1')
|
86 |
|
87 |
# Tokenize sentences
|
88 |
encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
|