Add Support for AutoTokenizer and Pipeline

README.md

@@ -4,6 +4,12 @@ language: ko
 
 # Bert base model for Korean
 
+## Update
+
+- Update at 2021.11.17 : Add Native Support for BERT Tokenizer (works with AutoTokenizer, pipeline)
+
+---
+
 * 70GB Korean text dataset and 42000 lower-cased subwords are used
 * Check the model performance and other language models for Korean in [github](https://github.com/kiyoungkim1/LM-kor)
 
@@ -13,4 +19,4 @@ tokenizer_gpt3 = BertTokenizerFast.from_pretrained("kykim/gpt3-kor-small_based_o
 input_ids = tokenizer_gpt3.encode("text to tokenize")[1:]  # remove cls token
         
 model_gpt3 = GPT2LMHeadModel.from_pretrained("kykim/gpt3-kor-small_based_on_gpt2")
-```
+```

config.json

@@ -1,7 +1,7 @@
 {
   "activation_function": "gelu_new",
   "architectures": [
-    "GPT2Model"
+    "GPT2LMHeadModel"
   ],
   "attn_pdrop": 0.1,
   "bos_token_id": 3,
@@ -25,4 +25,4 @@
   "summary_use_proj": true,
   "use_cache": true,
   "vocab_size": 42000
-}
+}

tokenizer_config.json

@@ -1,5 +1,6 @@
 {
   "do_lower_case": true,
   "strip_accents": false,
-  "model_max_length": 2048
-}
+  "model_max_length": 2048,
+  "tokenizer_class": "BertTokenizer"
+}