Upload 6 files

config.json

@@ -60,5 +60,5 @@
   "transformers_version": "4.38.1",
   "type_vocab_size": 2,
   "use_cache": true,
-  "vocab_size": 54343
+  "vocab_size": 54397
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a89744c63112c91969ff6e072e1a4cb6dcd7d38474a9971eb46ffc2bc6b7578
-size 511174280
+oid sha256:f1a96fd496b7abaf6e12d4d751401d5035f31c163ac1843276526e6259196bf4
+size 511340168

special_tokens_map.json

@@ -1,7 +1,37 @@
 {
-  "cls_token": "[CLS]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": "[UNK]"
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -54499,7 +54499,61 @@
       "방석": 54339,
       "미네르바": 54340,
       "변별력": 54341,
-      "프로게이머": 54342
+      "프로게이머": 54342,
+      "ㅇㅌㅎ": 54343,
+      "ㅂㅂㅈ": 54344,
+      "ㅂㅎㅈ": 54345,
+      "ㅂㄱㅇ": 54346,
+      "ㅇㅎㅎ": 54347,
+      "ㄱㄷㅎ": 54348,
+      "ㅈㅇㅅ": 54349,
+      "ㄱㅈㅎ": 54350,
+      "ㅎㅇㅊ": 54351,
+      "ㅊㅈㅎ": 54352,
+      "ㅈㅅㄱ": 54353,
+      "ㅇㅈㅇ": 54354,
+      "ㅇㅎㅈ": 54355,
+      "ㅊㅈㅅ": 54356,
+      "ㅊㅈㅇ": 54357,
+      "ㄱㅂㄱ": 54358,
+      "ㅇㅈㅎ": 54359,
+      "ㅂㄱㄴ": 54360,
+      "ㅇㅅㅇ": 54361,
+      "ㅈㅇㅈ": 54362,
+      "ㅇㅈㅁ": 54363,
+      "ㅎㅇㅌ": 54364,
+      "ㄱㅅㅇ": 54365,
+      "ㅇㄷㄴ": 54366,
+      "ㄱㅇㅈ": 54367,
+      "ㅇㅇㅅ": 54368,
+      "ㅇㅇㅎ": 54369,
+      "ㅇㅇㅈ": 54370,
+      "ㅅㅇㅎ": 54371,
+      "ㅈㅇㅊ": 54372,
+      "ㅎㅇㅅ": 54373,
+      "ㄱㅈㄱ": 54374,
+      "ㅂㅁㅈ": 54375,
+      "ㅈㅎㅇ": 54376,
+      "ㄱㅎㅎ": 54377,
+      "ㅇㅇㄱ": 54378,
+      "ㄱㅅㅎ": 54379,
+      "ㄱㅎㅅ": 54380,
+      "ㅇㅈㅅ": 54381,
+      "ㅇㅅㅎ": 54382,
+      "ㅇㄱㅇ": 54383,
+      "ㄱㅁㄷ": 54384,
+      "ㅈㅈㅎ": 54385,
+      "ㅎㅅㅎ": 54386,
+      "ㅈㄱㅅ": 54387,
+      "ㅍㅌㅊ": 54388,
+      "ㄱㅈㅇ": 54389,
+      "ㅂㅅㅇ": 54390,
+      "ㅊㅅㅎ": 54391,
+      "ㅇㄷㅎ": 54392,
+      "ㅊㅇㅊ": 54393,
+      "ㅇㄱㅎ": 54394,
+      "ㄱㅇㅇ": 54395,
+      "ㄱㅇㅎ": 54396
     }
   }
 }

tokenizer_config.json

@@ -43,7 +43,7 @@
   },
   "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
-  "do_basic_tokenize": true,
+  "do_basic_tokenize": false,
   "do_lower_case": false,
   "mask_token": "[MASK]",
   "model_max_length": 512,

vocab.txt

@@ -54341,3 +54341,57 @@ str
 미네르바
 변별력
 프로게이머
+ㅇㅌㅎ
+ㅂㅂㅈ
+ㅂㅎㅈ
+ㅂㄱㅇ
+ㅇㅎㅎ
+ㄱㄷㅎ
+ㅈㅇㅅ
+ㄱㅈㅎ
+ㅎㅇㅊ
+ㅊㅈㅎ
+ㅈㅅㄱ
+ㅇㅈㅇ
+ㅇㅎㅈ
+ㅊㅈㅅ
+ㅊㅈㅇ
+ㄱㅂㄱ
+ㅇㅈㅎ
+ㅂㄱㄴ
+ㅇㅅㅇ
+ㅈㅇㅈ
+ㅇㅈㅁ
+ㅎㅇㅌ
+ㄱㅅㅇ
+ㅇㄷㄴ
+ㄱㅇㅈ
+ㅇㅇㅅ
+ㅇㅇㅎ
+ㅇㅇㅈ
+ㅅㅇㅎ
+ㅈㅇㅊ
+ㅎㅇㅅ
+ㄱㅈㄱ
+ㅂㅁㅈ
+ㅈㅎㅇ
+ㄱㅎㅎ
+ㅇㅇㄱ
+ㄱㅅㅎ
+ㄱㅎㅅ
+ㅇㅈㅅ
+ㅇㅅㅎ
+ㅇㄱㅇ
+ㄱㅁㄷ
+ㅈㅈㅎ
+ㅎㅅㅎ
+ㅈㄱㅅ
+ㅍㅌㅊ
+ㄱㅈㅇ
+ㅂㅅㅇ
+ㅊㅅㅎ
+ㅇㄷㅎ
+ㅊㅇㅊ
+ㅇㄱㅎ
+ㄱㅇㅇ
+ㄱㅇㅎ