New version

char_tokenizer.py

@@ -3,7 +3,7 @@ from typing import Optional, Tuple, List
 from collections import OrderedDict
 
 from torch.utils.data import Dataset
-from transformers import PreTrainedTokenizer
+from transformers import PreTrainedTokenizer, AutoTokenizer
 
 
 def load_vocab(vocab_file):
@@ -22,10 +22,11 @@ class CharTokenizer(PreTrainedTokenizer):
     def __init__(
         self,
         vocab_file=None,
-        pad_token="[PAD]",
-        unk_token="[UNK]",
-        bos_token="[BOS]",
-        eos_token="[EOS]",
+        pad_token="[pad]",
+        unk_token="[unk]",
+        bos_token="[bos]",
+        eos_token="[eos]",
+        do_lower_case=False,
         *args,
         **kwargs
     ):
@@ -34,8 +35,10 @@ class CharTokenizer(PreTrainedTokenizer):
             unk_token=unk_token,
             bos_token=bos_token,
             eos_token=eos_token,
+            do_lower_case=do_lower_case,
             **kwargs
         )
+        self.do_lower_case = do_lower_case
 
         if not vocab_file or not os.path.isfile(vocab_file):
             self.vocab = OrderedDict()
@@ -49,6 +52,8 @@ class CharTokenizer(PreTrainedTokenizer):
         with open(file_path) as r:
             for line in r:
                 word = line.strip()
+                if self.do_lower_case:
+                    word = word.lower()
                 vocab |= set(word)
         vocab = list(vocab)
         vocab.sort()
@@ -67,12 +72,16 @@ class CharTokenizer(PreTrainedTokenizer):
         return self.vocab
 
     def _convert_token_to_id(self, token):
-        return self.vocab.get(token)
+        if self.do_lower_case:
+            token = token.lower()
+        return self.vocab.get(token, self.vocab[self.unk_token])
 
     def _convert_id_to_token(self, index):
         return self.ids_to_tokens[index]
 
     def _tokenize(self, text):
+        if self.do_lower_case:
+            text = text.lower()
         return list(text)
 
     def convert_tokens_to_string(self, tokens):
@@ -119,3 +128,5 @@ class CharTokenizer(PreTrainedTokenizer):
                 writer.write(token + "\n")
                 index += 1
         return (vocab_file,)
+
+AutoTokenizer.register("char_tokenizer", CharTokenizer)

config.json

@@ -2,17 +2,17 @@
   "architectures": [
     "DebertaV2ForTokenClassification"
   ],
-  "attention_probs_dropout_prob": 0.1,
+  "attention_probs_dropout_prob": 0.2,
   "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.1,
-  "hidden_size": 128,
+  "hidden_dropout_prob": 0.2,
+  "hidden_size": 256,
   "id2label": {
     "0": "NO",
     "1": "PRIMARY",
     "2": "SECONDARY"
   },
   "initializer_range": 0.02,
-  "intermediate_size": 512,
+  "intermediate_size": 1024,
   "label2id": {
     "NO": 0,
     "PRIMARY": 1,
@@ -20,20 +20,20 @@
   },
   "layer_norm_eps": 1e-07,
   "max_length": 40,
-  "max_position_embeddings": 64,
-  "max_relative_positions": -1,
+  "max_position_embeddings": 42,
+  "max_relative_positions": 42,
   "model_type": "deberta-v2",
-  "num_attention_heads": 4,
+  "num_attention_heads": 8,
   "num_hidden_layers": 4,
   "pad_token_id": 0,
   "pooler_dropout": 0,
   "pooler_hidden_act": "gelu",
-  "pooler_hidden_size": 128,
+  "pooler_hidden_size": 256,
   "pos_att_type": null,
   "position_biased_input": true,
-  "relative_attention": false,
+  "relative_attention": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.20.1",
+  "transformers_version": "4.25.1",
   "type_vocab_size": 0,
-  "vocab_size": 70
+  "vocab_size": 40
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f168b92d2481d3e465ef0c3579cd771c3dbde8188b90efe582cc4468085a9807
-size 3267088
+oid sha256:b93628caf4493a15351b7b17bfb6c4d77a26960f08ee247f8959b6eb70e7db24
+size 12835213

special_tokens_map.json

@@ -1,6 +1,6 @@
 {
-  "bos_token": "[BOS]",
-  "eos_token": "[EOS]",
-  "pad_token": "[PAD]",
-  "unk_token": "[UNK]"
+  "bos_token": "[bos]",
+  "eos_token": "[eos]",
+  "pad_token": "[pad]",
+  "unk_token": "[unk]"
 }

tokenizer_config.json

@@ -1,11 +1,9 @@
 {
-  "bos_token": "[BOS]",
-  "eos_token": "[EOS]",
-  "pad_token": "[PAD]",
-  "unk_token": "[UNK]",
-  "model_max_length": 40,
+  "bos_token": "[bos]",
+  "do_lower_case": true,
+  "eos_token": "[eos]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[pad]",
   "tokenizer_class": "CharTokenizer",
-  "auto_map": {
-    "AutoTokenizer": ["char_tokenizer.CharTokenizer", null]
-  }
+  "unk_token": "[unk]"
 }

vocab.txt

@@ -1,40 +1,10 @@
-[PAD]
-[UNK]
-[BOS]
-[EOS]
+[pad]
+[unk]
+[bos]
+[eos]
 '
 -
 `
-А
-Б
-В
-Г
-Д
-Е
-Ж
-З
-И
-Й
-К
-Л
-М
-Н
-О
-П
-Р
-С
-Т
-У
-Ф
-Х
-Ц
-Ч
-Ш
-Щ
-Ы
-Э
-Ю
-Я
 а
 б
 в