model improved for transformers 4.42

config.json

@@ -4,22 +4,11 @@
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoModelForTokenClassification": "upos.LlamaForTokenClassification"
-  },
   "bos_token_id": 1,
   "custom_pipelines": {
     "upos": {
       "impl": "upos.BellmanFordTokenClassificationPipeline",
       "pt": "AutoModelForTokenClassification"
-    },
-    "token-classification": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
-    },
-    "ner": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
     }
   },
   "eos_token_id": 2,
@@ -153,6 +142,7 @@
   },
   "max_position_embeddings": 4096,
   "max_sequence_length": 4096,
+  "mlp_bias": false,
   "model_type": "llama",
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
@@ -165,7 +155,7 @@
   "tie_word_embeddings": false,
   "tokenizer_class": "LlamaTokenizerFast",
   "torch_dtype": "float32",
-  "transformers_version": "4.38.1",
+  "transformers_version": "4.42.4",
   "use_cache": true,
   "vocab_size": 43744
 }

maker.sh

@@ -48,45 +48,9 @@ cat << 'EOF' > $TMPB
 #! /usr/bin/env deepspeed
 src="exSwallow-7b-plus-hf"
 tgt="KoichiYasuoka/Swallow-7b-plus-upos"
-from transformers import LlamaTokenizerFast,LlamaModel,LlamaPreTrainedModel,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
-from transformers.modeling_outputs import TokenClassifierOutput
+from transformers import LlamaTokenizerFast,LlamaForTokenClassification,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
 from tokenizers.normalizers import Replace
 
-class LlamaForTokenClassification(LlamaPreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=LlamaModel(config)
-    if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None:
-      classifier_dropout=config.classifier_dropout
-    elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.classifier=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    hidden_states=transformer_outputs[0]
-    hidden_states=self.dropout(hidden_states)
-    logits=self.classifier(hidden_states)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+transformer_outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions)
-
 class UPOSFileDataset(object):
   def __init__(self,conllu,tokenizer):
     self.conllu=open(conllu,"r",encoding="utf-8")

pytorch_model-00001-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:febbf717e4d094a9f32b65b84a0dc1e76d671b6c09dd22e1729ea30cc1e72f26
+oid sha256:15b7ecb40dc994c01d584b8f6061aafb937adb91d3ea88d190874393ef152b77
 size 4965712452

pytorch_model-00002-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0df6860eb51bbd2ab136351a39f295d4a2853a74a7719e1ea83a296b0af3e9d1
+oid sha256:7ee7cc71f1d19b1403f8f7fdefee9c2fdc7db7fb2d793fb1676bd9440335cc78
 size 4924328556

pytorch_model-00003-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9680466f58012f08f43e5f12372f3d48769f70e387debe22aaefba6d154682f
+oid sha256:504a5638d62594b8c0025a56083668458291dcd1137455a403efa35093b7c15c
 size 4857219294

pytorch_model-00004-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:161461dcdc9717b80393bfa008d903aaa92122f584fb287047d4e78b2c68433a
+oid sha256:a1e030bf6ba8296546621ba35acdc1486c0695b500141aa20f242bbc514002a2
 size 4857219294

pytorch_model-00005-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5dbfcec466897e051cbcf40dff7961a632a3836e68ae494edb2ce90e94a920f0
+oid sha256:c068f31f763a9235aeef0f60407db1fb5ffde971ede030ccf9787eb7490bb8d8
 size 4857219294

pytorch_model-00006-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c75d142cb23ea767435988d9cff5daca87f8f22b2755b4424cfc9aa9a7f78df
+oid sha256:4fb6cbefff2a23a4edc4414e8b48e6d7d336f9cb4e3338fe21b55273bd827dff
 size 2161173694

pytorch_model.bin.index.json

@@ -3,8 +3,6 @@
     "total_size": 26622771440
   },
   "weight_map": {
-    "classifier.bias": "pytorch_model-00006-of-00006.bin",
-    "classifier.weight": "pytorch_model-00006-of-00006.bin",
     "model.embed_tokens.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00006.bin",
@@ -294,6 +292,8 @@
     "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
-    "model.norm.weight": "pytorch_model-00006-of-00006.bin"
+    "model.norm.weight": "pytorch_model-00006-of-00006.bin",
+    "score.bias": "pytorch_model-00006-of-00006.bin",
+    "score.weight": "pytorch_model-00006-of-00006.bin"
   }
 }

tokenizer.json

@@ -38,7 +38,12 @@
     },
     "content": "▁"
   },
-  "pre_tokenizer": null,
+  "pre_tokenizer": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "prepend_scheme": "first",
+    "split": false
+  },
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [
@@ -125,6 +130,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": false,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,
@@ -43302,574 +43308,574 @@
       "趙": 43173,
       "弛": 43174,
       "徊": 43175,
-      "搆": 43176,
-      "槓": 43177,
-      "淇": 43178,
-      "呵": 43179,
-      "糀": 43180,
-      "遁": 43181,
-      "鄙": 43182,
-      "寶": 43183,
-      "禎": 43184,
-      "儕": 43185,
-      "嚼": 43186,
-      "僞": 43187,
-      "吏": 43188,
-      "勒": 43189,
-      "饗": 43190,
-      "舒": 43191,
-      "梃": 43192,
-      "朧": 43193,
-      "殷": 43194,
-      "爲": 43195,
-      "巍": 43196,
-      "專": 43197,
-      "拮": 43198,
-      "貽": 43199,
-      "勸": 43200,
-      "毆": 43201,
-      "竭": 43202,
-      "賣": 43203,
-      "汀": 43204,
-      "豕": 43205,
-      "籾": 43206,
-      "恤": 43207,
-      "觀": 43208,
-      "麾": 43209,
-      "跋": 43210,
-      "勵": 43211,
-      "毘": 43212,
-      "證": 43213,
-      "壓": 43214,
-      "彗": 43215,
-      "倹": 43216,
-      "諛": 43217,
-      "狡": 43218,
-      "裴": 43219,
-      "覽": 43220,
-      "燮": 43221,
-      "黨": 43222,
-      "咀": 43223,
-      "濤": 43224,
-      "埴": 43225,
-      "亥": 43226,
-      "熾": 43227,
-      "袈": 43228,
-      "尤": 43229,
-      "讓": 43230,
-      "惟": 43231,
-      "駭": 43232,
-      "價": 43233,
-      "咎": 43234,
-      "駢": 43235,
-      "魏": 43236,
-      "酉": 43237,
-      "錮": 43238,
-      "唖": 43239,
-      "饜": 43240,
-      "况": 43241,
-      "恪": 43242,
-      "墮": 43243,
-      "戾": 43244,
-      "縣": 43245,
-      "諮": 43246,
-      "攝": 43247,
-      "俄": 43248,
-      "屬": 43249,
-      "翹": 43250,
-      "瑣": 43251,
-      "蛾": 43252,
-      "闍": 43253,
-      "勅": 43254,
-      "驅": 43255,
-      "詔": 43256,
-      "囮": 43257,
-      "拔": 43258,
-      "悖": 43259,
-      "衷": 43260,
-      "膽": 43261,
-      "碩": 43262,
-      "濟": 43263,
-      "逕": 43264,
-      "羸": 43265,
-      "浹": 43266,
-      "覊": 43267,
-      "諂": 43268,
-      "隱": 43269,
-      "傅": 43270,
-      "醨": 43271,
-      "餘": 43272,
-      "頗": 43273,
-      "舍": 43274,
-      "緡": 43275,
-      "滊": 43276,
-      "菑": 43277,
-      "媢": 43278,
-      "壯": 43279,
-      "游": 43280,
-      "藥": 43281,
-      "楔": 43282,
-      "讀": 43283,
-      "內": 43284,
-      "麿": 43285,
-      "亂": 43286,
-      "蘊": 43287,
-      "廢": 43288,
-      "籌": 43289,
-      "竝": 43290,
-      "蓉": 43291,
-      "憚": 43292,
-      "孛": 43293,
-      "澳": 43294,
-      "恂": 43295,
-      "擧": 43296,
-      "鵠": 43297,
-      "纉": 43298,
-      "帥": 43299,
-      "嘲": 43300,
-      "揠": 43301,
-      "肅": 43302,
-      "僩": 43303,
-      "斤": 43304,
-      "埠": 43305,
-      "靜": 43306,
-      "朕": 43307,
-      "乎": 43308,
-      "禽": 43309,
-      "庸": 43310,
-      "黽": 43311,
-      "纂": 43312,
-      "僨": 43313,
-      "騁": 43314,
-      "羹": 43315,
-      "𠮟": 43316,
-      "欷": 43317,
-      "皈": 43318,
-      "鋒": 43319,
-      "對": 43320,
-      "歟": 43321,
-      "雜": 43322,
-      "醉": 43323,
-      "虧": 43324,
-      "沮": 43325,
-      "榮": 43326,
-      "僥": 43327,
-      "傀": 43328,
-      "姦": 43329,
-      "駸": 43330,
-      "稱": 43331,
-      "薛": 43332,
-      "龕": 43333,
-      "嬰": 43334,
-      "攬": 43335,
-      "掩": 43336,
-      "聚": 43337,
-      "閻": 43338,
-      "舅": 43339,
-      "輕": 43340,
-      "綮": 43341,
-      "變": 43342,
-      "飮": 43343,
-      "匈": 43344,
-      "蓁": 43345,
-      "岨": 43346,
-      "貶": 43347,
-      "畧": 43348,
-      "胥": 43349,
-      "闢": 43350,
-      "厘": 43351,
-      "隋": 43352,
-      "聲": 43353,
-      "塡": 43354,
-      "弐": 43355,
-      "勞": 43356,
-      "參": 43357,
-      "濶": 43358,
-      "罷": 43359,
-      "穆": 43360,
-      "允": 43361,
-      "榜": 43362,
-      "蜻": 43363,
-      "貮": 43364,
-      "詭": 43365,
-      "憾": 43366,
-      "伍": 43367,
-      "隧": 43368,
-      "擕": 43369,
-      "諱": 43370,
-      "汝": 43371,
-      "傳": 43372,
-      "錐": 43373,
-      "舷": 43374,
-      "裨": 43375,
-      "冰": 43376,
-      "營": 43377,
-      "聘": 43378,
-      "欣": 43379,
-      "饒": 43380,
-      "恆": 43381,
-      "僻": 43382,
-      "駑": 43383,
-      "滌": 43384,
-      "舊": 43385,
-      "鵬": 43386,
-      "肚": 43387,
-      "遑": 43388,
-      "淺": 43389,
-      "彥": 43390,
-      "臾": 43391,
-      "獻": 43392,
-      "丙": 43393,
-      "發": 43394,
-      "皺": 43395,
-      "墾": 43396,
-      "俟": 43397,
-      "嚮": 43398,
-      "擔": 43399,
-      "畫": 43400,
-      "誥": 43401,
-      "諺": 43402,
-      "驕": 43403,
-      "甍": 43404,
-      "沃": 43405,
-      "謨": 43406,
-      "蹄": 43407,
-      "黠": 43408,
-      "闊": 43409,
-      "關": 43410,
-      "痍": 43411,
-      "悌": 43412,
-      "繼": 43413,
-      "篩": 43414,
-      "體": 43415,
-      "鸞": 43416,
-      "畢": 43417,
-      "恰": 43418,
-      "頒": 43419,
-      "毋": 43420,
-      "歸": 43421,
-      "决": 43422,
-      "蠻": 43423,
-      "愼": 43424,
-      "橙": 43425,
-      "賈": 43426,
-      "迭": 43427,
-      "悉": 43428,
-      "辟": 43429,
-      "壹": 43430,
-      "脹": 43431,
-      "淫": 43432,
-      "芒": 43433,
-      "絜": 43434,
-      "竊": 43435,
-      "猗": 43436,
-      "諠": 43437,
-      "實": 43438,
-      "舜": 43439,
-      "釀": 43440,
-      "盖": 43441,
-      "揆": 43442,
-      "擇": 43443,
-      "摧": 43444,
-      "尸": 43445,
-      "巖": 43446,
-      "罕": 43447,
-      "膠": 43448,
-      "臘": 43449,
-      "愈": 43450,
-      "躓": 43451,
-      "逓": 43452,
-      "盈": 43453,
-      "璽": 43454,
-      "峻": 43455,
-      "鐸": 43456,
-      "翰": 43457,
-      "謁": 43458,
-      "揖": 43459,
-      "忒": 43460,
-      "賂": 43461,
-      "趨": 43462,
-      "蔗": 43463,
-      "從": 43464,
+      "戲": 43176,
+      "舍": 43177,
+      "猾": 43178,
+      "專": 43179,
+      "爲": 43180,
+      "黠": 43181,
+      "爰": 43182,
+      "橙": 43183,
+      "鰭": 43184,
+      "榜": 43185,
+      "摧": 43186,
+      "飜": 43187,
+      "關": 43188,
+      "朧": 43189,
+      "畢": 43190,
+      "汝": 43191,
+      "挈": 43192,
+      "吻": 43193,
+      "懺": 43194,
+      "僻": 43195,
+      "毋": 43196,
+      "盡": 43197,
+      "羣": 43198,
+      "嫡": 43199,
+      "煥": 43200,
+      "鐸": 43201,
+      "忿": 43202,
+      "參": 43203,
+      "欵": 43204,
+      "聘": 43205,
+      "攝": 43206,
+      "纂": 43207,
+      "扣": 43208,
+      "險": 43209,
+      "儘": 43210,
+      "尙": 43211,
+      "臥": 43212,
+      "淇": 43213,
+      "籾": 43214,
+      "豫": 43215,
+      "龕": 43216,
+      "佩": 43217,
+      "鍾": 43218,
+      "歟": 43219,
+      "饜": 43220,
+      "權": 43221,
+      "雖": 43222,
+      "狡": 43223,
+      "翹": 43224,
+      "吏": 43225,
+      "繼": 43226,
+      "鄙": 43227,
+      "儕": 43228,
+      "袈": 43229,
+      "轍": 43230,
+      "袁": 43231,
+      "囮": 43232,
+      "謭": 43233,
+      "皺": 43234,
+      "梃": 43235,
+      "餘": 43236,
+      "彝": 43237,
+      "岨": 43238,
+      "舒": 43239,
+      "埸": 43240,
+      "碍": 43241,
+      "鸞": 43242,
+      "胤": 43243,
+      "屡": 43244,
+      "崗": 43245,
+      "靜": 43246,
+      "禀": 43247,
+      "聯": 43248,
+      "皓": 43249,
+      "騷": 43250,
+      "擔": 43251,
+      "逼": 43252,
+      "盜": 43253,
+      "譯": 43254,
+      "駢": 43255,
+      "勅": 43256,
+      "錐": 43257,
+      "僞": 43258,
+      "厭": 43259,
+      "丞": 43260,
+      "臂": 43261,
+      "憮": 43262,
+      "轉": 43263,
+      "蜻": 43264,
+      "竊": 43265,
+      "攘": 43266,
+      "內": 43267,
+      "鄭": 43268,
+      "嚴": 43269,
+      "靱": 43270,
+      "縣": 43271,
+      "茫": 43272,
+      "俟": 43273,
+      "菑": 43274,
+      "隋": 43275,
+      "倩": 43276,
+      "冰": 43277,
+      "輕": 43278,
+      "盍": 43279,
+      "豈": 43280,
+      "據": 43281,
+      "實": 43282,
+      "紂": 43283,
+      "滊": 43284,
+      "墮": 43285,
+      "臘": 43286,
+      "濶": 43287,
+      "歸": 43288,
+      "驗": 43289,
+      "寔": 43290,
+      "脩": 43291,
+      "糀": 43292,
+      "禎": 43293,
+      "藥": 43294,
+      "絜": 43295,
+      "羞": 43296,
+      "寃": 43297,
+      "亨": 43298,
+      "濤": 43299,
+      "恤": 43300,
+      "鑒": 43301,
+      "朕": 43302,
+      "嚼": 43303,
+      "拔": 43304,
+      "蓁": 43305,
+      "臾": 43306,
+      "滿": 43307,
+      "蹈": 43308,
+      "擇": 43309,
+      "兒": 43310,
+      "兩": 43311,
+      "嬰": 43312,
+      "揠": 43313,
+      "脹": 43314,
+      "總": 43315,
+      "詔": 43316,
+      "恊": 43317,
+      "肚": 43318,
+      "悖": 43319,
+      "沮": 43320,
+      "續": 43321,
+      "媢": 43322,
+      "駸": 43323,
+      "慘": 43324,
+      "賂": 43325,
+      "礦": 43326,
+      "哨": 43327,
+      "蠻": 43328,
+      "舅": 43329,
+      "纉": 43330,
+      "傅": 43331,
+      "埠": 43332,
+      "鵬": 43333,
+      "儁": 43334,
+      "篆": 43335,
+      "勵": 43336,
+      "體": 43337,
+      "隱": 43338,
+      "盈": 43339,
+      "竭": 43340,
+      "澳": 43341,
+      "呵": 43342,
+      "耘": 43343,
+      "閻": 43344,
+      "衷": 43345,
+      "胥": 43346,
+      "蛾": 43347,
+      "諺": 43348,
+      "薛": 43349,
+      "戊": 43350,
+      "咎": 43351,
+      "剝": 43352,
+      "處": 43353,
+      "孛": 43354,
+      "逈": 43355,
+      "游": 43356,
+      "尤": 43357,
+      "遑": 43358,
+      "雜": 43359,
+      "嘲": 43360,
+      "憚": 43361,
+      "釀": 43362,
+      "葢": 43363,
+      "唖": 43364,
+      "傳": 43365,
+      "黽": 43366,
+      "騁": 43367,
+      "頽": 43368,
+      "殉": 43369,
+      "榮": 43370,
+      "頒": 43371,
+      "蘊": 43372,
+      "澁": 43373,
+      "汀": 43374,
+      "畫": 43375,
+      "趨": 43376,
+      "峻": 43377,
+      "饒": 43378,
+      "菉": 43379,
+      "宍": 43380,
+      "諮": 43381,
+      "證": 43382,
+      "玻": 43383,
+      "麿": 43384,
+      "嚮": 43385,
+      "窘": 43386,
+      "碇": 43387,
+      "鬢": 43388,
+      "緡": 43389,
+      "蘐": 43390,
+      "贖": 43391,
+      "誣": 43392,
+      "獨": 43393,
+      "醉": 43394,
+      "單": 43395,
+      "丙": 43396,
+      "戌": 43397,
+      "覽": 43398,
+      "燮": 43399,
+      "撿": 43400,
+      "允": 43401,
+      "逬": 43402,
+      "痍": 43403,
+      "恣": 43404,
+      "厘": 43405,
+      "緝": 43406,
+      "稱": 43407,
+      "斤": 43408,
+      "羸": 43409,
+      "酉": 43410,
+      "惠": 43411,
+      "蹄": 43412,
+      "胖": 43413,
+      "錮": 43414,
+      "璽": 43415,
+      "愈": 43416,
+      "斷": 43417,
+      "戮": 43418,
+      "竝": 43419,
+      "彥": 43420,
+      "棍": 43421,
+      "菰": 43422,
+      "亥": 43423,
+      "懥": 43424,
+      "攬": 43425,
+      "殲": 43426,
+      "殘": 43427,
+      "駑": 43428,
+      "俄": 43429,
+      "恕": 43430,
+      "茲": 43431,
+      "悌": 43432,
+      "蜀": 43433,
+      "舷": 43434,
+      "筭": 43435,
+      "揖": 43436,
+      "姦": 43437,
+      "堡": 43438,
+      "聲": 43439,
+      "歎": 43440,
+      "廢": 43441,
+      "肆": 43442,
+      "數": 43443,
+      "滌": 43444,
+      "鳶": 43445,
+      "沃": 43446,
+      "隨": 43447,
+      "决": 43448,
+      "屬": 43449,
+      "蛉": 43450,
+      "靡": 43451,
+      "桀": 43452,
+      "愧": 43453,
+      "謨": 43454,
+      "儼": 43455,
+      "瑟": 43456,
+      "勞": 43457,
+      "虧": 43458,
+      "匈": 43459,
+      "歔": 43460,
+      "恙": 43461,
+      "啜": 43462,
+      "匍": 43463,
+      "逓": 43464,
       "鎭": 43465,
-      "煥": 43466,
-      "宋": 43467,
-      "乘": 43468,
-      "裔": 43469,
-      "雖": 43470,
-      "桀": 43471,
-      "胖": 43472,
-      "渺": 43473,
-      "扈": 43474,
-      "儁": 43475,
-      "攘": 43476,
-      "斥": 43477,
-      "耘": 43478,
-      "缺": 43479,
-      "沒": 43480,
-      "當": 43481,
-      "諟": 43482,
-      "羣": 43483,
-      "憮": 43484,
-      "畏": 43485,
-      "處": 43486,
-      "厭": 43487,
-      "禀": 43488,
-      "倖": 43489,
-      "蜀": 43490,
-      "鏃": 43491,
-      "傚": 43492,
-      "誣": 43493,
-      "騷": 43494,
-      "譬": 43495,
-      "惠": 43496,
-      "烟": 43497,
-      "爰": 43498,
-      "滿": 43499,
-      "險": 43500,
-      "廠": 43501,
-      "憬": 43502,
-      "嗚": 43503,
-      "甞": 43504,
-      "崗": 43505,
-      "玻": 43506,
-      "盜": 43507,
-      "獸": 43508,
-      "愧": 43509,
-      "碇": 43510,
-      "憊": 43511,
-      "隨": 43512,
-      "魯": 43513,
-      "亞": 43514,
-      "孟": 43515,
-      "耻": 43516,
-      "雉": 43517,
-      "緝": 43518,
-      "埸": 43519,
-      "闡": 43520,
-      "殉": 43521,
-      "丞": 43522,
-      "瀑": 43523,
-      "轉": 43524,
-      "與": 43525,
-      "諧": 43526,
-      "艱": 43527,
-      "澁": 43528,
-      "糢": 43529,
-      "辨": 43530,
-      "權": 43531,
-      "抔": 43532,
-      "尹": 43533,
-      "哨": 43534,
-      "矩": 43535,
-      "樣": 43536,
-      "筵": 43537,
-      "豈": 43538,
-      "辭": 43539,
-      "戊": 43540,
-      "熙": 43541,
-      "恙": 43542,
-      "宍": 43543,
-      "痘": 43544,
-      "斂": 43545,
-      "陷": 43546,
-      "縱": 43547,
-      "臀": 43548,
-      "倩": 43549,
-      "圓": 43550,
-      "醵": 43551,
-      "菉": 43552,
-      "懥": 43553,
-      "葢": 43554,
-      "儼": 43555,
-      "陋": 43556,
-      "柘": 43557,
-      "脩": 43558,
-      "窩": 43559,
-      "吻": 43560,
-      "豫": 43561,
-      "茲": 43562,
-      "糾": 43563,
-      "僇": 43564,
-      "犂": 43565,
-      "咤": 43566,
-      "逈": 43567,
-      "袁": 43568,
-      "據": 43569,
-      "飜": 43570,
-      "狸": 43571,
-      "酋": 43572,
-      "撹": 43573,
-      "亨": 43574,
-      "觸": 43575,
-      "礦": 43576,
-      "繭": 43577,
-      "堡": 43578,
-      "陪": 43579,
-      "婿": 43580,
-      "篆": 43581,
-      "驗": 43582,
-      "皷": 43583,
-      "慘": 43584,
-      "圖": 43585,
-      "假": 43586,
-      "棍": 43587,
-      "襄": 43588,
-      "戮": 43589,
-      "耆": 43590,
-      "���": 43591,
-      "佩": 43592,
-      "蘐": 43593,
-      "恕": 43594,
-      "鑒": 43595,
-      "儘": 43596,
-      "鳶": 43597,
-      "恣": 43598,
-      "鬢": 43599,
-      "盡": 43600,
-      "堯": 43601,
-      "諷": 43602,
-      "疆": 43603,
-      "續": 43604,
-      "劾": 43605,
-      "歡": 43606,
-      "稍": 43607,
-      "嫡": 43608,
-      "顯": 43609,
-      "亦": 43610,
-      "猾": 43611,
-      "竢": 43612,
-      "聊": 43613,
-      "蛉": 43614,
-      "珊": 43615,
-      "虔": 43616,
-      "臂": 43617,
-      "臥": 43618,
-      "戲": 43619,
-      "逬": 43620,
-      "匐": 43621,
-      "寵": 43622,
-      "蹊": 43623,
-      "歎": 43624,
-      "蕃": 43625,
-      "簪": 43626,
-      "挈": 43627,
-      "斷": 43628,
-      "筭": 43629,
-      "獨": 43630,
-      "盍": 43631,
-      "尙": 43632,
-      "賤": 43633,
-      "虞": 43634,
-      "竟": 43635,
-      "贖": 43636,
-      "巓": 43637,
-      "屡": 43638,
-      "聽": 43639,
-      "團": 43640,
-      "蛯": 43641,
-      "竄": 43642,
-      "將": 43643,
-      "轍": 43644,
-      "扣": 43645,
-      "鰭": 43646,
-      "頽": 43647,
-      "懼": 43648,
-      "碍": 43649,
-      "數": 43650,
-      "馭": 43651,
-      "搖": 43652,
-      "惡": 43653,
-      "啻": 43654,
-      "綬": 43655,
-      "禮": 43656,
-      "紂": 43657,
-      "懺": 43658,
-      "儒": 43659,
-      "楯": 43660,
-      "覺": 43661,
-      "諫": 43662,
-      "拷": 43663,
-      "謭": 43664,
-      "爨": 43665,
-      "蹈": 43666,
-      "碎": 43667,
-      "嚴": 43668,
-      "誦": 43669,
-      "辜": 43670,
-      "單": 43671,
-      "鉤": 43672,
-      "剝": 43673,
-      "兒": 43674,
-      "宥": 43675,
-      "毫": 43676,
-      "槙": 43677,
-      "擾": 43678,
-      "楷": 43679,
-      "慄": 43680,
-      "寔": 43681,
-      "茫": 43682,
-      "壤": 43683,
-      "菰": 43684,
-      "忿": 43685,
-      "爭": 43686,
-      "遲": 43687,
-      "彭": 43688,
-      "譯": 43689,
-      "雞": 43690,
-      "殲": 43691,
-      "瑟": 43692,
-      "頴": 43693,
-      "聯": 43694,
-      "禊": 43695,
-      "窘": 43696,
-      "恊": 43697,
-      "醗": 43698,
-      "奧": 43699,
-      "皓": 43700,
-      "胤": 43701,
-      "儡": 43702,
-      "葦": 43703,
-      "靡": 43704,
-      "瞻": 43705,
-      "赫": 43706,
-      "苟": 43707,
-      "灼": 43708,
-      "歐": 43709,
-      "揜": 43710,
-      "敖": 43711,
-      "苅": 43712,
-      "寃": 43713,
-      "殘": 43714,
-      "巌": 43715,
-      "兩": 43716,
-      "洽": 43717,
-      "撿": 43718,
-      "總": 43719,
-      "夭": 43720,
-      "邑": 43721,
-      "岌": 43722,
-      "歔": 43723,
-      "徽": 43724,
-      "鐵": 43725,
-      "檣": 43726,
-      "肆": 43727,
-      "號": 43728,
-      "羞": 43729,
-      "靱": 43730,
-      "逼": 43731,
-      "鄭": 43732,
-      "啜": 43733,
-      "頰": 43734,
-      "戌": 43735,
-      "矜": 43736,
-      "鍾": 43737,
-      "妾": 43738,
-      "彝": 43739,
-      "欵": 43740,
-      "漸": 43741,
-      "敎": 43742,
-      "匍": 43743
+      "漸": 43466,
+      "岌": 43467,
+      "懼": 43468,
+      "覺": 43469,
+      "迭": 43470,
+      "穆": 43471,
+      "逕": 43472,
+      "繭": 43473,
+      "蓉": 43474,
+      "詭": 43475,
+      "酋": 43476,
+      "儡": 43477,
+      "將": 43478,
+      "隧": 43479,
+      "爭": 43480,
+      "辟": 43481,
+      "痘": 43482,
+      "宋": 43483,
+      "揆": 43484,
+      "誥": 43485,
+      "槙": 43486,
+      "濟": 43487,
+      "辭": 43488,
+      "埴": 43489,
+      "耆": 43490,
+      "顯": 43491,
+      "孟": 43492,
+      "諧": 43493,
+      "搖": 43494,
+      "恂": 43495,
+      "爨": 43496,
+      "弐": 43497,
+      "壤": 43498,
+      "諷": 43499,
+      "臀": 43500,
+      "遲": 43501,
+      "陷": 43502,
+      "號": 43503,
+      "辨": 43504,
+      "聽": 43505,
+      "堯": 43506,
+      "亂": 43507,
+      "烟": 43508,
+      "碩": 43509,
+      "遁": 43510,
+      "瞻": 43511,
+      "瀑": 43512,
+      "倹": 43513,
+      "讀": 43514,
+      "缺": 43515,
+      "竄": 43516,
+      "矩": 43517,
+      "庸": 43518,
+      "赫": 43519,
+      "扈": 43520,
+      "瑣": 43521,
+      "毆": 43522,
+      "柘": 43523,
+      "猗": 43524,
+      "駭": 43525,
+      "糾": 43526,
+      "裔": 43527,
+      "圓": 43528,
+      "尸": 43529,
+      "楔": 43530,
+      "賤": 43531,
+      "頴": 43532,
+      "禊": 43533,
+      "墾": 43534,
+      "惟": 43535,
+      "籌": 43536,
+      "珊": 43537,
+      "壓": 43538,
+      "寶": 43539,
+      "從": 43540,
+      "諛": 43541,
+      "憊": 43542,
+      "咤": 43543,
+      "窩": 43544,
+      "忒": 43545,
+      "咀": 43546,
+      "貶": 43547,
+      "魯": 43548,
+      "舜": 43549,
+      "愼": 43550,
+      "啻": 43551,
+      "綬": 43552,
+      "灼": 43553,
+      "芒": 43554,
+      "撹": 43555,
+      "楷": 43556,
+      "毫": 43557,
+      "苅": 43558,
+      "鉤": 43559,
+      "楯": 43560,
+      "馭": 43561,
+      "綮": 43562,
+      "戾": 43563,
+      "勸": 43564,
+      "渺": 43565,
+      "矜": 43566,
+      "敎": 43567,
+      "擧": 43568,
+      "欷": 43569,
+      "賈": 43570,
+      "筵": 43571,
+      "廟": 43572,
+      "豕": 43573,
+      "彗": 43574,
+      "諟": 43575,
+      "樣": 43576,
+      "貽": 43577,
+      "蛯": 43578,
+      "譬": 43579,
+      "鵠": 43580,
+      "皈": 43581,
+      "跋": 43582,
+      "疆": 43583,
+      "皷": 43584,
+      "恰": 43585,
+      "襄": 43586,
+      "亞": 43587,
+      "洽": 43588,
+      "躓": 43589,
+      "團": 43590,
+      "膠": 43591,
+      "麾": 43592,
+      "斥": 43593,
+      "艱": 43594,
+      "敖": 43595,
+      "篩": 43596,
+      "糢": 43597,
+      "熙": 43598,
+      "拷": 43599,
+      "𠮟": 43600,
+      "當": 43601,
+      "圖": 43602,
+      "浹": 43603,
+      "辜": 43604,
+      "熾": 43605,
+      "覊": 43606,
+      "蕃": 43607,
+      "僩": 43608,
+      "驅": 43609,
+      "宥": 43610,
+      "簪": 43611,
+      "變": 43612,
+      "乘": 43613,
+      "諠": 43614,
+      "蔗": 43615,
+      "僥": 43616,
+      "狸": 43617,
+      "巖": 43618,
+      "甍": 43619,
+      "醨": 43620,
+      "巍": 43621,
+      "翰": 43622,
+      "闢": 43623,
+      "擕": 43624,
+      "抔": 43625,
+      "裴": 43626,
+      "頗": 43627,
+      "禮": 43628,
+      "雞": 43629,
+      "雉": 43630,
+      "毘": 43631,
+      "僇": 43632,
+      "陋": 43633,
+      "儒": 43634,
+      "讓": 43635,
+      "闍": 43636,
+      "虞": 43637,
+      "伍": 43638,
+      "沒": 43639,
+      "婿": 43640,
+      "聊": 43641,
+      "劾": 43642,
+      "檣": 43643,
+      "嗚": 43644,
+      "盖": 43645,
+      "淫": 43646,
+      "勒": 43647,
+      "竢": 43648,
+      "淺": 43649,
+      "聚": 43650,
+      "僨": 43651,
+      "欣": 43652,
+      "闡": 43653,
+      "亦": 43654,
+      "壯": 43655,
+      "奧": 43656,
+      "傀": 43657,
+      "耻": 43658,
+      "碎": 43659,
+      "寵": 43660,
+      "觀": 43661,
+      "搆": 43662,
+      "甞": 43663,
+      "驕": 43664,
+      "陪": 43665,
+      "慄": 43666,
+      "巌": 43667,
+      "與": 43668,
+      "尹": 43669,
+      "揜": 43670,
+      "畧": 43671,
+      "竟": 43672,
+      "徽": 43673,
+      "魏": 43674,
+      "醗": 43675,
+      "頰": 43676,
+      "拮": 43677,
+      "罷": 43678,
+      "况": 43679,
+      "惡": 43680,
+      "恪": 43681,
+      "闊": 43682,
+      "倖": 43683,
+      "諫": 43684,
+      "肅": 43685,
+      "憾": 43686,
+      "價": 43687,
+      "巓": 43688,
+      "蹊": 43689,
+      "醵": 43690,
+      "槓": 43691,
+      "葦": 43692,
+      "飮": 43693,
+      "獻": 43694,
+      "賣": 43695,
+      "夭": 43696,
+      "犂": 43697,
+      "禽": 43698,
+      "塡": 43699,
+      "謁": 43700,
+      "羹": 43701,
+      "壹": 43702,
+      "苟": 43703,
+      "誦": 43704,
+      "舊": 43705,
+      "觸": 43706,
+      "妾": 43707,
+      "膽": 43708,
+      "鐵": 43709,
+      "邑": 43710,
+      "乎": 43711,
+      "發": 43712,
+      "殷": 43713,
+      "恆": 43714,
+      "貮": 43715,
+      "罕": 43716,
+      "掩": 43717,
+      "傚": 43718,
+      "歐": 43719,
+      "縱": 43720,
+      "廠": 43721,
+      "畏": 43722,
+      "彭": 43723,
+      "虔": 43724,
+      "帥": 43725,
+      "鏃": 43726,
+      "憬": 43727,
+      "鋒": 43728,
+      "諂": 43729,
+      "悉": 43730,
+      "歡": 43731,
+      "裨": 43732,
+      "斂": 43733,
+      "營": 43734,
+      "諱": 43735,
+      "假": 43736,
+      "稍": 43737,
+      "擾": 43738,
+      "饗": 43739,
+      "黨": 43740,
+      "對": 43741,
+      "匐": 43742,
+      "獸": 43743
     },
     "merges": [
       "▁ t",
@@ -115374,4 +115380,4 @@
       "▁ ▁▁▁▁▁▁▁▁▁▁▁▁▁▁"
     ]
   }
-}
+}

tokenizer_config.json

@@ -1,6 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -31,6 +32,7 @@
   "clean_up_tokenization_spaces": false,
   "cls_token": "<s>",
   "eos_token": "</s>",
+  "legacy": true,
   "mask_token": "<unk>",
   "model_max_length": 4096,
   "pad_token": "</s>",

upos.py

@@ -1,5 +1,4 @@
-from transformers import TokenClassificationPipeline,LlamaModel,LlamaPreTrainedModel
-from transformers.modeling_outputs import TokenClassifierOutput
+from transformers import TokenClassificationPipeline
 
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
@@ -40,41 +39,3 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
       t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
     return w
 
-class RawTokenClassificationPipeline(TokenClassificationPipeline):
-  def check_model_type(self,supported_models):
-    pass
-
-class LlamaForTokenClassification(LlamaPreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=LlamaModel(config)
-    if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None:
-      classifier_dropout=config.classifier_dropout
-    elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.classifier=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    hidden_states=transformer_outputs[0]
-    hidden_states=self.dropout(hidden_states)
-    logits=self.classifier(hidden_states)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+transformer_outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions)