model improved for transformers 4.42

config.json

@@ -3,22 +3,11 @@
     "Qwen2ForTokenClassification"
   ],
   "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoModelForTokenClassification": "upos.Qwen2ForTokenClassification"
-  },
   "bos_token_id": 151643,
   "custom_pipelines": {
     "upos": {
       "impl": "upos.BellmanFordTokenClassificationPipeline",
       "pt": "AutoModelForTokenClassification"
-    },
-    "token-classification":{
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
-    },
-    "ner":{
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
     }
   },
   "eos_token_id": 151643,
@@ -376,9 +365,9 @@
   "rope_theta": 5000000.0,
   "sliding_window": 32768,
   "tie_word_embeddings": false,
-  "tokenizer_class": "Qwen2Tokenizer",
   "torch_dtype": "float32",
-  "transformers_version": "4.41.2",
+  "tokenizer_class": "Qwen2Tokenizer",
+  "transformers_version": "4.42.4",
   "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 151936

maker.sh

@@ -13,43 +13,7 @@ TMP=./maker$$.py
   echo 'tgt="KoichiYasuoka/'$S'-upos"'
 ) > $TMP
 cat << 'EOF' >> $TMP
-from transformers import AutoTokenizer,Qwen2Model,Qwen2PreTrainedModel,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
-from transformers.modeling_outputs import TokenClassifierOutput
-
-class Qwen2ForTokenClassification(Qwen2PreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=Qwen2Model(config)
-    if getattr(config,"classifier_dropout",None) is not None:
-      classifier_dropout=config.classifier_dropout
-    elif getattr(config,"hidden_dropout",None) is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.score=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    sequence_output=outputs[0]
-    sequence_output=self.dropout(sequence_output)
-    logits=self.score(sequence_output)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=outputs.hidden_states,attentions=outputs.attentions)
+from transformers import AutoTokenizer,Qwen2ForTokenClassification,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
 
 class UPOSFileDataset(object):
   def __init__(self,conllu,tokenizer):

pytorch_model-00001-of-00003.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c5185783530ae8fed3b241173fa869e56fbdb2e134dc9154d20c08057c958db2
+oid sha256:47333ed441637dc876fad1a283080dbf29c00970c0570d14f4293ac5d9382723
 size 4974769352

pytorch_model-00002-of-00003.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:db08d2ff094e9c86a2bf683e97297da33031011d9e070cebf9b9ead1cad98ea4
+oid sha256:db2ebda5f396ded9ba823c3b86e46fd751d8e208822c9b94da333c5c3ababd89
 size 4934433952

pytorch_model-00003-of-00003.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ad91e73ed770e4a05528e3713b41a4c39b0796ebb5f68a3fb78cd13497b5c7e
+oid sha256:8d64c6d1dc4366f38ac560fa2413115f565dc7bff0ad64eb9ae55e7b0476ca23
 size 4338334558

tokenizer_config.json

@@ -31,6 +31,7 @@
     "<|im_end|>"
   ],
   "bos_token": null,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content + '<|endoftext|>' }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",

upos.py

@@ -1,5 +1,4 @@
-from transformers import TokenClassificationPipeline,Qwen2Model,Qwen2PreTrainedModel
-from transformers.modeling_outputs import TokenClassifierOutput
+from transformers import TokenClassificationPipeline
 
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
@@ -17,6 +16,7 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
     import numpy
     if "logits" not in model_outputs:
       return self.postprocess(model_outputs[0],**kwargs)
+    print(model_outputs["logits"].size())
     m=model_outputs["logits"][0].numpy()
     e=numpy.exp(m-numpy.max(m,axis=-1,keepdims=True))
     z=e/e.sum(axis=-1,keepdims=True)
@@ -40,42 +40,3 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
       t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
     return w
 
-class RawTokenClassificationPipeline(TokenClassificationPipeline):
-  def check_model_type(self,supported_models):
-    pass
-
-class Qwen2ForTokenClassification(Qwen2PreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=Qwen2Model(config)
-    if getattr(config,"classifier_dropout",None) is not None:
-      classifier_dropout=config.classifier_dropout
-    elif getattr(config,"hidden_dropout",None) is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.score=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    sequence_output=outputs[0]
-    sequence_output=self.dropout(sequence_output)
-    logits=self.score(sequence_output)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=outputs.hidden_states,attentions=outputs.attentions)
-