#! /bin/sh test -f ja_gsd_modern.conllu || curl -LO https://github.com/KoichiYasuoka/SuPar-UniDic/raw/main/suparunidic/suparmodels/ja_gsd_modern.conllu curl -L https://huggingface.co/KoichiYasuoka/Swallow-7b-plus-upos/resolve/main/tokenizer.json | env LANG=ja_JP.utf8 egrep -v '"[ぁ-ん] [ぁ-ん]",$' > newtokenizer.json TMP=./maker$$.py cat << 'EOF' > $TMP #! /usr/bin/env deepspeed src="KoichiYasuoka/Swallow-7b-plus-upos" tgt="KoichiYasuoka/Swallow-7b-plus-char-upos" from transformers import LlamaTokenizerFast,LlamaModel,LlamaPreTrainedModel,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer from transformers.modeling_outputs import TokenClassifierOutput class LlamaForTokenClassification(LlamaPreTrainedModel): def __init__(self,config): from torch import nn super().__init__(config) self.num_labels=config.num_labels self.model=LlamaModel(config) if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None: classifier_dropout=config.classifier_dropout elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None: classifier_dropout=config.hidden_dropout else: classifier_dropout=0.1 self.dropout=nn.Dropout(classifier_dropout) self.classifier=nn.Linear(config.hidden_size,config.num_labels) self.post_init() def get_input_embeddings(self): return self.model.embed_tokens def set_input_embeddings(self,value): self.model.embed_tokens=value def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None): return_dict=return_dict if return_dict is not None else self.config.use_return_dict transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict) hidden_states=transformer_outputs[0] hidden_states=self.dropout(hidden_states) logits=self.classifier(hidden_states) loss=None if labels is not None: from torch import nn loss_fct=nn.CrossEntropyLoss() loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1)) if not return_dict: output=(logits,)+transformer_outputs[1:] return ((loss,)+output) if loss is not None else output return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions) class UPOSFileDataset(object): def __init__(self,conllu,tokenizer): self.conllu=open(conllu,"r",encoding="utf-8") self.tokenizer=tokenizer self.seeks=[0] self.multiword={} label=set(["SYM"]) s=self.conllu.readline() while s!="": if s=="\n": self.seeks.append(self.conllu.tell()) else: w=s.split("\t") if len(w)==10: if w[0].isdecimal(): label.add(w[3] if w[5]=="_" else w[3]+"|"+w[5]) elif w[0].find("-")>0: t=w[0].split("-") f,j,k=w[1],[],[] for i in range(int(t[0]),int(t[1])+1): w=self.conllu.readline().split("\t") j.append(w[3] if w[5]=="_" else w[3]+"|"+w[5]) k.append(w[1]) p="+".join(j) label.add(p) if p in self.multiword: self.multiword[p][f]=list(k) else: self.multiword[p]={f:list(k)} s=self.conllu.readline() lid={} for i,l in enumerate(sorted(label)): lid[l],lid["B-"+l],lid["I-"+l]=i*3,i*3+1,i*3+2 self.label2id=lid def __call__(*args): lid={l:i for i,l in enumerate(sorted(set(sum([list(t.label2id) for t in args],[]))))} for t in args: t.label2id=lid return lid def __del__(self): self.conllu.close() __len__=lambda self:len(self.seeks)-1 def __getitem__(self,i): self.conllu.seek(self.seeks[i]) form,upos=[],[] while self.conllu.tell()0: t=w[0].split("-") u=[] for j in range(int(t[0]),int(t[1])+1): k=self.conllu.readline().split("\t") u.append(k[3] if k[5]=="_" else k[3]+"|"+k[5]) upos.append("+".join(u)) v=self.tokenizer(form,add_special_tokens=False) i,u=[],[] for j,(x,y) in enumerate(zip(v["input_ids"],upos)): if x!=[]: i+=x u+=[y] if len(x)==1 else ["B-"+y]+["I-"+y]*(len(x)-1) if len(i)