#! /usr/bin/python3
src="KoichiYasuoka/deberta-base-ainu-upos"
tgt="KoichiYasuoka/deberta-base-ainu-ud-goeswith"
import os
url="https://github.com/KoichiYasuoka/UD-Ainu"
d=os.path.basename(url)
os.system("test -d {} || git clone --depth=1 {}".format(d,url))
s='{if($0==""){if(u~/\\t0\\troot\\t/)print u;u=""}else u=u$0"\\n"}'
os.system("nawk -F'\\t' '{}' {}/ain_*-ud-*.conllu > train.conllu".format(s,d))
class UDgoeswithDataset(object):
  def __init__(self,conllu,tokenizer):
    self.ids,self.tags,label=[],[],set()
    with open(conllu,"r",encoding="utf-8") as r:
      cls,sep,msk=tokenizer.cls_token_id,tokenizer.sep_token_id,tokenizer.mask_token_id
      dep,c="-|_|dep",[]
      for s in r:
        t=s.split("\t")
        if len(t)==10:
          if t[0].isdecimal():
            c.append(t)
        elif c!=[]:
          for x in [1,2]:
            d=list(c)
            v=tokenizer([t[x] for t in d],add_special_tokens=False)["input_ids"]
            for i in range(len(v)-1,-1,-1):
              for j in range(1,len(v[i])):
                d.insert(i+1,[d[i][0],"_","_","X","_","_",d[i][0],"goeswith","_","_"])
            y=["0"]+[t[0] for t in d]
            h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(d,1)]
            p,v=[t[3]+"|"+t[4]+"|"+t[7] for t in d],sum(v,[])
            if len(v)<tokenizer.model_max_length-3:
              self.ids.append([cls]+v+[sep])
              self.tags.append([dep]+p+[dep])
              label=set(sum([self.tags[-1],list(label)],[]))
              for i,k in enumerate(v):
                self.ids.append([cls]+v[0:i]+[msk]+v[i+1:]+[sep,k])
                self.tags.append([dep]+[t if h[j]==i+1 else dep for j,t in enumerate(p)]+[dep,dep])
          c=[]
    self.label2id={l:i for i,l in enumerate(sorted(label))}
  __len__=lambda self:len(self.ids)
  __getitem__=lambda self,i:{"input_ids":self.ids[i],"labels":[self.label2id[t] for t in self.tags[i]]}
from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
tkz=AutoTokenizer.from_pretrained(src)
trainDS=UDgoeswithDataset("train.conllu",tkz)
lid=trainDS.label2id
cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True,task_specific_params=None)
arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=16,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1)
trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True),train_dataset=trainDS)
trn.train()
trn.save_model(tgt)
tkz.save_pretrained(tgt)