File size: 3,963 Bytes
3efdd53 615aa9b 3efdd53 ed74159 3efdd53 615aa9b 3efdd53 0a226a4 615aa9b 3efdd53 615aa9b 3efdd53 615aa9b 3efdd53 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 |
#! /usr/bin/python3
src="KoichiYasuoka/roberta-base-vietnamese-upos"
tgt="KoichiYasuoka/roberta-base-vietnamese-ud-goeswith"
import os
url="https://github.com/UniversalDependencies/UD_Vietnamese-VTB"
d=os.path.basename(url)
os.system("test -d "+d+" || git clone --depth=1 "+url)
os.system("for F in train dev test ; do cp "+d+"/*-$F.conllu $F.conllu ; done")
url="https://github.com/datquocnguyen/VnDT"
d=os.path.basename(url)
os.system("test -d "+d+" || git clone --depth=1 "+url)
os.system("for F in train dev test ; do cp "+d+"/*-gold-*-$F.conll pre-$F.conll ; done")
class UDgoeswithDataset(object):
def __init__(self,conllu,tokenizer):
self.ids,self.tags,label=[],[],set()
with open(conllu,"r",encoding="utf-8") as r:
cls,sep,msk=tokenizer.cls_token_id,tokenizer.sep_token_id,tokenizer.mask_token_id
dep,c="-|_|dep",[]
for s in r:
t=s.split("\t")
if len(t)==10:
if t[0].isdecimal():
c.append(t)
elif c!=[]:
v=tokenizer([t[1].replace("_"," ") for t in c],add_special_tokens=False)["input_ids"]
for i in range(len(v)-1,-1,-1):
for j in range(1,len(v[i])):
c.insert(i+1,[c[i][0],"_","_","X","_","_",c[i][0],"goeswith","_","_"])
y=["0"]+[t[0] for t in c]
h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)]
p,v=[t[3]+"|"+t[5]+"|"+t[7] for t in c],sum(v,[])
if len(v)<tokenizer.model_max_length-3:
self.ids.append([cls]+v+[sep])
self.tags.append([dep]+p+[dep])
label=set(sum([self.tags[-1],list(label)],[]))
for i,k in enumerate(v):
self.ids.append([cls]+v[0:i]+[msk]+v[i+1:]+[sep,k])
self.tags.append([dep]+[t if h[j]==i+1 else dep for j,t in enumerate(p)]+[dep,dep])
c=[]
self.label2id={l:i for i,l in enumerate(sorted(label))}
def __call__(*args):
label=set(sum([list(t.label2id) for t in args],[]))
lid={l:i for i,l in enumerate(sorted(label))}
for t in args:
t.label2id=lid
return lid
__len__=lambda self:len(self.ids)
__getitem__=lambda self,i:{"input_ids":self.ids[i],"labels":[self.label2id[t] for t in self.tags[i]]}
from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
tkz=AutoTokenizer.from_pretrained(src)
trainDS=UDgoeswithDataset("pre-train.conll",tkz)
devDS=UDgoeswithDataset("pre-dev.conll",tkz)
testDS=UDgoeswithDataset("pre-test.conll",tkz)
lid=trainDS(devDS,testDS)
cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,evaluation_strategy="epoch",learning_rate=5e-05,warmup_ratio=0.1)
trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True),train_dataset=trainDS,eval_dataset=devDS)
trn.train()
trn.save_model("tmpdir")
tkz.save_pretrained("tmpdir")
trainDS=UDgoeswithDataset("train.conllu",tkz)
devDS=UDgoeswithDataset("dev.conllu",tkz)
testDS=UDgoeswithDataset("test.conllu",tkz)
lid=trainDS(devDS,testDS)
cfg=AutoConfig.from_pretrained("tmpdir",num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,evaluation_strategy="epoch",learning_rate=5e-05,warmup_ratio=0.1)
trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained("tmpdir",config=cfg,ignore_mismatched_sizes=True),train_dataset=trainDS,eval_dataset=devDS)
trn.train()
trn.save_model(tgt)
tkz.save_pretrained(tgt)
|