File size: 3,979 Bytes
afec0a4 df327a7 afec0a4 df327a7 afec0a4 df327a7 afec0a4 df327a7 afec0a4 df327a7 afec0a4 df327a7 afec0a4 df327a7 afec0a4 df327a7 afec0a4 df327a7 afec0a4 df327a7 afec0a4 df327a7 afec0a4 df327a7 afec0a4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 |
#! /usr/bin/python3
src="KoichiYasuoka/modernbert-large-japanese-wikipedia-upos"
tgt="KoichiYasuoka/modernbert-large-japanese-wikipedia-ud-square"
url="https://github.com/UniversalDependencies/UD_Japanese-GSDLUW"
import os,numpy
d=os.path.basename(url)
os.system("test -d "+d+" || git clone --depth=1 "+url)
os.system("for F in train dev test ; do cp "+d+"/*-$F.conllu $F.conllu ; done")
class UDSquareDataset(object):
def __init__(self,conllu,tokenizer):
self.conllu=open(conllu,"r",encoding="utf-8")
self.tokenizer=tokenizer
self.seeks=[0]
label=set(["SYM.","X.","X.|[goeswith]"])
s=self.conllu.readline()
while s!="":
if s=="\n":
self.seeks.append(self.conllu.tell())
else:
w=s.split("\t")
if len(w)==10:
if w[0].isdecimal():
p=w[3]
q="" if w[5]=="_" else "|"+w[5]
r="|["+w[7]+"]"
label.add(p+q)
label.add(p+"."+q)
label.add(p+q+r)
label.add(p+"."+q+r)
s=self.conllu.readline()
self.label2id={l:i for i,l in enumerate(sorted(label))}
def __call__(*args):
lid={l:i for i,l in enumerate(sorted(set(sum([list(t.label2id) for t in args],[]))))}
for t in args:
t.label2id=lid
return lid
def __del__(self):
self.conllu.close()
__len__=lambda self:len(self.seeks)-1
def __getitem__(self,i):
self.conllu.seek(self.seeks[i])
c,t=[],[""]
while t[0]!="\n":
t=self.conllu.readline().split("\t")
if len(t)==10 and t[0].isdecimal():
c.append(t)
h={t[6] for t in c}
for t in c:
if t[6]!="0" and t[0] not in h:
t[3]+="."
v=self.tokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"]
for i in range(len(v)-1,-1,-1):
if v[i]==[]:
v[i]=[self.tokenizer.unk_token_id]
if len(v[i])>1:
c[i][3]=c[i][3].replace(".","")
for j in range(1,len(v[i])):
c.insert(i+1,[c[i][0],"_","_","X.","_","_",c[i][0],"goeswith","_","_"])
y=["0"]+[t[0] for t in c]
h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)]
p=[t[3] if t[5]=="_" else t[3]+"|"+t[5] for t in c]
d=["|["+t[7]+"]" for t in c]
x=[not t[3].endswith(".") for t in c]
if len(x)<90:
x=[True]*len(x)
else:
w=(sum([1 for b in x if b])+1)*(len(x)+1)+1
for i in numpy.argsort([-abs(j-i-1) for i,j in enumerate(h)]):
if w+len(x)>8191:
break
if not x[i]:
x[i]=True
w+=len(x)+1
v=sum(v,[])
ids=[self.tokenizer.cls_token_id]+v+[self.tokenizer.sep_token_id]
upos=["SYM."]+p+["SYM."]
for i in range(len(v)):
if x[i]:
for j in range(len(v)):
ids.append(self.tokenizer.mask_token_id if i==j else v[j])
upos.append(p[j]+d[j] if h[j]==i+1 else p[j])
ids.append(self.tokenizer.sep_token_id)
upos.append("SYM.")
return {"input_ids":ids,"labels":[self.label2id[p] for p in upos]}
from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
tkz=AutoTokenizer.from_pretrained(src)
trainDS=UDSquareDataset("train.conllu",tkz)
devDS=UDSquareDataset("dev.conllu",tkz)
testDS=UDSquareDataset("test.conllu",tkz)
lid=trainDS(devDS,testDS)
cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True,trust_remote_code=True)
mdl=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True,trust_remote_code=True)
arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=1,dataloader_pin_memory=False,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False)
trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=mdl,train_dataset=trainDS)
trn.train()
trn.save_model(tgt)
tkz.save_pretrained(tgt)
|