|
|
|
src="KoichiYasuoka/modernbert-base-japanese-aozora-upos" |
|
tgt="KoichiYasuoka/modernbert-base-japanese-aozora-ud-square" |
|
url="https://github.com/UniversalDependencies/UD_Japanese-GSDLUW" |
|
import os,numpy |
|
d=os.path.basename(url) |
|
os.system("test -d "+d+" || git clone --depth=1 "+url) |
|
os.system("for F in train dev test ; do cp "+d+"/*-$F.conllu $F.conllu ; done") |
|
class UDSquareDataset(object): |
|
def __init__(self,conllu,tokenizer): |
|
self.conllu=open(conllu,"r",encoding="utf-8") |
|
self.tokenizer=tokenizer |
|
self.seeks=[0] |
|
label=set(["SYM.","X.","X.|[goeswith]"]) |
|
s=self.conllu.readline() |
|
while s!="": |
|
if s=="\n": |
|
self.seeks.append(self.conllu.tell()) |
|
else: |
|
w=s.split("\t") |
|
if len(w)==10: |
|
if w[0].isdecimal(): |
|
p=w[3] |
|
q="" if w[5]=="_" else "|"+w[5] |
|
r="|["+w[7]+"]" |
|
label.add(p+q) |
|
label.add(p+"."+q) |
|
label.add(p+q+r) |
|
label.add(p+"."+q+r) |
|
s=self.conllu.readline() |
|
self.label2id={l:i for i,l in enumerate(sorted(label))} |
|
def __call__(*args): |
|
lid={l:i for i,l in enumerate(sorted(set(sum([list(t.label2id) for t in args],[]))))} |
|
for t in args: |
|
t.label2id=lid |
|
return lid |
|
def __del__(self): |
|
self.conllu.close() |
|
__len__=lambda self:len(self.seeks)-1 |
|
def __getitem__(self,i): |
|
self.conllu.seek(self.seeks[i]) |
|
c,t=[],[""] |
|
while t[0]!="\n": |
|
t=self.conllu.readline().split("\t") |
|
if len(t)==10 and t[0].isdecimal(): |
|
c.append(t) |
|
h={t[6] for t in c} |
|
for t in c: |
|
if t[6]!="0" and t[0] not in h: |
|
t[3]+="." |
|
v=self.tokenizer([t[1] for t in c],add_special_tokens=False)["input_ids"] |
|
for i in range(len(v)-1,-1,-1): |
|
if v[i]==[]: |
|
v[i]=[self.tokenizer.unk_token_id] |
|
if len(v[i])>1: |
|
c[i][3]=c[i][3].replace(".","") |
|
for j in range(1,len(v[i])): |
|
c.insert(i+1,[c[i][0],"_","_","X.","_","_",c[i][0],"goeswith","_","_"]) |
|
y=["0"]+[t[0] for t in c] |
|
h=[i if t[6]=="0" else y.index(t[6]) for i,t in enumerate(c,1)] |
|
p=[t[3] if t[5]=="_" else t[3]+"|"+t[5] for t in c] |
|
d=["|["+t[7]+"]" for t in c] |
|
x=[not t[3].endswith(".") for t in c] |
|
if len(x)<90: |
|
x=[True]*len(x) |
|
else: |
|
w=(sum([1 for b in x if b])+1)*(len(x)+1)+1 |
|
for i in numpy.argsort([-abs(j-i-1) for i,j in enumerate(h)]): |
|
if w+len(x)>8191: |
|
break |
|
if not x[i]: |
|
x[i]=True |
|
w+=len(x)+1 |
|
v=sum(v,[]) |
|
ids=[self.tokenizer.cls_token_id]+v+[self.tokenizer.sep_token_id] |
|
upos=["SYM."]+p+["SYM."] |
|
for i in range(len(v)): |
|
if x[i]: |
|
for j in range(len(v)): |
|
ids.append(self.tokenizer.mask_token_id if i==j else v[j]) |
|
upos.append(p[j]+d[j] if h[j]==i+1 else p[j]) |
|
ids.append(self.tokenizer.sep_token_id) |
|
upos.append("SYM.") |
|
return {"input_ids":ids,"labels":[self.label2id[p] for p in upos]} |
|
from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer |
|
tkz=AutoTokenizer.from_pretrained(src) |
|
trainDS=UDSquareDataset("train.conllu",tkz) |
|
devDS=UDSquareDataset("dev.conllu",tkz) |
|
testDS=UDSquareDataset("test.conllu",tkz) |
|
lid=trainDS(devDS,testDS) |
|
cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True,trust_remote_code=True) |
|
mdl=AutoModelForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True,trust_remote_code=True) |
|
arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=1,dataloader_pin_memory=False,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False) |
|
trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=mdl,train_dataset=trainDS) |
|
trn.train() |
|
trn.save_model(tgt) |
|
tkz.save_pretrained(tgt) |
|
|
