model improved

config.json

@@ -20,12 +20,15 @@
   "pooler_dropout": 0,
   "pooler_hidden_act": "gelu",
   "pooler_hidden_size": 768,
-  "pos_att_type": null,
-  "position_biased_input": true,
-  "relative_attention": false,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "relative_attention": true,
   "tokenizer_class": "DebertaV2TokenizerFast",
   "torch_dtype": "float32",
-  "transformers_version": "4.19.4",
+  "transformers_version": "4.22.1",
   "type_vocab_size": 0,
   "vocab_size": 32000
 }

deprel/config.json

@@ -5,7 +5,6 @@
   "attention_probs_dropout_prob": 0.1,
   "bos_token_id": 0,
   "eos_token_id": 2,
-  "finetuning_task": "pos",
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
@@ -21,43 +20,45 @@
     "8": "B-compound",
     "9": "B-cop",
     "10": "B-csubj",
-    "11": "B-dep",
-    "12": "B-det",
-    "13": "B-discourse",
-    "14": "B-dislocated",
+    "11": "B-csubj:outer",
+    "12": "B-dep",
+    "13": "B-det",
+    "14": "B-discourse",
     "15": "B-fixed",
     "16": "B-mark",
     "17": "B-nmod",
     "18": "B-nsubj",
-    "19": "B-nummod",
-    "20": "B-obj",
-    "21": "B-obl",
-    "22": "B-punct",
-    "23": "B-root",
-    "24": "I-acl",
-    "25": "I-advcl",
-    "26": "I-advmod",
-    "27": "I-amod",
-    "28": "I-aux",
-    "29": "I-case",
-    "30": "I-cc",
-    "31": "I-ccomp",
-    "32": "I-compound",
-    "33": "I-cop",
-    "34": "I-csubj",
-    "35": "I-dep",
-    "36": "I-det",
-    "37": "I-discourse",
-    "38": "I-dislocated",
-    "39": "I-fixed",
-    "40": "I-mark",
-    "41": "I-nmod",
-    "42": "I-nsubj",
-    "43": "I-nummod",
-    "44": "I-obj",
-    "45": "I-obl",
-    "46": "I-punct",
-    "47": "I-root"
+    "19": "B-nsubj:outer",
+    "20": "B-nummod",
+    "21": "B-obj",
+    "22": "B-obl",
+    "23": "B-punct",
+    "24": "B-root",
+    "25": "I-acl",
+    "26": "I-advcl",
+    "27": "I-advmod",
+    "28": "I-amod",
+    "29": "I-aux",
+    "30": "I-case",
+    "31": "I-cc",
+    "32": "I-ccomp",
+    "33": "I-compound",
+    "34": "I-cop",
+    "35": "I-csubj",
+    "36": "I-csubj:outer",
+    "37": "I-dep",
+    "38": "I-det",
+    "39": "I-discourse",
+    "40": "I-fixed",
+    "41": "I-mark",
+    "42": "I-nmod",
+    "43": "I-nsubj",
+    "44": "I-nsubj:outer",
+    "45": "I-nummod",
+    "46": "I-obj",
+    "47": "I-obl",
+    "48": "I-punct",
+    "49": "I-root"
   },
   "initializer_range": 0.02,
   "intermediate_size": 3072,
@@ -73,43 +74,45 @@
     "B-compound": 8,
     "B-cop": 9,
     "B-csubj": 10,
-    "B-dep": 11,
-    "B-det": 12,
-    "B-discourse": 13,
-    "B-dislocated": 14,
+    "B-csubj:outer": 11,
+    "B-dep": 12,
+    "B-det": 13,
+    "B-discourse": 14,
     "B-fixed": 15,
     "B-mark": 16,
     "B-nmod": 17,
     "B-nsubj": 18,
-    "B-nummod": 19,
-    "B-obj": 20,
-    "B-obl": 21,
-    "B-punct": 22,
-    "B-root": 23,
-    "I-acl": 24,
-    "I-advcl": 25,
-    "I-advmod": 26,
-    "I-amod": 27,
-    "I-aux": 28,
-    "I-case": 29,
-    "I-cc": 30,
-    "I-ccomp": 31,
-    "I-compound": 32,
-    "I-cop": 33,
-    "I-csubj": 34,
-    "I-dep": 35,
-    "I-det": 36,
-    "I-discourse": 37,
-    "I-dislocated": 38,
-    "I-fixed": 39,
-    "I-mark": 40,
-    "I-nmod": 41,
-    "I-nsubj": 42,
-    "I-nummod": 43,
-    "I-obj": 44,
-    "I-obl": 45,
-    "I-punct": 46,
-    "I-root": 47
+    "B-nsubj:outer": 19,
+    "B-nummod": 20,
+    "B-obj": 21,
+    "B-obl": 22,
+    "B-punct": 23,
+    "B-root": 24,
+    "I-acl": 25,
+    "I-advcl": 26,
+    "I-advmod": 27,
+    "I-amod": 28,
+    "I-aux": 29,
+    "I-case": 30,
+    "I-cc": 31,
+    "I-ccomp": 32,
+    "I-compound": 33,
+    "I-cop": 34,
+    "I-csubj": 35,
+    "I-csubj:outer": 36,
+    "I-dep": 37,
+    "I-det": 38,
+    "I-discourse": 39,
+    "I-fixed": 40,
+    "I-mark": 41,
+    "I-nmod": 42,
+    "I-nsubj": 43,
+    "I-nsubj:outer": 44,
+    "I-nummod": 45,
+    "I-obj": 46,
+    "I-obl": 47,
+    "I-punct": 48,
+    "I-root": 49
   },
   "layer_norm_eps": 1e-07,
   "max_position_embeddings": 512,
@@ -121,12 +124,15 @@
   "pooler_dropout": 0,
   "pooler_hidden_act": "gelu",
   "pooler_hidden_size": 768,
-  "pos_att_type": null,
-  "position_biased_input": true,
-  "relative_attention": false,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "relative_attention": true,
   "tokenizer_class": "DebertaV2TokenizerFast",
   "torch_dtype": "float32",
-  "transformers_version": "4.19.4",
+  "transformers_version": "4.22.1",
   "type_vocab_size": 0,
   "vocab_size": 32000
 }

deprel/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:423343f1b03611981be584d5b49b2d9903fb3e977298416953716ab89f5ba110
-size 440319475
+oid sha256:1f79b5773908f6496ab08a9c0d54cca6da1170aa301142f7affe2cf27abbc18c
+size 498612115

deprel/special_tokens_map.json

@@ -1 +1,9 @@
-{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

deprel/tokenizer.json

@@ -1,11 +1,6 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 512,
-    "strategy": "LongestFirst",
-    "stride": 0
-  },
+  "truncation": null,
   "padding": null,
   "added_tokens": [
     {

deprel/tokenizer_config.json

@@ -1 +1,14 @@
-{"do_lower_case": false, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "split_by_punct": true, "keep_accents": true, "model_max_length": 512, "tokenizer_class": "DebertaV2TokenizerFast"}
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "keep_accents": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "split_by_punct": true,
+  "tokenizer_class": "DebertaV2TokenizerFast",
+  "unk_token": "[UNK]"
+}

maker.py

@@ -0,0 +1,110 @@
+#! /usr/bin/python3
+import os
+src="KoichiYasuoka/deberta-base-japanese-wikipedia"
+tgt="KoichiYasuoka/deberta-base-japanese-wikipedia-ud-head"
+url="https://github.com/UniversalDependencies/UD_Japanese-GSDLUW"
+d=os.path.basename(url)
+os.system("test -d {} || git clone --depth=1 {}".format(d,url))
+os.system("for F in train dev test ; do cp "+d+"/*-$F*.conllu $F.conllu ; done")
+from transformers import (AutoTokenizer,AutoModelForQuestionAnswering,
+  AutoModelForTokenClassification,AutoConfig,DefaultDataCollator,
+  DataCollatorForTokenClassification,TrainingArguments,Trainer)
+class HEADDataset(object):
+  def __init__(self,conllu,tokenizer,augment=False,length=384):
+    self.qa,self.pad,self.length=[],tokenizer.pad_token_id,length
+    with open(conllu,"r",encoding="utf-8") as r:
+      form,head=[],[]
+      for t in r:
+        w=t.split("\t")
+        if len(w)==10 and w[0].isdecimal():
+          form.append(w[1])
+          head.append(len(head) if w[6]=="0" else int(w[6])-1)
+        elif t.strip()=="" and form!=[]:
+          v=tokenizer(form,add_special_tokens=False)["input_ids"]
+          for i,t in enumerate(v):
+            q=[tokenizer.cls_token_id]+t+[tokenizer.sep_token_id]
+            c=[q]+v[0:i]+[[tokenizer.mask_token_id]]+v[i+1:]+[[q[-1]]]
+            b=[len(sum(c[0:j+1],[])) for j in range(len(c))]
+            if b[-1]<length:
+              self.qa.append((sum(c,[]),head[i],b))
+            if augment and [1 for x in v if t==x]==[1]:
+              c[i+1]=t
+              b=[len(sum(c[0:j+1],[])) for j in range(len(c))]
+              if b[-1]<length:
+                self.qa.append((sum(c,[]),head[i],b))
+          form,head=[],[]
+  __len__=lambda self:len(self.qa)
+  def __getitem__(self,i):
+    (v,h,b),k=self.qa[i],self.length-self.qa[i][2][-1]
+    return {"input_ids":v+[self.pad]*k,"attention_mask":[1]*b[-1]+[0]*k,
+      "token_type_ids":[0]*b[0]+[1]*(b[-1]-b[0])+[0]*k,
+      "start_positions":b[h],"end_positions":b[h+1]-1}
+class UPOSDataset(object):
+  def __init__(self,conllu,tokenizer,fields=[3]):
+    self.ids,self.upos=[],[]
+    label,cls,sep=set(),tokenizer.cls_token_id,tokenizer.sep_token_id
+    with open(conllu,"r",encoding="utf-8") as r:
+      form,upos=[],[]
+      for t in r:
+        w=t.split("\t")
+        if len(w)==10 and w[0].isdecimal():
+          form.append(w[1])
+          upos.append("|".join(w[i] for i in fields))
+        elif t.strip()=="" and form!=[]:
+          v,u=tokenizer(form,add_special_tokens=False)["input_ids"],[]
+          for x,y in zip(v,upos):
+            u.extend(["B-"+y]*min(len(x),1)+["I-"+y]*(len(x)-1))
+          if len(u)>tokenizer.model_max_length-4:
+            self.ids.append(sum(v,[])[0:tokenizer.model_max_length-2])
+            self.upos.append(u[0:tokenizer.model_max_length-2])
+          elif len(u)>0:
+            self.ids.append([cls]+sum(v,[])+[sep])
+            self.upos.append([u[0]]+u+[u[0]])
+          label=set(sum([self.upos[-1],list(label)],[]))
+          form,upos=[],[]
+    self.label2id={l:i for i,l in enumerate(sorted(label))}
+  def __call__(*args):
+    label=set(sum([list(t.label2id) for t in args],[]))
+    lid={l:i for i,l in enumerate(sorted(label))}
+    for t in args:
+      t.label2id=lid
+    return lid
+  __len__=lambda self:len(self.ids)
+  __getitem__=lambda self,i:{"input_ids":self.ids[i],
+    "labels":[self.label2id[t] for t in self.upos[i]]}
+tkz=AutoTokenizer.from_pretrained(src)
+trainDS=HEADDataset("train.conllu",tkz,True)
+devDS=HEADDataset("dev.conllu",tkz)
+arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=8,
+  output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,
+  evaluation_strategy="epoch",learning_rate=5e-05,warmup_ratio=0.1)
+trn=Trainer(args=arg,data_collator=DefaultDataCollator(),
+  model=AutoModelForQuestionAnswering.from_pretrained(src),
+  train_dataset=trainDS,eval_dataset=devDS)
+trn.train()
+trn.save_model(tgt)
+tkz.save_pretrained(tgt)
+trainDS=UPOSDataset("train.conllu",tkz,[7])
+devDS=UPOSDataset("dev.conllu",tkz,[7])
+testDS=UPOSDataset("test.conllu",tkz,[7])
+lid=trainDS(devDS,testDS)
+cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,
+  id2label={i:l for l,i in lid.items()})
+trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),
+  model=AutoModelForTokenClassification.from_pretrained(src,config=cfg),
+  train_dataset=trainDS,eval_dataset=devDS)
+trn.train()
+trn.save_model(tgt+"/deprel")
+tkz.save_pretrained(tgt+"/deprel")
+trainDS=UPOSDataset("train.conllu",tkz,[3,5])
+devDS=UPOSDataset("dev.conllu",tkz,[3,5])
+testDS=UPOSDataset("test.conllu",tkz,[3,5])
+lid=trainDS(devDS,testDS)
+cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,
+  id2label={i:l for l,i in lid.items()})
+trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),
+  model=AutoModelForTokenClassification.from_pretrained(src,config=cfg),
+  train_dataset=trainDS,eval_dataset=devDS)
+trn.train()
+trn.save_model(tgt+"/tagger")
+tkz.save_pretrained(tgt+"/tagger")

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4d0c84d139b98520551dc35b55ee63a40bf6e3eed3983a986234cb463feabdb
-size 440178035
+oid sha256:688b9a34440a937c6a5aa93bbe75b24a474a768c0980b0218dd9f20d25f53f30
+size 498464467

special_tokens_map.json

@@ -1 +1,9 @@
-{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tagger/config.json

@@ -5,7 +5,6 @@
   "attention_probs_dropout_prob": 0.1,
   "bos_token_id": 0,
   "eos_token_id": 2,
-  "finetuning_task": "pos",
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
@@ -101,12 +100,15 @@
   "pooler_dropout": 0,
   "pooler_hidden_act": "gelu",
   "pooler_hidden_size": 768,
-  "pos_att_type": null,
-  "position_biased_input": true,
-  "relative_attention": false,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "relative_attention": true,
   "tokenizer_class": "DebertaV2TokenizerFast",
   "torch_dtype": "float32",
-  "transformers_version": "4.19.4",
+  "transformers_version": "4.22.1",
   "type_vocab_size": 0,
   "vocab_size": 32000
 }

tagger/pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:74d7b076bf46d851cb9643edb9b8c5b0dd16cc8ceffb1864bf0f692949653f44
-size 440288755
+oid sha256:4551b4b329972065979303c79f1addac688f1ce5a6e4bbc0007ca52739d3ed91
+size 498575187

tagger/special_tokens_map.json

@@ -1 +1,9 @@
-{"bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tagger/tokenizer.json

@@ -1,11 +1,6 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 512,
-    "strategy": "LongestFirst",
-    "stride": 0
-  },
+  "truncation": null,
   "padding": null,
   "added_tokens": [
     {

tagger/tokenizer_config.json

@@ -1 +1,14 @@
-{"do_lower_case": false, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "split_by_punct": true, "keep_accents": true, "model_max_length": 512, "tokenizer_class": "DebertaV2TokenizerFast"}
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "keep_accents": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "split_by_punct": true,
+  "tokenizer_class": "DebertaV2TokenizerFast",
+  "unk_token": "[UNK]"
+}

tokenizer.json

@@ -1,21 +1,7 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 384,
-    "strategy": "OnlySecond",
-    "stride": 128
-  },
-  "padding": {
-    "strategy": {
-      "Fixed": 384
-    },
-    "direction": "Right",
-    "pad_to_multiple_of": null,
-    "pad_id": 1,
-    "pad_type_id": 0,
-    "pad_token": "[PAD]"
-  },
+  "truncation": null,
+  "padding": null,
   "added_tokens": [
     {
       "id": 0,

tokenizer_config.json

@@ -1 +1,14 @@
-{"do_lower_case": false, "bos_token": "[CLS]", "eos_token": "[SEP]", "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "split_by_punct": true, "keep_accents": true, "model_max_length": 512, "tokenizer_class": "DebertaV2TokenizerFast"}
+{
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "keep_accents": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "split_by_punct": true,
+  "tokenizer_class": "DebertaV2TokenizerFast",
+  "unk_token": "[UNK]"
+}