re-initialize

suparkanbun/models/labelPOS.txt

@@ -0,0 +1,129 @@
+n,代名詞,人称,他,PRON,Person=1|PronType=Prs
+n,代名詞,人称,他,PRON,Person=2|PronType=Prs
+n,代名詞,人称,他,PRON,Person=3|PronType=Prs
+n,代名詞,人称,他,PRON,PronType=Prs
+n,代名詞,人称,他,PRON,PronType=Prs|Reflex=Yes
+n,代名詞,人称,止格,PRON,Person=1|PronType=Prs
+n,代名詞,人称,止格,PRON,Person=2|PronType=Prs
+n,代名詞,人称,止格,PRON,Person=3|PronType=Prs
+n,代名詞,人称,止格,PRON,PronType=Prs
+n,代名詞,人称,起格,PRON,Person=1|PronType=Prs
+n,代名詞,人称,起格,PRON,Person=2|PronType=Prs
+n,代名詞,人称,起格,PRON,Person=3|PronType=Prs
+n,代名詞,人称,起格,PRON,PronType=Prs
+n,代名詞,指示,*,PRON,PronType=Dem
+n,代名詞,疑問,*,PRON,PronType=Int
+n,名詞,不可譲,属性,NOUN,_
+n,名詞,不可譲,疾病,NOUN,_
+n,名詞,不可譲,身体,NOUN,_
+n,名詞,主体,動物,NOUN,_
+n,名詞,主体,国名,PROPN,Case=Loc|NameType=Nat
+n,名詞,主体,書物,NOUN,_
+n,名詞,主体,機関,NOUN,_
+n,名詞,主体,集団,NOUN,_
+n,名詞,人,その他の人名,PROPN,NameType=Prs
+n,名詞,人,人,NOUN,_
+n,名詞,人,名,PROPN,NameType=Giv
+n,名詞,人,姓氏,PROPN,NameType=Sur
+n,名詞,人,役割,NOUN,_
+n,名詞,人,複合的人名,PROPN,NameType=Prs
+n,名詞,人,関係,NOUN,_
+n,名詞,制度,儀礼,NOUN,_
+n,名詞,制度,場,NOUN,Case=Loc
+n,名詞,可搬,乗り物,NOUN,_
+n,名詞,可搬,伝達,NOUN,_
+n,名詞,可搬,成果物,NOUN,_
+n,名詞,可搬,糧食,NOUN,_
+n,名詞,可搬,道具,NOUN,_
+n,名詞,固定物,地名,PROPN,Case=Loc|NameType=Geo
+n,名詞,固定物,地形,NOUN,Case=Loc
+n,名詞,固定物,建造物,NOUN,Case=Loc
+n,名詞,固定物,樹木,NOUN,_
+n,名詞,固定物,関係,NOUN,Case=Loc
+n,名詞,外観,人,NOUN,_
+n,名詞,天象,天文,NOUN,_
+n,名詞,天象,怪異,NOUN,_
+n,名詞,天象,気象,NOUN,_
+n,名詞,度量衡,*,NOUN,NounType=Clf
+n,名詞,思考,*,NOUN,_
+n,名詞,描写,形質,NOUN,_
+n,名詞,描写,態度,NOUN,_
+n,名詞,数量,*,NOUN,_
+n,名詞,時,*,NOUN,Case=Tem
+n,名詞,行為,*,NOUN,_
+n,数詞,干支,*,NUM,NumType=Ord
+n,数詞,数,*,NUM,_
+n,数詞,数字,*,NUM,_
+p,助詞,句末,*,PART,_
+p,助詞,句頭,*,PART,_
+p,助詞,接続,並列,CCONJ,_
+p,助詞,接続,体言化,PART,_
+p,助詞,接続,属格,SCONJ,_
+p,助詞,提示,*,PART,_
+p,感嘆詞,*,*,INTJ,_
+p,接尾辞,*,*,PART,_
+s,文字,*,*,SYM,_
+s,記号,一般,*,SYM,_
+s,記号,句点,*,PUNCT,_
+s,記号,読点,*,PUNCT,_
+v,前置詞,基盤,*,ADP,_
+v,前置詞,源泉,*,ADP,_
+v,前置詞,経由,*,ADP,_
+v,前置詞,関係,*,ADP,_
+v,副詞,判断,推定,ADV,_
+v,副詞,判断,確定,ADV,_
+v,副詞,判断,逆接,ADV,_
+v,副詞,否定,体言否定,ADV,Polarity=Neg
+v,副詞,否定,有界,ADV,Polarity=Neg
+v,副詞,否定,無界,ADV,Polarity=Neg
+v,副詞,否定,禁止,ADV,Polarity=Neg
+v,副詞,描写,*,ADV,_
+v,副詞,時相,変化,ADV,AdvType=Tim
+v,副詞,時相,完了,ADV,AdvType=Tim|Aspect=Perf
+v,副詞,時相,将来,ADV,AdvType=Tim|Tense=Fut
+v,副詞,時相,恒常,ADV,AdvType=Tim
+v,副詞,時相,現在,ADV,AdvType=Tim|Tense=Pres
+v,副詞,時相,終局,ADV,AdvType=Tim
+v,副詞,時相,継起,ADV,AdvType=Tim
+v,副詞,時相,緊接,ADV,AdvType=Tim
+v,副詞,時相,過去,ADV,AdvType=Tim|Tense=Past
+v,副詞,疑問,原因,ADV,AdvType=Cau
+v,副詞,疑問,反語,ADV,_
+v,副詞,疑問,所在,ADV,_
+v,副詞,程度,やや高度,ADV,AdvType=Deg|Degree=Cmp
+v,副詞,程度,極度,ADV,AdvType=Deg|Degree=Sup
+v,副詞,程度,軽度,ADV,AdvType=Deg|Degree=Pos
+v,副詞,範囲,共同,ADV,_
+v,副詞,範囲,総括,ADV,_
+v,副詞,範囲,限定,ADV,_
+v,副詞,頻度,偶発,ADV,_
+v,副詞,頻度,重複,ADV,_
+v,副詞,頻度,頻繁,ADV,_
+v,助動詞,受動,*,AUX,Voice=Pass
+v,助動詞,可能,*,AUX,Mood=Pot
+v,助動詞,必要,*,AUX,Mood=Nec
+v,助動詞,願望,*,AUX,Mood=Des
+v,動詞,変化,制度,VERB,_
+v,動詞,変化,性質,VERB,_
+v,動詞,変化,生物,VERB,_
+v,動詞,存在,存在,VERB,Polarity=Neg
+v,動詞,存在,存在,VERB,VerbType=Cop
+v,動詞,存在,存在,VERB,_
+v,動詞,描写,境遇,VERB,Degree=Pos
+v,動詞,描写,形質,VERB,Degree=Pos
+v,動詞,描写,態度,VERB,Degree=Pos
+v,動詞,描写,量,VERB,Degree=Pos
+v,動詞,行為,交流,VERB,_
+v,動詞,行為,伝達,VERB,_
+v,動詞,行為,使役,VERB,_
+v,動詞,行為,儀礼,VERB,_
+v,動詞,行為,分類,VERB,Degree=Equ
+v,動詞,行為,動作,VERB,_
+v,動詞,行為,姿勢,VERB,_
+v,動詞,行為,役割,VERB,_
+v,動詞,行為,得失,VERB,_
+v,動詞,行為,態度,VERB,_
+v,動詞,行為,生産,VERB,_
+v,動詞,行為,移動,VERB,_
+v,動詞,行為,設置,VERB,_
+v,動詞,行為,飲食,VERB,_

suparkanbun/models/lzh_kyoto.conllu

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86c46887798cd5d93f500ef99674897876501177366ad2b3e4ad861f3a362beb
+size 24744523

suparkanbun/models/mkmodel.sh

@@ -0,0 +1,341 @@
+#! /bin/sh
+# pip3 install transformers seqeval datasets supar
+test -f run_ner.py || curl -LO https://raw.githubusercontent.com/huggingface/transformers/v4.0.1/examples/token-classification/run_ner.py
+
+python3 -c '
+from suparkanbun.simplify import simplify
+c=[]
+h=[0]
+while True:
+  try:
+    s=input()
+  except:
+    quit()
+  t=s.strip().split("\t")
+  if len(t)==10:
+    if t[0]!="#":
+      t[0]=str(len(c)+1)
+      i=len(t[1])
+      if i>1:
+        form=t[1]
+        lemma=t[2]
+        head=t[6]
+        deprel=t[7]
+        for j in range(0,i-1):
+          t[1]=form[j]
+          if t[1] in simplify:
+            t[1]=simplify[t[1]]
+          t[2]=lemma[j]
+          t[6]="-1"
+          t[7]="compound"
+          c.append(list(t))
+          t[0]=str(len(c)+1)
+        t[1]=form[i-1]
+        t[2]=lemma[i-1]
+        t[6]=head
+        t[7]=deprel
+      if t[1] in simplify:
+        t[1]=simplify[t[1]]
+      c.append(list(t))
+      h.append(len(c))
+  elif s.strip()=="":
+    for t in c:
+      t[6]=str(int(t[0])+1 if t[6]=="-1" else h[int(t[6])])
+      print("\t".join(t))
+    print("")
+    c=[]
+    h=[0]
+' < lzh_kyoto.conllu | tee simplified.conllu | python3 -c '
+tokens=[]
+tags=[]
+while True:
+  try:
+    s=input()
+  except:
+    if len(tokens)>0:
+      print("{\"tokens\":[\""+"\",\"".join(tokens)+"\"],\"tags\":[\""+"\",\"".join(tags)+"\"]}")
+    quit()
+  t=s.split("\t")
+  if len(t)==10:
+    p=t[4]+","+t[3]+","+t[5]
+    for c in t[1]:
+      tokens.append(c)
+      tags.append(p)
+  elif len(tokens)>80:
+    print("{\"tokens\":[\""+"\",\"".join(tokens)+"\"],\"tags\":[\""+"\",\"".join(tags)+"\"]}")
+    tokens=[]
+    tags=[]
+' | tee simplifiedPOS.json | nawk '
+{
+  if(NR%10>0)
+    printf("%s\n",$0)>"trainPOS.json";
+  else
+    printf("%s\n",$0)>"validPOS.json";
+}'
+sed 's/^.*"tags":\[//' trainPOS.json | tr '"' '\012' | sort -u | egrep '^[nvps],' > labelPOS.txt
+if [ ! -d guwenbert-base.pos ]
+then mkdir -p guwenbert-base.pos
+     python3 run_ner.py --model_name_or_path ethanyt/guwenbert-base --train_file trainPOS.json --validation_file validPOS.json --output_dir guwenbert-base.pos --do_train --do_eval
+fi
+if [ ! -d guwenbert-large.pos ]
+then mkdir -p guwenbert-large.pos
+     python3 run_ner.py --model_name_or_path ethanyt/guwenbert-large --train_file trainPOS.json --validation_file validPOS.json --output_dir guwenbert-large.pos --do_train --do_eval
+fi
+
+nawk '
+BEGIN{
+  f[0]="test.conllu";
+  f[1]="dev.conllu";
+  for(i=2;i<10;i++)
+    f[i]="train.conllu";
+}
+{
+  printf("%s\n",$0)>f[i%10];
+  if($0=="")
+    i++;
+}' simplified.conllu
+if [ ! -f guwenbert-base.pos/guwenbert-base.supar ]
+then python3 -m supar.cmds.biaffine_dep train -b -d 0 -p guwenbert-base.pos/guwenbert-base.supar -c biaffine-dep-en -f bert --bert ethanyt/guwenbert-base --train train.conllu --dev dev.conllu --test test.conllu --embed='' --proj
+fi
+if [ ! -f guwenbert-large.pos/guwenbert-large.supar ]
+then python3 -m supar.cmds.biaffine_dep train -b -d 0 -p guwenbert-large.pos/guwenbert-large.supar -c biaffine-dep-en -f bert --bert ethanyt/guwenbert-large --train train.conllu --dev dev.conllu --test test.conllu --embed='' --proj
+fi
+
+python3 -c '
+tokens=[]
+tags=[]
+i=0
+while True:
+  try:
+    s=input()
+  except:
+    if len(tokens)>0:
+      print("{\"tokens\":[\""+"\",\"".join(tokens)+"\"],\"tags\":[\""+"\",\"".join(tags)+"\"]}")
+    quit()
+  t=s.split("\t")
+  if len(t)==10:
+    for c in t[1]:
+      tokens.append(c)
+      i+=1
+  else:
+    if i==1:
+      tags.append("S")
+    elif i==2:
+      tags+=["B","E"]
+    elif i==3:
+      tags+=["B","E2","E"]
+    else:
+      tags+=["B"]+["M"]*(i-4)+["E3","E2","E"]
+    i=0
+    if len(tokens)>80:
+      print("{\"tokens\":[\""+"\",\"".join(tokens)+"\"],\"tags\":[\""+"\",\"".join(tags)+"\"]}")
+      tokens=[]
+      tags=[]
+' < simplified.conllu | tee simplifiedDanku.json | nawk '
+{
+  if(NR%10>0)
+    printf("%s\n",$0)>"trainDanku.json";
+  else
+    printf("%s\n",$0)>"validDanku.json";
+}'
+sed 's/^.*"tags":\[//' trainDanku.json | tr '"' '\012' | sort -u | egrep '^[A-Z]' > labelDanku.txt
+if [ ! -d guwenbert-base.danku ]
+then mkdir -p guwenbert-base.danku
+     python3 run_ner.py --model_name_or_path ethanyt/guwenbert-base --train_file trainDanku.json --validation_file validDanku.json --output_dir guwenbert-base.danku --do_train --do_eval
+fi
+if [ ! -d guwenbert-large.danku ]
+then mkdir -p guwenbert-large.danku
+     python3 run_ner.py --model_name_or_path ethanyt/guwenbert-large --train_file trainDanku.json --validation_file validDanku.json --output_dir guwenbert-large.danku --do_train --do_eval
+fi
+
+python3 -c '
+c=[]
+h=[0]
+while True:
+  try:
+    s=input()
+  except:
+    quit()
+  t=s.strip().split("\t")
+  if len(t)==10:
+    if t[0]!="#":
+      t[0]=str(len(c)+1)
+      i=len(t[1])
+      if i>1:
+        form=t[1]
+        lemma=t[2]
+        head=t[6]
+        deprel=t[7]
+        for j in range(0,i-1):
+          t[1]=form[j]
+          t[2]=lemma[j]
+          t[6]="-1"
+          t[7]="compound"
+          c.append(list(t))
+          t[0]=str(len(c)+1)
+        t[1]=form[i-1]
+        t[2]=lemma[i-1]
+        t[6]=head
+        t[7]=deprel
+      c.append(list(t))
+      h.append(len(c))
+  elif s.strip()=="":
+    for t in c:
+      t[6]=str(int(t[0])+1 if t[6]=="-1" else h[int(t[6])])
+      print("\t".join(t))
+    print("")
+    c=[]
+    h=[0]
+' < lzh_kyoto.conllu | tee traditional.conllu | python3 -c '
+tokens=[]
+tags=[]
+while True:
+  try:
+    s=input()
+  except:
+    if len(tokens)>0:
+      print("{\"tokens\":[\""+"\",\"".join(tokens)+"\"],\"tags\":[\""+"\",\"".join(tags)+"\"]}")
+    quit()
+  t=s.split("\t")
+  if len(t)==10:
+    p=t[4]+","+t[3]+","+t[5]
+    for c in t[1]:
+      tokens.append(c)
+      tags.append(p)
+  elif len(tokens)>80:
+    print("{\"tokens\":[\""+"\",\"".join(tokens)+"\"],\"tags\":[\""+"\",\"".join(tags)+"\"]}")
+    tokens=[]
+    tags=[]
+' | tee traditionalPOS.json | nawk '
+{
+  if(NR%10>0)
+    printf("%s\n",$0)>>"trainPOS.json";
+  else
+    printf("%s\n",$0)>>"validPOS.json";
+}'
+if [ ! -d roberta-classical-chinese-base-char.pos ]
+then mkdir -p roberta-classical-chinese-base-char.pos
+     python3 run_ner.py --model_name_or_path KoichiYasuoka/roberta-classical-chinese-base-char --train_file trainPOS.json --validation_file validPOS.json --output_dir roberta-classical-chinese-base-char.pos --do_train --do_eval
+fi
+if [ ! -d roberta-classical-chinese-large-char.pos ]
+then mkdir -p roberta-classical-chinese-large-char.pos
+     python3 run_ner.py --model_name_or_path KoichiYasuoka/roberta-classical-chinese-large-char --train_file trainPOS.json --validation_file validPOS.json --output_dir roberta-classical-chinese-large-char.pos --do_train --do_eval
+fi
+
+nawk '
+BEGIN{
+  f[0]="test.conllu";
+  f[1]="dev.conllu";
+  for(i=2;i<10;i++)
+    f[i]="train.conllu";
+}
+{
+  printf("%s\n",$0)>>f[i%10];
+  if($0=="")
+    i++;
+}' traditional.conllu
+if [ ! -f roberta-classical-chinese-base-char.pos/roberta-classical-chinese-base-char.supar ]
+then python3 -m supar.cmds.biaffine_dep train -b -d 0 -p roberta-classical-chinese-base-char.pos/roberta-classical-chinese-base-char.supar -c biaffine-dep-en -f bert --bert KoichiYasuoka/roberta-classical-chinese-base-char --train train.conllu --dev dev.conllu --test test.conllu --embed='' --proj
+fi
+if [ ! -f roberta-classical-chinese-large-char.pos/roberta-classical-chinese-large-char.supar ]
+then python3 -m supar.cmds.biaffine_dep train -b -d 0 -p roberta-classical-chinese-large-char.pos/roberta-classical-chinese-large-char.supar -c biaffine-dep-en -f bert --bert KoichiYasuoka/roberta-classical-chinese-large-char --train train.conllu --dev dev.conllu --test test.conllu --embed='' --proj
+fi
+
+python3 -c '
+tokens=[]
+tags=[]
+i=0
+while True:
+  try:
+    s=input()
+  except:
+    if len(tokens)>0:
+      print("{\"tokens\":[\""+"\",\"".join(tokens)+"\"],\"tags\":[\""+"\",\"".join(tags)+"\"]}")
+    quit()
+  t=s.split("\t")
+  if len(t)==10:
+    for c in t[1]:
+      tokens.append(c)
+      i+=1
+  else:
+    if i==1:
+      tags.append("S")
+    elif i==2:
+      tags+=["B","E"]
+    elif i==3:
+      tags+=["B","E2","E"]
+    else:
+      tags+=["B"]+["M"]*(i-4)+["E3","E2","E"]
+    i=0
+    if len(tokens)>80:
+      print("{\"tokens\":[\""+"\",\"".join(tokens)+"\"],\"tags\":[\""+"\",\"".join(tags)+"\"]}")
+      tokens=[]
+      tags=[]
+' < traditional.conllu | tee traditionalDanku.json | nawk '
+{
+  if(NR%10>0)
+    printf("%s\n",$0)>>"trainDanku.json";
+  else
+    printf("%s\n",$0)>>"validDanku.json";
+}'
+if [ ! -d roberta-classical-chinese-base-char.danku ]
+then mkdir -p roberta-classical-chinese-base-char.danku
+     python3 run_ner.py --model_name_or_path KoichiYasuoka/roberta-classical-chinese-base-char --train_file trainDanku.json --validation_file validDanku.json --output_dir roberta-classical-chinese-base-char.danku --do_train --do_eval
+fi
+if [ ! -d roberta-classical-chinese-large-char.danku ]
+then mkdir -p roberta-classical-chinese-large-char.danku
+     python3 run_ner.py --model_name_or_path KoichiYasuoka/roberta-classical-chinese-large-char --train_file trainDanku.json --validation_file validDanku.json --output_dir roberta-classical-chinese-large-char.danku --do_train --do_eval
+fi
+
+nawk '
+{
+  if(NR%10>0)
+    printf("%s\n",$0)>"trainPOS.json";
+  else
+    printf("%s\n",$0)>"validPOS.json";
+}' traditionalPOS.json
+if [ ! -d sikubert.pos ]
+then mkdir -p sikubert.pos
+     python3 run_ner.py --model_name_or_path SIKU-BERT/sikubert --train_file trainPOS.json --validation_file validPOS.json --output_dir sikubert.pos --do_train --do_eval
+fi
+if [ ! -d sikuroberta.pos ]
+then mkdir -p sikuroberta.pos
+     python3 run_ner.py --model_name_or_path SIKU-BERT/sikuroberta --train_file trainPOS.json --validation_file validPOS.json --output_dir sikuroberta.pos --do_train --do_eval
+fi
+
+nawk '
+BEGIN{
+  f[0]="test.conllu";
+  f[1]="dev.conllu";
+  for(i=2;i<10;i++)
+    f[i]="train.conllu";
+}
+{
+  printf("%s\n",$0)>f[i%10];
+  if($0=="")
+    i++;
+}' traditional.conllu
+if [ ! -f sikubert.pos/sikubert.supar ]
+then python3 -m supar.cmds.biaffine_dep train -b -d 0 -p sikubert.pos/sikubert.supar -c biaffine-dep-en -f bert --bert SIKU-BERT/sikubert --train train.conllu --dev dev.conllu --test test.conllu --embed='' --proj
+fi
+if [ ! -f sikuroberta.pos/sikuroberta.supar ]
+then python3 -m supar.cmds.biaffine_dep train -b -d 0 -p sikuroberta.pos/sikuroberta.supar -c biaffine-dep-en -f bert --bert SIKU-BERT/sikuroberta --train train.conllu --dev dev.conllu --test test.conllu --embed='' --proj
+fi
+
+nawk '
+{
+  if(NR%10>0)
+    printf("%s\n",$0)>"trainDanku.json";
+  else
+    printf("%s\n",$0)>"validDanku.json";
+}' traditionalDanku.json
+if [ ! -d sikubert.danku ]
+then mkdir -p sikubert.danku
+     python3 run_ner.py --model_name_or_path SIKU-BERT/sikubert --train_file trainDanku.json --validation_file validDanku.json --output_dir sikubert.danku --do_train --do_eval
+fi
+if [ ! -d sikuroberta.danku ]
+then mkdir -p sikuroberta.danku
+     python3 run_ner.py --model_name_or_path SIKU-BERT/sikuroberta --train_file trainDanku.json --validation_file validDanku.json --output_dir sikuroberta.danku --do_train --do_eval
+fi
+
+exit 0

suparkanbun/models/splitter.sh

@@ -0,0 +1,6 @@
+#! /bin/sh
+for F
+do split -a 2 -b 83886080 --numeric-suffixes=01 $F $F.
+   ls -1 $F.0[1-9] | sed 's/^\(.*\)0\([1-9]\)$/mv & \1\2/' | sh
+done
+exit 0