|
#!/bin/sh |
|
|
|
wget https://lindat.mff.cuni.cz/repository/xmlui/bitstream/handle/11234/1-4611/ud-treebanks-v2.9.tgz |
|
tar xf ud-treebanks-v2.9.tgz |
|
rm ud-treebanks-v2.9.tgz |
|
for tb in ud-treebanks-v2.9/*/*-ud-train.conllu; do |
|
dir=`dirname $tb` |
|
long_name=`basename $dir` |
|
code=`basename $tb` |
|
code=${code%%-*} |
|
|
|
case $long_name in |
|
UD_Arabic-NYUAD|UD_English-ESL|UD_English-GUMReddit|UD_French-FTB|UD_Hindi_English-HIENCS|UD_Japanese-BCCWJ|UD_Japanese-BCCWJLUW) echo Skipping treebank $long_name without forms; continue;; |
|
esac |
|
|
|
mkdir -p "$code" |
|
cp "$dir"/* "$code" |
|
if [ -f "$dir"/"$code"-ud-dev.conllu ]; then |
|
cp "$dir"/"$code"-ud-train.conllu "$dir"/"$code"-ud-dev.conllu "$code" |
|
else |
|
perl conllu_split.pl "$code" "$code" <"$dir"/"$code"-ud-train.conllu |
|
fi |
|
|
|
train_words=$(grep -c '^[0-9]' $code/*-ud-train.conllu) |
|
[ "$train_words" -lt 1000 ] && { echo Skipping treebank $code with $train_words training words. >&2; rm -r "$code/"; continue; } |
|
echo $code $long_name | tee -a iso_names.txt |
|
|
|
for conllu in "$code"/*.conllu; do |
|
sed "1i# variant = $code" -i "$conllu" |
|
case "$conllu" in |
|
*train.conllu) rm "${conllu%.conllu}.txt";; |
|
*) perl conllu_to_text.pl --language="$code" <"$conllu" >"${conllu%.conllu}.txt";; |
|
esac |
|
done |
|
done |
|
rm -r ud-treebanks-v2.9 |
|
|
|
for code in $(for d in */; do echo ${d%%[_/]*}; done | sort | uniq -d); do |
|
mkdir "${code}_all" |
|
cat "$code"*/*train.conllu >"${code}_all/${code}_all-ud-train.conllu" |
|
done |
|
|
