fairseq/examples/multilingual/data_scripts/download_wmt20.sh

#!/bin/bash
# Copyright (c) Facebook, Inc. and its affiliates.
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.

if [ -z $WORKDIR_ROOT ] ;
then
        echo "please specify your working directory root in environment variable WORKDIR_ROOT. Exitting..."
        exit
fi



set -x -e

# TODO update the workdir and dest dir name
# put fasttext model
WORKDIR=$WORKDIR_ROOT
# put intermediate files
TMP_DIR=$WORKDIR_ROOT/tmp/tmp_wmt20_lowres_download
# output {train,valid,test} files to dest
DEST=$WORKDIR_ROOT/ML50/raw

UTILS=$PWD/utils

# per dataset locations
COMMONCRAWL_DIR=$TMP_DIR/commoncrawl
YANDEX_CORPUS=$WORKDIR_ROOT/wmt20/official/ru/yandex/1mcorpus.zip
# unzipped
CZENG_CORPUS=$WORKDIR_ROOT/wmt20/official/cs/czeng/czeng20-train
CCMT_DIR=$WORKDIR_ROOT/wmt20/official/zh/ccmt/parallel

download_and_select() {
  SUBFOLDER=$1
  URL=$2
  UNCOMPRESS_CMD=$3
  LANG=$4
  INPUT_FILEPATH=$5
  if [[ $# -gt 5 ]]; then
    LANG_COL=$6
    EN_COL=$7
  fi

  mkdir -p $SUBFOLDER
  cd $SUBFOLDER
  wget -nc --content-disposition $URL
  $UNCOMPRESS_CMD

  if [[ $# -gt 5 ]]; then
    cut -f$LANG_COL $INPUT_FILEPATH > $INPUT_FILEPATH.$LANG
    cut -f$EN_COL $INPUT_FILEPATH > $INPUT_FILEPATH.en
  fi
  cd ..

  ln -sf $SUBFOLDER/$INPUT_FILEPATH.$LANG $SUBFOLDER.$LANG
  ln -sf $SUBFOLDER/$INPUT_FILEPATH.en $SUBFOLDER.en
}

prepare_lid() {
  pip install fasttext

  # TODO specify global workdir
  MODEL=$WORKDIR/fasttext/lid.176.bin
  LID_MULTI=$UTILS/fasttext_multi_filter.py

  if [ ! -f "$MODEL" ]; then
    echo "downloading fasttext lid model..."
    mkdir -p $WORKDIR/fasttext
    wget -nc https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin -O $MODEL
  fi
}

prepare_moses() {
  pushd $UTILS
  echo 'Cloning Moses github repository (for tokenization scripts)...'
  git clone https://github.com/moses-smt/mosesdecoder.git  
  popd
}

lid_filter() {
  # TODO specify global workdir
  MODEL=$WORKDIR/fasttext/lid.176.bin
  LID_MULTI=$UTILS/fasttext_multi_filter.py

  prepare_lid

  SRC=$1
  SRC_FILE=$2
  SRC_OUTPUT=$3
  TGT=$4
  TGT_FILE=$5
  TGT_OUTPUT=$6
  python $LID_MULTI --model $MODEL --inputs $SRC_FILE $TGT_FILE --langs $SRC $TGT --outputs $SRC_OUTPUT $TGT_OUTPUT
}

prepare_ja_ted() {
  mkdir -p ted
  cd ted

  wget -nc https://wit3.fbk.eu/archive/2017-01-trnted//texts/en/ja/en-ja.tgz
  tar -zxvf en-ja.tgz
  cat en-ja/train.tags.en-ja.en | grep -v -P "^[ ]*\<" | sed 's/^[ \t]*//g' | sed 's/[ \t]*$//g' > en-ja/train.en-ja.en
  cat en-ja/train.tags.en-ja.ja | grep -v -P "^[ ]*\<" | sed 's/^[ \t]*//g' | sed 's/[ \t]*$//g' > en-ja/train.en-ja.ja

  cd ..
  ln -sf ted/en-ja/train.en-ja.ja ted.ja
  ln -sf ted/en-ja/train.en-ja.en ted.en
}

prepare_ja() {
  OUTPUT_DIR=$TMP_DIR/ja
  mkdir -p $OUTPUT_DIR
  cd $OUTPUT_DIR

  download_and_select paracrawl "http://www.kecl.ntt.co.jp/icl/lirg/jparacrawl/release/2.0/bitext/en-ja.tar.gz" "tar -zxvf en-ja.tar.gz" ja en-ja/en-ja.bicleaner05.txt 4 3 &
  download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.en-ja.tsv.gz" "gunzip -f news-commentary-v15.en-ja.tsv.gz" ja news-commentary-v15.en-ja.tsv 2 1 &
  download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.ja-en.tsv.gz" "gunzip -f wikititles-v2.ja-en.tsv.gz" ja wikititles-v2.ja-en.tsv 1 2 &
  download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-ja.langid.tsv.gz" "gunzip -f WikiMatrix.v1.en-ja.langid.tsv.gz" ja WikiMatrix.v1.en-ja.langid.tsv 3 2 &
  download_and_select subtitle "https://nlp.stanford.edu/projects/jesc/data/split.tar.gz" "tar -zxvf split.tar.gz" ja split/train 2 1 &
  download_and_select kftt "http://www.phontron.com/kftt/download/kftt-data-1.0.tar.gz" "tar -zxvf kftt-data-1.0.tar.gz" ja kftt-data-1.0/data/orig/kyoto-train &

  prepare_ja_ted &

  # ted data needs to 

  wait

  # remove previous results
  rm -f all.??
  find ./ -maxdepth 1 -name "*.ja" | sort -V | xargs cat > all.ja
  find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
  lid_filter ja all.ja $DEST/train.ja_XX-en_XX.ja_XX en all.en $DEST/train.ja_XX-en_XX.en_XX
}

prepare_ta() {
  OUTPUT_DIR=$TMP_DIR/ta
  mkdir -p $OUTPUT_DIR
  cd $OUTPUT_DIR

  download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.ta-en.tsv.gz" "gunzip -f wikititles-v2.ta-en.tsv.gz" ta wikititles-v2.ta-en.tsv 1 2 &
  download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-ta.langid.tsv.gz" "gunzip -f WikiMatrix.v1.en-ta.langid.tsv.gz" ta WikiMatrix.v1.en-ta.langid.tsv 3 2 &
  download_and_select pmindia "http://data.statmt.org/pmindia/v1/parallel/pmindia.v1.ta-en.tsv" "" ta pmindia.v1.ta-en.tsv 2 1 &
  download_and_select tanzil "https://object.pouta.csc.fi/OPUS-Tanzil/v1/moses/en-ta.txt.zip" "unzip en-ta.txt.zip" ta Tanzil.en-ta &
  download_and_select pib "http://preon.iiit.ac.in/~jerin/resources/datasets/pib-v0.tar" "tar -xvf pib-v0.tar" ta pib/en-ta/train &
  download_and_select mkb "http://preon.iiit.ac.in/~jerin/resources/datasets/mkb-v0.tar" "tar -xvf mkb-v0.tar" ta mkb/en-ta/mkb &
  download_and_select ufal "http://ufal.mff.cuni.cz/~ramasamy/parallel/data/v2/en-ta-parallel-v2.tar.gz" "tar -zxvf en-ta-parallel-v2.tar.gz" ta en-ta-parallel-v2/corpus.bcn.train &

  wait

  # need special handling for nlpc
  mkdir -p nlpc
  cd nlpc
  wget -nc https://raw.githubusercontent.com/nlpc-uom/English-Tamil-Parallel-Corpus/master/En-Ta%20Corpus/En-Ta%20English.txt
  wget -nc https://github.com/nlpc-uom/English-Tamil-Parallel-Corpus/raw/master/En-Ta%20Corpus/En-Ta%20Tamil.txt
  tail -n +4 "En-Ta English.txt" > en-ta.en
  tail -n +4 "En-Ta Tamil.txt" > en-ta.ta
  cd ..
  ln -sf nlpc/en-ta.en nlpc.en
  ln -sf nlpc/en-ta.ta nlpc.ta

  # remove previous results
  rm -f all.??
  find ./ -maxdepth 1 -name "*.ta" | sort -V | xargs cat > all.ta
  find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
  lid_filter ta all.ta $DEST/train.ta_IN-en_XX.ta_IN en all.en $DEST/train.ta_IN-en_XX.en_XX
}

prepare_iu() {
  OUTPUT_DIR=$TMP_DIR/iu
  mkdir -p $OUTPUT_DIR
  cd $OUTPUT_DIR
  
  download_and_select nh "https://nrc-digital-repository.canada.ca/eng/view/dataset/?id=c7e34fa7-7629-43c2-bd6d-19b32bf64f60" "tar -zxvf Nunavut-Hansard-Inuktitut-English-Parallel-Corpus-3.0.1.tgz" iu Nunavut-Hansard-Inuktitut-English-Parallel-Corpus-3.0/NunavutHansard > /dev/null &
  download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.iu-en.tsv.gz" "gunzip -f wikititles-v2.iu-en.tsv.gz" iu wikititles-v2.iu-en.tsv 1 2 &

  wait

  # remove previous results
  rm -f all.??
  find ./ -maxdepth 1 -name "*.iu" | sort -V | xargs cat | nh/Nunavut-Hansard-Inuktitut-English-Parallel-Corpus-3.0/scripts/normalize-iu-spelling.pl > all.iu
  find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
  paste all.iu all.en | awk -F $'\t' '$1!=""&&$2!=""' > all.iuen
  cut -f1 all.iuen > $DEST/train.iu_CA-en_XX.iu_CA
  cut -f2 all.iuen > $DEST/train.iu_CA-en_XX.en_XX
}

prepare_km() {
  OUTPUT_DIR=$TMP_DIR/km
  mkdir -p $OUTPUT_DIR
  cd $OUTPUT_DIR

  download_and_select paracrawl "http://data.statmt.org/wmt20/translation-task/ps-km/wmt20-sent.en-km.xz" "unxz wmt20-sent.en-km.zx" km wmt20-sent.en-km 2 1 &

  # km-parallel has multiple sets, concat all of them together
  mkdir -p opus
  cd opus
  wget -nc "http://data.statmt.org/wmt20/translation-task/ps-km/km-parallel.tgz"
  tar -zxvf km-parallel.tgz
  find ./km-parallel -maxdepth 1 -name "*.km" | sort -V | xargs cat > opus.km
  find ./km-parallel -maxdepth 1 -name "*.en" | sort -V | xargs cat > opus.en
  cd ..
  ln -sf opus/opus.km .
  ln -sf opus/opus.en .

  wait

  # remove previous results
  rm -f all.??
  find ./ -maxdepth 1 -name "*.km" | sort -V | xargs cat > all.km
  find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
  lid_filter km all.km $DEST/train.km_KH-en_XX.km_KH en all.en $DEST/train.km_KH-en_XX.en_XX
}

prepare_ps() {
  OUTPUT_DIR=$TMP_DIR/ps
  mkdir -p $OUTPUT_DIR
  cd $OUTPUT_DIR

  download_and_select paracrawl "http://data.statmt.org/wmt20/translation-task/ps-km/wmt20-sent.en-ps.xz" "unxz wmt20-sent.en-ps.xz" ps wmt20-sent.en-ps 2 1 &
  download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.ps-en.tsv.gz" "gunzip -f wikititles-v2.ps-en.tsv.gz" ps wikititles-v2.ps-en.tsv 1 2 &
  # ps-parallel has multiple sets, concat all of them together
  mkdir -p opus
  cd opus
  wget -nc "http://data.statmt.org/wmt20/translation-task/ps-km/ps-parallel.tgz"
  tar -zxvf ps-parallel.tgz
  find ./ps-parallel -maxdepth 1 -name "*.ps" | sort -V | xargs cat > opus.ps
  find ./ps-parallel -maxdepth 1 -name "*.en" | sort -V | xargs cat > opus.en
  cd ..
  ln -sf opus/opus.ps opus.ps
  ln -sf opus/opus.en opus.en

  wait

  # remove previous results
  rm -f all.??
  find ./ -maxdepth 1 -name "*.ps" | sort -V | xargs cat > all.ps
  find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
  lid_filter ps all.ps $DEST/train.ps_AF-en_XX.ps_AF en all.en $DEST/train.ps_AF-en_XX.en_XX
}

download_commoncrawl() {
  mkdir -p $COMMONCRAWL_DIR
  cd $COMMONCRAWL_DIR

  wget -nc "http://www.statmt.org/wmt13/training-parallel-commoncrawl.tgz"
  tar -zxvf training-parallel-commoncrawl.tgz
}
link_commoncrawl() {
  LANG=$1
  ln -sf $COMMONCRAWL_DIR/commoncrawl.$LANG-en.en commoncrawl.en
  ln -sf $COMMONCRAWL_DIR/commoncrawl.$LANG-en.$LANG commoncrawl.$LANG
}

strip_xlf() {
  INPUT_FILE=$1
  SRC=$2
  TGT=$3
  grep '<source xml:lang=' $INPUT_FILE | sed 's/^<[^<>]*>//g' | sed 's/<[^<>]*>$//g' > $INPUT_FILE.$SRC
  grep '<target xml:lang=' $INPUT_FILE | sed 's/^<[^<>]*>//g' | sed 's/<[^<>]*>$//g' > $INPUT_FILE.$TGT
}

download_and_process_tilde() {
  URL=$1
  UNCOMPRESS_CMD=$2
  FILENAME=$3
  LANG=$4
  PROCESS_CMD=$5

  mkdir -p tilde
  cd tilde
  wget -nc $URL
  $UNCOMPRESS_CMD
  echo "executing cmd"
  echo $PROCESS_CMD
  $PROCESS_CMD
  cd ..
  ln -sf tilde/$FILENAME.$LANG tilde.$LANG
  ln -sf tilde/$FILENAME.en tilde.en
}

prepare_cs() {
  OUTPUT_DIR=$TMP_DIR/cs
  mkdir -p $OUTPUT_DIR
  cd $OUTPUT_DIR

  #download_and_select europarl "http://www.statmt.org/europarl/v10/training/europarl-v10.cs-en.tsv.gz" "gunzip europarl-v10.cs-en.tsv.gz" cs europarl-v10.cs-en.tsv 1 2 &
  #download_and_select paracrawl "https://s3.amazonaws.com/web-language-models/paracrawl/release5.1/en-cs.txt.gz" "gunzip en-cs.txt.gz" cs en-cs.txt 2 1 &
  #link_commoncrawl cs
  #download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.cs-en.tsv.gz" "gunzip news-commentary-v15.cs-en.tsv.gz" cs news-commentary-v15.cs-en.tsv 1 2 &
  #download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.cs-en.tsv.gz" "gunzip wikititles-v2.cs-en.tsv.gz" cs wikititles-v2.cs-en.tsv 1 2 &
  #download_and_process_tilde "http://data.statmt.org/wmt20/translation-task/rapid/RAPID_2019.cs-en.xlf.gz" "gunzip RAPID_2019.cs-en.xlf.gz" RAPID_2019.cs-en.xlf cs "strip_xlf RAPID_2019.cs-en.xlf cs en" &
  #download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.cs-en.langid.tsv.gz" "gunzip WikiMatrix.v1.cs-en.langid.tsv.gz" cs WikiMatrix.v1.cs-en.langid.tsv 2 3 &

  #wait

  # remove previous results
  #rm -f all.??
  #find ./ -maxdepth 1 -name "*.cs" | sort -V | xargs cat > all.cs
  #find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
  if [ -z $CZENG_CORPUS ] ;
  then
          echo "Please download CZENG_CORPUS manually and place them at $CZENG_CORPUS. Exitting..."
          exit
  fi  
  cat $CZENG_CORPUS | sed '/^$/d' | cut -f5 > all.cs
  cat $CZENG_CORPUS | sed '/^$/d' | cut -f6 > all.en

  lid_filter cs all.cs $DEST/train.cs_CZ-en_XX.cs_CZ en all.en $DEST/train.cs_CZ-en_XX.en_XX
}

prepare_de() {
  OUTPUT_DIR=$TMP_DIR/de
  mkdir -p $OUTPUT_DIR
  cd $OUTPUT_DIR

  download_and_select europarl "http://www.statmt.org/europarl/v10/training/europarl-v10.de-en.tsv.gz" "gunzip europarl-v10.de-en.tsv.gz" de europarl-v10.de-en.tsv 1 2 &
  download_and_select paracrawl "https://s3.amazonaws.com/web-language-models/paracrawl/release5.1/en-de.txt.gz"  "gunzip en-de.txt.gz" de en-de.txt 2 1 &
  link_commoncrawl de
  download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.de-en.tsv.gz" "gunzip news-commentary-v15.de-en.tsv.gz" de news-commentary-v15.de-en.tsv 1 2 &
  download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.de-en.tsv.gz" "gunzip wikititles-v2.de-en.tsv.gz" de wikititles-v2.de-en.tsv 1 2 &
  download_and_process_tilde "http://data.statmt.org/wmt20/translation-task/rapid/RAPID_2019.de-en.xlf.gz" "gunzip RAPID_2019.de-en.xlf.gz" RAPID_2019.de-en.xlf de "strip_xlf RAPID_2019.de-en.xlf de en" &
  download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.de-en.langid.tsv.gz" "gunzip WikiMatrix.v1.de-en.langid.tsv.gz" de WikiMatrix.v1.de-en.langid.tsv 2 3 &

  wait

  # remove previous results
  rm -f all.??
  find ./ -maxdepth 1 -name "*.de" | sort -V | xargs cat > all.de
  find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
  lid_filter de all.de $DEST/train.de_DE-en_XX.de_DE en all.en $DEST/train.de_DE-en_XX.en_XX
}

prepare_tmx() {
  TMX_FILE=$1
  git clone https://github.com/amake/TMX2Corpus $UTILS/tmx2corpus
  pip install tinysegmenter

  python $UTILS/tmx2corpus/tmx2corpus.py $TMX_FILE
}

prepare_pl() {
  OUTPUT_DIR=$TMP_DIR/pl
  mkdir -p $OUTPUT_DIR
  cd $OUTPUT_DIR

  # download_and_select europarl "http://www.statmt.org/europarl/v10/training/europarl-v10.pl-en.tsv.gz" "gunzip europarl-v10.pl-en.tsv.gz" pl europarl-v10.pl-en.tsv 1 2 &
  # download_and_select paracrawl "https://s3.amazonaws.com/web-language-models/paracrawl/release5.1/en-pl.txt.gz" "gunzip en-pl.txt.gz" pl en-pl.txt 2 1 &
  # download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.pl-en.tsv.gz" "gunzip wikititles-v2.pl-en.tsv.gz" pl wikititles-v2.pl-en.tsv 1 2 &
  download_and_select tilde "https://tilde-model.s3-eu-west-1.amazonaws.com/rapid2019.en-pl.tmx.zip" "gunzip rapid2019.en-pl.tmx.zip" bitext pl "prepare_tmx RAPID_2019.UNIQUE.en-pl.tmx" &
  # download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-pl.langid.tsv.gz" "gunzip WikiMatrix.v1.en-pl.langid.tsv.gz" pl WikiMatrix.v1.en-pl.langid.tsv 3 2 &

  wait

  # remove previous results
  rm -f all.??
  find ./ -maxdepth 1 -name "*.pl" | sort -V | xargs cat > all.pl
  find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
  lid_filter pl all.pl $DEST/train.pl_PL-en_XX.pl_PL en all.en $DEST/train.pl_PL-en_XX.en_XX
}

prepare_uncorpus() {
  $URLS=$1
  $FILES=$2

  mkdir -p uncorpus
  cd uncorpus

  for URL in $URLS; do
    wget -nc $URL
  done
  cat $FILES > uncorpus.tar.gz
  tar -zxvf uncorpus.tar.gz

  cd ..
  ln -sf uncorpus/en-$LANG/UNv1.0.en-$LANG.$LANG uncorpus.$LANG
  ln -sf uncorpus/en-$LANG/UNv1.0.en-$LANG.en uncorpus.en
}

prepare_yandex() {
  mkdir -p yandex
  cd yandex
  unzip $YANDEX_CORPUS ./
  cd ..
  ln -s yandex/corpus.en_ru.1m.en yandex.en
  ln -s yandex/corpus.en_ru.1m.ru yandex.ru
}

prepare_ru() {
  OUTPUT_DIR=$TMP_DIR/ru
  mkdir -p $OUTPUT_DIR
  cd $OUTPUT_DIR

  download_and_select paracrawl "https://s3.amazonaws.com/web-language-models/paracrawl/release1/paracrawl-release1.en-ru.zipporah0-dedup-clean.tgz" "tar -zxvf paracrawl-release1.en-ru.zipporah0-dedup-clean.tgz" ru paracrawl-release1.en-ru.zipporah0-dedup-clean &
  link_commoncrawl ru
  download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.en-ru.tsv.gz" "gunzip news-commentary-v15.en-ru.tsv.gz" ru news-commentary-v15.en-ru.tsv 2 1 &
  prepare_yandex &
  download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.ru-en.tsv.gz" "gunzip wikititles-v2.ru-en.tsv.gz" ru wikititles-v2.ru-en.tsv 1 2 &
  prepare_uncorpus "https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.00 https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.01 https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-ru.tar.gz.02" "UNv1.0.en-ru.tar.gz.00 UNv1.0.en-ru.tar.gz.01 UNv1.0.en-ru.tar.gz.02" &
  download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-ru.langid.tsv.gz" "gunzip WikiMatrix.v1.en-ru.langid.tsv.gz" ru WikiMatrix.v1.en-ru.langid.tsv 3 2 &

  wait

  # remove previous results
  rm -f all.??
  find ./ -maxdepth 1 -name "*.ru" | sort -V | xargs cat > all.ru
  find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
  lid_filter ru all.ru $DEST/train.ru_RU-en_XX.ru_RU en all.en $DEST/train.ru_RU-en_XX.en_XX
}

prepare_ccmt() {
  mkdir -p ccmt
  cd ccmt
  # assume ccmt data is already unzipped under CCMT_DIR folder
  cat $CCMT_DIR/datum2017/Book*_cn.txt | sed 's/ //g' > datum2017.detok.zh
  cat $CCMT_DIR/datum2017/Book*_en.txt > datum2017.detok.en
  cat $CCMT_DIR/casict2011/casict-A_ch.txt $CCMT_DIR/casict2011/casict-B_ch.txt $CCMT_DIR/casict2015/casict2015_ch.txt $CCMT_DIR/datum2015/datum_ch.txt $CCMT_DIR/neu2017/NEU_cn.txt datum2017.detok.zh > ccmt.zh
  cat $CCMT_DIR/casict2011/casict-A_en.txt $CCMT_DIR/casict2011/casict-B_en.txt $CCMT_DIR/casict2015/casict2015_en.txt $CCMT_DIR/datum2015/datum_en.txt $CCMT_DIR/neu2017/NEU_en.txt datum2017.detok.en > ccmt.en
  cd ..
  ln -sf ccmt/ccmt.zh ccmt.zh
  ln -sf ccmt/ccmt.en ccmt.en
}

prepare_zh() {
  OUTPUT_DIR=$TMP_DIR/zh
  mkdir -p $OUTPUT_DIR
  cd $OUTPUT_DIR

  download_and_select newscommentary "http://data.statmt.org/news-commentary/v15/training/news-commentary-v15.en-zh.tsv.gz" "gunzip news-commentary-v15.en-zh.tsv.gz" zh news-commentary-v15.en-zh.tsv 2 1 &
  download_and_select wikititles "http://data.statmt.org/wikititles/v2/wikititles-v2.zh-en.tsv.gz" "gunzip wikititles-v2.zh-en.tsv.gz" zh wikititles-v2.zh-en.tsv 1 2 &
  prepare_uncorpus "https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-zh.tar.gz.00 https://stuncorpusprod.blob.core.windows.net/corpusfiles/UNv1.0.en-zh.tar.gz.01" "UNv1.0.en-zh.tar.gz.00 UNv1.0.en-zh.tar.gz.01" &
  prepare_ccmt &
  download_and_select wikimatrix "http://data.statmt.org/wmt20/translation-task/WikiMatrix/WikiMatrix.v1.en-zh.langid.tsv.gz" "gunzip WikiMatrix.v1.en-zh.langid.tsv.gz" zh WikiMatrix.v1.en-zh.langid.tsv 3 2 &

  wait

  # remove previous results
  rm -f all.??
  find ./ -maxdepth 1 -name "*.zh" | sort -V | xargs cat > all.zh
  find ./ -maxdepth 1 -name "*.en" | sort -V | xargs cat > all.en
  lid_filter zh all.zh $DEST/train.zh_CN-en_XX.zh_CN en all.en $DEST/train.zh_CN-en_XX.en_XX
}

prepare_tests() {
  OUTPUT_DIR=$TMP_DIR
  mkdir -p $OUTPUT_DIR
  cd $OUTPUT_DIR
  wget -nc http://data.statmt.org/wmt20/translation-task/dev.tgz
  tar -zxvf dev.tgz
  cd dev

  cat newsdev2020-jaen-src.ja.sgm | $UTILS/strip_sgm.sh > newsdev2020-jaen.ja
  cat newsdev2020-jaen-ref.en.sgm | $UTILS/strip_sgm.sh > newsdev2020-jaen.en
  split newsdev2020-jaen.ja -a 0 -n r/1/2 > $DEST/valid.ja_XX-en_XX.ja_XX
  split newsdev2020-jaen.en -a 0 -n r/1/2 > $DEST/valid.ja_XX-en_XX.en_XX
  split newsdev2020-jaen.ja -a 0 -n r/2/2 > $DEST/test.ja_XX-en_XX.ja_XX
  split newsdev2020-jaen.en -a 0 -n r/2/2 > $DEST/test.ja_XX-en_XX.en_XX

  cat newsdev2020-iuen-src.iu.sgm | strip_sgm.sh > newsdev2020-iuen.iu
  cat newsdev2020-iuen-ref.en.sgm | strip_sgm.sh > newsdev2020-iuen.en
  split newsdev2020-iuen.iu -a 0 -n r/1/2 > $DEST/valid.iu_CA-en_XX.iu_CA
  split newsdev2020-iuen.en -a 0 -n r/1/2 > $DEST/valid.iu_CA-en_XX.en_XX
  split newsdev2020-iuen.iu -a 0 -n r/2/2 > $DEST/test.iu_CA-en_XX.iu_CA
  split newsdev2020-iuen.en -a 0 -n r/2/2 > $DEST/test.iu_CA-en_XX.en_XX

  cat newsdev2020-taen-src.ta.sgm | strip_sgm.sh > newsdev2020-taen.ta
  cat newsdev2020-taen-ref.en.sgm | strip_sgm.sh > newsdev2020-taen.en
  split newsdev2020-taen.ta -a 0 -n r/1/2 > $DEST/valid.ta_IN-en_XX.ta_IN
  split newsdev2020-taen.en -a 0 -n r/1/2 > $DEST/valid.ta_IN-en_XX.en_XX
  split newsdev2020-taen.ta -a 0 -n r/2/2 > $DEST/test.ta_IN-en_XX.ta_IN
  split newsdev2020-taen.en -a 0 -n r/2/2 > $DEST/test.ta_IN-en_XX.en_XX

  cp wikipedia.dev.km-en.km $DEST/valid.km_KH-en_XX.km_KH
  cp wikipedia.dev.km-en.en $DEST/valid.km_KH-en_XX.en_XX
  cp wikipedia.devtest.km-en.km $DEST/test.km_KH-en_XX.km_KH
  cp wikipedia.devtest.km-en.en $DEST/test.km_KH-en_XX.en_XX

  cp wikipedia.dev.ps-en.ps $DEST/valid.ps_AF-en_XX.ps_AF
  cp wikipedia.dev.ps-en.en $DEST/valid.ps_AF-en_XX.en_XX
  cp wikipedia.devtest.ps-en.ps $DEST/test.ps_AF-en_XX.ps_AF
  cp wikipedia.devtest.ps-en.en $DEST/test.ps_AF-en_XX.en_XX

  cat newsdev2020-plen-src.pl.sgm | strip_sgm.sh > newsdev2020-plen.pl
  cat newsdev2020-plen-ref.en.sgm | strip_sgm.sh > newsdev2020-plen.en
  split newsdev2020-plen.pl -a 0 -n r/1/2 > $DEST/valid.pl_PL-en_XX.pl_PL
  split newsdev2020-plen.en -a 0 -n r/1/2 > $DEST/valid.pl_PL-en_XX.en_XX
  split newsdev2020-plen.pl -a 0 -n r/2/2 > $DEST/test.pl_PL-en_XX.pl_PL
  split newsdev2020-plen.en -a 0 -n r/2/2 > $DEST/test.pl_PL-en_XX.en_XX

  cat newstest2018-encs-src.en.sgm | strip_sgm.sh > $DEST/valid.en_XX-cs_CZ.en_XX
  cat newstest2018-encs-ref.cs.sgm | strip_sgm.sh > $DEST/valid.en_XX-cs_CZ.cs_CZ
  cat newstest2019-encs-src.en.sgm | strip_sgm.sh > $DEST/test.en_XX-cs_CZ.en_XX
  cat newstest2019-encs-ref.cs.sgm | strip_sgm.sh > $DEST/test.en_XX-cs_CZ.cs_CZ

  cat newstest2018-deen-src.de.sgm | strip_sgm.sh > $DEST/valid.de_DE-en_XX.de_DE
  cat newstest2018-deen-ref.en.sgm | strip_sgm.sh > $DEST/valid.de_DE-en_XX.en_XX
  cat newstest2018-ende-src.en.sgm | strip_sgm.sh > $DEST/valid.en_XX-de_DE.en_XX
  cat newstest2018-ende-ref.de.sgm | strip_sgm.sh > $DEST/valid.en_XX-de_DE.de_DE
  cat newstest2019-deen-src.de.sgm | strip_sgm.sh > $DEST/test.de_DE-en_XX.de_DE
  cat newstest2019-deen-ref.en.sgm | strip_sgm.sh > $DEST/test.de_DE-en_XX.en_XX
  cat newstest2019-ende-src.en.sgm | strip_sgm.sh > $DEST/test.en_XX-de_DE.en_XX
  cat newstest2019-ende-ref.de.sgm | strip_sgm.sh > $DEST/test.en_XX-de_DE.de_DE

  cat newstest2018-ruen-src.ru.sgm | strip_sgm.sh > $DEST/valid.ru_RU-en_XX.ru_RU
  cat newstest2018-ruen-ref.en.sgm | strip_sgm.sh > $DEST/valid.ru_RU-en_XX.en_XX
  cat newstest2018-enru-src.en.sgm | strip_sgm.sh > $DEST/valid.en_XX-ru_RU.en_XX
  cat newstest2018-enru-ref.ru.sgm | strip_sgm.sh > $DEST/valid.en_XX-ru_RU.ru_RU
  cat newstest2019-ruen-src.ru.sgm | strip_sgm.sh > $DEST/test.ru_RU-en_XX.ru_RU
  cat newstest2019-ruen-ref.en.sgm | strip_sgm.sh > $DEST/test.ru_RU-en_XX.en_XX
  cat newstest2019-enru-src.en.sgm | strip_sgm.sh > $DEST/test.en_XX-ru_RU.en_XX
  cat newstest2019-enru-ref.ru.sgm | strip_sgm.sh > $DEST/test.en_XX-ru_RU.ru_RU

  cat newstest2018-zhen-src.zh.sgm | strip_sgm.sh > $DEST/valid.zh_CN-en_XX.zh_CN
  cat newstest2018-zhen-ref.en.sgm | strip_sgm.sh > $DEST/valid.zh_CN-en_XX.en_XX
  cat newstest2018-enzh-src.en.sgm | strip_sgm.sh > $DEST/valid.en_XX-zh_CN.en_XX
  cat newstest2018-enzh-ref.zh.sgm | strip_sgm.sh > $DEST/valid.en_XX-zh_CN.zh_CN
  cat newstest2019-zhen-src.zh.sgm | strip_sgm.sh > $DEST/test.zh_CN-en_XX.zh_CN
  cat newstest2019-zhen-ref.en.sgm | strip_sgm.sh > $DEST/test.zh_CN-en_XX.en_XX
  cat newstest2019-enzh-src.en.sgm | strip_sgm.sh > $DEST/test.en_XX-zh_CN.en_XX
  cat newstest2019-enzh-ref.zh.sgm | strip_sgm.sh > $DEST/test.en_XX-zh_CN.zh_CN
}

mkdir -p $DEST

prepare_lid
prepare_moses
download_commoncrawl

prepare_ja &
prepare_ta &
prepare_km &
prepare_ps &
prepare_iu &
prepare_cs &
prepare_de &
prepare_pl &
prepare_ru &
prepare_zh &

# prepare valid/test set
prepare_tests &

# wait

# TODO remove intermediate files
# rm -rf $TMP_DIR