Spaces:
Runtime error
Runtime error
| # Copyright (c) 2019-present, Facebook, Inc. | |
| # All rights reserved. | |
| # | |
| # This source code is licensed under the license found in the | |
| # LICENSE file in the root directory of this source tree. | |
| # | |
| set -e | |
| TOKENIZERS_SCRIPTS=tokenizers | |
| INSTALL_PATH=$TOKENIZERS_SCRIPTS/thirdparty | |
| N_THREADS=8 | |
| lg=$1 | |
| MOSES=$INSTALL_PATH/mosesdecoder | |
| REPLACE_UNICODE_PUNCT=$MOSES/scripts/tokenizer/replace-unicode-punctuation.perl | |
| NORM_PUNC=$MOSES/scripts/tokenizer/normalize-punctuation.perl | |
| REM_NON_PRINT_CHAR=$MOSES/scripts/tokenizer/remove-non-printing-char.perl | |
| TOKENIZER=$MOSES/scripts/tokenizer/tokenizer.perl | |
| # special tokenization for Romanian | |
| WMT16_SCRIPTS=$INSTALL_PATH/wmt16-scripts | |
| NORMALIZE_ROMANIAN=$WMT16_SCRIPTS/preprocess/normalise-romanian.py | |
| REMOVE_DIACRITICS=$WMT16_SCRIPTS/preprocess/remove-diacritics.py | |
| # Burmese | |
| MY_SEGMENT=$INSTALL_PATH/seg_my.py | |
| # Arabic | |
| AR_TOKENIZER=$TOKENIZERS_SCRIPTS/tokenizer_ar.sh | |
| # Korean | |
| KO_SEGMENT=$TOKENIZERS_SCRIPTS/seg_ko.sh | |
| # Japanese | |
| JA_SEGMENT=$TOKENIZERS_SCRIPTS/seg_ja.sh | |
| # Indic | |
| IN_TOKENIZER=$TOKENIZERS_SCRIPTS/tokenize_indic.py | |
| INDIC_RESOURCES_PATH=$INSTALL_PATH/indic_nlp_resources | |
| # Thai | |
| THAI_TOKENIZER=$TOKENIZERS_SCRIPTS/tokenize_thai.py | |
| # Chinese | |
| CHINESE_TOKENIZER=$TOKENIZERS_SCRIPTS/tokenize_zh.py | |
| # Chinese | |
| if [ "$lg" = "zh" ]; then | |
| cat - | $REPLACE_UNICODE_PUNCT | $NORM_PUNC -l $lg | $REM_NON_PRINT_CHAR | python $CHINESE_TOKENIZER | |
| # Thai | |
| elif [ "$lg" = "th" ]; then | |
| cat - | python $THAI_TOKENIZER | |
| # Japanese | |
| elif [ "$lg" = "ja" ]; then | |
| cat - | $REPLACE_UNICODE_PUNCT | $NORM_PUNC -l $lg | $REM_NON_PRINT_CHAR | ${JA_SEGMENT} | |
| # Korean | |
| elif [ "$lg" = "ko" ]; then | |
| cat - | $REM_NON_PRINT_CHAR | ${KO_SEGMENT} | |
| # Romanian | |
| elif [ "$lg" = "ro" ]; then | |
| cat - | $REPLACE_UNICODE_PUNCT | $NORM_PUNC -l $lg | $REM_NON_PRINT_CHAR | $NORMALIZE_ROMANIAN | $REMOVE_DIACRITICS | $TOKENIZER -no-escape -threads $N_THREADS -l $lg | |
| # Burmese | |
| elif [ "$lg" = "my" ]; then | |
| cat - | python ${MY_SEGMENT} | |
| # Arabic | |
| elif [ "$lg" = "ar" ]; then | |
| cat - | ${AR_TOKENIZER} | |
| # Indic | |
| elif [ "$lg" = "ne" ]; then | |
| cat - | python ${IN_TOKENIZER} $lg | |
| elif [ "$lg" = "si" ]; then | |
| cat - | python ${IN_TOKENIZER} $lg | |
| elif [ "$lg" = "hi" ]; then | |
| cat - | python ${IN_TOKENIZER} $lg | |
| # other languages | |
| else | |
| cat - | $REPLACE_UNICODE_PUNCT | $NORM_PUNC -l $lg | $REM_NON_PRINT_CHAR | $TOKENIZER -no-escape -threads $N_THREADS -l $lg | |
| fi | |