Spaces:
Runtime error
Runtime error
File size: 1,260 Bytes
e50fe35 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 |
#!/bin/bash
expdir=$1 # EXPDIR
num_operations=${2:-32000}
#`dirname $0`/env.sh
SUBWORD_NMT_DIR="subword-nmt"
data_dir="$expdir/data"
train_file=$data_dir/train
# num_operations=32000
echo Input file: $train_file
mkdir -p $expdir/vocab
echo "learning joint BPE"
cat $train_file.SRC $train_file.TGT > $train_file.ALL
python $SUBWORD_NMT_DIR/subword_nmt/learn_bpe.py \
--input $train_file.ALL \
-s $num_operations \
-o $expdir/vocab/bpe_codes.32k.SRC_TGT \
--num-workers -1
echo "computing SRC vocab"
python $SUBWORD_NMT_DIR/subword_nmt/apply_bpe.py \
-c $expdir/vocab/bpe_codes.32k.SRC_TGT \
--num-workers -1 \
-i $train_file.SRC | \
python $SUBWORD_NMT_DIR/subword_nmt/get_vocab.py \
> $expdir/vocab/vocab.tmp.SRC
python scripts/clean_vocab.py $expdir/vocab/vocab.tmp.SRC $expdir/vocab/vocab.SRC
#rm $expdir/vocab/vocab.tmp.SRC
echo "computing TGT vocab"
python $SUBWORD_NMT_DIR/subword_nmt/apply_bpe.py \
-c $expdir/vocab/bpe_codes.32k.SRC_TGT \
--num-workers -1 \
-i $train_file.TGT | \
python $SUBWORD_NMT_DIR/subword_nmt/get_vocab.py \
> $expdir/vocab/vocab.tmp.TGT
python scripts/clean_vocab.py $expdir/vocab/vocab.tmp.TGT $expdir/vocab/vocab.TGT
#rm $expdir/vocab/vocab.tmp.TGT
rm $train_file.ALL
|