Spaces:
Runtime error
Runtime error
File size: 1,212 Bytes
e50fe35 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 |
#!/bin/bash
expdir=$1 # EXPDIR
SUBWORD_NMT_DIR="subword-nmt"
data_dir="$expdir/data"
mkdir -p $expdir/bpe
for dset in `echo train dev test`
do
echo $dset
in_dset_dir="$data_dir/$dset"
out_dset_dir="$expdir/bpe/$dset"
# out_dset_dir="$expdir/final/$dset"
echo "Apply to SRC corpus"
# for very large datasets, use gnu-parallel to speed up applying bpe
# uncomment the below line if the apply bpe is slow
# parallel --pipe --keep-order \
python $SUBWORD_NMT_DIR/subword_nmt/apply_bpe.py \
-c $expdir/vocab/bpe_codes.32k.SRC \
--vocabulary $expdir/vocab/vocab.SRC \
--vocabulary-threshold 5 \
--num-workers "-1" \
< $in_dset_dir.SRC \
> $out_dset_dir.SRC
echo "Apply to TGT corpus"
# for very large datasets, use gnu-parallel to speed up applying bpe
# uncomment the below line if the apply bpe is slow
# parallel --pipe --keep-order \
python $SUBWORD_NMT_DIR/subword_nmt/apply_bpe.py \
-c $expdir/vocab/bpe_codes.32k.TGT \
--vocabulary $expdir/vocab/vocab.TGT \
--vocabulary-threshold 5 \
--num-workers "-1" \
< $in_dset_dir.TGT \
> $out_dset_dir.TGT
done
|