|
|
#!/bin/bash |
|
|
|
|
|
""" |
|
|
This file is adapted from |
|
|
https://github.com/salesforce/awd-lstm-lm/blob/master/getdata.sh |
|
|
Copyright by the AWD LSTM authors. |
|
|
""" |
|
|
DATA_DIR=$1 |
|
|
echo "- Downloading WikiText-2" |
|
|
|
|
|
wget --continue -P $DATA_DIR https://s3.amazonaws.com/research.metamind.io/wikitext/wikitext-2-v1.zip |
|
|
unzip -q $DATA_DIR/wikitext-2-v1.zip -d $DATA_DIR |
|
|
cd $DATA_DIR/wikitext-2 |
|
|
mv wiki.train.tokens train.txt |
|
|
sed -i -e "s/<unk>/[UNK]/g" train.txt |
|
|
mv wiki.valid.tokens valid.txt |
|
|
sed -i -e "s/<unk>/[UNK]/g" valid.txt |
|
|
mv wiki.test.tokens test.txt |
|
|
sed -i -e "s/<unk>/[UNK]/g" test.txt |
|
|
cd .. |
|
|
rm wikitext-2-v1.zip |
|
|
|
|
|
echo "- WikiText-2 saved at $DATA_DIR/wikitext-2" |
|
|
|
