Repository: facebookresearch/ParlAI

Issue:
Building Dict is too slow
**Bug description**
```
from parlai.scripts.train_model import TrainModel

TrainModel.main(
    # similar to before
    task='amazon_qa',
    model='projects.wizard_of_wikipedia.generator.agents:EndToEndAgent',
    model_file='/tmp/end2end_generator/model',

    # initialize with a pretrained model
    init_model='zoo:wizard_of_wikipedia/end2end_generator/model',

    # arguments we get from the pretrained model.
    # Unfortunately, these must be looked up separately for each model.
    # eps
    dict_lower=True,
    dict_tokenizer='bpe',
    n_layers=5,
    n_heads=2,
    dropout=0.20,
    ffn_size=512,
    embedding_size=256,
    log_every_n_secs=10,
    validation_patience=12,
    validation_metric='ppl',
    validation_metric_mode='min',
    validation_every_n_epochs=0.5,
    n_positions=128,
    truncate=128,
    max_knowledge=32,
    knowledge_alpha=0.95,
    knowledge_truncate=32,
    learningrate=5e-4,
    warmup_updates=5000,
    clip=0.1,
    lr_scheduler='invsqrt',
    embedding_type='fasttext',
    beam_size=1,
    skip_generation=False,
    batchsize=64,
)

```
I am trying to train amazon_qa task on wizard of Wikipedia model, just to experiment it out, I am not sure if it will work but when I run this script it says creating a task and goes to next stage building dictionary it just becomes to slow of a process.

<img width="1021" alt="Screenshot 2020-07-18 at 10 42 32 PM" src="https://user-images.githubusercontent.com/45225143/87858114-2f34b380-c949-11ea-9928-3bfc77fa91c8.png">

Like has been around 2 hrs and it hasn't crossed 0% yet.
Can anyone please point me out the error, thanks.



Discussion:
Hmm, it seems like it crashed. My own building of a BPE dictionary runs at about 8000ex/s. Which would take 10 minutes to build, but you can cache it.

Can you run `python -m parlai.scripts.display_data -t amazon_qa` ?

Resolving patch:
diff --git a/parlai/tasks/amazon_qa/agents.py b/parlai/tasks/amazon_qa/agents.py
--- a/parlai/tasks/amazon_qa/agents.py
+++ b/parlai/tasks/amazon_qa/agents.py
@@ -10,6 +10,22 @@
 import json
 
 
+EXISTING_KEYS = [
+    'question',
+    'answer',
+    'asin',
+    'questionType',
+    'questionTime',
+    'askerID',
+    'answerType',
+    'answerTime',
+    'unixTime',
+    'answererID',
+    'helpful',
+    'answerScore',
+]
+
+
 class DefaultTeacher(FixedDialogTeacher):
     def __init__(self, opt, shared=None):
         # store datatype
@@ -21,38 +37,27 @@ def __init__(self, opt, shared=None):
 
         if shared:
             self.data = shared['data']
+            self.num_ex = shared['num_ex']
+            self.num_ep = shared['num_ep']
         else:
             build(opt)
             self._setup_data()
+            self.num_ex = sum([len(x) for x in self.data])
+            self.num_ep = len(self.data)
         self.reset()
 
     def num_episodes(self):
-        return len(self.data)
+        return self.num_ep
 
     def num_examples(self):
-        return sum([len(x) for x in self.data])
+        return self.num_ex
 
     def _setup_data(self):
-        self.existing_keys = [
-            'question',
-            'answer',
-            'asin',
-            'questionType',
-            'questionTime',
-            'askerID',
-            'answerType',
-            'answerTime',
-            'unixTime',
-            'answererID',
-            'helpful',
-            'answerScore',
-        ]
-
         self.data = []
 
         def create_entry_single(episode):
             entry = []
-            for key in self.existing_keys:
+            for key in EXISTING_KEYS:
                 if key in episode:
                     entry.append(str(episode[key]))
                 else:
@@ -99,7 +104,7 @@ def get(self, episode_idx, entry_idx=0):
         entry = ep[entry_idx]
         action = dict()
         action['id'] = episode_idx
-        for i, key in enumerate(self.existing_keys):
+        for i, key in enumerate(EXISTING_KEYS):
             if i < 2:
                 continue
             action[key] = entry[i]
@@ -112,4 +117,6 @@ def get(self, episode_idx, entry_idx=0):
     def share(self):
         shared = super().share()
         shared['data'] = self.data
+        shared['num_ex'] = self.num_ex
+        shared['num_ep'] = self.num_ep
         return shared


Test patch: