Repository: facebookresearch/ParlAI

Issue:
Remaining slow unit tests
Following up from #2327, here are a few possible improvements that could be done to unit tests to bring them to sub 5 minutes:

- there's a test_context_blocking and test_beam_blocking. Both of those train a model to get 100% performance on a custom task, and then use blocking to ensure performance degrades in an expected manner. These are also somewhat flaky. By moving them to pretrained models (distributed with the other unit test models), we could remove the flakiness and speed up their inference from ~50s to well under 10.
- Deleting the pytorch data teacher tests
- ?


Discussion:
Hi, @stephenroller I am interested to solve this problem? Can I do mentioned task for improvement?
Absolutely! Thanks! If you find that you need to distribute some model files, we’ll have to upload them for you, but won’t be a problem.
@stephenroller Okay, I have started the work and will raise PR soon. Thanks
Hi, @stephenroller While I am looking for test `text_context_blocking` and `test_beam_blocking`, can you tell me the path for these function definitions as I didn't find these in `test`. I can move that into the pre-trained model for fast test performance.

Thanks and I am looking for your comment.
https://github.com/facebookresearch/ParlAI/blob/b4a442018d1a46f9374547ef0a9b8134b7bb4944/tests/test_transformers.py#L305
https://github.com/facebookresearch/ParlAI/blob/b4a442018d1a46f9374547ef0a9b8134b7bb4944/tests/test_transformers.py#L396

Resolving patch:
diff --git a/conftest.py b/conftest.py
--- a/conftest.py
+++ b/conftest.py
@@ -13,51 +13,102 @@
 
 import os
 import pathlib
-import random
-from pytest import ExitCode
+import collections
+import pytest
+import subprocess
 
 
 # TODO: rename the folders nicer so they make more sense, maybe even have
 # a 1:1 correspondance with the circleci name
 
 
-def pytest_collection_modifyitems(config, items):
-    # handle circleci parallelism
-    if 'CIRCLE_NODE_TOTAL' in os.environ:
-        total = int(os.environ['CIRCLE_NODE_TOTAL'])
-        index = int(os.environ['CIRCLE_NODE_INDEX'])
+# -----------------------------------------------------------------------
+# From https://github.com/ryanwilsonperkin/pytest-circleci-parallelized.
+# MIT licensed, Copyright Ryan Wilson-Perkin.
+# -----------------------------------------------------------------------
+def get_class_name(item):
+    class_name, module_name = None, None
+    for parent in reversed(item.listchain()):
+        if isinstance(parent, pytest.Class):
+            class_name = parent.name
+        elif isinstance(parent, pytest.Module):
+            module_name = parent.module.__name__
+            break
+
+    if class_name:
+        return "{}.{}".format(module_name, class_name)
     else:
-        total = 1
-        index = 0
+        return module_name
+
+
+def filter_tests_with_circleci(test_list):
+    circleci_input = "\n".join(test_list).encode("utf-8")
+    p = subprocess.Popen(
+        [
+            "circleci",
+            "tests",
+            "split",
+            "--split-by=timings",
+            "--timings-type=classname",
+        ],
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+    )
+    circleci_output, _ = p.communicate(circleci_input)
+    return [
+        line.strip() for line in circleci_output.decode("utf-8").strip().split("\n")
+    ]
+
+
+# -----------------------------------------------------------------------
+MARKER_RULES = [
+    ('parlai_internal', 'test'),
+    ('nightly/gpu', 'nightly_gpu'),
+    ('nightly/cpu/', 'nightly_cpu'),
+    ('datatests/', 'data'),
+    ('tasks/', 'tasks'),
+    ('parlai/mturk/core/test/', 'mturk'),
+]
+
+
+def pytest_collection_modifyitems(config, items):
+    marker_expr = config.getoption('markexpr')
 
+    deselected = []
+
+    # first add all the markers, possibly filtering
     # python 3.4/3.5 compat: rootdir = pathlib.Path(str(config.rootdir))
     rootdir = pathlib.Path(config.rootdir)
-    parallels = [i % total == index for i in range(len(items))]
-    random.Random(42).shuffle(parallels)
-    deselected = []
-    for parallel, item in zip(parallels, items):
+    for item in items:
         rel_path = str(pathlib.Path(item.fspath).relative_to(rootdir))
-        if not parallel:
-            deselected.append(item)
-        elif "parlai_internal" in rel_path:
-            item.add_marker("internal")
-        elif "nightly/gpu/" in rel_path:
-            item.add_marker("nightly_gpu")
-        elif "nightly/cpu/" in rel_path:
-            item.add_marker("nightly_cpu")
-        elif "datatests/" in rel_path:
-            item.add_marker("data")
-        elif "tasks/" in rel_path:
-            item.add_marker("tasks")
-        elif "parlai/mturk/core/test/" in rel_path:
-            item.add_marker("mturk")
-        elif "/" not in rel_path[6:]:
-            item.add_marker("unit")
+        for file_pattern, marker in MARKER_RULES:
+            if file_pattern in rel_path:
+                item.add_marker(marker)
+                if marker_expr and marker != marker_expr:
+                    deselected.append(item)
+                break
         else:
-            raise ValueError(f"Couldn't categorize '{rel_path}'")
-    config.hook.pytest_deselected(items=deselected)
-    for d in deselected:
-        items.remove(d)
+            assert "/" not in rel_path[6:], f"Couldn't categorize '{rel_path}'"
+            item.add_marker("unit")
+            if marker_expr != 'unit' and marker_expr != '':
+                deselected.append(item)
+
+    # kill everything that wasn't grabbed
+    for item in deselected:
+        items.remove(item)
+
+    if 'CIRCLE_NODE_TOTAL' in os.environ:
+        # circleci, split up the parallelism by classes
+        class_mapping = collections.defaultdict(list)
+        for item in items:
+            class_name = get_class_name(item)
+            class_mapping[class_name].append(item)
+
+        filtered_tests = filter_tests_with_circleci(class_mapping.keys())
+        new_items = []
+        for name in filtered_tests:
+            new_items.extend(class_mapping[name])
+            items[:] = new_items
 
 
 def pytest_sessionfinish(session, exitstatus):
@@ -67,5 +118,5 @@ def pytest_sessionfinish(session, exitstatus):
     This can sometimes happen due to the way we distribute tests across multiple circle
     nodes.
     """
-    if exitstatus == ExitCode.NO_TESTS_COLLECTED:
-        session.exitstatus = ExitCode.OK
+    if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED:
+        session.exitstatus = pytest.ExitCode.OK
diff --git a/parlai/zoo/unittest/build.py b/parlai/zoo/unittest/build.py
--- a/parlai/zoo/unittest/build.py
+++ b/parlai/zoo/unittest/build.py
@@ -8,11 +8,15 @@
 Pretrained models used by unit tests.
 """
 
-from parlai.core.build_data import download_models
+import os
+from parlai.core.build_data import download_models, built, get_model_dir
 
 
 def download(datapath):
     opt = {'datapath': datapath}
+    model_name = 'unittest'
+    mdir = os.path.join(get_model_dir(datapath), model_name)
+    version = 'v6.1'
     model_filenames = [
         'seq2seq.tar.gz',
         'transformer_ranker.tar.gz',
@@ -20,5 +24,8 @@ def download(datapath):
         'memnn.tar.gz',
         'apex_v1.tar.gz',
         'test_bytelevel_bpe_v2.tar.gz',
+        'beam_blocking1.tar.gz',
+        'context_blocking1.tar.gz',
     ]
-    download_models(opt, model_filenames, 'unittest', version='v5.0')
+    if not built(mdir, version):
+        download_models(opt, model_filenames, model_name, version=version)


Test patch:
diff --git a/tests/test_tra.py b/tests/test_tra.py
--- a/tests/test_tra.py
+++ b/tests/test_tra.py
@@ -31,10 +31,12 @@ def _get_args(self):
         return dict(
             task='integration_tests:candidate',
             optimizer='adamax',
+            candidates='batch',
             learningrate=7e-3,
             batchsize=16,
-            embedding_size=32,
+            embedding_size=16,
             num_epochs=4,
+            gradient_clip=0.0,
         )
 
     def _get_threshold(self):
@@ -149,13 +151,7 @@ def test_eval_vocab(self):
 class TestTransformerRanker(_AbstractTRATest):
     def _get_args(self):
         args = super()._get_args()
-        new_args = dict(
-            model='transformer/ranker',
-            n_layers=1,
-            n_heads=4,
-            ffn_size=64,
-            gradient_clip=0.5,
-        )
+        new_args = dict(model='transformer/ranker', n_layers=1, n_heads=4, ffn_size=32,)
         for k, v in new_args.items():
             args[k] = v
         return args
@@ -176,11 +172,7 @@ class TestPolyRanker(_AbstractTRATest):
     def _get_args(self):
         args = super()._get_args()
         new_args = dict(
-            model='transformer/polyencoder',
-            n_layers=1,
-            n_heads=4,
-            ffn_size=64,
-            gradient_clip=0.5,
+            model='transformer/polyencoder', n_layers=1, n_heads=4, ffn_size=32,
         )
         for k, v in new_args.items():
             args[k] = v
@@ -228,7 +220,7 @@ def test_eval_fixed_label_not_in_cands(self):
                 testing_utils.eval_model(args, skip_valid=True)
 
             args['add_label_to_fixed_cands'] = True
-            valid, test = testing_utils.eval_model(args, skip_valid=True)
+            _, test = testing_utils.eval_model(args, skip_valid=True)
             self.assertGreaterEqual(test['hits@100'], 0.0)
 
 
diff --git a/tests/test_transformers.py b/tests/test_transformers.py
--- a/tests/test_transformers.py
+++ b/tests/test_transformers.py
@@ -302,55 +302,48 @@ def test_beamsearch(self):
         self.assertLessEqual(test['ppl'], 1.50)
         self.assertGreaterEqual(test['bleu-4'], 0.90)
 
-    @testing_utils.retry(ntries=3)
     def test_beamsearch_blocking(self):
         """
         Test beamsearch blocking.
         """
         with testing_utils.tempdir() as tmpdir:
-            mf = os.path.join(tmpdir, 'model')
-            df = os.path.join(tmpdir, 'model.dict')
-            valid, test = testing_utils.train_model(
+            valid, _ = testing_utils.eval_model(
                 dict(
                     task='integration_tests:repeat_words',
-                    model='transformer/generator',
-                    model_file=mf,
-                    dict_file=df,
-                    optimizer='adamax',
-                    learningrate=7e-3,
-                    batchsize=32,
-                    num_epochs=20,
-                    n_layers=1,
-                    n_heads=1,
-                    ffn_size=32,
-                    embedding_size=32,
+                    model_file='zoo:unittest/beam_blocking/model',
+                    dict_file='zoo:unittest/beam_blocking/model.dict',
+                    batch_size=1,
                     inference='beam',
-                    beam_size=2,
-                )
+                    beam_size=5,
+                    skip_generation=False,
+                ),
+                skip_test=True,
             )
-            valid_beam_block, test_beam_block = testing_utils.eval_model(
+            valid_beam_block, _ = testing_utils.eval_model(
                 dict(
                     task='integration_tests:repeat_words',
-                    model_file=mf,
-                    dict_file=df,
+                    model_file='zoo:unittest/beam_blocking/model',
+                    dict_file='zoo:unittest/beam_blocking/model.dict',
                     batch_size=1,
                     inference='beam',
                     beam_size=5,
                     beam_block_ngram=1,
                     skip_generation=False,
-                )
+                ),
+                skip_test=True,
             )
-            valid_beam_block2, test_beam_block2 = testing_utils.eval_model(
+            valid_beam_block2, _ = testing_utils.eval_model(
                 dict(
                     task='integration_tests:repeat_words',
-                    model_file=mf,
-                    dict_file=df,
+                    model_file='zoo:unittest/beam_blocking/model',
+                    dict_file='zoo:unittest/beam_blocking/model.dict',
                     batch_size=1,
                     inference='beam',
                     beam_size=5,
                     beam_block_ngram=2,
                     skip_generation=False,
-                )
+                ),
+                skip_test=True,
             )
 
             with open(os.path.join(tmpdir, 'blacklist.txt'), 'w') as f:
@@ -359,8 +352,8 @@ def test_beamsearch_blocking(self):
             valid_beam_block3, _ = testing_utils.eval_model(
                 dict(
                     task='integration_tests:repeat_words',
-                    model_file=mf,
-                    dict_file=df,
+                    model_file='zoo:unittest/beam_blocking/model',
+                    dict_file='zoo:unittest/beam_blocking/model.dict',
                     batch_size=1,
                     inference='beam',
                     beam_size=5,
@@ -373,27 +366,19 @@ def test_beamsearch_blocking(self):
         self.assertLessEqual(valid['ppl'], 1.30)
         self.assertGreaterEqual(valid['f1'], 0.80)
         self.assertGreaterEqual(valid['bleu-4'], 0.5)
-        self.assertLessEqual(test['ppl'], 1.30)
-        self.assertGreaterEqual(test['f1'], 0.80)
-        self.assertGreaterEqual(test['bleu-4'], 0.5)
 
         # Beam Block 1
         self.assertLessEqual(valid_beam_block['f1'], 0.4)
         self.assertLessEqual(valid_beam_block['bleu-4'], 1e-9)
-        self.assertLessEqual(test_beam_block['f1'], 0.4)
-        self.assertLessEqual(test_beam_block['bleu-4'], 1e-9)
 
         # Beam Block 2
         self.assertLessEqual(valid_beam_block2['f1'], 0.6)
         self.assertLessEqual(valid_beam_block2['bleu-4'], 1e-6)
-        self.assertLessEqual(test_beam_block2['f1'], 0.6)
-        self.assertLessEqual(test_beam_block2['bleu-4'], 1e-6)
 
         # Beam Block blacklist
         self.assertLess(valid_beam_block3['bleu-4'], valid['bleu-4'])
         self.assertLess(valid_beam_block3['f1'], valid['f1'])
 
-    @testing_utils.retry(ntries=3)
     def test_beamsearch_contextblocking(self):
         """
         Test beamsearch context blocking.
@@ -402,69 +387,54 @@ def test_beamsearch_contextblocking(self):
         well. Measure how much context blocking affects performance.
         """
 
-        with testing_utils.tempdir() as tmpdir:
-            mf = os.path.join(tmpdir, 'model')
-            df = os.path.join(tmpdir, 'model.dict')
-            # we'll reuse these
-            args = dict(
-                task='integration_tests', model_file=mf, dict_file=df, metrics='all'
-            )
-            noblock_valid, _ = testing_utils.train_model(
-                dict(
-                    model='transformer/generator',
-                    optimizer='adamax',
-                    learningrate=7e-3,
-                    batchsize=32,
-                    num_epochs=20,
-                    n_layers=1,
-                    n_heads=1,
-                    ffn_size=32,
-                    embedding_size=32,
-                    inference='beam',
-                    beam_size=5,
-                    **args,
-                )
-            )
-            self.assertGreaterEqual(noblock_valid['f1'], 0.95)
-
-            # first confirm all is good without blocking
-            valid, test = testing_utils.eval_model(
-                dict(beam_context_block_ngram=-1, **args)
-            )
-            self.assertGreaterEqual(valid['f1'], 0.95)
-            self.assertGreaterEqual(valid['bleu-4'], 0.95)
-
-            # there's a special case for block == 1
-            valid, test = testing_utils.eval_model(
-                dict(beam_context_block_ngram=1, **args)
-            )
-            # bleu and f1 should be totally wrecked.
-            self.assertLess(valid['f1'], 0.01)
-            self.assertLess(valid['bleu-4'], 0.01)
-
-            # a couple general cases
-            valid, test = testing_utils.eval_model(
-                dict(beam_context_block_ngram=2, **args)
-            )
-            # should take a big hit here
-            self.assertLessEqual(valid['f1'], noblock_valid['f1'])
-            # bleu-1 should be relatively okay
-            self.assertLessEqual(valid['bleu-1'], noblock_valid['bleu-1'])
-            self.assertGreaterEqual(valid['bleu-1'], 0.45)
-            # and bleu-2 should be 0 at this point
-            self.assertLessEqual(valid['bleu-2'], 0.01)
-
-            # larger blocking, we can do better now
-            valid, test = testing_utils.eval_model(
-                dict(beam_context_block_ngram=3, **args)
-            )
-            # not as hard a hit from the larger hit
-            self.assertLessEqual(valid['f1'], 0.95)
-            # bleu-1 and bleu-2 should be relatively okay
-            self.assertGreaterEqual(valid['bleu-1'], 0.60)
-            self.assertGreaterEqual(valid['bleu-2'], 0.25)
-            # bleu-3 should be totally screwed
-            self.assertLessEqual(valid['bleu-3'], 0.01)
+        # we'll reuse these
+        args = dict(
+            task='integration_tests',
+            model_file='zoo:unittest/context_blocking/model',
+            dict_file='zoo:unittest/context_blocking/model.dict',
+            metrics='all',
+        )
+        noblock_valid, _ = testing_utils.eval_model(args,)
+        self.assertGreaterEqual(noblock_valid['f1'], 0.95)
+
+        # first confirm all is good without blocking
+        valid, _ = testing_utils.eval_model(
+            dict(beam_context_block_ngram=-1, **args), skip_test=True
+        )
+        self.assertGreaterEqual(valid['f1'], 0.95)
+        self.assertGreaterEqual(valid['bleu-4'], 0.95)
+
+        # there's a special case for block == 1
+        valid, _ = testing_utils.eval_model(
+            dict(beam_context_block_ngram=1, **args), skip_test=True,
+        )
+        # bleu and f1 should be totally wrecked.
+        self.assertLess(valid['f1'], 0.01)
+        self.assertLess(valid['bleu-4'], 0.01)
+
+        # a couple general cases
+        valid, _ = testing_utils.eval_model(
+            dict(beam_context_block_ngram=2, **args), skip_test=True
+        )
+        # should take a big hit here
+        self.assertLessEqual(valid['f1'], noblock_valid['f1'])
+        # bleu-1 should be relatively okay
+        self.assertLessEqual(valid['bleu-1'], noblock_valid['bleu-1'])
+        self.assertGreaterEqual(valid['bleu-1'], 0.45)
+        # and bleu-2 should be 0 at this point
+        self.assertLessEqual(valid['bleu-2'], 0.01)
+
+        # larger blocking, we can do better now
+        valid, _ = testing_utils.eval_model(
+            dict(beam_context_block_ngram=3, **args), skip_test=True
+        )
+        # not as hard a hit from the larger hit
+        self.assertLessEqual(valid['f1'], 0.95)
+        # bleu-1 and bleu-2 should be relatively okay
+        self.assertGreaterEqual(valid['bleu-1'], 0.60)
+        self.assertGreaterEqual(valid['bleu-2'], 0.25)
+        # bleu-3 should be totally screwed
+        self.assertLessEqual(valid['bleu-3'], 0.01)
 
     def test_nucleus(self):
         """