Repository: facebookresearch/ParlAI

Issue:
Legacy seq2seq doesn't output version requirement
**Bug description**
Attempting to evaluate the pretrained seq2seq convai2 model results in a bug.

**Reproduction steps**
`python examples/eval_model.py -mf models:convai2/seq2seq/convai2_self_seq2seq_model -t convai2`

**Expected behavior**
The model evaluates

**Logs**
Please paste the command line output:

```
python examples/eval_model.py -mf models:convai2/seq2seq/convai2_self_seq2seq_model -t convai2
[ warning: overriding opt['model_file'] to /Users/kshuster/ParlAI/data/models/convai2/seq2seq/convai2_self_seq2seq_model (previously: /Users/edinan/ParlAI/data/models/convai2/seq2seq/convai2_self_seq2seq_model )]
[ warning: overriding opt['task'] to convai2 (previously: convai2:self:no_cands )]
Traceback (most recent call last):
  File "examples/eval_model.py", line 17, in <module>
    eval_model(opt, print_parser=parser)
  File "/Users/kshuster/ParlAI/parlai/scripts/eval_model.py", line 108, in eval_model
    agent = create_agent(opt, requireModelExists=True)
  File "/Users/kshuster/ParlAI/parlai/core/agents.py", line 683, in create_agent
    model = load_agent_module(opt)
  File "/Users/kshuster/ParlAI/parlai/core/agents.py", line 502, in load_agent_module
    new_opt = model_class.upgrade_opt(new_opt)
  File "/Users/kshuster/ParlAI/parlai/core/torch_generator_agent.py", line 248, in upgrade_opt
    assert 'beam_size' in opt_from_disk
AssertionError
```



Discussion:
Why has this not been picked up by the gpu nightly test?
No idea, but I can still repro on master
It's because it needs `-m legacy:seq2seq:0`
So the real problem is the version checker isn't occurring early enough.

I'd really like to just get rid of the legacy copies.

Resolving patch:
diff --git a/parlai/core/agents.py b/parlai/core/agents.py
--- a/parlai/core/agents.py
+++ b/parlai/core/agents.py
@@ -498,27 +498,6 @@ def load_agent_module(opt):
 
         model_class = get_agent_module(new_opt['model'])
 
-        if hasattr(model_class, 'upgrade_opt'):
-            new_opt = model_class.upgrade_opt(new_opt)
-
-        # add model arguments to new_opt if they aren't in new_opt already
-        for k, v in opt.items():
-            if k not in new_opt:
-                new_opt[k] = v
-        new_opt['model_file'] = model_file
-        if not new_opt.get('dict_file'):
-            new_opt['dict_file'] = model_file + '.dict'
-        elif new_opt.get('dict_file') and not os.path.isfile(new_opt['dict_file']):
-            old_dict_file = new_opt['dict_file']
-            new_opt['dict_file'] = model_file + '.dict'
-        if not os.path.isfile(new_opt['dict_file']):
-            warn_once(
-                'WARNING: Neither the specified dict file ({}) nor the '
-                '`model_file`.dict file ({}) exists, check to make sure either '
-                'is correct. This may manifest as a shape mismatch later '
-                'on.'.format(old_dict_file, new_opt['dict_file'])
-            )
-
         # check for model version
         if hasattr(model_class, 'model_version'):
             curr_version = new_opt.get('model_version', 0)
@@ -542,6 +521,27 @@ def load_agent_module(opt):
                         m.format(m='modelname', v=curr_version, c='ModelAgent')
                     )
 
+        if hasattr(model_class, 'upgrade_opt'):
+            new_opt = model_class.upgrade_opt(new_opt)
+
+        # add model arguments to new_opt if they aren't in new_opt already
+        for k, v in opt.items():
+            if k not in new_opt:
+                new_opt[k] = v
+        new_opt['model_file'] = model_file
+        if not new_opt.get('dict_file'):
+            new_opt['dict_file'] = model_file + '.dict'
+        elif new_opt.get('dict_file') and not os.path.isfile(new_opt['dict_file']):
+            old_dict_file = new_opt['dict_file']
+            new_opt['dict_file'] = model_file + '.dict'
+        if not os.path.isfile(new_opt['dict_file']):
+            warn_once(
+                'WARNING: Neither the specified dict file ({}) nor the '
+                '`model_file`.dict file ({}) exists, check to make sure either '
+                'is correct. This may manifest as a shape mismatch later '
+                'on.'.format(old_dict_file, new_opt['dict_file'])
+            )
+
         # if we want to load weights from --init-model, compare opts with
         # loaded ones
         compare_init_model_opts(opt, new_opt)


Test patch:
diff --git a/parlai/core/testing_utils.py b/parlai/core/testing_utils.py
--- a/parlai/core/testing_utils.py
+++ b/parlai/core/testing_utils.py
@@ -13,6 +13,7 @@
 import tempfile
 import shutil
 import io
+from typing import Tuple
 
 
 try:
@@ -302,6 +303,30 @@ def display_data(opt):
     return (train_output.getvalue(), valid_output.getvalue(), test_output.getvalue())
 
 
+def display_model(opt) -> Tuple[str, str]:
+    """
+    Run display_model.py.
+
+    :return: (stdout_valid, stdout_test)
+    """
+    import parlai.scripts.display_model as dm
+
+    parser = dm.setup_args()
+    parser.set_params(**opt)
+    popt = parser.parse_args(print_args=False)
+    with capture_output() as train_output:
+        # evalmode so that we don't hit train_step
+        popt['datatype'] = 'train:evalmode:stream'
+        dm.display_model(popt)
+    with capture_output() as valid_output:
+        popt['datatype'] = 'valid:stream'
+        dm.display_model(popt)
+    with capture_output() as test_output:
+        popt['datatype'] = 'test:stream'
+        dm.display_model(popt)
+    return (train_output.getvalue(), valid_output.getvalue(), test_output.getvalue())
+
+
 def download_unittest_models():
     """Download the unittest pretrained models."""
     from parlai.core.params import ParlaiParser
diff --git a/tests/test_torch_agent.py b/tests/test_torch_agent.py
--- a/tests/test_torch_agent.py
+++ b/tests/test_torch_agent.py
@@ -982,5 +982,27 @@ def get_opt(init_mf, mf):
                 self.assertFalse(is_finetune)
 
 
+class TestLegacyVersioning(unittest.TestCase):
+    def test_legacy_version(self):
+        # simply tries to load and run some models with versioning attached
+        with self.assertRaises(RuntimeError):
+            testing_utils.display_model(
+                {
+                    'model_file': 'models:convai2/seq2seq/convai2_self_seq2seq_model',
+                    'task': 'convai2',
+                    'no_cuda': True,
+                }
+            )
+
+        testing_utils.display_model(
+            {
+                'model': 'legacy:seq2seq:0',
+                'model_file': 'models:convai2/seq2seq/convai2_self_seq2seq_model',
+                'task': 'convai2',
+                'no_cuda': True,
+            }
+        )
+
+
 if __name__ == '__main__':
     unittest.main()