Repository: facebookresearch/ParlAI

Issue:
lr scheduler does not reset during fine turning in local model
**Bug description**
When train_model.py is loading model with init-model argument with , lr scheduler seems to continue from the last learning rate recorded, based on model status stored in init-model location, rather than have a hard reset.

Notice that:
* When bert bi ranker model is first downloaded and loaded via --pretrained-path argument, the issue does NOT persist. Only when model is loaded locally.
* I am not sure this should be the expected behavior or not, I assume not. Main basis of my assumptions are based on @emilydinan advice at #823 for fine-turn. In addition, @stephenroller 's comment on #1734 indicates that LR scheduler are expected to be reset during fine-turn.

**Reproduction steps**
Model 1 is first trained locally on convai2 task (with bert bi ranker) and complete training. Later fine turning with --init-model argument point to model 1 path, combine with --model-file argument point to folder does not exist (and expect to create). The LR scheduler does not reset, but rather continue decay based on last learning rate recorded.

**Expected behavior**
LR scheduler expected to reset to default state based on arguments given at command line inputs.

**My silly opinions**
In my opinion, it should be expected to perform one of the followings:
* At command line interface for train_model.py, a new argument should be allow to pass in to control "hard_reset" on lr scheduler, or scheduler do automatically hard reset when --init-model paramter is used. 
* At [torch ranker](https://github.com/facebookresearch/ParlAI/blob/46f656abf28c4fba2818c62a2199b25d484e0b41/parlai/core/torch_ranker_agent.py#L118), the "[hard_reset](https://github.com/facebookresearch/ParlAI/blob/master/parlai/core/torch_agent.py#L686)" parameter should be controlled or accessed from outside of ranker.


Discussion:
You’re right. It’s actually okay in TGA:

https://github.com/facebookresearch/ParlAI/blob/72a17b3343f0c3b7ad8d7827d5a3d6a726ef377e/parlai/core/torch_generator_agent.py#L375

Resolving patch:
diff --git a/parlai/core/torch_ranker_agent.py b/parlai/core/torch_ranker_agent.py
--- a/parlai/core/torch_ranker_agent.py
+++ b/parlai/core/torch_ranker_agent.py
@@ -69,7 +69,7 @@ def add_cmdline_args(cls, argparser):
     def __init__(self, opt, shared=None):
         # Must call _get_init_model() first so that paths are updated if necessary
         # (e.g., a .dict file)
-        init_model, _ = self._get_init_model(opt, shared)
+        init_model, is_finetune = self._get_init_model(opt, shared)
         opt['rank_candidates'] = True
         super().__init__(opt, shared)
 
@@ -115,7 +115,7 @@ def __init__(self, opt, shared=None):
                 optim_params,
                 states.get('optimizer'), states.get('optimizer_type')
             )
-            self.build_lr_scheduler(states)
+            self.build_lr_scheduler(states, hard_reset=is_finetune)
 
         if shared is None and is_distributed():
             self.model = torch.nn.parallel.DistributedDataParallel(


Test patch:
diff --git a/tests/test_transformers.py b/tests/test_transformers.py
--- a/tests/test_transformers.py
+++ b/tests/test_transformers.py
@@ -368,9 +368,78 @@ def test_xlm(self):
         )
 
 
+def test_learning_rate_resuming(self, args):
+    mdl = args['model']
+    with testing_utils.tempdir() as tmpdir:
+        model_file = os.path.join(tmpdir, 'model')
+        stdout1, valid1, test1 = testing_utils.train_model(dict(
+            model_file=model_file,
+            lr_scheduler='invsqrt',
+            **args,
+        ))
+        stdout2, valid2, test2 = testing_utils.train_model(dict(
+            model_file=model_file,
+            lr_scheduler='invsqrt',
+            **args,
+        ))
+        # make sure the number of updates is being tracked correctly
+        self.assertGreater(
+            valid2['num_updates'],
+            valid1['num_updates'],
+            '({}) Number of updates is not increasing'.format(mdl)
+        )
+        # make sure the learning rate is decreasing
+        self.assertLess(
+            valid2['lr'],
+            valid1['lr'],
+            '({}) Learning rate is not decreasing'.format(mdl)
+        )
+        # but make sure we're not loading the scheduler if we're fine
+        # tuning
+        stdout3, valid3, test3 = testing_utils.train_model(dict(
+            init_model=os.path.join(tmpdir, 'model'),
+            model_file=os.path.join(tmpdir, 'newmodel'),
+            lr_scheduler='invsqrt',
+            **args,
+        ))
+        self.assertEqual(
+            valid3['num_updates'],
+            valid1['num_updates'],
+            '({}) Finetuning LR scheduler reset failed '
+            '(num_updates).'.format(mdl)
+        )
+        self.assertEqual(
+            valid3['lr'],
+            valid1['lr'],
+            '({}) Finetuning LR scheduler reset failed '
+            '(lr).'.format(mdl)
+        )
+        # and make sure we're not loading the scheduler if it changes
+        stdout4, valid4, test4 = testing_utils.train_model(dict(
+            init_model=os.path.join(tmpdir, 'model'),
+            model_file=os.path.join(tmpdir, 'newmodel2'),
+            lr_scheduler='reduceonplateau',
+            **args
+        ))
+        self.assertEqual(
+            valid4['num_updates'],
+            valid1['num_updates'],
+            '({}) LR scheduler change reset failed (num_updates).'
+            '\n{}'.format(mdl, stdout4)
+        )
+        self.assertEqual(
+            valid4['lr'],
+            1e-3,
+            '({}) LR is not correct in final resume.\n{}'.format(
+                mdl, stdout4)
+        )
+
+
 class TestLearningRateScheduler(unittest.TestCase):
-    def test_resuming(self):
-        BASE_ARGS = dict(
+    """Test learning rate scheduler for both generative and ranking
+    transformers."""
+    def test_resuming_generator(self):
+        GENERATOR_ARGS = dict(
             task='integration_tests:nocandidate',
             model='transformer/generator',
             optimizer='adamax',
@@ -384,66 +453,23 @@ def test_resuming(self):
             skip_generation=True,
             warmup_updates=1,
         )
+        test_learning_rate_resuming(self, GENERATOR_ARGS)
 
-        with testing_utils.tempdir() as tmpdir:
-            model_file = os.path.join(tmpdir, 'model')
-
-            stdout1, valid1, test1 = testing_utils.train_model(dict(
-                model_file=model_file,
-                lr_scheduler='invsqrt',
-                **BASE_ARGS,
-            ))
-            stdout2, valid2, test2 = testing_utils.train_model(dict(
-                model_file=model_file,
-                lr_scheduler='invsqrt',
-                **BASE_ARGS,
-            ))
-            # make sure the number of updates is being tracked correctly
-            self.assertGreater(
-                valid2['num_updates'],
-                valid1['num_updates'],
-                'Number of updates is not increasing'
-            )
-            # make sure the learning rate is decreasing
-            self.assertLess(
-                valid2['lr'],
-                valid1['lr'],
-                'Learning rate is not decreasing'
-            )
-            # but make sure we're not loading the scheduler if we're fine tuning
-            stdout3, valid3, test3 = testing_utils.train_model(dict(
-                init_model=os.path.join(tmpdir, 'model'),
-                model_file=os.path.join(tmpdir, 'newmodel'),
-                lr_scheduler='invsqrt',
-                **BASE_ARGS,
-            ))
-            self.assertEqual(
-                valid3['num_updates'],
-                valid1['num_updates'],
-                'Finetuning LR scheduler reset failed (num_updates).'
-            )
-            self.assertEqual(
-                valid3['lr'],
-                valid1['lr'],
-                'Finetuning LR scheduler reset failed (lr).'
-            )
-            # and make sure we're not loading the scheduler if it changes
-            stdout4, valid4, test4 = testing_utils.train_model(dict(
-                init_model=os.path.join(tmpdir, 'model'),
-                model_file=os.path.join(tmpdir, 'newmodel2'),
-                lr_scheduler='reduceonplateau',
-                **BASE_ARGS
-            ))
-            self.assertEqual(
-                valid4['num_updates'],
-                valid1['num_updates'],
-                'LR scheduler change reset failed (num_updates).\n' + stdout4
-            )
-            self.assertEqual(
-                valid4['lr'],
-                1e-3,
-                'LR is not correct in final resume.\n' + stdout4
-            )
+    def test_resuming_ranker(self):
+        RANKER_ARGS = dict(
+            task='integration_tests:candidate',
+            model='transformer/ranker',
+            optimizer='adamax',
+            learningrate=1e-3,
+            batchsize=32,
+            num_epochs=1,
+            n_layers=1,
+            n_heads=1,
+            ffn_size=32,
+            embedding_size=32,
+            warmup_updates=1,
+        )
+        test_learning_rate_resuming(self, RANKER_ARGS)
 
 
 if __name__ == '__main__':