Repository: Project-MONAI/MONAI

Issue:
Add LrScheduleHandler for engines
**Is your feature request related to a problem? Please describe.**
When developing a training program based on ignite, we need LrScheduleHandler to update the learning rate when "epoch completed".



Discussion:

Resolving patch:
diff --git a/monai/handlers/__init__.py b/monai/handlers/__init__.py
--- a/monai/handlers/__init__.py
+++ b/monai/handlers/__init__.py
@@ -17,4 +17,5 @@
 from .segmentation_saver import SegmentationSaver
 from .stats_handler import StatsHandler
 from .tensorboard_handlers import TensorBoardImageHandler, TensorBoardStatsHandler
+from .lr_schedule_handler import LrScheduleHandler
 from .utils import *
diff --git a/monai/handlers/lr_schedule_handler.py b/monai/handlers/lr_schedule_handler.py
new file mode 100644
--- /dev/null
+++ b/monai/handlers/lr_schedule_handler.py
@@ -0,0 +1,56 @@
+# Copyright 2020 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+from typing import Callable
+from ignite.engine import Events
+from monai.utils import ensure_tuple
+
+
+class LrScheduleHandler:
+    """
+    ignite handler to update the Learning Rate based on PyTorch LR scheduler.
+    """
+
+    def __init__(self, lr_scheduler, print_lr=True, name=None, epoch_level=True, step_transform=lambda engine: ()):
+        """
+        Args:
+            lr_scheduler (torch.optim.lr_scheduler): typically, lr_scheduler should be PyTorch
+                lr_scheduler object. if customized version, must have `step` and `get_last_lr` methods.
+            print_lr (bool): whether to print out the latest learning rate with logging.
+            name (str): identifier of logging.logger to use, if None, defaulting to ``engine.logger``.
+            epoch_level (bool): execute lr_scheduler.step() after every epoch or every iteration.
+                `True` is epoch level, `False` is iteration level.
+            step_transform (Callable): a callable that is used to transform the information from `engine`
+                to expected input data of lr_scheduler.step() function if neccessary.
+
+        """
+        self.lr_scheduler = lr_scheduler
+        self.print_lr = print_lr
+        self.logger = None if name is None else logging.getLogger(name)
+        self.epoch_level = epoch_level
+        if not isinstance(step_transform, Callable):
+            raise ValueError("argument `step_transform` must be a callable.")
+        self.step_transform = step_transform
+
+    def attach(self, engine):
+        if self.logger is None:
+            self.logger = engine.logger
+        if self.epoch_level:
+            engine.add_event_handler(Events.EPOCH_COMPLETED, self)
+        else:
+            engine.add_event_handler(Events.ITERATION_COMPLETED, self)
+
+    def __call__(self, engine):
+        args = ensure_tuple(self.step_transform(engine))
+        self.lr_scheduler.step(*args)
+        if self.print_lr:
+            self.logger.info(f"Current learning rate: {self.lr_scheduler._last_lr[0]}")


Test patch:
diff --git a/tests/test_handler_lr_scheduler.py b/tests/test_handler_lr_scheduler.py
new file mode 100644
--- /dev/null
+++ b/tests/test_handler_lr_scheduler.py
@@ -0,0 +1,63 @@
+# Copyright 2020 MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import unittest
+import numpy as np
+from ignite.engine import Engine, Events
+from monai.handlers import LrScheduleHandler
+import logging
+import sys
+
+
+class TestHandlerLrSchedule(unittest.TestCase):
+    def test_content(self):
+        logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+        data = [0] * 8
+
+        # set up engine
+        def _train_func(engine, batch):
+            pass
+
+        val_engine = Engine(_train_func)
+        train_engine = Engine(_train_func)
+
+        @train_engine.on(Events.EPOCH_COMPLETED)
+        def run_validation(engine):
+            val_engine.run(data)
+            val_engine.state.metrics["val_loss"] = 1
+
+        # set up testing handler
+        net = torch.nn.PReLU()
+
+        def _reduce_lr_on_plateau():
+            optimizer = torch.optim.SGD(net.parameters(), 0.1)
+            lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=1)
+            handler = LrScheduleHandler(lr_scheduler, step_transform=lambda x: val_engine.state.metrics["val_loss"])
+            handler.attach(train_engine)
+            return lr_scheduler
+
+        def _reduce_on_step():
+            optimizer = torch.optim.SGD(net.parameters(), 0.1)
+            lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.1)
+            handler = LrScheduleHandler(lr_scheduler)
+            handler.attach(train_engine)
+            return lr_scheduler
+
+        schedulers = _reduce_lr_on_plateau(), _reduce_on_step()
+
+        train_engine.run(data, max_epochs=5)
+        for scheduler in schedulers:
+            np.testing.assert_allclose(scheduler._last_lr[0], 0.001)
+
+
+if __name__ == "__main__":
+    unittest.main()