Repository: pytorch/ignite

Issue:
Add `start_lr` option to `LRFinder` 
## 🚀 Feature

Currently we choose the start learning rate for the Learning Rate Finder as the learning rate specified with the optimizer
in case of linear `step_mode`
https://github.com/pytorch/ignite/blob/master/ignite/handlers/lr_finder.py#L124
And also in case of exp `step_mode` (we use torch `_LRScheduler`) they specify their `base_lrs` as the given learning rates given with the optimizer also 
https://github.com/pytorch/pytorch/blob/c371542efc31b1abfe6f388042aa3ab0cef935f2/torch/optim/lr_scheduler.py#L35
Which is not the best choice if the learning rate with the optimizer is not a small value.

The idea is to add `start_lr` argument to the `attach` method in the `LRFinder` to be used in both modes (linear, exp), that would be a nice improvement to to get a better results from the LRFinder.



Discussion:

Resolving patch:
diff --git a/ignite/handlers/lr_finder.py b/ignite/handlers/lr_finder.py
--- a/ignite/handlers/lr_finder.py
+++ b/ignite/handlers/lr_finder.py
@@ -85,6 +85,7 @@ def _run(
         optimizer: Optimizer,
         output_transform: Callable,
         num_iter: int,
+        start_lr: float,
         end_lr: float,
         step_mode: str,
         smooth_f: float,
@@ -117,11 +118,13 @@ def _run(
             )
 
         self.logger.debug(f"Running LR finder for {num_iter} iterations")
+        if start_lr is None:
+            start_lr = optimizer.param_groups[0]["lr"]
         # Initialize the proper learning rate policy
         if step_mode.lower() == "exp":
-            self._lr_schedule = LRScheduler(_ExponentialLR(optimizer, end_lr, num_iter))
+            start_lr = [start_lr] * len(optimizer.param_groups)  # type: ignore
+            self._lr_schedule = LRScheduler(_ExponentialLR(optimizer, start_lr, end_lr, num_iter))
         else:
-            start_lr = optimizer.param_groups[0]["lr"]
             self._lr_schedule = PiecewiseLinear(
                 optimizer, param_name="lr", milestones_values=[(0, start_lr), (num_iter, end_lr)]
             )
@@ -362,6 +365,7 @@ def attach(
         to_save: Mapping,
         output_transform: Callable = lambda output: output,
         num_iter: Optional[int] = None,
+        start_lr: Optional[float] = None,
         end_lr: float = 10.0,
         step_mode: str = "exp",
         smooth_f: float = 0.05,
@@ -381,15 +385,17 @@ def attach(
             trainer: lr_finder is attached to this trainer. Please, keep in mind that all attached handlers
                 will be executed.
             to_save: dictionary with optimizer and other objects that needs to be restored after running
-                the LR finder. For example, ``to_save={'optimizer': optimizer, 'model': model}``. All objects should
-                implement ``state_dict`` and ``load_state_dict`` methods.
+                the LR finder. For example, ``to_save={'optimizer': optimizer, 'model': model}``.
+                It should contain "optimizer" key for the optimizer.
+                Also all objects should implement ``state_dict`` and ``load_state_dict`` methods.
             output_transform: function that transforms the trainer's ``state.output`` after each
                 iteration. It must return the loss of that iteration.
             num_iter: number of iterations for lr schedule between base lr and end_lr. Default, it will
                 run for ``trainer.state.epoch_length * trainer.state.max_epochs``.
+            start_lr: lower bound for lr search. Default, Learning Rate specified with the optimizer.
             end_lr: upper bound for lr search. Default, 10.0.
-            step_mode: "exp" or "linear", which way should the lr be increased from optimizer's initial
-                lr to ``end_lr``. Default, "exp".
+            step_mode: "exp" or "linear", which way should the lr be increased from ``start_lr``
+                to ``end_lr``. Default, "exp".
             smooth_f: loss smoothing factor in range ``[0, 1)``. Default, 0.05
             diverge_th: Used for stopping the search when ``current loss > diverge_th * best_loss``.
                 Default, 5.0.
@@ -425,6 +431,8 @@ def attach(
                 raise TypeError(f"if provided, num_iter should be an integer, but give {num_iter}")
             if num_iter <= 0:
                 raise ValueError(f"if provided, num_iter should be positive, but give {num_iter}")
+        if isinstance(start_lr, (float, int)) and start_lr >= end_lr:
+            raise ValueError(f"start_lr must be less than end_lr, start_lr={start_lr} vs end_lr={end_lr}")
 
         # store to_save
         with tempfile.TemporaryDirectory() as tmpdirname:
@@ -443,6 +451,7 @@ def attach(
                     optimizer,
                     output_transform,
                     num_iter,
+                    start_lr,
                     end_lr,
                     step_mode,
                     smooth_f,
@@ -477,11 +486,14 @@ class _ExponentialLR(_LRScheduler):
 
     """
 
-    def __init__(self, optimizer: Optimizer, end_lr: float, num_iter: int, last_epoch: int = -1):
+    def __init__(self, optimizer: Optimizer, start_lr: float, end_lr: float, num_iter: int, last_epoch: int = -1):
         self.end_lr = end_lr
         self.num_iter = num_iter
         super(_ExponentialLR, self).__init__(optimizer, last_epoch)
 
+        # override base_lrs
+        self.base_lrs = start_lr
+
     def get_lr(self) -> List[float]:  # type: ignore
         curr_iter = self.last_epoch + 1  # type: ignore[attr-defined]
         r = curr_iter / self.num_iter


Test patch:
diff --git a/tests/ignite/contrib/handlers/test_lr_finder.py b/tests/ignite/contrib/handlers/test_lr_finder.py
--- a/tests/ignite/contrib/handlers/test_lr_finder.py
+++ b/tests/ignite/contrib/handlers/test_lr_finder.py
@@ -231,6 +231,17 @@ def test_with_attach(lr_finder, to_save, dummy_engine, dataloader):
         assert len(dummy_engine._event_handlers[event]) == 0
 
 
+def test_wrong_values_start_lr_and_end_lr(lr_finder, to_save, dummy_engine, dataloader):
+
+    with pytest.raises(ValueError, match=r"start_lr must be less than end_lr"):
+        with lr_finder.attach(dummy_engine, to_save=to_save, start_lr=10, end_lr=1) as trainer_with_finder:
+            trainer_with_finder.run(dataloader)
+
+    with pytest.raises(ValueError, match=r"start_lr must be less than end_lr"):
+        with lr_finder.attach(dummy_engine, to_save=to_save, start_lr=10, end_lr=10) as trainer_with_finder:
+            trainer_with_finder.run(dataloader)
+
+
 def test_model_optimizer_reset(lr_finder, to_save, dummy_engine, dataloader):
     optimizer = to_save["optimizer"]
     model = to_save["model"]
@@ -308,6 +319,20 @@ def test_detach_terminates(lr_finder, to_save, dummy_engine, dataloader, recwarn
     assert len(recwarn) == 0
 
 
+@pytest.mark.parametrize("step_mode", ["exp", "linear"])
+def test_start_lr(lr_finder, to_save, dummy_engine, dataloader, step_mode):
+    with lr_finder.attach(
+        dummy_engine, to_save, start_lr=0.01, end_lr=0.1, num_iter=5, step_mode=step_mode
+    ) as trainer_with_finder:
+        trainer_with_finder.run(dataloader)
+    history = lr_finder.get_results()
+
+    if step_mode == "exp":
+        assert 0.01 < history["lr"][0] < 0.016
+    else:
+        assert pytest.approx(history["lr"][0]) == 0.01
+
+
 def test_engine_output_type(lr_finder, dummy_engine, optimizer):
     from ignite.handlers.param_scheduler import PiecewiseLinear