Repository: mosaicml/composer

Issue:
Scheduler hparams interval default value should be 'step' instead of 'epoch'
## 🚀 Feature Request
All of the [`SchedulerHparams` classes](https://github.com/mosaicml/composer/blob/main/composer/optim/scheduler.py) have `epoch` as the default value for `interval`. The default value seems like it should be `step`, because stepwise LR updates yield superior performance.


Discussion:
Soliciting @hanlint, @abhi-mosaic, @ravi-mosaicml for thoughts
Is the the pytorch convention where schedulers step every batch or epoch? I
think we should match it.

On Thu, Jan 27, 2022, 10:09 PM Matthew ***@***.***> wrote:

> Soliciting @hanlint <https://github.com/hanlint>, @abhi-mosaic
> <https://github.com/abhi-mosaic>, @ravi-mosaicml
> <https://github.com/ravi-mosaicml> for thoughts
>
> —
> Reply to this email directly, view it on GitHub
> <https://github.com/mosaicml/composer/issues/287#issuecomment-1023844529>,
> or unsubscribe
> <https://github.com/notifications/unsubscribe-auth/AUYBL6H6WLPO26PI6MNSKJ3UYICIPANCNFSM5M7HF4EQ>
> .
> Triage notifications on the go with GitHub Mobile for iOS
> <https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675>
> or Android
> <https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub>.
>
> You are receiving this because you were mentioned.Message ID:
> ***@***.***>
>

@ravi-mosaicml The Pytorch schedulers are pretty bad, I would prefer not to follow convention for convention's sake. I think step-wise as default makes sense as a best practice, and we've documented it in blog posts. And since we are handling the `.step()` calls within Trainer it won't impose anything on the user.
We originally set it to to default to `epoch`, since that's the pytorch default. However, I am OK with what is suggested here given our blog posts / best practices. 

Resolving patch:
diff --git a/composer/optim/scheduler.py b/composer/optim/scheduler.py
--- a/composer/optim/scheduler.py
+++ b/composer/optim/scheduler.py
@@ -68,7 +68,7 @@ def _convert_time_fields(interval: str,
 class SchedulerHparams(hp.Hparams, ABC):
 
     scheduler_object = None  # type: Optional[Callable[..., Scheduler]]
-    interval = 'epochs'  # type: str
+    interval = 'step'  # type: str
 
     def initialize_object(
         self,
@@ -178,7 +178,7 @@ class PolynomialLRHparams(SchedulerHparams):
     power: float = hp.required(doc='Power of LR schedule.')
     eta_min: float = hp.optional(default=0.0, doc='Minimum learning rate.')
     verbose: bool = hp.optional(default=False, doc='Prints message to stdout.')
-    interval: str = hp.optional(default='epoch', doc=_interval_doc)
+    interval: str = hp.optional(default='step', doc=_interval_doc)
 
     scheduler_object = PolynomialLR
 
@@ -187,7 +187,7 @@ class PolynomialLRHparams(SchedulerHparams):
 class ConstantLRHparams(SchedulerHparams):
     """Hyperparameters for the :class:`ConstantLR` scheduler."""
     verbose: bool = hp.optional(default=False, doc='prints message to stdout')
-    interval: str = hp.optional(default='epoch', doc=_interval_doc)
+    interval: str = hp.optional(default='step', doc=_interval_doc)
 
     scheduler_object = ConstantLR
 
@@ -201,7 +201,7 @@ class StepLRHparams(SchedulerHparams):
     step_size: str = hp.required(doc='Period of learning rate decay')
     gamma: float = hp.optional(default=0.1, doc='multiplicative factor of decay')
     verbose: bool = hp.optional(default=False, doc='prints message to stdout')
-    interval: str = hp.optional(default='epoch', doc=_interval_doc)
+    interval: str = hp.optional(default='step', doc=_interval_doc)
 
     scheduler_object = torch.optim.lr_scheduler.StepLR
 
@@ -215,7 +215,7 @@ class MultiStepLRHparams(SchedulerHparams):
     milestones: List[str] = hp.required(doc='List of milestone time strings')
     gamma: float = hp.optional(default=0.1, doc='multiplicative factor of decay')
     verbose: bool = hp.optional(default=False, doc='prints message to stdout')
-    interval: str = hp.optional(default='epoch', doc=_interval_doc)
+    interval: str = hp.optional(default='step', doc=_interval_doc)
 
     scheduler_object = torch.optim.lr_scheduler.MultiStepLR
 
@@ -228,7 +228,7 @@ class ExponentialLRHparams(SchedulerHparams):
 
     gamma: float = hp.required(doc='multiplicative factor of decay')
     verbose: bool = hp.optional(default=False, doc='prints message to stdout')
-    interval: str = hp.optional(default='epoch', doc=_interval_doc)
+    interval: str = hp.optional(default='step', doc=_interval_doc)
 
     scheduler_object = torch.optim.lr_scheduler.ExponentialLR
 
@@ -242,7 +242,7 @@ class CosineAnnealingLRHparams(SchedulerHparams):
     T_max: str = hp.required(doc="Maximum scheduler duration.")
     eta_min: float = hp.optional(default=0.0, doc='minimum learning rate.')
     verbose: bool = hp.optional(default=False, doc='prints message to stdout')
-    interval: str = hp.optional(default='epoch', doc=_interval_doc)
+    interval: str = hp.optional(default='step', doc=_interval_doc)
 
     scheduler_object = torch.optim.lr_scheduler.CosineAnnealingLR
 
@@ -256,7 +256,7 @@ class CosineAnnealingWarmRestartsHparams(SchedulerHparams):
     T_0: str = hp.required("Duration for the first restart.")
     eta_min: float = hp.optional(default=0.0, doc='minimum learning rate.')
     verbose: bool = hp.optional(default=False, doc='prints message to stdout')
-    interval: str = hp.optional(default='epoch', doc=_interval_doc)
+    interval: str = hp.optional(default='step', doc=_interval_doc)
     T_mult: int = hp.optional("A factor increases :math:`T_{i}` after a restart. Default: 1.", default=1)
 
     scheduler_object = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts
@@ -272,7 +272,7 @@ class LinearLRHparams(SchedulerHparams):
     end_factor: float = hp.optional("Number to multiply learning rate at the end .", default=1.0)
     total_iters: str = hp.optional("Duration of linear decay steps. Default: 5 iterations.", default="5ba")
     verbose: bool = hp.optional('Prints message to stdout', default=False)
-    interval: str = hp.optional(default='epoch', doc=_interval_doc)
+    interval: str = hp.optional(default='step', doc=_interval_doc)
 
     scheduler_object = LinearLR
 
@@ -288,7 +288,7 @@ class WarmUpLRHparams(SchedulerHparams):
     warmup_iters: str = hp.optional("Warmup duration. Default: 5 iterations.", default="5ba")
     warmup_method: str = hp.optional("Warmup method (linear or constant)", default='linear')
     verbose: bool = hp.optional('Prints message to stdout', default=False)
-    interval: str = hp.optional('Warmup the LR every step or epoch. Default: epoch', default='epoch')
+    interval: str = hp.optional('Warmup the LR every step or epoch. Default: epoch', default='step')
 
     scheduler_object = WarmUpLR
 


Test patch: