|
""" Scheduler Factory |
|
Hacked together by / Copyright 2020 Ross Wightman |
|
""" |
|
from .timm.cosine_lr import CosineLRScheduler |
|
from .timm.tanh_lr import TanhLRScheduler |
|
from .timm.step_lr import StepLRScheduler |
|
from .timm.plateau_lr import PlateauLRScheduler |
|
import torch |
|
|
|
def create_scheduler(args, optimizer, **kwargs): |
|
num_epochs = args.epochs |
|
|
|
if getattr(args, 'lr_noise', None) is not None: |
|
lr_noise = getattr(args, 'lr_noise') |
|
if isinstance(lr_noise, (list, tuple)): |
|
noise_range = [n * num_epochs for n in lr_noise] |
|
if len(noise_range) == 1: |
|
noise_range = noise_range[0] |
|
else: |
|
noise_range = lr_noise * num_epochs |
|
else: |
|
noise_range = None |
|
|
|
lr_scheduler = None |
|
if args.lr_policy == 'cosine': |
|
lr_scheduler = CosineLRScheduler( |
|
optimizer, |
|
t_initial=num_epochs, |
|
t_mul=getattr(args, 'lr_cycle_mul', 1.), |
|
lr_min=args.lr_min, |
|
decay_rate=args.decay_rate, |
|
warmup_lr_init=args.warmup_lr, |
|
warmup_t=args.warmup_epochs, |
|
cycle_limit=getattr(args, 'lr_cycle_limit', 1), |
|
t_in_epochs=True, |
|
noise_range_t=noise_range, |
|
noise_pct=getattr(args, 'lr_noise_pct', 0.67), |
|
noise_std=getattr(args, 'lr_noise_std', 1.), |
|
noise_seed=getattr(args, 'seed', 42), |
|
) |
|
num_epochs = lr_scheduler.get_cycle_length() + args.COOLDOWN_EPOCHS |
|
elif args.lr_policy == 'tanh': |
|
lr_scheduler = TanhLRScheduler( |
|
optimizer, |
|
t_initial=num_epochs, |
|
t_mul=getattr(args, 'lr_cycle_mul', 1.), |
|
lr_min=args.min_lr, |
|
warmup_lr_init=args.warmup_lr, |
|
warmup_t=args.warmup_epochs, |
|
cycle_limit=getattr(args, 'lr_cycle_limit', 1), |
|
t_in_epochs=True, |
|
noise_range_t=noise_range, |
|
noise_pct=getattr(args, 'lr_noise_pct', 0.67), |
|
noise_std=getattr(args, 'lr_noise_std', 1.), |
|
noise_seed=getattr(args, 'seed', 42), |
|
) |
|
num_epochs = lr_scheduler.get_cycle_length() + args.COOLDOWN_EPOCHS |
|
elif args.lr_policy == 'step': |
|
lr_scheduler = StepLRScheduler( |
|
optimizer, |
|
decay_t=args.decay_epochs - getattr(kwargs, 'init_epoch', 0), |
|
decay_rate=args.decay_rate, |
|
warmup_lr_init=args.warmup_lr, |
|
warmup_t=args.warmup_epochs, |
|
noise_range_t=noise_range, |
|
noise_pct=getattr(args, 'lr_noise_pct', 0.67), |
|
noise_std=getattr(args, 'lr_noise_std', 1.), |
|
noise_seed=getattr(args, 'seed', 42), |
|
) |
|
elif args.lr_policy == 'plateau': |
|
mode = 'min' if 'loss' in getattr(args, 'eval_metric', '') else 'max' |
|
lr_scheduler = PlateauLRScheduler( |
|
optimizer, |
|
decay_rate=args.decay_rate, |
|
patience_t=args.patience_epochs, |
|
lr_min=args.min_lr, |
|
mode=mode, |
|
warmup_lr_init=args.warmup_lr, |
|
warmup_t=args.warmup_epochs, |
|
cooldown_t=0, |
|
noise_range_t=noise_range, |
|
noise_pct=getattr(args, 'lr_noise_pct', 0.67), |
|
noise_std=getattr(args, 'lr_noise_std', 1.), |
|
noise_seed=getattr(args, 'seed', 42), |
|
) |
|
elif args.lr_policy == "onecyclelr": |
|
lr_scheduler = torch.optim.lr_scheduler.OneCycleLR( |
|
optimizer, |
|
max_lr=args.LR, |
|
total_steps=kwargs["total_steps"], |
|
pct_start=args.PCT_START, |
|
div_factor=args.DIV_FACTOR_ONECOS, |
|
final_div_factor=args.FIN_DACTOR_ONCCOS, |
|
) |
|
elif args.lr_policy == "cosinerestart": |
|
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( |
|
optimizer, |
|
T_0 = kwargs["total_steps"], |
|
T_mult=2, |
|
eta_min = 1e-6, |
|
last_epoch=-1, |
|
) |
|
return lr_scheduler |
