Repository: pytorch
Issue: Memory leak when using SequentialLR and ChainedLR schedulers
Labels: module: optimizer,module: tests,triaged,module: LrScheduler,oncall: pt2

### 🐛 Describe the bug

Looks like memory is allocated in the caching allocator which doesn't get freed. I checked refcycle and it doesn't appear to be due to a cyclical reference in the python code. The issue only occurs with the Sequential and Chained LR schedulers. Repro below.

Root cause of 
https://github.com/pytorch/pytorch/issues/125925 
https://github.com/pytorch/pytorch/issues/125925
https://github.com/pytorch/pytorch/issues/125924

```py
import torch
from torch.optim.lr_scheduler import LambdaLR, ChainedScheduler, ConstantLR, SequentialLR
from torch.testing._internal.common_utils import CudaMemoryLeakCheck, TestCase, run_tests


LR_SCHEDULER_TO_KWARGS = {
    LambdaLR: {"lr_lambda": lambda x: 10},
    ChainedScheduler: {"schedulers": None},
    SequentialLR: {"schedulers": None, "milestones": [1, 2]},
    ConstantLR: {"factor": 0.001},
}


def create_scheduler(scheduler, optim):
    kwargs = LR_SCHEDULER_TO_KWARGS[scheduler]
    if "schedulers" in kwargs:
        kwargs["schedulers"] = [
            create_scheduler(ConstantLR, optim)
            for _ in range(2)
        ] + [create_scheduler(LambdaLR, optim)]

    if scheduler == ChainedScheduler:
        return scheduler(**kwargs)
    else:
        return scheduler(optim, **kwargs)

def chained_fn():
    from torch.testing._internal.common_utils import CudaMemoryLeakCheck
    with CudaMemoryLeakCheck(None,name="hi"):
        device="cuda:0"
        dtype=torch.float32
        optim_cls = torch.optim.ASGD
        kwargs = {'lr': 0.001, 'weight_decay': 0.1, 'maximize': True, 'capturable': True, 'foreach': False}
        scheduler_cls = ChainedScheduler
        print(scheduler_cls)
        print(kwargs)
        input = torch.ones([10, 10], device=device)
        model_eager = torch.nn.Sequential(
            *[torch.nn.Linear(10, 10, device=device) for _ in range(1)]
        )
        model_eager(input).sum().backward()
        opt_eager = optim_cls(model_eager.parameters(), **kwargs)
        scheduler_eager = create_scheduler(scheduler_cls, opt_eager)


def sequential_fn():
    from torch.testing._internal.common_utils import CudaMemoryLeakCheck
    with CudaMemoryLeakCheck(None,name="hi"):
        device="cuda:0"
        dtype=torch.float32
        optim_cls = torch.optim.ASGD
        kwargs = {'lr': 0.001, 'weight_decay': 0.1, 'maximize': True, 'capturable': True, 'foreach': False}
        scheduler_cls = SequentialLR
        print(scheduler_cls)
        print(kwargs)
        input = torch.ones([10, 10], device=device)
        model_eager = torch.nn.Sequential(
            *[torch.nn.Linear(10, 10, device=device) for _ in range(1)]
        )
        model_eager(input).sum().backward()
        opt_eager = optim_cls(model_eager.parameters(), **kwargs)
        scheduler_eager = create_scheduler(scheduler_cls, opt_eager)

# both sequential_fn and chained_fn fail
chained_fn()
```

### Versions

nightly:
a5c93a6899c657832944cd2eeb5069449e28dbea

cc @vincentqb @jbschlosser @albanD @janeyx99 @crcrpar @mruberry @ZainRizvi @ezyang @msaroufim @bdhirsh @anijain2305 @chauhang @gchanan @zou3519 @kadeng