Ok lambda here is not doing anything. 

import torch
from torch.optim.optimizer import Optimizer
import math

class Fusedbun(Optimizer):
    def __init__(self, params, lr=1e-3, eps=1e-8, beta_decay=0.8, Lambda=0.01, momentum_beta=0.9, centralize=True, use_rms=True):
        defaults = dict(lr=lr, eps=eps, beta_decay=beta_decay, Lambda=Lambda, momentum_beta=momentum_beta, centralize=centralize, use_rms=use_rms)
        super(Fusedbun, self).__init__(params, defaults)
    
    @torch.no_grad()
    def step(self, closure=None):
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            lr = group['lr']
            eps = group['eps']
            beta_decay = group['beta_decay']
            Lambda = group['Lambda']
            momentum_beta = group['momentum_beta']
            centralize = group['centralize']
            use_rms = group['use_rms']
            
            for p in group['params']:
                if p.grad is None:
                    continue

                grad = p.grad.data
                if centralize and len(p.shape) > 1:
                    grad = grad.sub(grad.mean(dim=tuple(range(1, len(grad.shape))), keepdim=True))
                    
                state = self.state[p]

                if len(state) == 0:
                    state['step'] = 0
                    state['accumulator'] = torch.zeros_like(p.data)
                    if momentum_beta > 0:
                        state['momentum_buffer'] = torch.zeros_like(p.data)
                
                acc = state['accumulator']
                t = state['step'] + 1
                state['step'] = t
                
                if p.dim() > 1:
                    grad_squared = grad.square().mean(dim=0)
                else:
                    grad_squared = grad.square()
                
                beta_t = 1.0 - math.pow(t, -beta_decay)
                acc.mul_(beta_t).add_(grad_squared, alpha=1-beta_t)

                denom = acc.sqrt().add(eps)
                grad_normalized = grad / denom if use_rms else grad

                if momentum_beta > 0:
                    momentum_buffer = state['momentum_buffer']
                    momentum_buffer.mul_(momentum_beta).add_(grad_normalized, alpha=1-momentum_beta)
                    grad_normalized = momentum_buffer
                
                p.data.add_(grad_normalized, alpha=-lr)
        return loss

Make it better. Im making optimiser both memory efficient and better than others out there. I have taken inpiration from SM3 and Adalite. You must know SM3, im giving you adalite code. Make necessary changes.
import torch
import math
from dataclasses import dataclass, field
from typing import List, Optional
from collections import defaultdict

# Somewhat based on https://gist.github.com/albanD/18c240bd2e09f9d93f5c4a0c9ccda39e and LOMO

@dataclass
class OverlapOptimizer:
    model: torch.nn.Module
    lr: Optional[float] = None
    decay: Optional[float] = 0.0
    _acc_grads: Optional[List] = field(default_factory=lambda: [])

    def init(self):
        for p in self.model.parameters():
            if p.requires_grad:
                self.prepare(p)
                self.hook(p)

    def step(self, loss, lr):
        pass

    def hook(self, p):
        pass

@dataclass
class OverlapSGD(OverlapOptimizer):
    sign: bool = False

    def prepare(self, p):
        return

    def step(self, loss, lr):
        self.lr = lr
        loss.backward()

    def hook(self, p):
        ag = p.view_as(p).grad_fn.next_functions[0][0]
        p._acc_grads = [ag]

        @torch.no_grad()
        def gf(*_):
            if self.sign:
                p.add_(p.grad.sign(), alpha=-self.lr)
            else:
                p.add_(p.grad, alpha=-self.lr)
            p.grad = None

        ag.register_hook(gf)

@dataclass
class Adalite(OverlapOptimizer):
    eps: float = 1e-5
    Lambda: float = 0.01 # Akin to weight-decay
    beta_decay: float = 0.8
    centralize: bool = True
    use_rms: bool = True
    momentum: bool = False
    momentum_beta: float = 0.9
    _t: int = 0

    def step(self, loss, lr=None):
        self._t += 1
        self.lr = lr
        loss.backward()

    def prepare(self, p):
        if len(p.shape) == 2:
            p._c = torch.zeros(p.shape[1], device=p.device, dtype=p.dtype)
        else:
            p._v = torch.zeros_like(p)

        if self.momentum:
            p._m = torch.zeros_like(p)

    def hook(self, p):
        ag = p.view_as(p).grad_fn.next_functions[0][0]
        p._acc_grads = [ag]

        @torch.no_grad()
        def gf(*_):
            alpha = self.lr

            g = p.grad

            if self.centralize and sum(g.shape) > 1:
                g.sub_(g.mean(dim=tuple(range(1, len(g.shape))), keepdim=True))

            beta_t = 1.0 - math.pow(self._t, -self.beta_decay)
            u = g.square()

            if len(p.shape) == 2:
                u.mul_(1-beta_t).add_(p._c.unsqueeze(0).broadcast_to(g.shape), alpha=beta_t)
                u.add_(self.eps)
                p._c = u.mean(dim=0)
            else:
                u.mul_(1-beta_t).add_(p._v, alpha=beta_t)
                u.add_(self.eps)
                p._v = u

            m = u.rsqrt() * g

            if self.use_rms:
                m.div_(max(1.0, m.square().mean().sqrt()))

            p_norm = p.norm()
            g_norm = g.norm()

            if p_norm != 0 and g_norm != 0:
                m.mul_(p_norm / g_norm)
                m.add_(p - p/p_norm, alpha=self.Lambda)

            if self.momentum:
                p._m.mul_(self.momentum_beta).add_(m, alpha=1-self.momentum_beta)
                m = p._m

            p.add_(m, alpha=-alpha)
            p.grad = None

        ag.register_hook(gf)