I made an optimizer by fusing two optimizer implementations: 1. SM3 and 2. Adalite.
Code for adalite is as follows: 
import torch
import math
from dataclasses import dataclass, field
from typing import List, Optional
from collections import defaultdict

# Somewhat based on https://gist.github.com/albanD/18c240bd2e09f9d93f5c4a0c9ccda39e and LOMO

@dataclass
class OverlapOptimizer:
    model: torch.nn.Module
    lr: Optional[float] = None
    decay: Optional[float] = 0.0
    _acc_grads: Optional[List] = field(default_factory=lambda: [])

    def init(self):
        for p in self.model.parameters():
            if p.requires_grad:
                self.prepare(p)
                self.hook(p)

    def step(self, loss, lr):
        pass

    def hook(self, p):
        pass

@dataclass
class OverlapSGD(OverlapOptimizer):
    sign: bool = False

    def prepare(self, p):
        return

    def step(self, loss, lr):
        self.lr = lr
        loss.backward()

    def hook(self, p):
        ag = p.view_as(p).grad_fn.next_functions[0][0]
        p._acc_grads = [ag]

        @torch.no_grad()
        def gf(*_):
            if self.sign:
                p.add_(p.grad.sign(), alpha=-self.lr)
            else:
                p.add_(p.grad, alpha=-self.lr)
            p.grad = None

        ag.register_hook(gf)

@dataclass
class Adalite(OverlapOptimizer):
    eps: float = 1e-5
    Lambda: float = 0.01 # Akin to weight-decay
    beta_decay: float = 0.8
    centralize: bool = True
    use_rms: bool = True
    momentum: bool = False
    momentum_beta: float = 0.9
    _t: int = 0

    def step(self, loss, lr=None):
        self._t += 1
        self.lr = lr
        loss.backward()

    def prepare(self, p):
        if len(p.shape) == 2:
            p._c = torch.zeros(p.shape[1], device=p.device, dtype=p.dtype)
        else:
            p._v = torch.zeros_like(p)

        if self.momentum:
            p._m = torch.zeros_like(p)

    def hook(self, p):
        ag = p.view_as(p).grad_fn.next_functions[0][0]
        p._acc_grads = [ag]

        @torch.no_grad()
        def gf(*_):
            alpha = self.lr

            g = p.grad

            if self.centralize and sum(g.shape) > 1:
                g.sub_(g.mean(dim=tuple(range(1, len(g.shape))), keepdim=True))

            beta_t = 1.0 - math.pow(self._t, -self.beta_decay)
            u = g.square()

            if len(p.shape) == 2:
                u.mul_(1-beta_t).add_(p._c.unsqueeze(0).broadcast_to(g.shape), alpha=beta_t)
                u.add_(self.eps)
                p._c = u.mean(dim=0)
            else:
                u.mul_(1-beta_t).add_(p._v, alpha=beta_t)
                u.add_(self.eps)
                p._v = u

            m = u.rsqrt() * g

            if self.use_rms:
                m.div_(max(1.0, m.square().mean().sqrt()))

            p_norm = p.norm()
            g_norm = g.norm()

            if p_norm != 0 and g_norm != 0:
                m.mul_(p_norm / g_norm)
                m.add_(p - p/p_norm, alpha=self.Lambda)

            if self.momentum:
                p._m.mul_(self.momentum_beta).add_(m, alpha=1-self.momentum_beta)
                m = p._m

            p.add_(m, alpha=-alpha)
            p.grad = None

        ag.register_hook(gf)

After i fused techniques from both adalite and SM3 optimizer. Here is my code for fusedbun optimizer which uses techniwues from both optimizers
import torch
from torch.optim.optimizer import Optimizer
import math

class Fusedbun(Optimizer):
    def __init__(self, params, lr=1e-3, eps=1e-8, beta_decay=0.8, Lambda=0.01, momentum_beta=0.9, centralize=True, use_rms=True):
        """
        Args:
            params (iterable): An iterable of parameters.
            lr (float, optional): The learning rate (default: 1e-3).
            eps (float, optional): A small value for numerical stability (default: 1e-8).
            beta_decay (float, optional): A parameter controlling the rate of decay for the moving average of squared gradients (default: 0.8).
            Lambda (float, optional): default: 0.01
            momentum_beta (float, optional): Coefficient for the moving average of gradients (momentum) (default: 0.9).
            centralize (bool, optional): Boolean indicating whether to centralize gradients to have zero mean (default: True).
            use_rms (bool, optional): Boolean indicating whether to use RMSprop-like denominator normalization (default: True).
        """
        defaults = dict(lr=lr, eps=eps, beta_decay=beta_decay, Lambda=Lambda, momentum_beta=momentum_beta, centralize=centralize, use_rms=use_rms)
        super(Fusedbun, self).__init__(params, defaults)
    
    @torch.no_grad()
    def step(self, closure=None):
        """
        Args:
            closure (callable, optional): A closure that reevaluates the model and returns the loss.
        Returns:
            loss (Tensor, optional): The loss tensor, if the closure is provided.
        """
        loss = None
        if closure is not None:
            # compute the loss if a closure is provided
            loss = closure()

        for group in self.param_groups:
            lr = group['lr']
            eps = group['eps']
            beta_decay = group['beta_decay']
            Lambda = group['Lambda']
            momentum_beta = group['momentum_beta']
            centralize = group['centralize']
            use_rms = group['use_rms']
            
            for p in group['params']:
                if p.grad is None:
                    continue

                grad = p.grad.data
                if centralize and len(p.shape) > 1:
                    # centralize gradients for non-scalar parameters (from adalite)
                    grad = grad.sub(grad.mean(dim=tuple(range(1, len(grad.shape))), keepdim=True))
                    
                state = self.state[p]

                if len(state) == 0:
                    state['step'] = 0
                    state['exp_avg_sq'] = torch.zeros_like(p, memory_format=torch.preserve_format)
                    if momentum_beta > 0:
                        state['momentum_buffer'] = torch.zeros_like(p, memory_format=torch.preserve_format)
                
                exp_avg_sq = state['exp_avg_sq']
                state['step'] += 1

                if momentum_beta > 0:
                    momentum_buffer = state['momentum_buffer']
                    momentum_buffer.mul_(momentum_beta).add_(grad)
                    grad = momentum_buffer

                #sparse update mechanism
                if p.dim() > 1:
                    mask = grad.abs() > eps
                    grad = grad * mask
                    exp_avg_sq = torch.where(mask, exp_avg_sq*beta_decay + (1-beta_decay)*grad.pow(2), exp_avg_sq)
                else:
                    exp_avg_sq.mul_(beta_decay).addcmul_(grad, grad, value=1-beta_decay)

                denom = exp_avg_sq.sqrt().add_(eps)
                grad_normalized = grad / denom if use_rms else grad

                if Lambda != 0:
                    p.data.mul_(1 - lr * Lambda)

                p.data.add_(grad_normalized, alpha=-lr)
        return loss

Now, implement, hessian approximation matrix technique into my fusedbun optimizer. 
Keep comments and explain what is happening with the code.