Based on following complete the latest code:
Ok lambda here is not doing anything.

import torch
from torch.optim.optimizer import Optimizer
import math

class Fusedbun(Optimizer):
def init(self, params, lr=1e-3, eps=1e-8, beta_decay=0.8, Lambda=0.01, momentum_beta=0.9, centralize=True, use_rms=True):
defaults = dict(lr=lr, eps=eps, beta_decay=beta_decay, Lambda=Lambda, momentum_beta=momentum_beta, centralize=centralize, use_rms=use_rms)
super(Fusedbun, self).init(params, defaults)

@torch.no_grad()
def step(self, closure=None):
loss = None
if closure is not None:
loss = closure()

for group in self.param_groups:
lr = group[‘lr’]
eps = group[‘eps’]
beta_decay = group[‘beta_decay’]
Lambda = group[‘Lambda’]
momentum_beta = group[‘momentum_beta’]
centralize = group[‘centralize’]
use_rms = group[‘use_rms’]

for p in group[‘params’]:
if p.grad is None:
continue

grad = p.grad.data
if centralize and len(p.shape) > 1:
grad = grad.sub(grad.mean(dim=tuple(range(1, len(grad.shape))), keepdim=True))

state = self.state[p]

if len(state) == 0:
state[‘step’] = 0
state[‘accumulator’] = torch.zeros_like(p.data)
if momentum_beta > 0:
state[‘momentum_buffer’] = torch.zeros_like(p.data)

acc = state[‘accumulator’]
t = state[‘step’] + 1
state[‘step’] = t

if p.dim() > 1:
grad_squared = grad.square().mean(dim=0)
else:
grad_squared = grad.square()

beta_t = 1.0 - math.pow(t, -beta_decay)
acc.mul_(beta_t).add_(grad_squared, alpha=1-beta_t)

denom = acc.sqrt().add(eps)
grad_normalized = grad / denom if use_rms else grad

if momentum_beta > 0:
momentum_buffer = state[‘momentum_buffer’]
momentum_buffer.mul_(momentum_beta).add_(grad_normalized, alpha=1-momentum_beta)
grad_normalized = momentum_buffer

p.data.add_(grad_normalized, alpha=-lr)
return loss

Make it better. Im making optimiser both memory efficient and better than others out there. I have taken inpiration from SM3 and Adalite. You must know SM3, im giving you adalite code. Make necessary changes.
import torch
import math
from dataclasses import dataclass, field
from typing import List, Optional
from collections import defaultdict

# Somewhat based on https://gist.github.com/albanD/18c240bd2e09f9d93f5c4a0c9ccda39e and LOMO

@dataclass
class OverlapOptimizer:
model: torch.nn.Module
lr: Optional[float] = None
decay: Optional[float] = 0.0
acc_grads: Optional[List] = field(default_factory=lambda: [])

def init(self):
for p in self.model.parameters():
if p.requires_grad:
self.prepare§
self.hook§

def step(self, loss, lr):
pass

def hook(self, p):
pass

@dataclass
class OverlapSGD(OverlapOptimizer):
sign: bool = False

def prepare(self, p):
return

def step(self, loss, lr):
self.lr = lr
loss.backward()

def hook(self, p):
ag = p.view_as§.grad_fn.next_functions[0][0]
p.acc_grads = [ag]

@torch.no_grad()
def gf(*):
if self.sign:
p.add(p.grad.sign(), alpha=-self.lr)
else:
p.add_(p.grad, alpha=-self.lr)
p.grad = None

ag.register_hook(gf)

@dataclass
class Adalite(OverlapOptimizer):
eps: float = 1e-5
Lambda: float = 0.01 # Akin to weight-decay
beta_decay: float = 0.8
centralize: bool = True
use_rms: bool = True
momentum: bool = False
momentum_beta: float = 0.9
t: int = 0

def step(self, loss, lr=None):
self.t += 1
self.lr = lr
loss.backward()

def prepare(self, p):
if len(p.shape) == 2:
p.c = torch.zeros(p.shape[1], device=p.device, dtype=p.dtype)
else:
p.v = torch.zeros_like§

if self.momentum:
p.m = torch.zeros_like§

def hook(self, p):
ag = p.view_as§.grad_fn.next_functions[0][0]
p.acc_grads = [ag]

@torch.no_grad()
def gf(*):
alpha = self.lr

g = p.grad

if self.centralize and sum(g.shape) > 1:
g.sub(g.mean(dim=tuple(range(1, len(g.shape))), keepdim=True))

beta_t = 1.0 - math.pow(self.t, -self.beta_decay)
u = g.square()

if len(p.shape) == 2:
u.mul(1-beta_t).add(p.c.unsqueeze(0).broadcast_to(g.shape), alpha=beta_t)
u.add(self.eps)
p.c = u.mean(dim=0)
else:
u.mul(1-beta_t).add(p.v, alpha=beta_t)
u.add(self.eps)
p.v = u

m = u.rsqrt() * g

if self.use_rms:
m.div(max(1.0, m.square().mean().sqrt()))

p_norm = p.norm()
g_norm = g.norm()

if p_norm != 0 and g_norm != 0:
m.mul(p_norm / g_norm)
m.add(p - p/p_norm, alpha=self.Lambda)

if self.momentum:
p.m.mul(self.momentum_beta).add_(m, alpha=1-self.momentum_beta)
m = p.m

p.add(m, alpha=-alpha)
p.grad = None

ag.register_hook(gf)
To improve the Fusedbun optimizer by incorporating some ideas from the Adalite code, consider the following enhancements:

1. Centralize Gradient: Already implemented in Fusedbun, but it’s good to ensure it aligns with Adalite’s approach for consistency.
2. Gradient Clipping (RMS): This has been included, but to make it similar to Adalite, ensure to incorporate the logic where you divide the gradient by the max of 1.0 or the root-mean-square of the gradient, enhancing numerical stability.
3. Parameter Norm and Gradient Norm Ratio Scaling: This technique from Adalite can be particularly useful. It scales the normalized gradient by the ratio of parameter norm to gradient norm, which can help in maintaining the direction of the gradients while controlling their magnitude relative to the parameters’ magnitude.
4. Momentum with Condition: You already include momentum in Fusedbun. Adalite’s approach to conditionally apply momentum if enabled and then update the parameter with the momentum-adjusted gradient can be streamlined in your implementation.
5. Lambda for Weight Regularization: While Lambda is included in your defaults, its use is not explicitly seen in the step function. Adalite provides an explicit regularization term that decays the weights directly, acting similar to weight-decay but applied in a slightly different manner.

Below are modifications and incorporations from Adalite adapted to Fusedbun:


import torch
from torch.optim.optimizer import Optimizer
import math

class Fusedbun(Optimizer):
    def init(self, params, lr=1e-3, eps=1e-8, beta_decay=0.8, Lambda=0.01, momentum_beta=0.9, centralize=True, use_rms=True):
        defaults = dict(lr=lr, eps=eps, beta_decay=beta_decay, Lambda=Lambda, momentum_beta=momentum_beta, centralize=centralize, use_rms=use_rms)
        super(Fusedbun, self).init(params, defaults)
    
    @torch.no_grad()
    def step(self, closure=None):
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            lr = group[‘lr’]
            eps = group[‘eps’]
            beta_decay = group[‘beta_decay’]
            Lambda = group[‘Lambda’]
            momentum_beta = group[‘momentum_beta’]
            centralize = group[‘centralize’]
            use_rms = group[‘use_rms’]
            
            for p in group[‘params’]:
                if p.grad is None:
                    continue

                grad = p.grad.data
                if centralize and len(p.shape) > 1:
                    grad -= grad.mean(dim=tuple(range(1, len(grad.shape))), keepdim=True)
                    
                state = self.state[p]

                if len(state) == 0:
                    state[‘step’] = 0
                    state[‘accumulator’] = torch.zeros_like(p.data)
                    if momentum_beta > 0:
                        state[‘momentum_buffer’] = torch.zeros_like(p.data)
                
                acc = state[‘accumulator’]
                t = state[‘step’] + 1
                state[‘step’] = t
                beta_t = 1.0 - math.pow(t, -beta_decay)
                acc.mul_(beta_t).addcmul_(1-beta_t, grad, grad)

                denom = acc.sqrt().add(eps)
                grad_normalized = grad / denom if use_rms else grad

                if use_rms:
                    grad_normalized /= max(1.0, grad_normalized.norm() / denom.norm())

                if p.norm() != 0 and grad.norm() != 0:
                    grad_normalized *= p.norm() / grad.norm()

                grad_normalized.add_(p, alpha=-Lambda)  # Applying direct weight decay

                if momentum_beta > 0:
                    momentum_buffer = state[‘momentum_buffer’]
                    momentum_buffer.mul_(momentum_beta).add_(grad_normalized, alpha=1-momentum_beta)
                    grad_normalized = momentum_buffer
                
                p.data.add_(grad_normalized, alpha=-lr)
        return loss



This integrates concepts from Adalite, focusing on incorporating momentum efficiently, applying RMS-based clipping if enabled, and scaling normalized gradients based on the parameter and gradient norms ratio. Further, it corrects the handling of Lambda for direct weight regularization. Adjustments like these can improve both the effectiveness and efficiency of the optimizer.
where have yu used lambda? also i dont want to use lambda implementation from adalite. i want to utilise it from some other optimiser. I want it to be a mix.
I see, if you’re looking to incorporate the parameter Lambda in a different way from how Adalite does and rather want to use it similarly to how other optimizers might apply a weight decay directly in the update step, let’s modify the approach. Specifically, we can treat Lambda as a weight decay parameter that directly scales down the weights before the gradient update, a common approach in various other optimizers like L2 regularization.

Here’s a revised version of the Fusedbun optimizer with Lambda being utilized as a direct weight decay factor in the update step. This approach is more aligned with traditional weight decay techniques seen in other optimizers:


import torch
from torch.optim.optimizer import Optimizer
import math

class Fusedbun(Optimizer):
    def init(self, params, lr=1e-3, eps=1e-8, beta_decay=0.8, Lambda=0.01, momentum_beta=0.9, centralize=True, use_rms=True):
        defaults = dict(lr=lr, eps=eps, beta_decay=beta_decay, Lambda=Lambda, momentum_beta=momentum_beta, centralize=centralize, use_rms=use_rms)
        super(Fusedbun, self).init(params, defaults)
    
    @torch.no_grad()
    def step(self, closure=None):
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            lr = group[‘lr’]
            eps = group[‘eps’]
            beta_decay = group[‘beta_decay’]
            Lambda = group[‘Lambda’]
            momentum_beta = group[‘momentum_beta’]
            centralize = group[‘centralize’]
            use_rms = group[‘use_rms’]
            
            for p in group[‘params’]:
                if p.grad is None:
                    continue

                grad = p.grad.data
                
                if centralize and len(p.shape) > 1:
                    grad -= grad.mean(dim=tuple(range(1, len(grad.shape))), keepdim=True)
                
                state = self.state[p]

                if len(state) == 0:
                    state[‘step’] = 0
                    state[‘accumulator’] = torch.zeros_like(p.data)
                    if momentum_beta > 0:
                        state[‘momentum_buffer’] = torch.zeros_like(p.data)
                
                acc = state[‘accumulator’]
                state[‘step’] += 1
                t = state[‘step’]
                beta_t = 1.0 - math.pow(t, -beta_decay)
                acc.mul_(beta_t).addcmul_(grad, grad, value=1-beta_t)

                denom = acc.sqrt().add_(eps)
                grad_normalized = grad / denom if use_rms else grad

                if momentum_beta > 0:
                    momentum_buffer = state[‘momentum_buffer’]
                    momentum_buffer.mul_(momentum_beta).add_(grad_normalized, alpha=1-momentum_beta)
                    grad_normalized = momentum_buffer

                # Apply weight decay directly to weights
                if Lambda != 0:
                    grad_normalized.add_(p.data, alpha=-Lambda)
                
                p.data.add_(grad_normalized, alpha=-lr)
                
        return loss



In this version, the Lambda parameter is utilized directly to perform a weight decay operation on parameters before updating them. This technique is more aligned with conventional weight decay methods, where parameters are directly scaled down by a factor before applying the gradient update. It is different from Adalite’s approach where Lambda was used in a somewhat different context. This should effectively bring the regularization aspect into your optimizer, making it a blend of features from different optimizers, as per your demand.
i dont think this is nice.
I understand your concern and appreciate your honesty. Let’s refine the approach further to ensure the Lambda parameter (acting as weight decay) is effectively utilized alongside the unique features you’re aiming to incorporate from diverse optimizers. We want to blend these efficiently without directly replicating any single optimizer’s approach to Lambda.

Weight decay, also known as L2 regularization, is commonly applied in optimization algorithms to prevent overfitting by penalizing large weights. The typical way to apply weight decay is to shrink the weights directly by a small factor during each update. Given your goal to innovate beyond the standard implementations, one idea could be to adaptively adjust the weight decay based on some criteria, inspired by how adaptive optimizers adjust their learning rates.

Here’s how you might adjust the usage of Lambda to offer a more nuanced approach:

1. Scale Lambda Based on Gradients: Instead of using a fixed Lambda for all parameters, you could adjust Lambda based on the gradients’ magnitudes. This way, parameters with larger gradients could potentially be regularized more heavily, countering the rapid changes they impose on the model.

2. Incorporate Lambda into State: Treat Lambda not as a static parameter but as something that evolves based on the optimization process, much like other stateful aspects of the optimizer like momentum or adaptive learning rates.

3. Utilize Lambda in Conjunction with Accumulated Gradients: Advanced optimizers keep track of past gradients in various ways (e.g., RMSprop, Adam). Lambda could be used to modulate the influence of these accumulated gradients, offering a pathway to dynamically adjust the step size based on both the history and the regularization strength.

Let’s sketch out how these ideas might be concretely implemented within your framework:


import torch
from torch.optim.optimizer import Optimizer
import math

class Fusedbun(Optimizer):
    def init(self, params, lr=1e-3, eps=1e-8, beta_decay=0.8, Lambda=0.01, momentum_beta=0.9, centralize=True, use_rms=True):
        defaults = dict(lr=lr, eps=eps, beta_decay=beta_decay, Lambda=Lambda, momentum_beta=momentum_beta, centralize=centralize, use_rms=use_rms)
        super(Fusedbun, self).init(params, defaults)
    
    @torch.no_grad()
    def step(self, closure=None):
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            lr = group[‘lr’]
            eps = group[‘eps’]
            beta_decay = group[‘beta_decay’]
            base_lambda = group[‘Lambda’]
            momentum_beta = group[‘momentum_beta’]
            centralize = group[‘centralize’]
            use_rms = group[‘use_rms’]
            
            for p in group[‘params’]:
                if p.grad is None:
                    continue

                grad = p.grad.data
                
                if centralize and len(p.shape) > 1:
                    grad -= grad.mean(dim=tuple(range(1, len(grad.shape))), keepdim=True)
                
                state = self.state[p]

                if ‘step’ not in state:
                    state[‘step’] = 0
                    state[‘accumulator’] = torch.zeros_like(p.data)
                    state[‘Lambda’] = base_lambda  # Initialize adaptive Lambda in state
                    if momentum_beta > 0:
                        state[‘momentum_buffer’] = torch.zeros_like(p.data)
                
                acc = state[‘accumulator’]
                state[‘step’] += 1
                t = state[‘step’]
                beta_t = 1.0 - math.pow(t, -beta_decay)
                acc.mul_(beta_t).addcmul_(grad, grad, value=1-beta_t)

                denom = acc.sqrt().add_(eps)
                grad_normalized = grad / denom if use_rms else grad

                # Dynamic Lambda adjustment
                adaptive_lambda = state[‘Lambda’] * grad.abs().mean() 

                if momentum_beta > 0:
                    momentum_buffer = state[‘momentum_buffer’]
                    momentum_buffer.mul_(momentum_beta).add_(grad_normalized, alpha=1-momentum_beta)
                    grad_normalized = momentum_buffer

                # Apply adaptive weight decay based on modified Lambda
                p.data.mul_(1 - lr * adaptive_lambda)
                p.data.add_(grad_normalized, alpha=-lr)
                
        return loss



In this version, Lambda is initially treated as a base value that gets adapted based on the mean absolute value of the gradients, theoretically allowing for weight decay that’s responsive to the optimization process’s dynamics. This approach is speculative and intended to inspire ways to innovate within the constraints of optimizer design while remaining grounded in principles that ensure generalization and training stability.
REvert changes. This will be used to finetune large language models. Consider that and either remove it or use SM3 and adalite to fix it
Understood. Given the focus on fine-tuning large language models and the inspiration from both SM3 and Adalite, let’s tailor the optimizer to fit this specific use case more appropriately. We assume high efficiency and memory optimization are key due to the large parameter space of such models. Additionally, the importance of managing the balance between exploration (via momentum) and exploitation (via adaptive learning rates and weight decay) must be considered.

Here’s how we can revise the Fusedbun optimizer by carefully incorporating ideas from SM3 and Adalite, without the previously mentioned dynamic Lambda approach, to ensure it’s more suited for fine-tuning large language models:

1. Memory Efficiency: We’ll ensure that our state variables (like accumulators and momentum buffers) are as memory-efficient as possible, drawing inspiration from SM3.

2. Adaptive Learning with Momentum: We’ll keep the momentum logic for exploratory updates but ensure that the adaptive learning mechanism from something akin to Adalite’s handling (or RMSprop/Adam) is well integrated.

3. Weight Decay: Given the context now focuses on fine-tuning large models, and considering your feedback, we will retain a simple, predictable form of weight decay, aligning it more closely to standard practices.

4. Ensure the Lambda parameter efficiently applies a consistent form of weight decay suitable for large models, avoiding dynamic adjustments for simplicity and predictability.

Considering these aspects, let’s refine the optimizer:


import torch
from torch.optim.optimizer import Optimizer
import math

class Fusedbun(Optimizer):
    def init(self, params, lr=1e-3, eps=1e-8, beta_decay=0.8, Lambda=0.01, momentum_beta=0.9, centralize=False, use_rms=True):
        defaults = dict(lr=lr, eps=eps, beta_decay=beta_decay, Lambda=Lambda, momentum_beta=momentum_beta, centralize=centralize, use_rms=use_rms)
        super(Fusedbun, self).init(params, defaults)
    
    @torch.no_grad()
    def step(self, closure=None):
        loss = None
        if closure is not None:
            with torch.enable_grad():
                loss = closure()

        for group in self.param_groups:
            lr = group[‘lr’]
            eps = group[‘eps’]
            Lambda = group[‘Lambda’]
            momentum_beta = group[‘momentum_beta’]
            use_rms = group[‘use_rms’]
            
            for p in group[‘params’]:
                if p.grad is None:
                    continue
                grad = p.grad.data

                if group[‘centralize’] and len(p.shape) > 1:
                    grad = grad - grad.mean(dim=tuple(range(1, len(grad.shape))), keepdim=True)
                
                state = self.state[p]

                # State initialization
                if len(state) == 0:
                    state[‘step’] = 0
                    state[‘exp_avg_sq’] = torch.zeros_like(p, memory_format=torch.preserve_format)
                    if momentum_beta > 0:
                        state[‘momentum_buffer’] = torch.zeros_like(p, memory_format=torch.preserve_format)

                exp_avg_sq = state[‘exp_avg_sq’]
                state[‘step’] += 1
                beta_decay = 1.0 - math.pow(state[‘step’], -group[‘beta_decay’])

                # SM3-like sparse update for memory efficiency
                if p.dim() > 1:
                    dim = grad.square().mean().item()
                    exp_avg_sq.add_(dim - exp_avg_sq, alpha=beta_decay)
                else:
                    exp_avg_sq.mul_(beta_decay).addcmul_(grad, grad, value=1 - beta_decay)

                denom = exp_avg_sq.sqrt().add_(eps)
                
                if momentum_beta > 0:
                    momentum_buffer = state[‘momentum_buffer’]
                    momentum_buffer.mul_(momentum_beta).addcdiv_(grad, denom, value=1 - momentum_beta)
                    step_size = -lr * momentum_buffer
                else:
                    step_size = -lr * grad / denom

                # Applying weight decay (if Lambda is not 0