Sabir-2-100M

Sabir-2-100M is a high-efficiency 100.8M parameter autoregressive causal language model designed for dense representation and optimized inference.

The model implements modern architectural advancements inspired by recent state-of-the-art architectures (such as DeepSeek-V2/V3), specifically utilizing Multi-Head Latent Attention (MLA) for compressed Key-Value representations, SwiGLU non-linear feed-forward networks, and Decoupled Rotary Position Embeddings (RoPE).

Architecture Specifications

• Total Parameters: 100,883,712 (~100.8M)
• Attention Mechanism: Multi-Head Latent Attention (MLA)
  • q_lora_rank: 256
  • kv_lora_rank: 128 (Low-rank Key-Value compression)
  • qk_rope_head_dim: 32 (Decoupled positional dimension)
  • v_head_dim: 64
• Hidden Dimension ($d_{model}$): 768
• Number of Layers: 16
• Number of Attention Heads: 12
• Intermediate Size ($d_{ff}$): 2048 (SwiGLU activation)
• Normalization: RMSNorm ($\epsilon = 10^{-6}$)
• Positional Encoding: Rotary Position Embedding (RoPE)
• Context Length ($L_{ctx}$): 128 Tokens
• Vocabulary Size: 8,192 (Byte-Pair Encoding with Metaspace pre-tokenization)
• Weight Tying: Enabled ($W_{embeddings} = W_{lm_head}$)

Key Architectural Advantages

1. Multi-Head Latent Attention (MLA)

Unlike standard Multi-Head Attention (MHA) or Grouped-Query Attention (GQA), MLA compresses Key-Value projections into a low-dimensional latent space ($kv_lora_rank = 128$), significantly reducing inference memory footprint while preserving full multi-head expressive capability.

2. SwiGLU Non-Linearity

The feed-forward network uses the Swish-Gated Linear Unit (SwiGLU), providing a smoother gradient flow and superior representation density compared to standard ReLU or GeLU variants.

3. Root Mean Square Normalization (RMSNorm)

RMSNorm replaces standard LayerNorm, discarding the mean-centering operation to achieve higher computational throughput on modern hardware accelerators.

Inference & Usage

import torch
import torch.nn as nn
import torch.nn.functional as F
from safetensors.torch import load_file
from tokenizers import Tokenizer

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

# 1. Load Tokenizer
tokenizer = Tokenizer.from_file("tokenizer.json")

# 2. Load Weights
weights = load_file("model.safetensors")

# Define Sabir-2-100M Architecture
class RMSNorm(nn.Module):
    def __init__(self, dim, eps=1e-6):
        super().__init__()
        self.eps = eps
        self.weight = nn.Parameter(torch.ones(dim))
    def forward(self, x):
        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) * self.weight

def apply_rotary_emb(x):
    B, H, T, D = x.shape
    inv_freq = 1.0 / (10000 ** (torch.arange(0, D, 2, device=x.device).float() / D))
    t = torch.arange(T, device=x.device, dtype=torch.float32)
    freqs = torch.outer(t, inv_freq)
    cos, sin = freqs.cos().unsqueeze(0).unsqueeze(1), freqs.sin().unsqueeze(0).unsqueeze(1)
    x1, x2 = x[..., 0::2], x[..., 1::2]
    return torch.cat([x1 * cos - x2 * sin, x1 * sin + x2 * cos], dim=-1)

class MultiHeadLatentAttention(nn.Module):
    def __init__(self, d_model=768, n_heads=12, q_lora_rank=256, kv_lora_rank=128, qk_rope_head_dim=32, v_head_dim=64):
        super().__init__()
        self.n_heads = n_heads
        self.qk_rope_head_dim = qk_rope_head_dim
        self.v_head_dim = v_head_dim
        self.q_down_proj = nn.Linear(d_model, q_lora_rank, bias=False)
        self.q_down_norm = RMSNorm(q_lora_rank)
        self.q_up_proj = nn.Linear(q_lora_rank, n_heads * (v_head_dim + qk_rope_head_dim), bias=False)
        self.kv_down_proj = nn.Linear(d_model, kv_lora_rank, bias=False)
        self.kv_down_norm = RMSNorm(kv_lora_rank)
        self.kv_up_proj = nn.Linear(kv_lora_rank, n_heads * v_head_dim, bias=False)
        self.k_rope_proj = nn.Linear(d_model, n_heads * qk_rope_head_dim, bias=False)
        self.out_proj = nn.Linear(n_heads * v_head_dim, d_model, bias=False)

    def forward(self, x):
        B, T, _ = x.shape
        q = self.q_up_proj(self.q_down_norm(self.q_down_proj(x)))
        q = q.view(B, T, self.n_heads, self.v_head_dim + self.qk_rope_head_dim).transpose(1, 2)
        q_val, q_rope = torch.split(q, [self.v_head_dim, self.qk_rope_head_dim], dim=-1)
        kv_latent = self.kv_down_norm(self.kv_down_proj(x))
        v = self.kv_up_proj(kv_latent).view(B, T, self.n_heads, self.v_head_dim).transpose(1, 2)
        k_val = v
        k_rope = self.k_rope_proj(x).view(B, T, self.n_heads, self.qk_rope_head_dim).transpose(1, 2)
        q_rope = apply_rotary_emb(q_rope)
        k_rope = apply_rotary_emb(k_rope)
        out = F.scaled_dot_product_attention(torch.cat([q_val, q_rope], dim=-1), torch.cat([k_val, k_rope], dim=-1), v, is_causal=True)
        return self.out_proj(out.transpose(1, 2).contiguous().view(B, T, self.n_heads * self.v_head_dim))

class SwiGLU(nn.Module):
    def __init__(self, d_model, hidden_dim):
        super().__init__()
        self.w1 = nn.Linear(d_model, hidden_dim, bias=False)
        self.w2 = nn.Linear(hidden_dim, d_model, bias=False)
        self.w3 = nn.Linear(d_model, hidden_dim, bias=False)
    def forward(self, x):
        return self.w2(F.silu(self.w1(x)) * self.w3(x))

class MLABlock(nn.Module):
    def __init__(self, d_model, n_heads, hidden_dim):
        super().__init__()
        self.attn_norm = RMSNorm(d_model)
        self.attn = MultiHeadLatentAttention(d_model, n_heads)
        self.ffn_norm = RMSNorm(d_model)
        self.ffn = SwiGLU(d_model, hidden_dim)
    def forward(self, x):
        return x + self.ffn(self.ffn_norm(x + self.attn(self.attn_norm(x))))

class Sabir2ForCausalLM(nn.Module):
    def __init__(self, vocab_size=8192, d_model=768, n_layers=16, n_heads=12, hidden_dim=2048):
        super().__init__()
        self.tok_embeddings = nn.Embedding(vocab_size, d_model)
        self.layers = nn.ModuleList([MLABlock(d_model, n_heads, hidden_dim) for _ in range(n_layers)])
        self.norm = RMSNorm(d_model)
        self.lm_head = nn.Linear(d_model, vocab_size, bias=False)
    def forward(self, idx):
        x = self.tok_embeddings(idx)
        for layer in self.layers:
            x = layer(x)
        return self.lm_head(self.norm(x))

# Instantiate and load
model = Sabir2ForCausalLM().to(device)
model.load_state_dict(weights)
model.eval()

# Generate function
def generate(prompt, max_new_tokens=40, temp=0.7):
    ids = tokenizer.encode(prompt).ids
    x = torch.tensor([ids], dtype=torch.long).to(device)
    with torch.no_grad():
        for _ in range(max_new_tokens):
            logits = model(x[:, -128:])[:, -1, :] / temp
            probs = F.softmax(logits, dim=-1)
            nxt = torch.multinomial(probs, 1)
            x = torch.cat((x, nxt), dim=1)
    return tokenizer.decode(x[0].tolist())

print(generate("Zaman neden tek yöne akar? Çünkü "))