tiny-random/raon-1-mini

This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from KRAFTON/Raon-1-mini-preview.

Structural comparison with KRAFTON/Raon-1-mini-preview:

Feature	Source	Tiny
Vocabulary / tied embeddings	259,840 / untied	same
Layers	25	8; two 3-RDA + 1-MLA cycles
RDA / NoPE MLA layer cadence	18 / 7	6 / 2
Raon mHC streams	4	same
Dense prefix / MoE layers	layer 0 / 24 layers	layer 0 / 7 layers
Routed experts / active experts	128 / 6	64 / 6
Hidden width / attention head dimension	2,560 / 128	32 / 32
RDA, MLA and LoRA dimensions	source widths	32-wide compatible projections
Routed-expert storage	MXFP8 E4M3 + E8M0 1x32 scales	same
Router / shared expert / output gate	sigmoid top-k / present / present	same

Transformers constructs Raon's architecture, then the definition rewrites only routed-expert tensors to the source MXFP8 storage representation. This is necessary because random CPU initialization of float8_e4m3fn parameters is not implemented by PyTorch; it does not change the serialized precision.

File path	Size
model.safetensors	35.3MB

Example usage:

• Transformers (✅ verified)

import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

model_id = "tiny-random/raon-1-mini"
device = "cuda" if torch.cuda.is_available() else "cpu"
model = AutoModelForCausalLM.from_pretrained(model_id, trust_remote_code=True).to(device).eval()
tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
inputs = tokenizer("Hello", return_tensors="pt").to(device)
output = model.generate(**inputs, do_sample=False, min_new_tokens=10, max_new_tokens=10)
print(tokenizer.decode(output[0, inputs["input_ids"].shape[1]:], skip_special_tokens=True))

Codes to create this repo (✅ verified):

Click to expand

import json
from pathlib import Path
from shutil import copyfile

import torch
from huggingface_hub import hf_hub_download, list_repo_files
from safetensors.torch import save_file
from transformers import AutoConfig, AutoModelForCausalLM, set_seed

source_model_id = "KRAFTON/Raon-1-mini-preview"
save_folder = "/tmp/tiny-random/raon-1-mini"
source_revision = "7bf87c45fc7b658d3c9532305f3aaed106bbac4f"
set_seed(42)
folder = Path(save_folder)
folder.mkdir(parents=True, exist_ok=True)
raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
raw.update(
    hidden_size=32, head_dim=32, num_attention_heads=1, num_hidden_layers=8, intermediate_size=128,
    num_experts=64, num_experts_per_token=6, moe_intermediate_size=32,
    q_lora_rank=32, kv_lora_rank=32, qk_nope_head_dim=32,
    qk_rope_head_dim=32, v_head_dim=32, rda_decay_rank=32,
    max_position_embeddings=4096,
)
raw["rda_config"].update(
    num_heads=1, head_dim=32,
    rda_layers=[1, 2, 3, 5, 6, 7], mla_layers=[4, 8],
)
# Build the public remote-code graph in BF16 first.  The source's MXFP8
# modules cannot initialize random float8 parameters directly, then the
# routed experts are converted back to the source's E4M3 + E8M0 format.
construction_raw = json.loads(json.dumps(raw))
construction_raw["quantization_config"]["dequantize"] = True
(folder / "config.json").write_text(json.dumps(construction_raw, indent=2) + "\n", encoding="utf-8")
for filename in ("configuration_raon1_mini.py", "modeling_raon1_mini.py"):
    copyfile(hf_hub_download(source_model_id, filename, revision=source_revision), folder / filename)
config = AutoConfig.from_pretrained(folder, trust_remote_code=True)
model = AutoModelForCausalLM.from_config(config, trust_remote_code=True).eval()
with torch.no_grad():
    for name, parameter in sorted(model.named_parameters()):
        if name.endswith(("weight", "phi_pre", "phi_post", "phi_z")) and not name.endswith("norm.weight"):
            torch.nn.init.normal_(parameter, mean=0, std=0.02)
        elif name.endswith("norm.weight") or name.endswith("layernorm.weight"):
            parameter.fill_(1)

def quantize_mxfp8(weight: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
    rows, columns = weight.shape
    assert columns % 32 == 0, f"MXFP8 columns must divide into 32-value blocks, got {columns}"
    blocks = weight.float().reshape(rows, columns // 32, 32)
    bits = (blocks.abs().amax(-1) / 448.0).view(torch.int32)
    exponent, mantissa = (bits >> 23) & 0xFF, bits & 0x7FFFFF
    round_up = (mantissa > 0) & (exponent != 0xFE) & ~((exponent == 0) & (mantissa <= 0x400000))
    scale = (exponent + round_up.int()).clamp(max=254)
    reciprocal = ((254 - scale) << 23).view(torch.float32)
    reciprocal = torch.where(scale == 254, torch.full_like(reciprocal, 2.0**-127), reciprocal)
    values = (blocks * reciprocal.unsqueeze(-1)).clamp(-448.0, 448.0)
    return values.to(torch.float8_e4m3fn).reshape(rows, columns), scale.to(torch.uint8)

state = model.state_dict()
for name in sorted(tuple(state)):
    if ".moe.experts." in name and name.endswith(".weight"):
        values, scales = quantize_mxfp8(state[name])
        state[name] = values
        state[name.removesuffix(".weight") + ".weight_scale_inv"] = scales
raw["quantization_config"]["dequantize"] = False
(folder / "config.json").write_text(json.dumps(raw, indent=2) + "\n", encoding="utf-8")
save_file(state, folder / "model.safetensors", metadata={"format": "pt"})
assets = {"LICENSE", "chat_template.jinja", "generation_config.json", "tokenizer.json", "tokenizer_config.json"}
for filename in sorted(set(list_repo_files(source_model_id, revision=source_revision)) & assets):
    copyfile(hf_hub_download(source_model_id, filename, revision=source_revision), folder / filename)

Printing the model:

Click to expand

Raon1MiniForCausalLM(
  (model): Raon1MiniModel(
    (embed_tokens): Embedding(259840, 32, padding_idx=259200)
    (layers): ModuleList(
      (0): Raon1MiniDecoderLayer(
        (input_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
        (post_attention_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
        (self_attn): Raon1MiniRDA(
          (q_proj): Linear(in_features=32, out_features=32, bias=False)
          (k_proj): Linear(in_features=32, out_features=32, bias=False)
          (v_proj): Linear(in_features=32, out_features=32, bias=False)
          (q_conv1d): Raon1MiniShortConv()
          (k_conv1d): Raon1MiniShortConv()
          (v_conv1d): Raon1MiniShortConv()
          (f_a_proj): Linear(in_features=32, out_features=32, bias=False)
          (f_b_proj): Linear(in_features=32, out_features=32, bias=False)
          (b_proj): Linear(in_features=32, out_features=1, bias=False)
          (rda_w_proj): Linear(in_features=32, out_features=1, bias=False)
          (g_proj): Linear(in_features=32, out_features=32, bias=False)
          (o_norm): Raon1MiniRMSNorm((32,), eps=1e-05)
          (o_proj): Linear(in_features=32, out_features=32, bias=False)
        )
        (self_attn_raon_mhc): RaonMHC(
          (res): RaonMHCStreamMix()
        )
        (mlp_raon_mhc): RaonMHC(
          (res): RaonMHCStreamMix()
        )
        (mlp): Raon1MiniMLP(
          (gate_proj): Linear(in_features=32, out_features=128, bias=False)
          (up_proj): Linear(in_features=32, out_features=128, bias=False)
          (down_proj): Linear(in_features=128, out_features=32, bias=False)
        )
      )
      (1-2): 2 x Raon1MiniDecoderLayer(
        (input_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
        (post_attention_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
        (self_attn): Raon1MiniRDA(
          (q_proj): Linear(in_features=32, out_features=32, bias=False)
          (k_proj): Linear(in_features=32, out_features=32, bias=False)
          (v_proj): Linear(in_features=32, out_features=32, bias=False)
          (q_conv1d): Raon1MiniShortConv()
          (k_conv1d): Raon1MiniShortConv()
          (v_conv1d): Raon1MiniShortConv()
          (f_a_proj): Linear(in_features=32, out_features=32, bias=False)
          (f_b_proj): Linear(in_features=32, out_features=32, bias=False)
          (b_proj): Linear(in_features=32, out_features=1, bias=False)
          (rda_w_proj): Linear(in_features=32, out_features=1, bias=False)
          (g_proj): Linear(in_features=32, out_features=32, bias=False)
          (o_norm): Raon1MiniRMSNorm((32,), eps=1e-05)
          (o_proj): Linear(in_features=32, out_features=32, bias=False)
        )
        (self_attn_raon_mhc): RaonMHC(
          (res): RaonMHCStreamMix()
        )
        (mlp_raon_mhc): RaonMHC(
          (res): RaonMHCStreamMix()
        )
        (moe): Raon1MiniMoE(
          (gate): Raon1MiniRouter()
          (experts): ModuleList(
            (0-63): 64 x Raon1MiniExpert(
              (gate_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
              (up_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
              (down_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
            )
          )
          (shared_experts): Raon1MiniMLP(
            (gate_proj): Linear(in_features=32, out_features=32, bias=False)
            (up_proj): Linear(in_features=32, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=32, bias=False)
          )
        )
      )
      (3): Raon1MiniDecoderLayer(
        (input_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
        (post_attention_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
        (self_attn): Raon1MiniMLA(
          (q_a_proj): Linear(in_features=32, out_features=32, bias=False)
          (q_a_layernorm): Raon1MiniRMSNorm((32,), eps=1e-06)
          (q_b_proj): Linear(in_features=32, out_features=64, bias=False)
          (kv_a_proj_with_mqa): Linear(in_features=32, out_features=64, bias=False)
          (kv_a_layernorm): Raon1MiniRMSNorm((32,), eps=1e-06)
          (kv_b_proj): Linear(in_features=32, out_features=64, bias=False)
          (g_proj): Linear(in_features=32, out_features=32, bias=False)
          (o_proj): Linear(in_features=32, out_features=32, bias=False)
        )
        (self_attn_raon_mhc): RaonMHC(
          (res): RaonMHCStreamMix()
        )
        (mlp_raon_mhc): RaonMHC(
          (res): RaonMHCStreamMix()
        )
        (moe): Raon1MiniMoE(
          (gate): Raon1MiniRouter()
          (experts): ModuleList(
            (0-63): 64 x Raon1MiniExpert(
              (gate_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
              (up_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
              (down_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
            )
          )
          (shared_experts): Raon1MiniMLP(
            (gate_proj): Linear(in_features=32, out_features=32, bias=False)
            (up_proj): Linear(in_features=32, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=32, bias=False)
          )
        )
      )
      (4-6): 3 x Raon1MiniDecoderLayer(
        (input_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
        (post_attention_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
        (self_attn): Raon1MiniRDA(
          (q_proj): Linear(in_features=32, out_features=32, bias=False)
          (k_proj): Linear(in_features=32, out_features=32, bias=False)
          (v_proj): Linear(in_features=32, out_features=32, bias=False)
          (q_conv1d): Raon1MiniShortConv()
          (k_conv1d): Raon1MiniShortConv()
          (v_conv1d): Raon1MiniShortConv()
          (f_a_proj): Linear(in_features=32, out_features=32, bias=False)
          (f_b_proj): Linear(in_features=32, out_features=32, bias=False)
          (b_proj): Linear(in_features=32, out_features=1, bias=False)
          (rda_w_proj): Linear(in_features=32, out_features=1, bias=False)
          (g_proj): Linear(in_features=32, out_features=32, bias=False)
          (o_norm): Raon1MiniRMSNorm((32,), eps=1e-05)
          (o_proj): Linear(in_features=32, out_features=32, bias=False)
        )
        (self_attn_raon_mhc): RaonMHC(
          (res): RaonMHCStreamMix()
        )
        (mlp_raon_mhc): RaonMHC(
          (res): RaonMHCStreamMix()
        )
        (moe): Raon1MiniMoE(
          (gate): Raon1MiniRouter()
          (experts): ModuleList(
            (0-63): 64 x Raon1MiniExpert(
              (gate_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
              (up_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
              (down_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
            )
          )
          (shared_experts): Raon1MiniMLP(
            (gate_proj): Linear(in_features=32, out_features=32, bias=False)
            (up_proj): Linear(in_features=32, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=32, bias=False)
          )
        )
      )
      (7): Raon1MiniDecoderLayer(
        (input_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
        (post_attention_layernorm): Raon1MiniRMSNorm((32,), eps=1e-05)
        (self_attn): Raon1MiniMLA(
          (q_a_proj): Linear(in_features=32, out_features=32, bias=False)
          (q_a_layernorm): Raon1MiniRMSNorm((32,), eps=1e-06)
          (q_b_proj): Linear(in_features=32, out_features=64, bias=False)
          (kv_a_proj_with_mqa): Linear(in_features=32, out_features=64, bias=False)
          (kv_a_layernorm): Raon1MiniRMSNorm((32,), eps=1e-06)
          (kv_b_proj): Linear(in_features=32, out_features=64, bias=False)
          (g_proj): Linear(in_features=32, out_features=32, bias=False)
          (o_proj): Linear(in_features=32, out_features=32, bias=False)
        )
        (self_attn_raon_mhc): RaonMHC(
          (res): RaonMHCStreamMix()
        )
        (mlp_raon_mhc): RaonMHC(
          (res): RaonMHCStreamMix()
        )
        (moe): Raon1MiniMoE(
          (gate): Raon1MiniRouter()
          (experts): ModuleList(
            (0-63): 64 x Raon1MiniExpert(
              (gate_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
              (up_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
              (down_proj): Raon1MiniRoutedLinear(in_features=32, out_features=32, bias=False)
            )
          )
          (shared_experts): Raon1MiniMLP(
            (gate_proj): Linear(in_features=32, out_features=32, bias=False)
            (up_proj): Linear(in_features=32, out_features=32, bias=False)
            (down_proj): Linear(in_features=32, out_features=32, bias=False)
          )
        )
      )
    )
    (norm): Raon1MiniRMSNorm((32,), eps=1e-05)
  )
  (lm_head): Linear(in_features=32, out_features=259840, bias=False)
)

Test environment:

• huggingface_hub: 1.27.0
• safetensors: 0.8.0
• tokenizers: 0.23.0rc0
• torch: 2.13.0+cu126
• transformers: 5.15.0.dev0

Change log:

• 2026-10-23: Initial version.

This repo is generated by GPT-6.1-Sol