---
library_name: transformers
base_model:
- thinkingmachines/Inkling
---
This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [thinkingmachines/Inkling](https://huggingface.co/thinkingmachines/Inkling).
| File path | Size |
|------|------|
| model.safetensors | 7.3MB |
### Example usage:
```python
import numpy as np
import torch
from PIL import Image
from transformers import AutoModelForMultimodalLM, AutoProcessor
model_id = "tiny-random/inkling"
processor = AutoProcessor.from_pretrained(model_id)
model = AutoModelForMultimodalLM.from_pretrained(
model_id,
dtype=torch.bfloat16,
device_map="cuda" if torch.cuda.is_available() else "cpu",
)
# Synthetic multimodal inputs — no network fetch.
image = Image.fromarray(np.random.randint(0, 255, (80, 80, 3), dtype=np.uint8))
sampling_rate = processor.feature_extractor.sampling_rate
t = np.linspace(0, 0.2, int(sampling_rate * 0.2), endpoint=False)
audio = (0.1 * np.sin(2 * np.pi * 440 * t)).astype(np.float32)
messages = [
{
"role": "user",
"content": [
{"type": "image", "image": image},
{"type": "audio", "audio": audio},
{"type": "text", "text": "Describe the image and audio briefly."},
],
},
]
inputs = processor.apply_chat_template(
messages,
add_generation_prompt=True,
tokenize=True,
return_dict=True,
return_tensors="pt",
reasoning_effort="none",
processor_kwargs={"sampling_rate": sampling_rate},
).to(model.device, dtype=model.dtype)
input_len = inputs["input_ids"].shape[-1]
outputs = model.generate(**inputs, max_new_tokens=16)
print(processor.decode(outputs[0], skip_special_tokens=False))
```
### Codes to create this repo:
Click to expand
```python
import json
from pathlib import Path
import torch
from huggingface_hub import file_exists, hf_hub_download
from safetensors.torch import load_file, save_file
from transformers import (
AutoConfig,
AutoProcessor,
GenerationConfig,
InklingForConditionalGeneration,
set_seed,
)
source_model_id = "thinkingmachines/Inkling"
save_folder = "/tmp/tiny-random/inkling"
processor = AutoProcessor.from_pretrained(source_model_id)
processor.save_pretrained(save_folder)
with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f:
config_json = json.load(f)
# Only shrink size-critical dims. Keep kernel-sensitive knobs (d_rel, rel_extent,
# sliding_window_size, num_experts_per_tok, n_shared_experts, ...) as upstream.
hidden_size = 8
num_mtp_layers = 1
config_json['text_config'].update({
'hidden_size': hidden_size,
'num_hidden_layers': 2,
'num_attention_heads': 8,
'num_key_value_heads': 4,
'head_dim': 32,
'swa_num_attention_heads': 8,
'swa_num_key_value_heads': 4,
'swa_head_dim': 32,
'local_layer_ids': [0], # keep 1 sliding + 1 global with 2 layers
'dense_mlp_idx': 1, # 1 dense + 1 sparse
'dense_intermediate_size': 32,
'intermediate_size': 32,
'moe_intermediate_size': 32,
})
config_json['vision_config'].update({
'decoder_dmodel': hidden_size,
'n_layers': 2,
})
config_json['audio_config'].update({
'decoder_dmodel': hidden_size,
})
config_json['mtp_config'].update({
'num_nextn_predict_layers': num_mtp_layers,
'local_layer_ids': [0],
})
with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
json.dump(config_json, f, indent=2)
config = AutoConfig.from_pretrained(save_folder)
print(config)
torch.set_default_dtype(torch.bfloat16)
model = InklingForConditionalGeneration(config)
torch.set_default_dtype(torch.float32)
if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
model.generation_config = GenerationConfig.from_pretrained(
source_model_id, trust_remote_code=True,
)
set_seed(42)
model = model.cpu()
num_params = sum(p.numel() for p in model.parameters())
with torch.no_grad():
for name, p in sorted(model.named_parameters()):
torch.nn.init.normal_(p, 0, 0.2)
print(name, p.shape, f'{p.numel() / num_params:.2%}', f'{p.numel() * p.element_size() / 1024**2:.2f}MB')
# Upstream MoE gate bias / global_scale are F32; sconv stays BF16 in the checkpoint.
for name, module in model.named_modules():
if hasattr(module, "e_score_correction_bias"):
module.e_score_correction_bias = torch.nn.Parameter(
module.e_score_correction_bias.detach().float()
)
if name.endswith(".mlp.gate") and hasattr(module, "global_scale"):
module.global_scale = torch.nn.Parameter(module.global_scale.detach().float())
model.save_pretrained(save_folder)
# HF ignores `model.mtp.*` on main load; write them with original checkpoint naming.
set_seed(42)
path = Path(save_folder) / "model.safetensors"
state = load_file(str(path))
dense_prefix = "model.llm.layers.0." # MTP blocks are dense
dense_keys = {k: v for k, v in state.items() if k.startswith(dense_prefix)}
for i in range(num_mtp_layers):
block_prefix = f"model.mtp.layers.{i}.transformer_block."
for src_key, tensor in dense_keys.items():
dst_key = block_prefix + src_key[len(dense_prefix):]
state[dst_key] = torch.empty_like(tensor)
torch.nn.init.normal_(state[dst_key], 0, 0.2)
print(dst_key, tuple(state[dst_key].shape))
for name, shape in (
(f"model.mtp.layers.{i}.embed_norm.weight", (hidden_size,)),
(f"model.mtp.layers.{i}.hidden_norm.weight", (hidden_size,)),
(f"model.mtp.layers.{i}.input_proj.weight", (hidden_size, hidden_size * 2)),
):
state[name] = torch.empty(shape, dtype=torch.bfloat16)
torch.nn.init.normal_(state[name], 0, 0.2)
print(name, shape)
# Keep checkpoint key dtypes aligned even if save_pretrained downcasts.
for key, tensor in list(state.items()):
if key.endswith(".mlp.gate.bias") or key.endswith(".mlp.gate.global_scale"):
state[key] = tensor.float()
save_file(state, str(path))
```
Click to expand
```text
InklingForConditionalGeneration(
(model): InklingModel(
(language_model): InklingTextModel(
(embed_tokens): Embedding(201024, 8)
(layers): ModuleList(
(0): InklingDecoderLayer(
(self_attn): InklingAttention(
(q_proj): Linear(in_features=8, out_features=256, bias=False)
(k_proj): Linear(in_features=8, out_features=128, bias=False)
(v_proj): Linear(in_features=8, out_features=128, bias=False)
(r_proj): Linear(in_features=8, out_features=128, bias=False)
(o_proj): Linear(in_features=256, out_features=8, bias=False)
(k_sconv): InklingShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
(v_sconv): InklingShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
(q_norm): InklingRMSNorm((32,), eps=1e-06)
(k_norm): InklingRMSNorm((32,), eps=1e-06)
(rel_logits_proj): InklingRelativeLogits()
)
(mlp): InklingMLP(
(gate_proj): Linear(in_features=8, out_features=32, bias=False)
(up_proj): Linear(in_features=8, out_features=32, bias=False)
(down_proj): Linear(in_features=32, out_features=8, bias=False)
(act_fn): SiLUActivation()
)
(input_layernorm): InklingRMSNorm((8,), eps=1e-06)
(post_attention_layernorm): InklingRMSNorm((8,), eps=1e-06)
(attn_sconv): InklingShortConvolution(
(conv1d): Conv1d(8, 8, kernel_size=(4,), stride=(1,), padding=(3,), groups=8, bias=False)
)
(mlp_sconv): InklingShortConvolution(
(conv1d): Conv1d(8, 8, kernel_size=(4,), stride=(1,), padding=(3,), groups=8, bias=False)
)
)
(1): InklingDecoderLayer(
(self_attn): InklingAttention(
(q_proj): Linear(in_features=8, out_features=256, bias=False)
(k_proj): Linear(in_features=8, out_features=128, bias=False)
(v_proj): Linear(in_features=8, out_features=128, bias=False)
(r_proj): Linear(in_features=8, out_features=128, bias=False)
(o_proj): Linear(in_features=256, out_features=8, bias=False)
(k_sconv): InklingShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
(v_sconv): InklingShortConvolution(
(conv1d): Conv1d(128, 128, kernel_size=(4,), stride=(1,), padding=(3,), groups=128, bias=False)
)
(q_norm): InklingRMSNorm((32,), eps=1e-06)
(k_norm): InklingRMSNorm((32,), eps=1e-06)
(rel_logits_proj): InklingRelativeLogits()
)
(mlp): InklingMoE(
(gate): InklingTopkRouter()
(experts): InklingExperts(
(act_fn): SiLUActivation()
)
(shared_experts): InklingSharedExperts(
(act_fn): SiLUActivation()
)
)
(input_layernorm): InklingRMSNorm((8,), eps=1e-06)
(post_attention_layernorm): InklingRMSNorm((8,), eps=1e-06)
(attn_sconv): InklingShortConvolution(
(conv1d): Conv1d(8, 8, kernel_size=(4,), stride=(1,), padding=(3,), groups=8, bias=False)
)
(mlp_sconv): InklingShortConvolution(
(conv1d): Conv1d(8, 8, kernel_size=(4,), stride=(1,), padding=(3,), groups=8, bias=False)
)
)
)
(norm): InklingRMSNorm((8,), eps=1e-06)
(embed_norm): InklingRMSNorm((8,), eps=1e-06)
)
(audio_tower): InklingAudioModel(
(embed_audio_tokens): InklingAudioModelEmbeddings(
(embed_audio_tokens): Embedding(1280, 8)
)
(norm): InklingRMSNorm((8,), eps=1e-06)
)
(vision_tower): InklingVisionModel(
(encoder_layers): ModuleList(
(0): InklingVisionEncoderLayer(
(projection): Linear(in_features=300, out_features=320, bias=False)
(layer_norm): InklingRMSNorm((320,), eps=1e-06)
)
(1): InklingVisionEncoderLayer(
(projection): Linear(in_features=10240, out_features=8, bias=False)
)
)
(final_norm): InklingRMSNorm((8,), eps=1e-06)
)
)
(lm_head): Linear(in_features=8, out_features=201024, bias=False)
)
```