tiny-random/muse-spark-open

This is a randomly initialized tiny model based on meta-models/Muse-Spark-Open-816B-A42B for testing and debugging.

Feature	Source	Tiny
Checkpoint tensor elements	821,906,490,368 BF16, 130,039,296 F32	3,580,264 BF16, 2,304 F32
Safetensors weight files	1,644,333,393,864 bytes (65 files)	7,211,288 bytes (model.safetensors)
Text layers / attention cycle	62 / sliding, full, sliding, sliding	8 / two source cycles
Residual / expert pool / expert intermediate width	8192 / 4096 / 4096	8 / 4 / 16
Q heads / KV heads / head dimension	128 / 16 / 64	8 / 1 / 32
Experts / activated experts	256 / 8	32 / 8
Vocabulary / tying	202048 / untied	Same
Vision layers / width / temporal patch / merge	50 / 1536 / 2 / 2	8 / 32 / 2 / 2
Audio stages / block depths / sample rate	4 / [2, 2, 2, 2] / 24 kHz	Same
Checkpoint precision	BF16; router weights/biases FP32	Same
Modalities	Text + image + video + audio → text	Same

File path	Size
model.safetensors	7.2MB

Example usage:

• Transformers (✅ verified)

This example uses text, an image, predecoded video frames and a 24 kHz waveform. SDPA remains enabled; batched expert GEMMs support the tiny expert pool without grouped-GEMM alignment padding.

def run_transformers(model_id: str):
    import numpy as np
    import torch
    from PIL import Image
    from transformers import AutoProcessor, MuseSparkForConditionalGeneration
    from transformers.video_utils import VideoMetadata

    processor = AutoProcessor.from_pretrained(model_id)
    model = MuseSparkForConditionalGeneration.from_pretrained(
        model_id,
        dtype=torch.bfloat16,
        attn_implementation="sdpa",
        experts_implementation="batched_mm",
        device_map="cuda",
    )
    image = Image.new("RGB", (112, 112), "blue")
    video = np.stack([np.full((112, 112, 3), value, dtype=np.uint8) for value in (40, 80, 120, 160)])
    audio = (0.1 * np.sin(2 * np.pi * 440 * np.arange(24000) / 24000)).astype(np.float32)
    messages = [
        {
            "role": "user",
            "content": [
                {"type": "image"},
                {"type": "video"},
                {"type": "audio"},
                {"type": "text", "text": "Describe the image, video and sound."},
            ],
        }
    ]
    prompt = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True, reasoning_effort="low")
    inputs = processor(
        text=[prompt],
        images=[image],
        videos=[video],
        audio=[audio],
        sampling_rate=24000,
        video_metadata=[VideoMetadata(total_num_frames=4, fps=2, frames_indices=[0, 1, 2, 3])],
        do_sample_frames=False,
        return_tensors="pt",
    ).to(model.device, dtype=model.dtype)
    outputs = model.generate(**inputs, max_new_tokens=10, do_sample=False)
    print(model)
    print(processor.batch_decode(outputs[:, inputs.input_ids.shape[1] :], skip_special_tokens=True))


if __name__ == "__main__":
    run_transformers(model_id="tiny-random/muse-spark-open")

• SGLang (✅ verified)

Muse Spark's BF16 experts require EP=TP, so this single-node launch uses TP=4 and EP=4. Install FFmpeg shared libraries for video decoding.

sglang serve \
  --model-path tiny-random/muse-spark-open \
  --tp-size 4 \
  --ep-size 4 \
  --attention-backend triton \
  --context-length 4096 \
  --max-running-requests 4 \
  --max-total-tokens 4096 \
  --mem-fraction-static 0.2 \
  --disable-custom-all-reduce

Codes to create this repo:

Click to expand

def create_model(source_model_id: str, source_revision: str, save_folder: str) -> str:
    import json
    from pathlib import Path
    import torch
    from huggingface_hub import hf_hub_download
    from transformers import MuseSparkConfig, MuseSparkForConditionalGeneration, set_seed

    folder = Path(save_folder)
    folder.mkdir(parents=True, exist_ok=True)
    config_json = json.loads(
        Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text()
    )
    text = config_json["text_config"]
    text.update(
        hidden_size=8,
        num_attention_heads=8,
        num_key_value_heads=1,
        head_dim=32,
        num_hidden_layers=8,
        intermediate_size=32,
        moe_hidden_size=4,
        moe_intermediate_size=16,
        num_local_experts=32,
    )
    for name in ("layer_types", "mlp_layer_types", "layer_rope_theta"):
        text[name] = text[name][:8]
    vision = config_json["vision_config"]
    vision.update(hidden_size=32, intermediate_size=64, num_attention_heads=4, num_hidden_layers=8)
    vision["layer_types"] = vision["layer_types"][:8]
    config_json["audio_config"].update(hidden_sizes=[8, 16, 32, 64], out_hidden_size=8)
    config_json.update(out_hidden_size=128, projector_hidden_size=16)
    config = MuseSparkConfig.from_dict(config_json)
    set_seed(42)
    model = MuseSparkForConditionalGeneration._from_config(config, dtype=torch.bfloat16, attn_implementation="sdpa")
    # The source checkpoint uses FP32 only for routed-expert router weights and
    # correction biases; residual gates and all other parameters remain BF16.
    for module in model.modules():
        if module.__class__.__name__ == "MuseSparkTopkRouter":
            module.float()
    model.save_pretrained(folder, safe_serialization=True)
    # Restore the deliberately edited raw config and immutable shared assets.
    (folder / "config.json").write_text(json.dumps(config_json, indent=2) + "\n")
    for name in (
        "tokenizer.json",
        "tokenizer_config.json",
        "special_tokens_map.json",
        "chat_template.jinja",
        "processor_config.json",
        "generation_config.json",
        "l4_200k_base",
        "LICENSE.md",
        "USE_POLICY.md",
    ):
        (folder / name).write_bytes(Path(hf_hub_download(source_model_id, name, revision=source_revision)).read_bytes())
    return str(model)


create_model(
    source_model_id="meta-models/Muse-Spark-Open-816B-A42B",
    source_revision="0fb02478b02565b2ce7fc57e6051c91172774ac9",
    save_folder="/tmp/muse-spark-open",
)

Printing the model:

Click to expand

MuseSparkForConditionalGeneration(
  (model): MuseSparkModel(
    (vision_tower): MuseSparkVisionModel(
      (patch_embedder): MuseSparkVisionPatchEmbedder(
        (patch_embedding): Linear(in_features=1176, out_features=32, bias=False)
        (position_embedding_table): Embedding(1024, 32)
      )
      (rotary_emb): MuseSparkVisionRotaryEmbedding()
      (ln_pre): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
      (layers): ModuleList(
        (0-7): 8 x MuseSparkVisionEncoderLayer(
          (norm1): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
          (norm2): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
          (attn): MuseSparkVisionAttention(
            (proj): Linear(in_features=32, out_features=32, bias=True)
            (q_proj): Linear(in_features=32, out_features=32, bias=True)
            (k_proj): Linear(in_features=32, out_features=32, bias=True)
            (v_proj): Linear(in_features=32, out_features=32, bias=True)
          )
          (mlp): MuseSparkVisionMLP(
            (fc1): Linear(in_features=32, out_features=64, bias=True)
            (act): GELUActivation()
            (fc2): Linear(in_features=64, out_features=32, bias=True)
          )
        )
      )
      (ln_post): LayerNorm((32,), eps=1e-05, elementwise_affine=True, bias=True)
    )
    (language_model): MuseSparkTextModel(
      (embed_tokens): MuseSparkTextNormedEmbedding(
        202048, 8, padding_idx=200018
        (embed_norm): MuseSparkRMSNorm()
      )
      (layers): ModuleList(
        (0-7): 8 x MuseSparkTextDecoderLayer(
          (self_attn): MuseSparkTextAttention(
            (q_proj): Linear(in_features=8, out_features=256, bias=False)
            (k_proj): Linear(in_features=8, out_features=32, bias=False)
            (v_proj): Linear(in_features=8, out_features=32, bias=False)
            (o_proj): Linear(in_features=256, out_features=8, bias=False)
            (gate_proj): Linear(in_features=8, out_features=256, bias=False)
            (qk_norm): MuseSparkRMSNorm()
            (o_norm): MuseSparkRMSNorm()
          )
          (mlp): MuseSparkMoE(
            (gate): MuseSparkTopkRouter(
              (scoring_function): Sigmoid()
            )
            (pre_expert_proj): Linear(in_features=8, out_features=4, bias=False)
            (pre_expert_norm): MuseSparkTextCenteredRMSNorm((4,), eps=1e-05)
            (experts): MuseSparkExperts(
              (act_fn): SiLUActivation()
              (post_expert_norm): MuseSparkTextInputScaledRMSNorm((4,), eps=1e-08)
            )
            (post_expert_proj): Linear(in_features=4, out_features=8, bias=False)
          )
          (input_layernorm): MuseSparkTextCenteredRMSNorm((8,), eps=1e-05)
          (post_attention_layernorm): MuseSparkRMSNorm()
          (pre_feedforward_layernorm): MuseSparkTextCenteredRMSNorm((8,), eps=1e-05)
          (post_feedforward_layernorm): MuseSparkTextInputScaledRMSNorm((8,), eps=1e-08)
          (post_attention_residual_gate): MuseSparkResidualGate()
          (post_feedforward_residual_gate): MuseSparkResidualGate()
        )
      )
      (norm): MuseSparkRMSNorm()
      (rotary_emb): MuseSparkTextRotaryEmbedding()
    )
    (vision_adapter): MuseSparkVisionAdapter(
      (fc1): Linear(in_features=128, out_features=16, bias=False)
      (act): GELUActivation()
      (fc2): Linear(in_features=16, out_features=16, bias=False)
    )
    (vision_projection): Linear(in_features=16, out_features=8, bias=False)
    (perception_emb_norm): MuseSparkRMSNorm()
    (audio_tower): MuseSparkAudioModel(
      (layers): ModuleList(
        (0): MuseSparkAudioLayer(
          (patch_embedder): MuseSparkAudioPatchEmbedding(
            (proj): Linear(in_features=16, out_features=8, bias=True)
            (norm): MuseSparkRMSNorm()
          )
          (conv_blocks): ModuleList(
            (0-1): 2 x MuseSparkAudioConvBlock(
              (conv): MuseSparkAudioCausalConv1d(8, 8, kernel_size=(7,), stride=(1,), groups=8)
              (norm): MuseSparkRMSNorm()
              (mlp): MuseSparkAudioMLP(
                (fc1): Linear(in_features=8, out_features=32, bias=True)
                (fc2): Linear(in_features=32, out_features=8, bias=True)
                (activation_fn): GELUActivation()
              )
            )
          )
        )
        (1): MuseSparkAudioLayer(
          (patch_embedder): MuseSparkAudioPatchEmbedding(
            (proj): Linear(in_features=48, out_features=16, bias=True)
            (norm): MuseSparkRMSNorm()
          )
          (conv_blocks): ModuleList(
            (0-1): 2 x MuseSparkAudioConvBlock(
              (conv): MuseSparkAudioCausalConv1d(16, 16, kernel_size=(7,), stride=(1,), groups=16)
              (norm): MuseSparkRMSNorm()
              (mlp): MuseSparkAudioMLP(
                (fc1): Linear(in_features=16, out_features=64, bias=True)
                (fc2): Linear(in_features=64, out_features=16, bias=True)
                (activation_fn): GELUActivation()
              )
            )
          )
        )
        (2): MuseSparkAudioLayer(
          (patch_embedder): MuseSparkAudioPatchEmbedding(
            (proj): Linear(in_features=80, out_features=32, bias=True)
            (norm): MuseSparkRMSNorm()
          )
          (conv_blocks): ModuleList(
            (0-1): 2 x MuseSparkAudioConvBlock(
              (conv): MuseSparkAudioCausalConv1d(32, 32, kernel_size=(7,), stride=(1,), groups=32)
              (norm): MuseSparkRMSNorm()
              (mlp): MuseSparkAudioMLP(
                (fc1): Linear(in_features=32, out_features=128, bias=True)
                (fc2): Linear(in_features=128, out_features=32, bias=True)
                (activation_fn): GELUActivation()
              )
            )
          )
        )
        (3): MuseSparkAudioLayer(
          (patch_embedder): MuseSparkAudioPatchEmbedding(
            (proj): Linear(in_features=128, out_features=64, bias=True)
            (norm): MuseSparkRMSNorm()
          )
          (conv_blocks): ModuleList(
            (0-1): 2 x MuseSparkAudioConvBlock(
              (conv): MuseSparkAudioCausalConv1d(64, 64, kernel_size=(7,), stride=(1,), groups=64)
              (norm): MuseSparkRMSNorm()
              (mlp): MuseSparkAudioMLP(
                (fc1): Linear(in_features=64, out_features=256, bias=True)
                (fc2): Linear(in_features=256, out_features=64, bias=True)
                (activation_fn): GELUActivation()
              )
            )
          )
        )
      )
      (norm): MuseSparkRMSNorm()
      (proj): Linear(in_features=64, out_features=8, bias=True)
      (post_proj_norm): MuseSparkRMSNorm()
    )
  )
  (lm_head): Linear(in_features=8, out_features=202048, bias=False)
)

Environment: .meta.json

Change log:

• 2026-10-26: Created a tiny Muse Spark Open checkpoint preserving text, image, video and audio inputs.

This repo is generated by GPT-6.1-Sol