tiny-random/embeddinggemma-2

This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from google/embeddinggemma-2.

Structural comparison with google/embeddinggemma-2:

Feature	Source	Tiny
Input modalities	text, image, video, audio	same
Vocabulary rows	262,144	same
Output embedding dimension	768	same
Text layers	24	12; two 5-sliding + 1-full cycles
Text width / local head dimension	512 / 256	16 / 32
Full-attention head dimension	512	64
Vision layers / width	16 / 768	2 / 64
Audio layers / width	12 / 1,024	2 / 64
PLE	projection-only	same
Stored precision	BF16	BF16

The output embedding dimension intentionally remains 768; it is a source-model interface rather than a vocabulary or hidden-width reduction target.

File path	Size
model.safetensors	12.0MB

Example usage:

• Transformers (✅ verified)

import numpy as np
import torch
from PIL import Image
from transformers import AutoProcessor, EmbeddingGemma2Model
from transformers.video_utils import VideoMetadata

model_id = "tiny-random/embeddinggemma-2"
device = "cuda" if torch.cuda.is_available() else "cpu"
dtype = torch.bfloat16 if device == "cuda" else torch.float32
model = EmbeddingGemma2Model.from_pretrained(model_id, dtype=dtype).to(device).eval()
processor = AutoProcessor.from_pretrained(model_id)
image = Image.new("RGB", (64, 64), "red")
video = np.stack([np.asarray(Image.new("RGB", (64, 64), color)) for color in ("red", "green", "blue", "yellow")])
audio = np.zeros(1_600, dtype=np.float32)
text = f"{processor.image_token} {processor.video_token} {processor.audio_token}"
inputs = processor(
    text=[text], images=[[image]], videos=[video], audio=[audio], return_tensors="pt",
    do_sample_frames=False,
    video_metadata=[VideoMetadata(total_num_frames=4, fps=2, frames_indices=[0, 1, 2, 3])],
).to(device)
with torch.inference_mode():
    embeddings = model(**inputs).last_hidden_state.mean(dim=1)
print(embeddings.shape)

Codes to create this repo (✅ verified):

Click to expand

import json
from pathlib import Path
from shutil import copyfile

import torch
from huggingface_hub import hf_hub_download, list_repo_files
from transformers import AutoConfig, EmbeddingGemma2Model, set_seed

source_model_id = "google/embeddinggemma-2"
save_folder = "/tmp/tiny-random/embeddinggemma-2"
source_revision = "914f7f89142e33e77833254d9c9b90c3cef7303b"
set_seed(42)
folder = Path(save_folder)
folder.mkdir(parents=True, exist_ok=True)
raw = json.loads(Path(hf_hub_download(source_model_id, "config.json", revision=source_revision)).read_text())
text = raw["text_config"]
text.update(
    hidden_size=16, hidden_size_per_layer_input=16, head_dim=32,
    num_attention_heads=1, num_key_value_heads=1, intermediate_size=64,
    num_hidden_layers=12,
    layer_types=["sliding_attention"] * 5 + ["full_attention"] + ["sliding_attention"] * 5 + ["full_attention"],
    per_layer_config={"5": {"head_dim": 64, "num_key_value_heads": 1}, "11": {"head_dim": 64, "num_key_value_heads": 1}},
)
raw["vision_config"].update(
    hidden_size=64, intermediate_size=128, num_hidden_layers=2,
    num_attention_heads=2, num_key_value_heads=2, head_dim=32, global_head_dim=32,
)
raw["audio_config"].update(
    hidden_size=64, num_attention_heads=2, num_hidden_layers=2,
    # Gemma4Audio flattens frequency bins after two stride-2 convolutions.
    # Its projection formula requires the first channel count to remain 128
    # for the source processor's 128-bin mel features.
    output_proj_dims=64, subsampling_conv_channels=[128, 8],
)
(folder / "config.json").write_text(json.dumps(raw, indent=2), encoding="utf-8")
config = AutoConfig.from_pretrained(folder)
model = EmbeddingGemma2Model(config).to(torch.bfloat16)
with torch.no_grad():
    for name, parameter in sorted(model.named_parameters()):
        if not parameter.is_floating_point():
            continue
        if name.endswith("norm.weight"):
            parameter.fill_(1)
        else:
            torch.nn.init.normal_(parameter, mean=0, std=0.02)
model.save_pretrained(folder, safe_serialization=True)
assets = {"1_Pooling/config.json", "2_Normalize/config.json", "chat_template.jinja", "config_sentence_transformers.json", "modules.json", "preprocessor_config.json", "processor_config.json", "sentence_bert_config.json", "tokenizer.json", "tokenizer.model", "tokenizer_config.json"}
for filename in sorted(set(list_repo_files(source_model_id, revision=source_revision)) & assets):
    destination = folder / filename
    destination.parent.mkdir(parents=True, exist_ok=True)
    copyfile(hf_hub_download(source_model_id, filename, revision=source_revision), destination)

Printing the model:

Click to expand

EmbeddingGemma2Model(
  (vision_tower): Gemma4VisionModel(
    (patch_embedder): Gemma4VisionPatchEmbedder(
      (input_proj): Linear(in_features=768, out_features=64, bias=False)
    )
    (encoder): Gemma4VisionEncoder(
      (rotary_emb): Gemma4VisionRotaryEmbedding()
      (layers): ModuleList(
        (0-1): 2 x Gemma4VisionEncoderLayer(
          (self_attn): Gemma4VisionAttention(
            (q_proj): Gemma4ClippableLinear(
              (linear): Linear(in_features=64, out_features=64, bias=False)
            )
            (k_proj): Gemma4ClippableLinear(
              (linear): Linear(in_features=64, out_features=64, bias=False)
            )
            (v_proj): Gemma4ClippableLinear(
              (linear): Linear(in_features=64, out_features=64, bias=False)
            )
            (o_proj): Gemma4ClippableLinear(
              (linear): Linear(in_features=64, out_features=64, bias=False)
            )
            (q_norm): Gemma4RMSNorm()
            (k_norm): Gemma4RMSNorm()
            (v_norm): Gemma4RMSNorm()
          )
          (mlp): Gemma4VisionMLP(
            (gate_proj): Gemma4ClippableLinear(
              (linear): Linear(in_features=64, out_features=128, bias=False)
            )
            (up_proj): Gemma4ClippableLinear(
              (linear): Linear(in_features=64, out_features=128, bias=False)
            )
            (down_proj): Gemma4ClippableLinear(
              (linear): Linear(in_features=128, out_features=64, bias=False)
            )
            (act_fn): GELUTanh()
          )
          (input_layernorm): Gemma4RMSNorm()
          (post_attention_layernorm): Gemma4RMSNorm()
          (pre_feedforward_layernorm): Gemma4RMSNorm()
          (post_feedforward_layernorm): Gemma4RMSNorm()
        )
      )
    )
    (pooler): Gemma4VisionPooler()
  )
  (language_model): EmbeddingGemma2TextModel(
    (embed_tokens): EmbeddingGemma2TextScaledWordEmbedding(262144, 16, padding_idx=0)
    (layers): ModuleList(
      (0-4): 5 x EmbeddingGemma2EncoderLayer(
        (self_attn): EmbeddingGemma2Attention(
          (q_proj): Linear(in_features=16, out_features=32, bias=False)
          (k_proj): Linear(in_features=16, out_features=32, bias=False)
          (v_proj): Linear(in_features=16, out_features=32, bias=False)
          (o_proj): Linear(in_features=32, out_features=16, bias=False)
          (q_norm): EmbeddingGemma2RMSNorm()
          (k_norm): EmbeddingGemma2RMSNorm()
          (v_norm): EmbeddingGemma2RMSNorm()
        )
        (mlp): EmbeddingGemma2MLP(
          (gate_proj): Linear(in_features=16, out_features=64, bias=False)
          (up_proj): Linear(in_features=16, out_features=64, bias=False)
          (down_proj): Linear(in_features=64, out_features=16, bias=False)
          (act_fn): GELUTanh()
        )
        (input_layernorm): EmbeddingGemma2RMSNorm()
        (post_attention_layernorm): EmbeddingGemma2RMSNorm()
        (pre_feedforward_layernorm): EmbeddingGemma2RMSNorm()
        (post_feedforward_layernorm): EmbeddingGemma2RMSNorm()
        (ple_block): EmbeddingGemma2TextPLEBlock(
          (act_fn): GELUTanh()
          (per_layer_input_gate): Linear(in_features=16, out_features=16, bias=False)
          (per_layer_projection): Linear(in_features=16, out_features=16, bias=False)
          (post_per_layer_input_norm): EmbeddingGemma2RMSNorm()
        )
      )
      (5): EmbeddingGemma2EncoderLayer(
        (self_attn): EmbeddingGemma2Attention(
          (q_proj): Linear(in_features=16, out_features=64, bias=False)
          (k_proj): Linear(in_features=16, out_features=64, bias=False)
          (v_proj): Linear(in_features=16, out_features=64, bias=False)
          (o_proj): Linear(in_features=64, out_features=16, bias=False)
          (q_norm): EmbeddingGemma2RMSNorm()
          (k_norm): EmbeddingGemma2RMSNorm()
          (v_norm): EmbeddingGemma2RMSNorm()
        )
        (mlp): EmbeddingGemma2MLP(
          (gate_proj): Linear(in_features=16, out_features=64, bias=False)
          (up_proj): Linear(in_features=16, out_features=64, bias=False)
          (down_proj): Linear(in_features=64, out_features=16, bias=False)
          (act_fn): GELUTanh()
        )
        (input_layernorm): EmbeddingGemma2RMSNorm()
        (post_attention_layernorm): EmbeddingGemma2RMSNorm()
        (pre_feedforward_layernorm): EmbeddingGemma2RMSNorm()
        (post_feedforward_layernorm): EmbeddingGemma2RMSNorm()
        (ple_block): EmbeddingGemma2TextPLEBlock(
          (act_fn): GELUTanh()
          (per_layer_input_gate): Linear(in_features=16, out_features=16, bias=False)
          (per_layer_projection): Linear(in_features=16, out_features=16, bias=False)
          (post_per_layer_input_norm): EmbeddingGemma2RMSNorm()
        )
      )
      (6-10): 5 x EmbeddingGemma2EncoderLayer(
        (self_attn): EmbeddingGemma2Attention(
          (q_proj): Linear(in_features=16, out_features=32, bias=False)
          (k_proj): Linear(in_features=16, out_features=32, bias=False)
          (v_proj): Linear(in_features=16, out_features=32, bias=False)
          (o_proj): Linear(in_features=32, out_features=16, bias=False)
          (q_norm): EmbeddingGemma2RMSNorm()
          (k_norm): EmbeddingGemma2RMSNorm()
          (v_norm): EmbeddingGemma2RMSNorm()
        )
        (mlp): EmbeddingGemma2MLP(
          (gate_proj): Linear(in_features=16, out_features=64, bias=False)
          (up_proj): Linear(in_features=16, out_features=64, bias=False)
          (down_proj): Linear(in_features=64, out_features=16, bias=False)
          (act_fn): GELUTanh()
        )
        (input_layernorm): EmbeddingGemma2RMSNorm()
        (post_attention_layernorm): EmbeddingGemma2RMSNorm()
        (pre_feedforward_layernorm): EmbeddingGemma2RMSNorm()
        (post_feedforward_layernorm): EmbeddingGemma2RMSNorm()
        (ple_block): EmbeddingGemma2TextPLEBlock(
          (act_fn): GELUTanh()
          (per_layer_input_gate): Linear(in_features=16, out_features=16, bias=False)
          (per_layer_projection): Linear(in_features=16, out_features=16, bias=False)
          (post_per_layer_input_norm): EmbeddingGemma2RMSNorm()
        )
      )
      (11): EmbeddingGemma2EncoderLayer(
        (self_attn): EmbeddingGemma2Attention(
          (q_proj): Linear(in_features=16, out_features=64, bias=False)
          (k_proj): Linear(in_features=16, out_features=64, bias=False)
          (v_proj): Linear(in_features=16, out_features=64, bias=False)
          (o_proj): Linear(in_features=64, out_features=16, bias=False)
          (q_norm): EmbeddingGemma2RMSNorm()
          (k_norm): EmbeddingGemma2RMSNorm()
          (v_norm): EmbeddingGemma2RMSNorm()
        )
        (mlp): EmbeddingGemma2MLP(
          (gate_proj): Linear(in_features=16, out_features=64, bias=False)
          (up_proj): Linear(in_features=16, out_features=64, bias=False)
          (down_proj): Linear(in_features=64, out_features=16, bias=False)
          (act_fn): GELUTanh()
        )
        (input_layernorm): EmbeddingGemma2RMSNorm()
        (post_attention_layernorm): EmbeddingGemma2RMSNorm()
        (pre_feedforward_layernorm): EmbeddingGemma2RMSNorm()
        (post_feedforward_layernorm): EmbeddingGemma2RMSNorm()
        (ple_block): EmbeddingGemma2TextPLEBlock(
          (act_fn): GELUTanh()
          (per_layer_input_gate): Linear(in_features=16, out_features=16, bias=False)
          (per_layer_projection): Linear(in_features=16, out_features=16, bias=False)
          (post_per_layer_input_norm): EmbeddingGemma2RMSNorm()
        )
      )
    )
    (norm): EmbeddingGemma2RMSNorm()
    (rotary_emb): EmbeddingGemma2RotaryEmbedding()
    (ple): EmbeddingGemma2TextPLE(
      (per_layer_model_projection): Linear(in_features=16, out_features=192, bias=False)
      (per_layer_projection_norm): EmbeddingGemma2RMSNorm()
    )
    (embedding_projection): Linear(in_features=16, out_features=768, bias=False)
  )
  (audio_tower): Gemma4AudioModel(
    (subsample_conv_projection): Gemma4AudioSubSampleConvProjection(
      (layer0): Gemma4AudioSubSampleConvProjectionLayer(
        (conv): Conv2d(1, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True, bias=False)
        (act): ReLU()
      )
      (layer1): Gemma4AudioSubSampleConvProjectionLayer(
        (conv): Conv2d(128, 8, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
        (norm): LayerNorm((8,), eps=1e-06, elementwise_affine=True, bias=False)
        (act): ReLU()
      )
      (input_proj_linear): Linear(in_features=256, out_features=64, bias=False)
    )
    (rel_pos_enc): Gemma4AudioRelPositionalEncoding()
    (layers): ModuleList(
      (0-1): 2 x Gemma4AudioLayer(
        (feed_forward1): Gemma4AudioFeedForward(
          (ffw_layer_1): Gemma4ClippableLinear(
            (linear): Linear(in_features=64, out_features=256, bias=False)
          )
          (ffw_layer_2): Gemma4ClippableLinear(
            (linear): Linear(in_features=256, out_features=64, bias=False)
          )
          (pre_layer_norm): Gemma4RMSNorm()
          (post_layer_norm): Gemma4RMSNorm()
          (act_fn): SiLUActivation()
        )
        (feed_forward2): Gemma4AudioFeedForward(
          (ffw_layer_1): Gemma4ClippableLinear(
            (linear): Linear(in_features=64, out_features=256, bias=False)
          )
          (ffw_layer_2): Gemma4ClippableLinear(
            (linear): Linear(in_features=256, out_features=64, bias=False)
          )
          (pre_layer_norm): Gemma4RMSNorm()
          (post_layer_norm): Gemma4RMSNorm()
          (act_fn): SiLUActivation()
        )
        (self_attn): Gemma4AudioAttention(
          (q_proj): Gemma4ClippableLinear(
            (linear): Linear(in_features=64, out_features=64, bias=False)
          )
          (k_proj): Gemma4ClippableLinear(
            (linear): Linear(in_features=64, out_features=64, bias=False)
          )
          (v_proj): Gemma4ClippableLinear(
            (linear): Linear(in_features=64, out_features=64, bias=False)
          )
          (post): Gemma4ClippableLinear(
            (linear): Linear(in_features=64, out_features=64, bias=False)
          )
          (relative_k_proj): Linear(in_features=64, out_features=64, bias=False)
        )
        (lconv1d): Gemma4AudioLightConv1d(
          (linear_start): Gemma4ClippableLinear(
            (linear): Linear(in_features=64, out_features=128, bias=False)
          )
          (linear_end): Gemma4ClippableLinear(
            (linear): Linear(in_features=64, out_features=64, bias=False)
          )
          (depthwise_conv1d): Gemma4AudioCausalConv1d(64, 64, kernel_size=(5,), stride=(1,), groups=64, bias=False)
          (pre_layer_norm): Gemma4RMSNorm()
          (conv_norm): Gemma4RMSNorm()
          (act_fn): SiLUActivation()
        )
        (norm_pre_attn): Gemma4RMSNorm()
        (norm_post_attn): Gemma4RMSNorm()
        (norm_out): Gemma4RMSNorm()
      )
    )
    (output_proj): Linear(in_features=64, out_features=64, bias=True)
  )
  (embed_vision): EmbeddingGemma2MultimodalEmbedder(
    (embedding_projection): Linear(in_features=64, out_features=16, bias=False)
    (embedding_pre_projection_norm): EmbeddingGemma2RMSNorm()
  )
  (embed_audio): EmbeddingGemma2MultimodalEmbedder(
    (embedding_projection): Linear(in_features=64, out_features=16, bias=False)
    (embedding_pre_projection_norm): EmbeddingGemma2RMSNorm()
  )
)

Test environment:

• Pillow: 13.0.0
• huggingface_hub: 2.3.0
• numpy: 2.3.5
• safetensors: 0.8.0
• tokenizers: 0.23.3
• torch: 2.13.0
• transformers: 5.19.0

Change log:

• 2026-10-23: Initial version.

This repo is generated by GPT-6.1-Sol