Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

.meta.json

@@ -0,0 +1,4 @@
+{
+  "torch": "2.11.0",
+  "transformers": "5.5.0"
+}

README.md

@@ -0,0 +1,276 @@
+---
+library_name: transformers
+base_model:
+- zai-org/GLM-5.1
+---
+
+This tiny model is intended for debugging. It is randomly initialized using the configuration adapted from [zai-org/GLM-5.1](https://huggingface.co/zai-org/GLM-5.1).
+
+| File path | Size |
+|------|------|
+| model.safetensors | 17.6MB |
+
+
+### Example usage:
+
+- vLLM
+
+```bash
+# Multi-token prediction is supported
+model_id=tiny-random/glm-5.1
+vllm serve $model_id \
+     --tensor-parallel-size 2 \
+     --speculative-config.method mtp \
+     --speculative-config.num_speculative_tokens 1 \
+     --tool-call-parser glm47 \
+     --reasoning-parser glm45 \
+     --enable-auto-tool-choice
+```
+
+- SGLang
+
+```bash
+# Multi-token prediction is supported
+model_id=tiny-random/glm-5.1
+python3 -m sglang.launch_server   --model-path $model_id   --tp-size 2 \
+  --tool-call-parser glm47  \
+  --reasoning-parser glm45 \
+  --speculative-algorithm EAGLE \
+  --speculative-num-steps 3 \
+  --speculative-eagle-topk 1 \
+  --speculative-num-draft-tokens 4
+```
+
+- Transformers
+
+```python
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+model_id = "tiny-random/glm-5.1"
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+input_ids = torch.randint(1000, 2000, size=(1, 2333), dtype=torch.long).to(device)  # trigger DSA
+model = AutoModelForCausalLM.from_pretrained(
+    model_id,
+    dtype=torch.bfloat16,
+    device_map=device,
+)
+generated_ids = model.generate(input_ids, max_new_tokens=8)
+output_text = tokenizer.decode(generated_ids[0][input_ids.shape[1]:])
+print(output_text)
+```
+
+### Codes to create this repo:
+
+<details>
+<summary>Click to expand</summary>
+
+```python
+import json
+from copy import deepcopy
+from pathlib import Path
+
+import accelerate
+import torch
+import torch.nn as nn
+from huggingface_hub import file_exists, hf_hub_download
+from transformers import (
+    AutoConfig,
+    AutoModelForCausalLM,
+    AutoProcessor,
+    GenerationConfig,
+    set_seed,
+)
+
+source_model_id = "zai-org/GLM-5.1"
+save_folder = "/tmp/tiny-random/glm-51"
+
+processor = AutoProcessor.from_pretrained(
+    source_model_id, trust_remote_code=True)
+processor.save_pretrained(save_folder)
+
+with open(hf_hub_download(source_model_id, filename='config.json', repo_type='model'), 'r', encoding='utf-8') as f:
+    config_json: dict = json.load(f)
+
+config_json.update({
+    "first_k_dense_replace": 1,
+    "mlp_layer_types": ['dense'] + ['sparse'],
+    "hidden_size": 8,
+    "index_n_heads": 4,
+    "intermediate_size": 32,
+    "moe_intermediate_size": 32,
+    "num_hidden_layers": 2,
+    "num_attention_heads": 8,
+    'num_key_value_heads': 8,
+    'q_lora_rank': 32,
+    'tie_word_embeddings': False,
+})
+with open(f"{save_folder}/config.json", "w", encoding='utf-8') as f:
+    json.dump(config_json, f, indent=2)
+
+config = AutoConfig.from_pretrained(
+    save_folder,
+    trust_remote_code=True,
+)
+print(config)
+torch.set_default_dtype(torch.bfloat16)
+model = AutoModelForCausalLM.from_config(config, dtype=torch.bfloat16)
+torch.set_default_dtype(torch.float32)
+
+if file_exists(filename="generation_config.json", repo_id=source_model_id, repo_type='model'):
+    model.generation_config = GenerationConfig.from_pretrained(
+        source_model_id, trust_remote_code=True,
+    )
+    model.generation_config.do_sample = True
+    print(model.generation_config)
+
+model = model.cpu()
+set_seed(42)
+n_params = sum(p.numel() for p in model.parameters())
+with torch.no_grad():
+    for name, p in sorted(model.named_parameters()):
+        torch.nn.init.normal_(p, 0, 0.2)
+        print(name, p.shape, p.numel() / n_params * 100, '%')
+# MTP
+set_seed(42)
+model.model.layers.append(nn.ModuleDict(dict(
+    shared_head=nn.ModuleDict(dict(
+        norm=nn.RMSNorm(config.hidden_size),
+        # head=deepcopy(model.model.embed_tokens),
+    )),
+    # embed_tokens=deepcopy(model.model.embed_tokens),
+    eh_proj=nn.Linear(config.hidden_size * 2,
+                      config.hidden_size, bias=False),
+    enorm=nn.RMSNorm(config.hidden_size),
+    hnorm=nn.RMSNorm(config.hidden_size),
+    input_layernorm=nn.RMSNorm(config.hidden_size),
+    post_attention_layernorm=nn.RMSNorm(config.hidden_size),
+    self_attn=deepcopy(model.model.layers[1].self_attn),
+    mlp=deepcopy(model.model.layers[1].mlp),
+)))
+for i in range(1, len(model.model.layers)):
+    model.model.layers[i].mlp.gate.e_score_correction_bias = torch.rand_like(
+        model.model.layers[i].mlp.gate.e_score_correction_bias).float()
+model.save_pretrained(save_folder)
+print(model)
+```
+
+</details>
+
+### Printing the model:
+
+<details><summary>Click to expand</summary>
+
+```text
+GlmMoeDsaForCausalLM(
+  (model): GlmMoeDsaModel(
+    (embed_tokens): Embedding(154880, 8, padding_idx=154820)
+    (layers): ModuleList(
+      (0): GlmMoeDsaDecoderLayer(
+        (self_attn): GlmMoeDsaAttention(
+          (q_a_proj): Linear(in_features=8, out_features=32, bias=False)
+          (q_a_layernorm): GlmMoeDsaRMSNorm((32,), eps=1e-06)
+          (q_b_proj): Linear(in_features=32, out_features=2048, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
+          (kv_a_layernorm): GlmMoeDsaRMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=3584, bias=False)
+          (o_proj): Linear(in_features=2048, out_features=8, bias=False)
+          (indexer): GlmMoeDsaIndexer(
+            (wq_b): Linear(in_features=32, out_features=512, bias=False)
+            (wk): Linear(in_features=8, out_features=128, bias=False)
+            (k_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True)
+            (weights_proj): Linear(in_features=8, out_features=4, bias=False)
+          )
+        )
+        (mlp): GlmMoeDsaMLP(
+          (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+          (up_proj): Linear(in_features=8, out_features=32, bias=False)
+          (down_proj): Linear(in_features=32, out_features=8, bias=False)
+          (act_fn): SiLUActivation()
+        )
+        (input_layernorm): GlmMoeDsaRMSNorm((8,), eps=1e-05)
+        (post_attention_layernorm): GlmMoeDsaRMSNorm((8,), eps=1e-05)
+      )
+      (1): GlmMoeDsaDecoderLayer(
+        (self_attn): GlmMoeDsaAttention(
+          (q_a_proj): Linear(in_features=8, out_features=32, bias=False)
+          (q_a_layernorm): GlmMoeDsaRMSNorm((32,), eps=1e-06)
+          (q_b_proj): Linear(in_features=32, out_features=2048, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
+          (kv_a_layernorm): GlmMoeDsaRMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=3584, bias=False)
+          (o_proj): Linear(in_features=2048, out_features=8, bias=False)
+          (indexer): GlmMoeDsaIndexer(
+            (wq_b): Linear(in_features=32, out_features=512, bias=False)
+            (wk): Linear(in_features=8, out_features=128, bias=False)
+            (k_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True)
+            (weights_proj): Linear(in_features=8, out_features=4, bias=False)
+          )
+        )
+        (mlp): GlmMoeDsaMoE(
+          (experts): GlmMoeDsaNaiveMoe(
+            (act_fn): SiLUActivation()
+          )
+          (gate): GlmMoeDsaTopkRouter()
+          (shared_experts): GlmMoeDsaMLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+        (input_layernorm): GlmMoeDsaRMSNorm((8,), eps=1e-05)
+        (post_attention_layernorm): GlmMoeDsaRMSNorm((8,), eps=1e-05)
+      )
+      (2): ModuleDict(
+        (shared_head): ModuleDict(
+          (norm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        )
+        (eh_proj): Linear(in_features=16, out_features=8, bias=False)
+        (enorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        (hnorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        (input_layernorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        (post_attention_layernorm): RMSNorm((8,), eps=None, elementwise_affine=True)
+        (self_attn): GlmMoeDsaAttention(
+          (q_a_proj): Linear(in_features=8, out_features=32, bias=False)
+          (q_a_layernorm): GlmMoeDsaRMSNorm((32,), eps=1e-06)
+          (q_b_proj): Linear(in_features=32, out_features=2048, bias=False)
+          (kv_a_proj_with_mqa): Linear(in_features=8, out_features=576, bias=False)
+          (kv_a_layernorm): GlmMoeDsaRMSNorm((512,), eps=1e-06)
+          (kv_b_proj): Linear(in_features=512, out_features=3584, bias=False)
+          (o_proj): Linear(in_features=2048, out_features=8, bias=False)
+          (indexer): GlmMoeDsaIndexer(
+            (wq_b): Linear(in_features=32, out_features=512, bias=False)
+            (wk): Linear(in_features=8, out_features=128, bias=False)
+            (k_norm): LayerNorm((128,), eps=1e-06, elementwise_affine=True)
+            (weights_proj): Linear(in_features=8, out_features=4, bias=False)
+          )
+        )
+        (mlp): GlmMoeDsaMoE(
+          (experts): GlmMoeDsaNaiveMoe(
+            (act_fn): SiLUActivation()
+          )
+          (gate): GlmMoeDsaTopkRouter()
+          (shared_experts): GlmMoeDsaMLP(
+            (gate_proj): Linear(in_features=8, out_features=32, bias=False)
+            (up_proj): Linear(in_features=8, out_features=32, bias=False)
+            (down_proj): Linear(in_features=32, out_features=8, bias=False)
+            (act_fn): SiLUActivation()
+          )
+        )
+      )
+    )
+    (norm): GlmMoeDsaRMSNorm((8,), eps=1e-05)
+    (rotary_emb): GlmMoeDsaRotaryEmbedding()
+  )
+  (lm_head): Linear(in_features=8, out_features=154880, bias=False)
+)
+```
+
+</details>
+
+### Test environment:
+
+- torch: 2.11.0
+- transformers: 5.5.0

chat_template.jinja

@@ -0,0 +1,119 @@
+[gMASK]<sop>
+{%- if tools -%}
+{%- macro tool_to_json(tool) -%}
+    {%- set ns_tool = namespace(first=true) -%}
+    {{ '{' -}}
+    {%- for k, v in tool.items() -%}
+        {%- if k != 'defer_loading' and k != 'strict' -%}
+            {%- if not ns_tool.first -%}{{- ', ' -}}{%- endif -%}
+            {%- set ns_tool.first = false -%}
+            "{{ k }}": {{ v | tojson(ensure_ascii=False) }}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- '}' -}}
+{%- endmacro -%}
+<|system|>
+# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{% for tool in tools %}
+{%- if 'function' in tool -%}
+    {%- set tool = tool['function'] -%}
+{%- endif -%}
+{% if tool.defer_loading is not defined or not tool.defer_loading %}
+{{ tool_to_json(tool) }}
+{% endif %}
+{% endfor %}
+</tools>
+
+For each function call, output the function name and arguments within the following XML format:
+<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>{%- endif -%}
+{%- macro visible_text(content) -%}
+    {%- if content is string -%}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping -%}
+        {%- for item in content -%}
+            {%- if item is mapping and item.type == 'text' -%}
+                {{- item.text }}
+            {%- elif item is string -%}
+                {{- item }}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- else -%}
+        {{- content }}
+    {%- endif -%}
+{%- endmacro -%}
+{%- set ns = namespace(last_user_index=-1, thinking_indices='') -%}
+{%- for m in messages %}
+    {%- if m.role == 'user' %}
+        {%- set ns.last_user_index = loop.index0 -%}
+    {%- elif m.role == 'assistant' %}
+        {%- if m.reasoning_content is string %}
+            {%- set ns.thinking_indices = ns.thinking_indices ~ ',' ~ ns.last_user_index ~ ',' -%}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- set ns.has_thinking = false -%}
+{%- for m in messages -%}
+{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }}{% set ns.has_thinking = (',' ~ loop.index0 ~ ',') in ns.thinking_indices -%}
+{%- elif m.role == 'assistant' -%}
+<|assistant|>
+{%- set content = visible_text(m.content) %}
+{%- if m.reasoning_content is string %}
+    {%- set reasoning_content = m.reasoning_content %}
+{%- elif '</think>' in content %}
+    {%- set reasoning_content = content.split('</think>')[0].split('<think>')[-1] %}
+    {%- set content = content.split('</think>')[-1] %}
+{%- elif loop.index0 > ns.last_user_index and not (enable_thinking is defined and not enable_thinking) %}
+    {%- set reasoning_content = '' %}
+{%- elif loop.index0 < ns.last_user_index and ns.has_thinking %}
+    {%- set reasoning_content = '' %}
+{%- endif %}
+{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content is defined -%}
+{{ '<think>' + reasoning_content +  '</think>'}}
+{%- else -%}
+{{ '</think>' }}
+{%- endif -%}
+{%- if content.strip() -%}
+{{ content.strip() }}
+{%- endif -%}
+{% if m.tool_calls %}
+{% for tc in m.tool_calls %}
+{%- if tc.function %}
+    {%- set tc = tc.function %}
+{%- endif %}
+{{- '<tool_call>' + tc.name -}}
+{% set _args = tc.arguments %}{% for k, v in _args.items() %}<arg_key>{{ k }}</arg_key><arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>{% endfor %}</tool_call>{% endfor %}
+{% endif %}
+{%- elif m.role == 'tool' -%}
+{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+    {{- '<|observation|>' -}}
+{%- endif %}
+{%- if m.content is string -%}
+    {{- '<tool_response>' + m.content + '</tool_response>' -}}
+{%- elif m.content is iterable and m.content is not mapping and m.content and m.content.0.type == "tool_reference" -%}
+    {{- '<tool_response><tools>\n' -}}
+    {% for tr in m.content %}
+        {%- for tool in tools -%}
+            {%- if 'function' in tool -%}
+                {%- set tool = tool['function'] -%}
+            {%- endif -%}
+            {%- if tool.name == tr.name -%}
+                {{- tool_to_json(tool) + '\n' -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- endfor -%}
+    {{- '</tools></tool_response>' -}}
+{%- else -%}
+    {{- '<tool_response>' + visible_text(m.content) + '</tool_response>' -}}
+{% endif -%}
+{%- elif m.role == 'system' -%}
+<|system|>{{ visible_text(m.content) }}
+{%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    <|assistant|>{{- '</think>' if (enable_thinking is defined and not enable_thinking) else '<think>' -}}
+{%- endif -%}

config.json

@@ -0,0 +1,63 @@
+{
+  "architectures": [
+    "GlmMoeDsaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dtype": "bfloat16",
+  "eos_token_id": [
+    154820,
+    154827,
+    154829
+  ],
+  "ep_size": 1,
+  "first_k_dense_replace": 1,
+  "hidden_act": "silu",
+  "hidden_size": 8,
+  "index_head_dim": 128,
+  "index_n_heads": 4,
+  "index_topk": 2048,
+  "indexer_rope_interleave": true,
+  "initializer_range": 0.02,
+  "intermediate_size": 32,
+  "kv_lora_rank": 512,
+  "max_position_embeddings": 202752,
+  "mlp_layer_types": [
+    "dense",
+    "sparse"
+  ],
+  "model_type": "glm_moe_dsa",
+  "moe_intermediate_size": 32,
+  "moe_layer_freq": 1,
+  "n_group": 1,
+  "n_routed_experts": 256,
+  "n_shared_experts": 1,
+  "norm_topk_prob": true,
+  "num_attention_heads": 8,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 8,
+  "num_nextn_predict_layers": 1,
+  "pad_token_id": 154820,
+  "pretraining_tp": 1,
+  "q_lora_rank": 32,
+  "qk_head_dim": 256,
+  "qk_nope_head_dim": 192,
+  "qk_rope_head_dim": 64,
+  "rms_norm_eps": 1e-05,
+  "rope_interleave": true,
+  "rope_parameters": {
+    "rope_theta": 1000000,
+    "rope_type": "default"
+  },
+  "routed_scaling_factor": 2.5,
+  "scoring_func": "sigmoid",
+  "tie_word_embeddings": false,
+  "topk_group": 1,
+  "topk_method": "noaux_tc",
+  "transformers_version": "5.5.0",
+  "use_cache": true,
+  "v_head_dim": 256,
+  "vocab_size": 154880
+}

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "_from_model_config": true,
+  "do_sample": true,
+  "eos_token_id": [
+    154820,
+    154827,
+    154829
+  ],
+  "pad_token_id": 154820,
+  "temperature": 1.0,
+  "top_p": 0.95,
+  "transformers_version": "5.5.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fde54d8d9a380c57a28e4871e487fe90b2b5a53b2c6c6d93d4f0bac83c875394
+size 17581680

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19e773648cb4e65de8660ea6365e10acca112d42a854923df93db4a6f333a82d
+size 20217442

tokenizer_config.json

@@ -0,0 +1,33 @@
+{
+  "backend": "tokenizers",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": [
+    "<|endoftext|>",
+    "[MASK]",
+    "[gMASK]",
+    "[sMASK]",
+    "<sop>",
+    "<eop>",
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|observation|>",
+    "<|begin_of_image|>",
+    "<|end_of_image|>",
+    "<|begin_of_video|>",
+    "<|end_of_video|>",
+    "<|begin_of_audio|>",
+    "<|end_of_audio|>",
+    "<|begin_of_transcription|>",
+    "<|end_of_transcription|>"
+  ],
+  "is_local": false,
+  "model_max_length": 202752,
+  "model_specific_special_tokens": {},
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "remove_space": false,
+  "tokenizer_class": "TokenizersBackend"
+}