lightseekorg commited on
Commit ·
7211e95
0
Parent(s):
Add Kimi-K3 DSpark checkpoint
Browse files- .gitattributes +35 -0
- README.md +147 -0
- config.json +55 -0
- model.safetensors +3 -0
.gitattributes
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: other
|
| 3 |
+
base_model: moonshotai/Kimi-K3
|
| 4 |
+
pipeline_tag: text-generation
|
| 5 |
+
tags:
|
| 6 |
+
- speculative-decoding
|
| 7 |
+
- dspark
|
| 8 |
+
- kimi-k3
|
| 9 |
+
- torchspec
|
| 10 |
+
- tokenspeed
|
| 11 |
+
---
|
| 12 |
+
|
| 13 |
+
# Kimi-K3 DSpark
|
| 14 |
+
|
| 15 |
+
Kimi-K3 DSpark is a speculative draft model for accelerating
|
| 16 |
+
[Kimi-K3](https://huggingface.co/moonshotai/Kimi-K3). It was trained with
|
| 17 |
+
[TorchSpec](https://github.com/torchspec-project/TorchSpec), using online target-model
|
| 18 |
+
inference and FSDP draft-model training. The architecture follows
|
| 19 |
+
[DSpark: Confidence-Scheduled Speculative Decoding with Semi-Autoregressive
|
| 20 |
+
Generation](https://arxiv.org/abs/2607.05147).
|
| 21 |
+
|
| 22 |
+
The draft uses a five-layer MLA backbone, five target hidden-state taps, a low-rank
|
| 23 |
+
Markov head, and a confidence head. It proposes up to seven draft tokens per speculative
|
| 24 |
+
decoding step.
|
| 25 |
+
|
| 26 |
+
## Training recipe
|
| 27 |
+
|
| 28 |
+
The training recipe was adapted from
|
| 29 |
+
[deepseek-ai/DeepSpec](https://github.com/deepseek-ai/DeepSpec) for online Kimi-K3
|
| 30 |
+
target inference and distributed TorchSpec training.
|
| 31 |
+
|
| 32 |
+
Due to limited compute resources, we trained on only approximately **150K selected
|
| 33 |
+
and regenerated samples**. We selected the prompts from
|
| 34 |
+
[lightseekorg/kimi-mtp-dataset](https://huggingface.co/datasets/lightseekorg/kimi-mtp-dataset)
|
| 35 |
+
and regenerated the responses with Kimi-K3. Regeneration is important for speculative
|
| 36 |
+
decoding because the draft must learn the token-level distribution of the exact target
|
| 37 |
+
model it accelerates.
|
| 38 |
+
|
| 39 |
+
The regenerated mix covers general instruction following, reasoning, coding and software
|
| 40 |
+
engineering, tool use, Chinese, and multilingual prompts. Training used the Kimi-K3
|
| 41 |
+
renderer with assistant-only loss on sequences up to 32K tokens.
|
| 42 |
+
|
| 43 |
+
| Setting | Value |
|
| 44 |
+
|---|---:|
|
| 45 |
+
| Training epochs | 5 |
|
| 46 |
+
| Optimizer steps | 6,891 |
|
| 47 |
+
| Global batch size | 128 |
|
| 48 |
+
| Maximum sequence length | 32,768 |
|
| 49 |
+
| Learning rate | 5e-5, cosine schedule |
|
| 50 |
+
| Warmup | 4% |
|
| 51 |
+
| Loss objective | Decayed DSpark loss, gamma=4.0 |
|
| 52 |
+
| Token loss weights | Cross-entropy=0.1, L1=0.9 |
|
| 53 |
+
| Confidence-head loss weight | 1.0 |
|
| 54 |
+
| Draft layers | 5 |
|
| 55 |
+
| Target hidden-state layers | 7, 31, 47, 63, 87 |
|
| 56 |
+
| Speculative tokens | 7 |
|
| 57 |
+
|
| 58 |
+
### Compute cost
|
| 59 |
+
|
| 60 |
+
The end-to-end online training run used **40 NVIDIA GB200 GPUs**: 8 GPUs for FSDP
|
| 61 |
+
training and 32 GPUs for online Kimi-K3 inference, equivalent to approximately
|
| 62 |
+
**1,104 GB200 GPU-hours**.
|
| 63 |
+
|
| 64 |
+
## Performance
|
| 65 |
+
|
| 66 |
+
The primary metric is `acc_len`, defined as `1 + accepted_draft_tokens / draft_steps`.
|
| 67 |
+
Higher is better. Evaluation used `num_speculative_tokens=7` and Kimi-K3 thinking mode.
|
| 68 |
+
|
| 69 |
+
| Benchmark | Samples | Greedy draft, t=0 | Probabilistic draft, t=1.0, top_p=0.95 |
|
| 70 |
+
|---|---:|---:|---:|
|
| 71 |
+
| GSM8K | 1,319 | 5.174 | 4.947 |
|
| 72 |
+
| HumanEval | 164 | 4.585 | 4.025 |
|
| 73 |
+
| SPEED-Bench coding | 80 | 3.986 | 3.745 |
|
| 74 |
+
| SPEED-Bench RAG | 80 | 3.373 | 3.265 |
|
| 75 |
+
| SPEED-Bench multilingual | 80 | 3.279 | 3.029 |
|
| 76 |
+
| MATH-500 | 500 | 3.098 | 3.076 |
|
| 77 |
+
| SPEED-Bench QA | 80 | 2.895 | 2.762 |
|
| 78 |
+
| MT-Bench | 80 | 2.871 | 2.699 |
|
| 79 |
+
| SPEED-Bench writing | 80 | 2.555 | 2.426 |
|
| 80 |
+
| AIME 2026 | 30 | 2.430 | 2.237 |
|
| 81 |
+
|
| 82 |
+
Requests were sent through `/v1/chat/completions` with:
|
| 83 |
+
|
| 84 |
+
```json
|
| 85 |
+
{
|
| 86 |
+
"thinking": true,
|
| 87 |
+
"thinking_effort": "max"
|
| 88 |
+
}
|
| 89 |
+
```
|
| 90 |
+
|
| 91 |
+
## Launch with TokenSpeed
|
| 92 |
+
|
| 93 |
+
This checkpoint uses native DSpark support in TokenSpeed. The auxiliary
|
| 94 |
+
AttnRes stream must be enabled for this training contract.
|
| 95 |
+
|
| 96 |
+
```bash
|
| 97 |
+
export TOKENSPEED_DFLASH_AUX_STREAM=attn_res
|
| 98 |
+
|
| 99 |
+
tokenspeed serve moonshotai/Kimi-K3 \
|
| 100 |
+
--served-model-name moonshotai/Kimi-K3 \
|
| 101 |
+
--tensor-parallel-size 16 \
|
| 102 |
+
--distributed-executor-backend ray \
|
| 103 |
+
--trust-remote-code \
|
| 104 |
+
--max-model-len 131072 \
|
| 105 |
+
--gpu-memory-utilization 0.92 \
|
| 106 |
+
--speculative-config '{"method":"dspark","model":"lightseekorg/kimi-k3-dspark","num_speculative_tokens":7,"attention_backend":"FLASHINFER_MLA","draft_sample_method":"greedy","rejection_sample_method":"block"}' \
|
| 107 |
+
--host 0.0.0.0 \
|
| 108 |
+
--port 8000
|
| 109 |
+
```
|
| 110 |
+
|
| 111 |
+
Use `"draft_sample_method":"greedy"` for temperature-zero workloads. For sampled
|
| 112 |
+
generation, launch a separate endpoint with `"draft_sample_method":"probabilistic"`.
|
| 113 |
+
The draft sampling method is a server-level setting.
|
| 114 |
+
|
| 115 |
+
## Important serving notes
|
| 116 |
+
|
| 117 |
+
1. The collected target hidden states must be captured **after the attention residual
|
| 118 |
+
(`attn_res`) has been applied and before the following pre-normalization**. Set
|
| 119 |
+
`TOKENSPEED_DFLASH_AUX_STREAM=attn_res`; using another residual stream changes the
|
| 120 |
+
draft model's input distribution.
|
| 121 |
+
2. This checkpoint's config declares **`fc_norm`**. Each collected hidden-state tap must
|
| 122 |
+
pass through its corresponding `fc_norm` before the five taps are concatenated and
|
| 123 |
+
passed to the linear context projection. Skipping this step is incompatible with the
|
| 124 |
+
training contract.
|
| 125 |
+
|
| 126 |
+
TokenSpeed support for both requirements is implemented in
|
| 127 |
+
[lightseekorg/tokenspeed#1016](https://github.com/lightseekorg/tokenspeed/pull/1016).
|
| 128 |
+
|
| 129 |
+
## Citation
|
| 130 |
+
|
| 131 |
+
```bibtex
|
| 132 |
+
@misc{cheng2026dspark,
|
| 133 |
+
title={DSpark: Confidence-Scheduled Speculative Decoding with Semi-Autoregressive Generation},
|
| 134 |
+
author={Xin Cheng and others},
|
| 135 |
+
year={2026},
|
| 136 |
+
eprint={2607.05147},
|
| 137 |
+
archivePrefix={arXiv},
|
| 138 |
+
primaryClass={cs.AI},
|
| 139 |
+
url={https://arxiv.org/abs/2607.05147}
|
| 140 |
+
}
|
| 141 |
+
|
| 142 |
+
@misc{torchspec2026,
|
| 143 |
+
title={TorchSpec: An Online Speculative Decoding Training Framework},
|
| 144 |
+
url={https://github.com/torchspec-project/TorchSpec},
|
| 145 |
+
year={2026}
|
| 146 |
+
}
|
| 147 |
+
```
|
config.json
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"architectures": [
|
| 3 |
+
"K3DSparkModel"
|
| 4 |
+
],
|
| 5 |
+
"model_type": "k3_dspark",
|
| 6 |
+
"hidden_size": 7168,
|
| 7 |
+
"intermediate_size": 14336,
|
| 8 |
+
"num_hidden_layers": 5,
|
| 9 |
+
"num_attention_heads": 64,
|
| 10 |
+
"num_key_value_heads": 64,
|
| 11 |
+
"q_lora_rank": 1536,
|
| 12 |
+
"kv_lora_rank": 512,
|
| 13 |
+
"qk_nope_head_dim": 128,
|
| 14 |
+
"qk_rope_head_dim": 64,
|
| 15 |
+
"v_head_dim": 128,
|
| 16 |
+
"mla_use_nope": false,
|
| 17 |
+
"mla_use_output_gate": false,
|
| 18 |
+
"vocab_size": 163840,
|
| 19 |
+
"rms_norm_eps": 1e-05,
|
| 20 |
+
"max_position_embeddings": 1048576,
|
| 21 |
+
"rope_theta": 50000.0,
|
| 22 |
+
"num_target_layers": 5,
|
| 23 |
+
"target_hidden_size": 7168,
|
| 24 |
+
"target_num_hidden_layers": 93,
|
| 25 |
+
"target_layer_ids": [
|
| 26 |
+
7,
|
| 27 |
+
31,
|
| 28 |
+
47,
|
| 29 |
+
63,
|
| 30 |
+
87
|
| 31 |
+
],
|
| 32 |
+
"fc_norm": true,
|
| 33 |
+
"mask_token_id": 163837,
|
| 34 |
+
"bos_token_id": 163584,
|
| 35 |
+
"eos_token_id": 163586,
|
| 36 |
+
"pad_token_id": 163839,
|
| 37 |
+
"markov_rank": 256,
|
| 38 |
+
"markov_head_type": "vanilla",
|
| 39 |
+
"enable_confidence_head": true,
|
| 40 |
+
"confidence_head_with_markov": true,
|
| 41 |
+
"tie_word_embeddings": false,
|
| 42 |
+
"draft_vocab_size": 163840,
|
| 43 |
+
"_torchspec_version": "0.1.0",
|
| 44 |
+
"torch_dtype": "bfloat16",
|
| 45 |
+
"rope_parameters": {
|
| 46 |
+
"rope_type": "yarn",
|
| 47 |
+
"factor": 32.0,
|
| 48 |
+
"original_max_position_embeddings": 32768,
|
| 49 |
+
"rope_theta": 50000.0,
|
| 50 |
+
"beta_fast": 32,
|
| 51 |
+
"beta_slow": 1,
|
| 52 |
+
"mscale": 1.0,
|
| 53 |
+
"mscale_all_dim": 1.0
|
| 54 |
+
}
|
| 55 |
+
}
|
model.safetensors
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:3c6a7a5ca1d020d2e77caf943ab93a0e6daf2f6a360d7713221fad2a50886fed
|
| 3 |
+
size 7124705578
|