Update modeling_glm4_moe.py
Browse files- modeling_glm4_moe.py +1 -1
modeling_glm4_moe.py
CHANGED
|
@@ -357,7 +357,7 @@ class Glm4MoeDecoderLayer(GradientCheckpointingLayer):
|
|
| 357 |
score_before_experts=False,
|
| 358 |
top_k=config.num_experts_per_tok,
|
| 359 |
use_grouped_mm=True,
|
| 360 |
-
load_balance_coeff=
|
| 361 |
)
|
| 362 |
|
| 363 |
if layer_idx >= config.first_k_dense_replace:
|
|
|
|
| 357 |
score_before_experts=False,
|
| 358 |
top_k=config.num_experts_per_tok,
|
| 359 |
use_grouped_mm=True,
|
| 360 |
+
load_balance_coeff=1e-3,
|
| 361 |
)
|
| 362 |
|
| 363 |
if layer_idx >= config.first_k_dense_replace:
|