Commit
•
470d8d3
1
Parent(s):
bcdea4c
Create sparsefinetuning.yaml
Browse files- sparsefinetuning.yaml +17 -0
sparsefinetuning.yaml
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
test_stage:
|
| 2 |
+
pruning_modifiers:
|
| 3 |
+
ConstantPruningModifier:
|
| 4 |
+
targets: [
|
| 5 |
+
"re:.*self_attn.q_proj",
|
| 6 |
+
"re:.*self_attn.k_proj",
|
| 7 |
+
"re:.*self_attn.v_proj",
|
| 8 |
+
"re:.*self_attn.o_proj",
|
| 9 |
+
"re:.*mlp.gate_proj",
|
| 10 |
+
"re:.*mlp.up_proj"
|
| 11 |
+
]
|
| 12 |
+
start: 0
|
| 13 |
+
distillation_modifiers:
|
| 14 |
+
OutputDistillationModifier:
|
| 15 |
+
targets: "__ALL__"
|
| 16 |
+
comparison: "square_head"
|
| 17 |
+
start: 0
|