Update modeling_llama.py

modeling_llama.py

@@ -91,7 +91,7 @@ class MightyLlamaModel(LlamaModel):
         all_self_attns = () if output_attentions else None
         next_decoder_cache = None
 
-        layer_order = list(range(0, 3)) + [x for _ in range(1) for x in range(4, 17)] + list(range(18, len(self.layers)))
+        layer_order = list(range(0, 3)) + [x for _ in range(16) for x in range(4, 17)] + list(range(18, len(self.layers)))
         for idx, layer_idx in enumerate(layer_order):
             decoder_layer = self.layers[layer_idx]
             if output_hidden_states: