crumb
/

Llama-p-small

Text Generation

Model card Files Files and versions Community

crumb commited on Feb 12

Commit

6514035

•

1 Parent(s): e5ff639

Update modeling_nano.py

Files changed (1) hide show

modeling_nano.py +1 -1

modeling_nano.py CHANGED Viewed

@@ -94,7 +94,7 @@ class NanoAttention(nn.Module):
             elif scaling_type == "dynamic":
                 self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding(
                     self.head_dim,
-                    max_position_embeddings=self.max_position_embeddings,
                     scaling_factor=scaling_factor,
                     base=self.config.rope_theta,
                 )

             elif scaling_type == "dynamic":
                 self.rotary_emb = LlamaDynamicNTKScalingRotaryEmbedding(
                     self.head_dim,
+                    max_position_embeddings=self.config.max_position_embeddings,
                     scaling_factor=scaling_factor,
                     base=self.config.rope_theta,
                 )