shunxing1234 commited on
Commit
3d19935
1 Parent(s): 72802bf

Update modeling_aquila.py

Browse files
Files changed (1) hide show
  1. modeling_aquila.py +3 -3
modeling_aquila.py CHANGED
@@ -127,7 +127,7 @@ class AquilaRotaryEmbedding(torch.nn.Module):
127
 
128
  # Copied from transformers.models.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding with Llama->Aquila
129
  class AquilaLinearScalingRotaryEmbedding(AquilaRotaryEmbedding):
130
- """LlamaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
131
 
132
  def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
133
  self.scaling_factor = scaling_factor
@@ -145,8 +145,8 @@ class AquilaLinearScalingRotaryEmbedding(AquilaRotaryEmbedding):
145
  self.register_buffer("sin_cached", emb.sin()[None, None, :, :].to(dtype), persistent=False)
146
 
147
  # Copied from transformers.models.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding with Llama->Aquila
148
- class AquilaDynamicNTKScalingRotaryEmbedding(LlamaRotaryEmbedding):
149
- """LlamaRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla"""
150
 
151
  def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
152
  self.scaling_factor = scaling_factor
 
127
 
128
  # Copied from transformers.models.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding with Llama->Aquila
129
  class AquilaLinearScalingRotaryEmbedding(AquilaRotaryEmbedding):
130
+ """AquilaRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
131
 
132
  def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
133
  self.scaling_factor = scaling_factor
 
145
  self.register_buffer("sin_cached", emb.sin()[None, None, :, :].to(dtype), persistent=False)
146
 
147
  # Copied from transformers.models.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding with Llama->Aquila
148
+ class AquilaDynamicNTKScalingRotaryEmbedding(AquilaRotaryEmbedding):
149
+ """AquilaRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla"""
150
 
151
  def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
152
  self.scaling_factor = scaling_factor