santacoder / modeling_gpt2_mq.py

:bug: fix past_length in santacoder

cbd1dd1 about 1 year ago

No virus

24 kB

	"""PyTorch OpenAI GPT-2 model modified with MultiQuery attention"""


	from typing import Optional, Tuple, Union

	import torch
	import torch.utils.checkpoint
	from torch import nn
	from torch.cuda.amp import autocast

	from transformers.modeling_outputs import BaseModelOutputWithPastAndCrossAttentions
	from transformers.pytorch_utils import Conv1D, find_pruneable_heads_and_indices, prune_conv1d_layer

	from transformers.utils import logging
	from transformers.models.gpt2.modeling_gpt2 import GPT2Model, GPT2Block, GPT2PreTrainedModel, GPT2LMHeadModel
	from .configuration_gpt2_mq import GPT2CustomConfig, MULTI_QUERY

	logger = logging.get_logger(__name__)


	class GPT2MQAttention(nn.Module):
	def __init__(self, config, is_cross_attention=False, layer_idx=None):
	super().__init__()
	assert config.attention_head_type == MULTI_QUERY

	max_positions = config.max_position_embeddings
	self.register_buffer(
	"bias",
	torch.tril(torch.ones((max_positions, max_positions), dtype=torch.uint8)).view(
	1, 1, max_positions, max_positions
	),
	)
	self.register_buffer("masked_bias", torch.tensor(-1e4))

	self.embed_dim = config.hidden_size
	self.num_heads = config.num_attention_heads
	self.head_dim = self.embed_dim // self.num_heads
	self.split_size = self.embed_dim
	if self.head_dim * self.num_heads != self.embed_dim:
	raise ValueError(
	f"`embed_dim` must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
	f" {self.num_heads})."
	)

	self.scale_attn_weights = config.scale_attn_weights
	if is_cross_attention:
	raise NotImplementedError("Cross-attention not implemented for MQA")
	self.is_cross_attention = is_cross_attention

	# Layer-wise attention scaling, reordering, and upcasting
	self.scale_attn_by_inverse_layer_idx = config.scale_attn_by_inverse_layer_idx
	self.layer_idx = layer_idx
	self.reorder_and_upcast_attn = config.reorder_and_upcast_attn

	if self.is_cross_attention:
	self.c_attn = Conv1D(2 * self.embed_dim, self.embed_dim)
	self.q_attn = Conv1D(self.embed_dim, self.embed_dim)
	else:
	# self.c_attn = Conv1D(3 * self.embed_dim, self.embed_dim)
	self.q_attn = Conv1D(self.embed_dim, self.embed_dim)
	# Keys and values are shared across heads
	self.kv_attn = Conv1D(2 * self.head_dim, self.embed_dim)
	self.c_proj = Conv1D(self.embed_dim, self.embed_dim)

	self.attn_dropout = nn.Dropout(config.attn_pdrop)
	self.resid_dropout = nn.Dropout(config.resid_pdrop)

	self.pruned_heads = set()

	def prune_heads(self, heads):
	if len(heads) == 0:
	return
	heads, index = find_pruneable_heads_and_indices(heads, self.num_heads, self.head_dim, self.pruned_heads)
	index_attn = torch.cat([index, index + self.split_size, index + (2 * self.split_size)])

	# Prune conv1d layers
	self.c_attn = prune_conv1d_layer(self.c_attn, index_attn, dim=1)
	self.c_proj = prune_conv1d_layer(self.c_proj, index, dim=0)

	# Update hyper params
	self.split_size = (self.split_size // self.num_heads) * (self.num_heads - len(heads))
	self.num_heads = self.num_heads - len(heads)
	self.pruned_heads = self.pruned_heads.union(heads)

	def _attn(self, query, key, value, attention_mask=None, head_mask=None):
	# query: (b, num_heads * sq, head_dim)
	# key: (b, head_dim, sk)
	# value: (b, sk, head_dim)
	batch_size = query.size(0)
	query_length = query.size(1) // self.num_heads
	key_length = key.size(2)
	# (b, num_heads * sq, head_dim) x (b, head_dim, sk) -> (b, num_heads * sq, sk)
	attn_weights = torch.bmm(query, key)
	# -> (b, num_heads, sq, sk)
	attn_weights = attn_weights.view(batch_size, self.num_heads, query_length, key_length)

	if self.scale_attn_weights:
	attn_weights = attn_weights / torch.tensor(
	value.size(-1) ** 0.5, dtype=attn_weights.dtype, device=attn_weights.device
	)

	# Layer-wise attention scaling
	if self.scale_attn_by_inverse_layer_idx:
	attn_weights = attn_weights / float(self.layer_idx + 1)

	if not self.is_cross_attention:
	# if only "normal" attention layer implements causal mask
	causal_mask = self.bias[:, :, key_length - query_length : key_length, :key_length].to(torch.bool)
	mask_value = torch.finfo(attn_weights.dtype).min
	# Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`.
	# Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device`
	mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(attn_weights.device)
	attn_weights = torch.where(causal_mask, attn_weights, mask_value)

	if attention_mask is not None:
	# Apply the attention mask
	attn_weights = attn_weights + attention_mask

	attn_weights = nn.functional.softmax(attn_weights, dim=-1)

	# Downcast (if necessary) back to V's dtype (if in mixed-precision) -- No-Op otherwise
	attn_weights = attn_weights.type(value.dtype)
	attn_weights = self.attn_dropout(attn_weights)

	# Mask heads if we want to
	if head_mask is not None:
	attn_weights = attn_weights * head_mask

	# (b, num_heads, sq, sk) -> (b, num_heads * sq, sk)
	_attn_weights = attn_weights.view(batch_size, self.num_heads * query_length, key_length)
	# (b, num_heads * sq, sk) x (b, sk, head_dim) -> (b, num_heads * sq, head_dim)
	attn_output = torch.bmm(_attn_weights, value)
	attn_output = attn_output.view(batch_size, self.num_heads, query_length, self.head_dim)

	return attn_output, attn_weights

	def _upcast_and_reordered_attn(self, query, key, value, attention_mask=None, head_mask=None):
	# Use `torch.baddbmm` (a bit more efficient w/ alpha param for scaling -- from Megatron-LM)
	bsz, num_heads, q_seq_len, dk = query.size()
	_, _, k_seq_len, _ = key.size()

	# Preallocate attn_weights for `baddbmm`
	attn_weights = torch.empty(bsz * num_heads, q_seq_len, k_seq_len, dtype=torch.float32, device=query.device)

	# Compute Scale Factor
	scale_factor = 1.0
	if self.scale_attn_weights:
	scale_factor /= float(value.size(-1)) ** 0.5

	if self.scale_attn_by_inverse_layer_idx:
	scale_factor /= float(self.layer_idx + 1)

	# Upcast (turn off autocast) and reorder (Scale K by 1 / root(dk))
	with autocast(enabled=False):
	q, k = query.reshape(-1, q_seq_len, dk), key.transpose(-1, -2).reshape(-1, dk, k_seq_len)
	attn_weights = torch.baddbmm(attn_weights, q.float(), k.float(), beta=0, alpha=scale_factor)
	attn_weights = attn_weights.reshape(bsz, num_heads, q_seq_len, k_seq_len)

	if not self.is_cross_attention:
	# if only "normal" attention layer implements causal mask
	query_length, key_length = query.size(-2), key.size(-2)
	causal_mask = self.bias[:, :, key_length - query_length : key_length, :key_length].bool()
	mask_value = torch.finfo(attn_weights.dtype).min
	# Need to be a tensor, otherwise we get error: `RuntimeError: expected scalar type float but found double`.
	# Need to be on the same device, otherwise `RuntimeError: ..., x and y to be on the same device`
	mask_value = torch.tensor(mask_value, dtype=attn_weights.dtype).to(attn_weights.device)
	attn_weights = torch.where(causal_mask, attn_weights, mask_value)

	if attention_mask is not None:
	# Apply the attention mask
	attn_weights = attn_weights + attention_mask

	attn_weights = nn.functional.softmax(attn_weights, dim=-1)

	# Downcast (if necessary) back to V's dtype (if in mixed-precision) -- No-Op if otherwise
	if attn_weights.dtype != torch.float32:
	raise RuntimeError("Error with upcasting, attn_weights does not have dtype torch.float32")
	attn_weights = attn_weights.type(value.dtype)
	attn_weights = self.attn_dropout(attn_weights)

	# Mask heads if we want to
	if head_mask is not None:
	attn_weights = attn_weights * head_mask

	attn_output = torch.matmul(attn_weights, value)

	return attn_output, attn_weights

	def _split_heads(self, tensor, num_heads, attn_head_size):
	"""
	Splits hidden_size dim into attn_head_size and num_heads
	"""
	new_shape = tensor.size()[:-1] + (num_heads, attn_head_size)
	tensor = tensor.view(new_shape)
	return tensor.permute(0, 2, 1, 3) # (batch, head, seq_length, head_features)

	def _merge_heads(self, tensor, num_heads, attn_head_size):
	"""
	Merges attn_head_size dim and num_attn_heads dim into hidden_size
	"""
	tensor = tensor.permute(0, 2, 1, 3).contiguous()
	new_shape = tensor.size()[:-2] + (num_heads * attn_head_size,)
	return tensor.view(new_shape)

	def forward(
	self,
	hidden_states: Optional[Tuple[torch.FloatTensor]],
	layer_past: Optional[Tuple[torch.Tensor]] = None,
	attention_mask: Optional[torch.FloatTensor] = None,
	head_mask: Optional[torch.FloatTensor] = None,
	encoder_hidden_states: Optional[torch.Tensor] = None,
	encoder_attention_mask: Optional[torch.FloatTensor] = None,
	use_cache: Optional[bool] = False,
	output_attentions: Optional[bool] = False,
	) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]], ...]:
	if encoder_hidden_states is not None:
	raise NotImplementedError("Cross-attention not implemented for MQA")
	if not hasattr(self, "q_attn"):
	raise ValueError(
	"If class is used as cross attention, the weights `q_attn` have to be defined. "
	"Please make sure to instantiate class with `GPT2Attention(..., is_cross_attention=True)`."
	)

	query = self.q_attn(hidden_states)
	key, value = self.c_attn(encoder_hidden_states).split(self.split_size, dim=2)
	attention_mask = encoder_attention_mask
	else:
	query = self.q_attn(hidden_states)
	key, value = self.kv_attn(hidden_states).split(self.head_dim, dim=2)


	batch_size, seq_length = query.shape[:2]
	# (query_length, batch, num_heads, head_dim)
	# (batch, num_heads * query_length, head_dim)\

	# (batch, query_length, hidden_size) -> (batch, num_heads, query_length, head_dim)
	query = query.view(batch_size, seq_length, self.num_heads, self.head_dim).permute([0, 2, 1, 3])
	# -> (batch, num_heads * query_length, head_dim)
	query = query.reshape(batch_size, self.num_heads * seq_length, self.head_dim)

	# (batch, query_length, hidden_size) -> (batch, query_length * num_heads, head_dim)
	# query = query.view(
	# batch_size, seq_length, self.num_heads, self.head_dim,
	# ).reshape(
	# batch_size, seq_length * self.num_heads, self.head_dim
	# )
	key = key.permute(0, 2, 1) # (batch_size, head_dim, seq_length)
	# value (batch_size, seq_length, head_dim)

	if layer_past is not None:
	past_key, past_value = layer_past
	# Concatenate on sequence dimension
	key = torch.cat((past_key, key), dim=-1)
	value = torch.cat((past_value, value), dim=-2)

	if use_cache is True:
	present = (key, value)
	else:
	present = None

	if self.reorder_and_upcast_attn:
	raise NotImplementedError("Reorder and upcast attention not implemented for MQA")
	attn_output, attn_weights = self._upcast_and_reordered_attn(query, key, value, attention_mask, head_mask)
	else:
	attn_output, attn_weights = self._attn(query, key, value, attention_mask, head_mask)

	attn_output = self._merge_heads(attn_output, self.num_heads, self.head_dim)
	attn_output = self.c_proj(attn_output)
	attn_output = self.resid_dropout(attn_output)

	outputs = (attn_output, present)
	if output_attentions:
	outputs += (attn_weights,)

	return outputs # a, present, (attentions)


	# inherit from gpt_modeling.py, and override `attn` module
	class GPT2CustomBlock(GPT2Block):

	def __init__(self, config: GPT2CustomConfig, layer_idx=None):
	super().__init__(config, layer_idx)
	# Override attention module if using multiquery
	if config.attention_head_type == MULTI_QUERY:
	self.attn = GPT2MQAttention(config, layer_idx=layer_idx)
	if config.add_cross_attention:
	raise NotImplementedError("Cross-attention not implemented for MQA")


	# inherit from gpt_modeling.py and override `__init__` method
	class GPT2CustomModel(GPT2Model):
	config_class = GPT2CustomConfig

	def __init__(self, config):
	GPT2PreTrainedModel.__init__(self, config)

	self.embed_dim = config.hidden_size

	self.wte = nn.Embedding(config.vocab_size, self.embed_dim)
	self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)

	self.drop = nn.Dropout(config.embd_pdrop)
	self.h = nn.ModuleList([GPT2CustomBlock(config, layer_idx=i) for i in range(config.num_hidden_layers)])
	self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)

	# Model parallel
	self.model_parallel = False
	self.device_map = None
	self.gradient_checkpointing = False

	# Initialize weights and apply final processing
	self.post_init()

	def forward(
	self,
	input_ids: Optional[torch.LongTensor] = None,
	past_key_values: Optional[Tuple[Tuple[torch.Tensor]]] = None,
	attention_mask: Optional[torch.FloatTensor] = None,
	token_type_ids: Optional[torch.LongTensor] = None,
	position_ids: Optional[torch.LongTensor] = None,
	head_mask: Optional[torch.FloatTensor] = None,
	inputs_embeds: Optional[torch.FloatTensor] = None,
	encoder_hidden_states: Optional[torch.Tensor] = None,
	encoder_attention_mask: Optional[torch.FloatTensor] = None,
	use_cache: Optional[bool] = None,
	output_attentions: Optional[bool] = None,
	output_hidden_states: Optional[bool] = None,
	return_dict: Optional[bool] = None,
	) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]:
	output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
	output_hidden_states = (
	output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
	)
	use_cache = use_cache if use_cache is not None else self.config.use_cache
	return_dict = return_dict if return_dict is not None else self.config.use_return_dict

	if input_ids is not None and inputs_embeds is not None:
	raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
	elif input_ids is not None:
	input_shape = input_ids.size()
	input_ids = input_ids.view(-1, input_shape[-1])
	batch_size = input_ids.shape[0]
	elif inputs_embeds is not None:
	input_shape = inputs_embeds.size()[:-1]
	batch_size = inputs_embeds.shape[0]
	else:
	raise ValueError("You have to specify either input_ids or inputs_embeds")

	device = input_ids.device if input_ids is not None else inputs_embeds.device

	if token_type_ids is not None:
	token_type_ids = token_type_ids.view(-1, input_shape[-1])
	if position_ids is not None:
	position_ids = position_ids.view(-1, input_shape[-1])

	if past_key_values is None:
	past_length = 0
	past_key_values = tuple([None] * len(self.h))
	else:
	# this is different from GPT2
	past_length = past_key_values[0][0].size(-1)
	if position_ids is None:
	position_ids = torch.arange(past_length, input_shape[-1] + past_length, dtype=torch.long, device=device)
	position_ids = position_ids.unsqueeze(0).view(-1, input_shape[-1])

	# GPT2Attention mask.
	if attention_mask is not None:
	if batch_size <= 0:
	raise ValueError("batch_size has to be defined and > 0")
	attention_mask = attention_mask.view(batch_size, -1)
	# We create a 3D attention mask from a 2D tensor mask.
	# Sizes are [batch_size, 1, 1, to_seq_length]
	# So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]
	# this attention mask is more simple than the triangular masking of causal attention
	# used in OpenAI GPT, we just need to prepare the broadcast dimension here.
	attention_mask = attention_mask[:, None, None, :]

	# Since attention_mask is 1.0 for positions we want to attend and 0.0 for
	# masked positions, this operation will create a tensor which is 0.0 for
	# positions we want to attend and the dtype's smallest value for masked positions.
	# Since we are adding it to the raw scores before the softmax, this is
	# effectively the same as removing these entirely.
	attention_mask = attention_mask.to(dtype=self.dtype) # fp16 compatibility
	attention_mask = (1.0 - attention_mask) * torch.finfo(self.dtype).min

	# If a 2D or 3D attention mask is provided for the cross-attention
	# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
	if self.config.add_cross_attention and encoder_hidden_states is not None:
	encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size()
	encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length)
	if encoder_attention_mask is None:
	encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device)
	encoder_attention_mask = self.invert_attention_mask(encoder_attention_mask)
	else:
	encoder_attention_mask = None

	# Prepare head mask if needed
	# 1.0 in head_mask indicate we keep the head
	# attention_probs has shape bsz x n_heads x N x N
	# head_mask has shape n_layer x batch x n_heads x N x N
	head_mask = self.get_head_mask(head_mask, self.config.n_layer)

	if inputs_embeds is None:
	inputs_embeds = self.wte(input_ids)
	position_embeds = self.wpe(position_ids)
	hidden_states = inputs_embeds + position_embeds

	if token_type_ids is not None:
	token_type_embeds = self.wte(token_type_ids)
	hidden_states = hidden_states + token_type_embeds

	hidden_states = self.drop(hidden_states)

	output_shape = input_shape + (hidden_states.size(-1),)

	presents = () if use_cache else None
	all_self_attentions = () if output_attentions else None
	all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None
	all_hidden_states = () if output_hidden_states else None
	for i, (block, layer_past) in enumerate(zip(self.h, past_key_values)):

	# Model parallel
	if self.model_parallel:
	torch.cuda.set_device(hidden_states.device)
	# Ensure layer_past is on same device as hidden_states (might not be correct)
	if layer_past is not None:
	layer_past = tuple(past_state.to(hidden_states.device) for past_state in layer_past)
	# Ensure that attention_mask is always on the same device as hidden_states
	if attention_mask is not None:
	attention_mask = attention_mask.to(hidden_states.device)
	if isinstance(head_mask, torch.Tensor):
	head_mask = head_mask.to(hidden_states.device)
	if output_hidden_states:
	all_hidden_states = all_hidden_states + (hidden_states,)

	if self.gradient_checkpointing and self.training:

	if use_cache:
	logger.warning(
	"`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
	)
	use_cache = False

	def create_custom_forward(module):
	def custom_forward(*inputs):
	# None for past_key_value
	return module(*inputs, use_cache, output_attentions)

	return custom_forward

	outputs = torch.utils.checkpoint.checkpoint(
	create_custom_forward(block),
	hidden_states,
	None,
	attention_mask,
	head_mask[i],
	encoder_hidden_states,
	encoder_attention_mask,
	)
	else:
	outputs = block(
	hidden_states,
	layer_past=layer_past,
	attention_mask=attention_mask,
	head_mask=head_mask[i],
	encoder_hidden_states=encoder_hidden_states,
	encoder_attention_mask=encoder_attention_mask,
	use_cache=use_cache,
	output_attentions=output_attentions,
	)

	hidden_states = outputs[0]
	if use_cache is True:
	presents = presents + (outputs[1],)

	if output_attentions:
	all_self_attentions = all_self_attentions + (outputs[2 if use_cache else 1],)
	if self.config.add_cross_attention:
	all_cross_attentions = all_cross_attentions + (outputs[3 if use_cache else 2],)

	# Model Parallel: If it's the last layer for that device, put things on the next device
	if self.model_parallel:
	for k, v in self.device_map.items():
	if i == v[-1] and "cuda:" + str(k) != self.last_device:
	hidden_states = hidden_states.to("cuda:" + str(k + 1))

	hidden_states = self.ln_f(hidden_states)

	hidden_states = hidden_states.view(output_shape)
	# Add last hidden state
	if output_hidden_states:
	all_hidden_states = all_hidden_states + (hidden_states,)

	if not return_dict:
	return tuple(
	v
	for v in [hidden_states, presents, all_hidden_states, all_self_attentions, all_cross_attentions]
	if v is not None
	)

	return BaseModelOutputWithPastAndCrossAttentions(
	last_hidden_state=hidden_states,
	past_key_values=presents,
	hidden_states=all_hidden_states,
	attentions=all_self_attentions,
	cross_attentions=all_cross_attentions,
	)


	class GPT2LMHeadCustomModel(GPT2LMHeadModel):
	config_class = GPT2CustomConfig

	def __init__(self, config):
	GPT2PreTrainedModel.__init__(self, config)
	self.transformer = GPT2CustomModel(config)
	self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)

	# Model parallel
	self.model_parallel = False
	self.device_map = None

	# Initialize weights and apply final processing
	self.post_init()