Files changed (1) hide show
  1. resampler.py +656 -6
resampler.py CHANGED
@@ -1,9 +1,18 @@
1
  from functools import partial
2
  import numpy as np
3
-
 
4
  import torch
5
  from torch import nn
 
 
 
 
 
6
  from torch.nn.init import trunc_normal_
 
 
 
7
 
8
  def get_2d_sincos_pos_embed(embed_dim, image_size):
9
  """
@@ -55,7 +64,7 @@ def get_1d_sincos_pos_embed_from_grid_new(embed_dim, pos):
55
  emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
56
  return emb
57
 
58
-
59
  class Resampler(nn.Module):
60
  """
61
  A 2D perceiver-resampler network with one cross attention layers by
@@ -82,14 +91,13 @@ class Resampler(nn.Module):
82
  self.max_size = max_size
83
 
84
  self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
85
- trunc_normal_(self.query, std=.02)
86
 
87
  if kv_dim is not None and kv_dim != embed_dim:
88
  self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
89
  else:
90
  self.kv_proj = nn.Identity()
91
 
92
- self.attn = nn.MultiheadAttention(embed_dim, num_heads)
93
  self.ln_q = norm_layer(embed_dim)
94
  self.ln_kv = norm_layer(embed_dim)
95
 
@@ -97,9 +105,10 @@ class Resampler(nn.Module):
97
  self.proj = nn.Parameter((embed_dim ** -0.5) * torch.randn(embed_dim, embed_dim))
98
 
99
  self._set_2d_pos_cache(self.max_size)
100
- self.apply(self._init_weights)
101
 
102
  def _set_2d_pos_cache(self, max_size, device='cpu'):
 
 
103
  pos_embed = torch.from_numpy(get_2d_sincos_pos_embed(self.embed_dim, max_size)).float().to(device)
104
  self.register_buffer("pos_embed", pos_embed, persistent=False)
105
 
@@ -160,4 +169,645 @@ class Resampler(nn.Module):
160
  return x
161
 
162
  def _repeat(self, query, N: int):
163
- return query.unsqueeze(1).repeat(1, N, 1)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  from functools import partial
2
  import numpy as np
3
+ import warnings
4
+ from typing import Optional, Tuple
5
  import torch
6
  from torch import nn
7
+ from torch import Tensor
8
+ import deepspeed
9
+ import torch.nn.functional as F
10
+ from torch.nn.functional import *
11
+ from torch.nn.modules.activation import *
12
  from torch.nn.init import trunc_normal_
13
+ from torch.nn.init import constant_, xavier_normal_, xavier_uniform_
14
+ from transformers import PreTrainedModel
15
+ from transformers.integrations import is_deepspeed_zero3_enabled
16
 
17
  def get_2d_sincos_pos_embed(embed_dim, image_size):
18
  """
 
64
  emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
65
  return emb
66
 
67
+
68
  class Resampler(nn.Module):
69
  """
70
  A 2D perceiver-resampler network with one cross attention layers by
 
91
  self.max_size = max_size
92
 
93
  self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
 
94
 
95
  if kv_dim is not None and kv_dim != embed_dim:
96
  self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
97
  else:
98
  self.kv_proj = nn.Identity()
99
 
100
+ self.attn = MultiheadAttention(embed_dim, num_heads)
101
  self.ln_q = norm_layer(embed_dim)
102
  self.ln_kv = norm_layer(embed_dim)
103
 
 
105
  self.proj = nn.Parameter((embed_dim ** -0.5) * torch.randn(embed_dim, embed_dim))
106
 
107
  self._set_2d_pos_cache(self.max_size)
 
108
 
109
  def _set_2d_pos_cache(self, max_size, device='cpu'):
110
+ if is_deepspeed_zero3_enabled():
111
+ device='cuda'
112
  pos_embed = torch.from_numpy(get_2d_sincos_pos_embed(self.embed_dim, max_size)).float().to(device)
113
  self.register_buffer("pos_embed", pos_embed, persistent=False)
114
 
 
169
  return x
170
 
171
  def _repeat(self, query, N: int):
172
+ return query.unsqueeze(1).repeat(1, N, 1)
173
+
174
+
175
+ class MultiheadAttention(nn.MultiheadAttention):
176
+ def __init__(self, embed_dim, num_heads, dropout=0., bias=True, add_bias_kv=False,
177
+ add_zero_attn=False, kdim=None, vdim=None, batch_first=False, device=None, dtype=None):
178
+ super().__init__(embed_dim, num_heads, dropout, bias, add_bias_kv, add_zero_attn, kdim, vdim, batch_first, device, dtype)
179
+
180
+ # rewrite out_proj layer,with nn.Linear
181
+ self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias, device=device, dtype=dtype)
182
+
183
+ def forward(
184
+ self,
185
+ query: Tensor,
186
+ key: Tensor,
187
+ value: Tensor,
188
+ key_padding_mask: Optional[Tensor] = None,
189
+ need_weights: bool = True,
190
+ attn_mask: Optional[Tensor] = None,
191
+ average_attn_weights: bool = True,
192
+ is_causal : bool = False) -> Tuple[Tensor, Optional[Tensor]]:
193
+ why_not_fast_path = ''
194
+ if ((attn_mask is not None and torch.is_floating_point(attn_mask))
195
+ or (key_padding_mask is not None) and torch.is_floating_point(key_padding_mask)):
196
+ why_not_fast_path = "floating-point masks are not supported for fast path."
197
+
198
+ is_batched = query.dim() == 3
199
+
200
+ key_padding_mask = F._canonical_mask(
201
+ mask=key_padding_mask,
202
+ mask_name="key_padding_mask",
203
+ other_type=F._none_or_dtype(attn_mask),
204
+ other_name="attn_mask",
205
+ target_type=query.dtype
206
+ )
207
+
208
+ attn_mask = F._canonical_mask(
209
+ mask=attn_mask,
210
+ mask_name="attn_mask",
211
+ other_type=None,
212
+ other_name="",
213
+ target_type=query.dtype,
214
+ check_other=False,
215
+ )
216
+
217
+
218
+ if not is_batched:
219
+ why_not_fast_path = f"input not batched; expected query.dim() of 3 but got {query.dim()}"
220
+ elif query is not key or key is not value:
221
+ # When lifting this restriction, don't forget to either
222
+ # enforce that the dtypes all match or test cases where
223
+ # they don't!
224
+ why_not_fast_path = "non-self attention was used (query, key, and value are not the same Tensor)"
225
+ elif self.in_proj_bias is not None and query.dtype != self.in_proj_bias.dtype:
226
+ why_not_fast_path = f"dtypes of query ({query.dtype}) and self.in_proj_bias ({self.in_proj_bias.dtype}) don't match"
227
+ elif self.in_proj_weight is None:
228
+ why_not_fast_path = "in_proj_weight was None"
229
+ elif query.dtype != self.in_proj_weight.dtype:
230
+ # this case will fail anyway, but at least they'll get a useful error message.
231
+ why_not_fast_path = f"dtypes of query ({query.dtype}) and self.in_proj_weight ({self.in_proj_weight.dtype}) don't match"
232
+ elif self.training:
233
+ why_not_fast_path = "training is enabled"
234
+ elif (self.num_heads % 2) != 0:
235
+ why_not_fast_path = "self.num_heads is not even"
236
+ elif not self.batch_first:
237
+ why_not_fast_path = "batch_first was not True"
238
+ elif self.bias_k is not None:
239
+ why_not_fast_path = "self.bias_k was not None"
240
+ elif self.bias_v is not None:
241
+ why_not_fast_path = "self.bias_v was not None"
242
+ elif self.add_zero_attn:
243
+ why_not_fast_path = "add_zero_attn was enabled"
244
+ elif not self._qkv_same_embed_dim:
245
+ why_not_fast_path = "_qkv_same_embed_dim was not True"
246
+ elif query.is_nested and (key_padding_mask is not None or attn_mask is not None):
247
+ why_not_fast_path = "supplying both src_key_padding_mask and src_mask at the same time \
248
+ is not supported with NestedTensor input"
249
+ elif torch.is_autocast_enabled():
250
+ why_not_fast_path = "autocast is enabled"
251
+
252
+ if not why_not_fast_path:
253
+ tensor_args = (
254
+ query,
255
+ key,
256
+ value,
257
+ self.in_proj_weight,
258
+ self.in_proj_bias,
259
+ self.out_proj.weight,
260
+ self.out_proj.bias,
261
+ )
262
+ # We have to use list comprehensions below because TorchScript does not support
263
+ # generator expressions.
264
+ if torch.overrides.has_torch_function(tensor_args):
265
+ why_not_fast_path = "some Tensor argument has_torch_function"
266
+ elif _is_make_fx_tracing():
267
+ why_not_fast_path = "we are running make_fx tracing"
268
+ elif not all(_check_arg_device(x) for x in tensor_args):
269
+ why_not_fast_path = ("some Tensor argument's device is neither one of "
270
+ f"cpu, cuda or {torch.utils.backend_registration._privateuse1_backend_name}")
271
+ elif torch.is_grad_enabled() and any(_arg_requires_grad(x) for x in tensor_args):
272
+ why_not_fast_path = ("grad is enabled and at least one of query or the "
273
+ "input/output projection weights or biases requires_grad")
274
+ if not why_not_fast_path:
275
+ merged_mask, mask_type = self.merge_masks(attn_mask, key_padding_mask, query)
276
+
277
+ if self.in_proj_bias is not None and self.in_proj_weight is not None:
278
+ return torch._native_multi_head_attention(
279
+ query,
280
+ key,
281
+ value,
282
+ self.embed_dim,
283
+ self.num_heads,
284
+ self.in_proj_weight,
285
+ self.in_proj_bias,
286
+ self.out_proj.weight,
287
+ self.out_proj.bias,
288
+ merged_mask,
289
+ need_weights,
290
+ average_attn_weights,
291
+ mask_type)
292
+
293
+ any_nested = query.is_nested or key.is_nested or value.is_nested
294
+ assert not any_nested, ("MultiheadAttention does not support NestedTensor outside of its fast path. " +
295
+ f"The fast path was not hit because {why_not_fast_path}")
296
+
297
+ if self.batch_first and is_batched:
298
+ # make sure that the transpose op does not affect the "is" property
299
+ if key is value:
300
+ if query is key:
301
+ query = key = value = query.transpose(1, 0)
302
+ else:
303
+ query, key = (x.transpose(1, 0) for x in (query, key))
304
+ value = key
305
+ else:
306
+ query, key, value = (x.transpose(1, 0) for x in (query, key, value))
307
+
308
+ if not self._qkv_same_embed_dim:
309
+ attn_output, attn_output_weights = self.multi_head_attention_forward(
310
+ query, key, value, self.embed_dim, self.num_heads,
311
+ self.in_proj_weight, self.in_proj_bias,
312
+ self.bias_k, self.bias_v, self.add_zero_attn,
313
+ self.dropout, self.out_proj.weight, self.out_proj.bias,
314
+ training=self.training,
315
+ key_padding_mask=key_padding_mask, need_weights=need_weights,
316
+ attn_mask=attn_mask,
317
+ use_separate_proj_weight=True,
318
+ q_proj_weight=self.q_proj_weight, k_proj_weight=self.k_proj_weight,
319
+ v_proj_weight=self.v_proj_weight,
320
+ average_attn_weights=average_attn_weights,
321
+ is_causal=is_causal)
322
+ else:
323
+ attn_output, attn_output_weights = self.multi_head_attention_forward(
324
+ query, key, value, self.embed_dim, self.num_heads,
325
+ self.in_proj_weight, self.in_proj_bias,
326
+ self.bias_k, self.bias_v, self.add_zero_attn,
327
+ self.dropout, self.out_proj.weight, self.out_proj.bias,
328
+ training=self.training,
329
+ key_padding_mask=key_padding_mask,
330
+ need_weights=need_weights,
331
+ attn_mask=attn_mask,
332
+ average_attn_weights=average_attn_weights,
333
+ is_causal=is_causal)
334
+ if self.batch_first and is_batched:
335
+ return attn_output.transpose(1, 0), attn_output_weights
336
+ else:
337
+ return attn_output, attn_output_weights
338
+
339
+ def multi_head_attention_forward(
340
+ self,
341
+ query: Tensor,
342
+ key: Tensor,
343
+ value: Tensor,
344
+ embed_dim_to_check: int,
345
+ num_heads: int,
346
+ in_proj_weight: Optional[Tensor],
347
+ in_proj_bias: Optional[Tensor],
348
+ bias_k: Optional[Tensor],
349
+ bias_v: Optional[Tensor],
350
+ add_zero_attn: bool,
351
+ dropout_p: float,
352
+ out_proj_weight: Tensor,
353
+ out_proj_bias: Optional[Tensor],
354
+ training: bool = True,
355
+ key_padding_mask: Optional[Tensor] = None,
356
+ need_weights: bool = True,
357
+ attn_mask: Optional[Tensor] = None,
358
+ use_separate_proj_weight: bool = False,
359
+ q_proj_weight: Optional[Tensor] = None,
360
+ k_proj_weight: Optional[Tensor] = None,
361
+ v_proj_weight: Optional[Tensor] = None,
362
+ static_k: Optional[Tensor] = None,
363
+ static_v: Optional[Tensor] = None,
364
+ average_attn_weights: bool = True,
365
+ is_causal: bool = False,
366
+ ) -> Tuple[Tensor, Optional[Tensor]]:
367
+ tens_ops = (query, key, value, in_proj_weight, in_proj_bias, bias_k, bias_v, out_proj_weight, out_proj_bias)
368
+ if has_torch_function(tens_ops):
369
+ return handle_torch_function(
370
+ multi_head_attention_forward,
371
+ tens_ops,
372
+ query,
373
+ key,
374
+ value,
375
+ embed_dim_to_check,
376
+ num_heads,
377
+ in_proj_weight,
378
+ in_proj_bias,
379
+ bias_k,
380
+ bias_v,
381
+ add_zero_attn,
382
+ dropout_p,
383
+ out_proj_weight,
384
+ out_proj_bias,
385
+ training=training,
386
+ key_padding_mask=key_padding_mask,
387
+ need_weights=need_weights,
388
+ attn_mask=attn_mask,
389
+ is_causal=is_causal,
390
+ use_separate_proj_weight=use_separate_proj_weight,
391
+ q_proj_weight=q_proj_weight,
392
+ k_proj_weight=k_proj_weight,
393
+ v_proj_weight=v_proj_weight,
394
+ static_k=static_k,
395
+ static_v=static_v,
396
+ average_attn_weights=average_attn_weights,
397
+ )
398
+
399
+ is_batched = _mha_shape_check(query, key, value, key_padding_mask, attn_mask, num_heads)
400
+
401
+ # For unbatched input, we unsqueeze at the expected batch-dim to pretend that the input
402
+ # is batched, run the computation and before returning squeeze the
403
+ # batch dimension so that the output doesn't carry this temporary batch dimension.
404
+ if not is_batched:
405
+ # unsqueeze if the input is unbatched
406
+ query = query.unsqueeze(1)
407
+ key = key.unsqueeze(1)
408
+ value = value.unsqueeze(1)
409
+ if key_padding_mask is not None:
410
+ key_padding_mask = key_padding_mask.unsqueeze(0)
411
+
412
+ # set up shape vars
413
+ tgt_len, bsz, embed_dim = query.shape
414
+ src_len, _, _ = key.shape
415
+
416
+ key_padding_mask = _canonical_mask(
417
+ mask=key_padding_mask,
418
+ mask_name="key_padding_mask",
419
+ other_type=_none_or_dtype(attn_mask),
420
+ other_name="attn_mask",
421
+ target_type=query.dtype
422
+ )
423
+
424
+ if is_causal and attn_mask is None:
425
+ raise RuntimeError(
426
+ "Need attn_mask if specifying the is_causal hint. "
427
+ "You may use the Transformer module method "
428
+ "`generate_square_subsequent_mask` to create this mask."
429
+ )
430
+
431
+ if is_causal and key_padding_mask is None and not need_weights:
432
+ # when we have a kpm or need weights, we need attn_mask
433
+ # Otherwise, we use the is_causal hint go as is_causal
434
+ # indicator to SDPA.
435
+ attn_mask = None
436
+ else:
437
+ attn_mask = _canonical_mask(
438
+ mask=attn_mask,
439
+ mask_name="attn_mask",
440
+ other_type=None,
441
+ other_name="",
442
+ target_type=query.dtype,
443
+ check_other=False,
444
+ )
445
+
446
+ if key_padding_mask is not None:
447
+ # We have the attn_mask, and use that to merge kpm into it.
448
+ # Turn off use of is_causal hint, as the merged mask is no
449
+ # longer causal.
450
+ is_causal = False
451
+
452
+ assert embed_dim == embed_dim_to_check, \
453
+ f"was expecting embedding dimension of {embed_dim_to_check}, but got {embed_dim}"
454
+ if isinstance(embed_dim, torch.Tensor):
455
+ # embed_dim can be a tensor when JIT tracing
456
+ head_dim = embed_dim.div(num_heads, rounding_mode='trunc')
457
+ else:
458
+ head_dim = embed_dim // num_heads
459
+ assert head_dim * num_heads == embed_dim, f"embed_dim {embed_dim} not divisible by num_heads {num_heads}"
460
+ if use_separate_proj_weight:
461
+ # allow MHA to have different embedding dimensions when separate projection weights are used
462
+ assert key.shape[:2] == value.shape[:2], \
463
+ f"key's sequence and batch dims {key.shape[:2]} do not match value's {value.shape[:2]}"
464
+ else:
465
+ assert key.shape == value.shape, f"key shape {key.shape} does not match value shape {value.shape}"
466
+
467
+ #
468
+ # compute in-projection
469
+ #
470
+ if not use_separate_proj_weight:
471
+ assert in_proj_weight is not None, "use_separate_proj_weight is False but in_proj_weight is None"
472
+ q, k, v = _in_projection_packed(query, key, value, in_proj_weight, in_proj_bias)
473
+ else:
474
+ assert q_proj_weight is not None, "use_separate_proj_weight is True but q_proj_weight is None"
475
+ assert k_proj_weight is not None, "use_separate_proj_weight is True but k_proj_weight is None"
476
+ assert v_proj_weight is not None, "use_separate_proj_weight is True but v_proj_weight is None"
477
+ if in_proj_bias is None:
478
+ b_q = b_k = b_v = None
479
+ else:
480
+ b_q, b_k, b_v = in_proj_bias.chunk(3)
481
+ q, k, v = _in_projection(query, key, value, q_proj_weight, k_proj_weight, v_proj_weight, b_q, b_k, b_v)
482
+
483
+ # prep attention mask
484
+
485
+ if attn_mask is not None:
486
+ # ensure attn_mask's dim is 3
487
+ if attn_mask.dim() == 2:
488
+ correct_2d_size = (tgt_len, src_len)
489
+ if attn_mask.shape != correct_2d_size:
490
+ raise RuntimeError(f"The shape of the 2D attn_mask is {attn_mask.shape}, but should be {correct_2d_size}.")
491
+ attn_mask = attn_mask.unsqueeze(0)
492
+ elif attn_mask.dim() == 3:
493
+ correct_3d_size = (bsz * num_heads, tgt_len, src_len)
494
+ if attn_mask.shape != correct_3d_size:
495
+ raise RuntimeError(f"The shape of the 3D attn_mask is {attn_mask.shape}, but should be {correct_3d_size}.")
496
+ else:
497
+ raise RuntimeError(f"attn_mask's dimension {attn_mask.dim()} is not supported")
498
+
499
+ # add bias along batch dimension (currently second)
500
+ if bias_k is not None and bias_v is not None:
501
+ assert static_k is None, "bias cannot be added to static key."
502
+ assert static_v is None, "bias cannot be added to static value."
503
+ k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
504
+ v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
505
+ if attn_mask is not None:
506
+ attn_mask = pad(attn_mask, (0, 1))
507
+ if key_padding_mask is not None:
508
+ key_padding_mask = pad(key_padding_mask, (0, 1))
509
+ else:
510
+ assert bias_k is None
511
+ assert bias_v is None
512
+
513
+ #
514
+ # reshape q, k, v for multihead attention and make em batch first
515
+ #
516
+ q = q.view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1)
517
+ if static_k is None:
518
+ k = k.view(k.shape[0], bsz * num_heads, head_dim).transpose(0, 1)
519
+ else:
520
+ # TODO finish disentangling control flow so we don't do in-projections when statics are passed
521
+ assert static_k.size(0) == bsz * num_heads, \
522
+ f"expecting static_k.size(0) of {bsz * num_heads}, but got {static_k.size(0)}"
523
+ assert static_k.size(2) == head_dim, \
524
+ f"expecting static_k.size(2) of {head_dim}, but got {static_k.size(2)}"
525
+ k = static_k
526
+ if static_v is None:
527
+ v = v.view(v.shape[0], bsz * num_heads, head_dim).transpose(0, 1)
528
+ else:
529
+ # TODO finish disentangling control flow so we don't do in-projections when statics are passed
530
+ assert static_v.size(0) == bsz * num_heads, \
531
+ f"expecting static_v.size(0) of {bsz * num_heads}, but got {static_v.size(0)}"
532
+ assert static_v.size(2) == head_dim, \
533
+ f"expecting static_v.size(2) of {head_dim}, but got {static_v.size(2)}"
534
+ v = static_v
535
+
536
+ # add zero attention along batch dimension (now first)
537
+ if add_zero_attn:
538
+ zero_attn_shape = (bsz * num_heads, 1, head_dim)
539
+ k = torch.cat([k, torch.zeros(zero_attn_shape, dtype=k.dtype, device=k.device)], dim=1)
540
+ v = torch.cat([v, torch.zeros(zero_attn_shape, dtype=v.dtype, device=v.device)], dim=1)
541
+ if attn_mask is not None:
542
+ attn_mask = pad(attn_mask, (0, 1))
543
+ if key_padding_mask is not None:
544
+ key_padding_mask = pad(key_padding_mask, (0, 1))
545
+
546
+ # update source sequence length after adjustments
547
+ src_len = k.size(1)
548
+
549
+ # merge key padding and attention masks
550
+ if key_padding_mask is not None:
551
+ assert key_padding_mask.shape == (bsz, src_len), \
552
+ f"expecting key_padding_mask shape of {(bsz, src_len)}, but got {key_padding_mask.shape}"
553
+ key_padding_mask = key_padding_mask.view(bsz, 1, 1, src_len). \
554
+ expand(-1, num_heads, -1, -1).reshape(bsz * num_heads, 1, src_len)
555
+ if attn_mask is None:
556
+ attn_mask = key_padding_mask
557
+ else:
558
+ attn_mask = attn_mask + key_padding_mask
559
+
560
+ # adjust dropout probability
561
+ if not training:
562
+ dropout_p = 0.0
563
+
564
+ #
565
+ # (deep breath) calculate attention and out projection
566
+ #
567
+
568
+ if need_weights:
569
+ B, Nt, E = q.shape
570
+ q_scaled = q / math.sqrt(E)
571
+
572
+ assert not (is_causal and attn_mask is None), "FIXME: is_causal not implemented for need_weights"
573
+
574
+ if attn_mask is not None:
575
+ attn_output_weights = torch.baddbmm(attn_mask, q_scaled, k.transpose(-2, -1))
576
+ else:
577
+ attn_output_weights = torch.bmm(q_scaled, k.transpose(-2, -1))
578
+ attn_output_weights = softmax(attn_output_weights, dim=-1)
579
+ if dropout_p > 0.0:
580
+ attn_output_weights = dropout(attn_output_weights, p=dropout_p)
581
+
582
+ attn_output = torch.bmm(attn_output_weights, v)
583
+
584
+ attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len * bsz, embed_dim)
585
+ attn_output = self.out_proj(attn_output)
586
+ attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))
587
+
588
+ # optionally average attention weights over heads
589
+ attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
590
+ if average_attn_weights:
591
+ attn_output_weights = attn_output_weights.mean(dim=1)
592
+
593
+ if not is_batched:
594
+ # squeeze the output if input was unbatched
595
+ attn_output = attn_output.squeeze(1)
596
+ attn_output_weights = attn_output_weights.squeeze(0)
597
+ return attn_output, attn_output_weights
598
+ else:
599
+ # attn_mask can be either (L,S) or (N*num_heads, L, S)
600
+ # if attn_mask's shape is (1, L, S) we need to unsqueeze to (1, 1, L, S)
601
+ # in order to match the input for SDPA of (N, num_heads, L, S)
602
+ if attn_mask is not None:
603
+ if attn_mask.size(0) == 1 and attn_mask.dim() == 3:
604
+ attn_mask = attn_mask.unsqueeze(0)
605
+ else:
606
+ attn_mask = attn_mask.view(bsz, num_heads, -1, src_len)
607
+
608
+ q = q.view(bsz, num_heads, tgt_len, head_dim)
609
+ k = k.view(bsz, num_heads, src_len, head_dim)
610
+ v = v.view(bsz, num_heads, src_len, head_dim)
611
+
612
+ attn_output = F.scaled_dot_product_attention(q, k, v, attn_mask, dropout_p, is_causal)
613
+ attn_output = attn_output.permute(2, 0, 1, 3).contiguous().view(bsz * tgt_len, embed_dim)
614
+
615
+ attn_output = self.out_proj(attn_output)
616
+ attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))
617
+ if not is_batched:
618
+ # squeeze the output if input was unbatched
619
+ attn_output = attn_output.squeeze(1)
620
+ return attn_output, None
621
+
622
+
623
+ def _mha_shape_check(query: Tensor, key: Tensor, value: Tensor,
624
+ key_padding_mask: Optional[Tensor], attn_mask: Optional[Tensor], num_heads: int):
625
+ # Verifies the expected shape for `query, `key`, `value`, `key_padding_mask` and `attn_mask`
626
+ # and returns if the input is batched or not.
627
+ # Raises an error if `query` is not 2-D (unbatched) or 3-D (batched) tensor.
628
+
629
+ # Shape check.
630
+ if query.dim() == 3:
631
+ # Batched Inputs
632
+ is_batched = True
633
+ assert key.dim() == 3 and value.dim() == 3, \
634
+ ("For batched (3-D) `query`, expected `key` and `value` to be 3-D"
635
+ f" but found {key.dim()}-D and {value.dim()}-D tensors respectively")
636
+ if key_padding_mask is not None:
637
+ assert key_padding_mask.dim() == 2, \
638
+ ("For batched (3-D) `query`, expected `key_padding_mask` to be `None` or 2-D"
639
+ f" but found {key_padding_mask.dim()}-D tensor instead")
640
+ if attn_mask is not None:
641
+ assert attn_mask.dim() in (2, 3), \
642
+ ("For batched (3-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D"
643
+ f" but found {attn_mask.dim()}-D tensor instead")
644
+ elif query.dim() == 2:
645
+ # Unbatched Inputs
646
+ is_batched = False
647
+ assert key.dim() == 2 and value.dim() == 2, \
648
+ ("For unbatched (2-D) `query`, expected `key` and `value` to be 2-D"
649
+ f" but found {key.dim()}-D and {value.dim()}-D tensors respectively")
650
+
651
+ if key_padding_mask is not None:
652
+ assert key_padding_mask.dim() == 1, \
653
+ ("For unbatched (2-D) `query`, expected `key_padding_mask` to be `None` or 1-D"
654
+ f" but found {key_padding_mask.dim()}-D tensor instead")
655
+
656
+ if attn_mask is not None:
657
+ assert attn_mask.dim() in (2, 3), \
658
+ ("For unbatched (2-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D"
659
+ f" but found {attn_mask.dim()}-D tensor instead")
660
+ if attn_mask.dim() == 3:
661
+ expected_shape = (num_heads, query.shape[0], key.shape[0])
662
+ assert attn_mask.shape == expected_shape, \
663
+ (f"Expected `attn_mask` shape to be {expected_shape} but got {attn_mask.shape}")
664
+ else:
665
+ raise AssertionError(
666
+ f"query should be unbatched 2D or batched 3D tensor but received {query.dim()}-D query tensor")
667
+
668
+ return is_batched
669
+
670
+
671
+ def _canonical_mask(
672
+ mask: Optional[Tensor],
673
+ mask_name: str,
674
+ other_type: Optional[DType],
675
+ other_name: str,
676
+ target_type: DType,
677
+ check_other: bool = True,
678
+ ) -> Optional[Tensor]:
679
+
680
+ if mask is not None:
681
+ _mask_dtype = mask.dtype
682
+ _mask_is_float = torch.is_floating_point(mask)
683
+ if _mask_dtype != torch.bool and not _mask_is_float:
684
+ raise AssertionError(
685
+ f"only bool and floating types of {mask_name} are supported")
686
+ if check_other and other_type is not None:
687
+ if _mask_dtype != other_type:
688
+ warnings.warn(
689
+ f"Support for mismatched {mask_name} and {other_name} "
690
+ "is deprecated. Use same type for both instead."
691
+ )
692
+ if not _mask_is_float:
693
+ mask = (
694
+ torch.zeros_like(mask, dtype=target_type)
695
+ .masked_fill_(mask, float("-inf"))
696
+ )
697
+ return mask
698
+
699
+
700
+ def _none_or_dtype(input: Optional[Tensor]) -> Optional[DType]:
701
+ if input is None:
702
+ return None
703
+ elif isinstance(input, torch.Tensor):
704
+ return input.dtype
705
+ raise RuntimeError("input to _none_or_dtype() must be None or torch.Tensor")
706
+
707
+ def _in_projection_packed(
708
+ q: Tensor,
709
+ k: Tensor,
710
+ v: Tensor,
711
+ w: Tensor,
712
+ b: Optional[Tensor] = None,
713
+ ) -> List[Tensor]:
714
+ r"""
715
+ Performs the in-projection step of the attention operation, using packed weights.
716
+ Output is a triple containing projection tensors for query, key and value.
717
+ Args:
718
+ q, k, v: query, key and value tensors to be projected. For self-attention,
719
+ these are typically the same tensor; for encoder-decoder attention,
720
+ k and v are typically the same tensor. (We take advantage of these
721
+ identities for performance if they are present.) Regardless, q, k and v
722
+ must share a common embedding dimension; otherwise their shapes may vary.
723
+ w: projection weights for q, k and v, packed into a single tensor. Weights
724
+ are packed along dimension 0, in q, k, v order.
725
+ b: optional projection biases for q, k and v, packed into a single tensor
726
+ in q, k, v order.
727
+ Shape:
728
+ Inputs:
729
+ - q: :math:`(..., E)` where E is the embedding dimension
730
+ - k: :math:`(..., E)` where E is the embedding dimension
731
+ - v: :math:`(..., E)` where E is the embedding dimension
732
+ - w: :math:`(E * 3, E)` where E is the embedding dimension
733
+ - b: :math:`E * 3` where E is the embedding dimension
734
+ Output:
735
+ - in output list :math:`[q', k', v']`, each output tensor will have the
736
+ same shape as the corresponding input tensor.
737
+ """
738
+ E = q.size(-1)
739
+ if k is v:
740
+ if q is k:
741
+ # self-attention
742
+ proj = linear(q, w, b)
743
+ # reshape to 3, E and not E, 3 is deliberate for better memory coalescing and keeping same order as chunk()
744
+ proj = proj.unflatten(-1, (3, E)).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()
745
+ return proj[0], proj[1], proj[2]
746
+ else:
747
+ # encoder-decoder attention
748
+ w_q, w_kv = w.split([E, E * 2])
749
+ if b is None:
750
+ b_q = b_kv = None
751
+ else:
752
+ b_q, b_kv = b.split([E, E * 2])
753
+ q_proj = linear(q, w_q, b_q)
754
+ kv_proj = linear(k, w_kv, b_kv)
755
+ # reshape to 2, E and not E, 2 is deliberate for better memory coalescing and keeping same order as chunk()
756
+ kv_proj = kv_proj.unflatten(-1, (2, E)).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()
757
+ return (q_proj, kv_proj[0], kv_proj[1])
758
+ else:
759
+ w_q, w_k, w_v = w.chunk(3)
760
+ if b is None:
761
+ b_q = b_k = b_v = None
762
+ else:
763
+ b_q, b_k, b_v = b.chunk(3)
764
+ return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v)
765
+
766
+
767
+ def _in_projection(
768
+ q: Tensor,
769
+ k: Tensor,
770
+ v: Tensor,
771
+ w_q: Tensor,
772
+ w_k: Tensor,
773
+ w_v: Tensor,
774
+ b_q: Optional[Tensor] = None,
775
+ b_k: Optional[Tensor] = None,
776
+ b_v: Optional[Tensor] = None,
777
+ ) -> Tuple[Tensor, Tensor, Tensor]:
778
+ r"""
779
+ Performs the in-projection step of the attention operation. This is simply
780
+ a triple of linear projections, with shape constraints on the weights which
781
+ ensure embedding dimension uniformity in the projected outputs.
782
+ Output is a triple containing projection tensors for query, key and value.
783
+ Args:
784
+ q, k, v: query, key and value tensors to be projected.
785
+ w_q, w_k, w_v: weights for q, k and v, respectively.
786
+ b_q, b_k, b_v: optional biases for q, k and v, respectively.
787
+ Shape:
788
+ Inputs:
789
+ - q: :math:`(Qdims..., Eq)` where Eq is the query embedding dimension and Qdims are any
790
+ number of leading dimensions.
791
+ - k: :math:`(Kdims..., Ek)` where Ek is the key embedding dimension and Kdims are any
792
+ number of leading dimensions.
793
+ - v: :math:`(Vdims..., Ev)` where Ev is the value embedding dimension and Vdims are any
794
+ number of leading dimensions.
795
+ - w_q: :math:`(Eq, Eq)`
796
+ - w_k: :math:`(Eq, Ek)`
797
+ - w_v: :math:`(Eq, Ev)`
798
+ - b_q: :math:`(Eq)`
799
+ - b_k: :math:`(Eq)`
800
+ - b_v: :math:`(Eq)`
801
+ Output: in output triple :math:`(q', k', v')`,
802
+ - q': :math:`[Qdims..., Eq]`
803
+ - k': :math:`[Kdims..., Eq]`
804
+ - v': :math:`[Vdims..., Eq]`
805
+ """
806
+ Eq, Ek, Ev = q.size(-1), k.size(-1), v.size(-1)
807
+ assert w_q.shape == (Eq, Eq), f"expecting query weights shape of {(Eq, Eq)}, but got {w_q.shape}"
808
+ assert w_k.shape == (Eq, Ek), f"expecting key weights shape of {(Eq, Ek)}, but got {w_k.shape}"
809
+ assert w_v.shape == (Eq, Ev), f"expecting value weights shape of {(Eq, Ev)}, but got {w_v.shape}"
810
+ assert b_q is None or b_q.shape == (Eq,), f"expecting query bias shape of {(Eq,)}, but got {b_q.shape}"
811
+ assert b_k is None or b_k.shape == (Eq,), f"expecting key bias shape of {(Eq,)}, but got {b_k.shape}"
812
+ assert b_v is None or b_v.shape == (Eq,), f"expecting value bias shape of {(Eq,)}, but got {b_v.shape}"
813
+ return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v)