Files changed (1) hide show
  1. resampler.py +663 -7
resampler.py CHANGED
@@ -19,6 +19,21 @@ from torch.nn.init import trunc_normal_
19
  from torchvision import transforms
20
  from torchvision.transforms import InterpolationMode
21
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
22
  def get_abs_pos(abs_pos, tgt_size):
23
  # abs_pos: L, C
24
  # tgt_size: (H, W)
@@ -117,24 +132,20 @@ class Resampler(nn.Module):
117
  self.pos_embed = nn.Parameter(
118
  torch.from_numpy(get_2d_sincos_pos_embed(embed_dim, grid_size)).float()
119
  ).requires_grad_(False)
120
-
121
  self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
122
- trunc_normal_(self.query, std=.02)
123
 
124
  if kv_dim is not None and kv_dim != embed_dim:
125
  self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
126
  else:
127
  self.kv_proj = nn.Identity()
128
 
129
- self.attn = nn.MultiheadAttention(embed_dim, num_heads)
130
  self.ln_q = norm_layer(embed_dim)
131
  self.ln_kv = norm_layer(embed_dim)
132
 
133
  self.ln_post = norm_layer(embed_dim)
134
  self.proj = nn.Parameter((embed_dim ** -0.5) * torch.randn(embed_dim, embed_dim))
135
 
136
- self.apply(self._init_weights)
137
-
138
  def _init_weights(self, m):
139
  if isinstance(m, nn.Linear):
140
  trunc_normal_(m.weight, std=.02)
@@ -149,22 +160,667 @@ class Resampler(nn.Module):
149
  pos_embed = torch.Tensor(get_2d_sincos_pos_embed(self.embed_dim, tgt_size)).float().to(device=x.device, dtype=x.dtype)
150
  else:
151
  pos_embed = get_abs_pos(self.pos_embed, tgt_size)
152
-
153
  x = self.kv_proj(x)
154
  x = self.ln_kv(x).permute(1, 0, 2)
155
 
156
  N = x.shape[1]
157
  q = self.ln_q(self.query)
 
158
  out = self.attn(
159
  self._repeat(q, N) + self.pos_embed.unsqueeze(1),
160
  x + pos_embed.unsqueeze(1),
161
  x,
162
  attn_mask=attn_mask)[0]
163
  x = out.permute(1, 0, 2)
164
-
165
  x = self.ln_post(x)
166
  x = x @ self.proj
167
  return x
168
 
169
  def _repeat(self, query, N: int):
170
  return query.unsqueeze(1).repeat(1, N, 1)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  from torchvision import transforms
20
  from torchvision.transforms import InterpolationMode
21
 
22
+ from functools import partial
23
+ import numpy as np
24
+ import warnings
25
+ from typing import Optional, Tuple
26
+ import torch
27
+ from torch import nn
28
+ from torch import Tensor
29
+ import deepspeed
30
+ import torch.nn.functional as F
31
+ from torch.nn.functional import *
32
+ from torch.nn.modules.activation import *
33
+ from torch.nn.init import trunc_normal_
34
+ from torch.nn.init import constant_, xavier_normal_, xavier_uniform_
35
+ from transformers import PreTrainedModel
36
+ from transformers.integrations import is_deepspeed_zero3_enabled
37
  def get_abs_pos(abs_pos, tgt_size):
38
  # abs_pos: L, C
39
  # tgt_size: (H, W)
 
132
  self.pos_embed = nn.Parameter(
133
  torch.from_numpy(get_2d_sincos_pos_embed(embed_dim, grid_size)).float()
134
  ).requires_grad_(False)
 
135
  self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
 
136
 
137
  if kv_dim is not None and kv_dim != embed_dim:
138
  self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
139
  else:
140
  self.kv_proj = nn.Identity()
141
 
142
+ self.attn = MultiheadAttention(embed_dim, num_heads)
143
  self.ln_q = norm_layer(embed_dim)
144
  self.ln_kv = norm_layer(embed_dim)
145
 
146
  self.ln_post = norm_layer(embed_dim)
147
  self.proj = nn.Parameter((embed_dim ** -0.5) * torch.randn(embed_dim, embed_dim))
148
 
 
 
149
  def _init_weights(self, m):
150
  if isinstance(m, nn.Linear):
151
  trunc_normal_(m.weight, std=.02)
 
160
  pos_embed = torch.Tensor(get_2d_sincos_pos_embed(self.embed_dim, tgt_size)).float().to(device=x.device, dtype=x.dtype)
161
  else:
162
  pos_embed = get_abs_pos(self.pos_embed, tgt_size)
163
+
164
  x = self.kv_proj(x)
165
  x = self.ln_kv(x).permute(1, 0, 2)
166
 
167
  N = x.shape[1]
168
  q = self.ln_q(self.query)
169
+
170
  out = self.attn(
171
  self._repeat(q, N) + self.pos_embed.unsqueeze(1),
172
  x + pos_embed.unsqueeze(1),
173
  x,
174
  attn_mask=attn_mask)[0]
175
  x = out.permute(1, 0, 2)
 
176
  x = self.ln_post(x)
177
  x = x @ self.proj
178
  return x
179
 
180
  def _repeat(self, query, N: int):
181
  return query.unsqueeze(1).repeat(1, N, 1)
182
+
183
+
184
+
185
+ class MultiheadAttention(nn.MultiheadAttention):
186
+ def __init__(self, embed_dim, num_heads, dropout=0., bias=True, add_bias_kv=False,
187
+ add_zero_attn=False, kdim=None, vdim=None, batch_first=False, device=None, dtype=None):
188
+ super().__init__(embed_dim, num_heads, dropout, bias, add_bias_kv, add_zero_attn, kdim, vdim, batch_first, device, dtype)
189
+
190
+ # rewrite out_proj layer,with nn.Linear
191
+ self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias,)
192
+
193
+ def forward(
194
+ self,
195
+ query: Tensor,
196
+ key: Tensor,
197
+ value: Tensor,
198
+ key_padding_mask: Optional[Tensor] = None,
199
+ need_weights: bool = True,
200
+ attn_mask: Optional[Tensor] = None,
201
+ average_attn_weights: bool = True,
202
+ is_causal : bool = False) -> Tuple[Tensor, Optional[Tensor]]:
203
+ why_not_fast_path = ''
204
+ if ((attn_mask is not None and torch.is_floating_point(attn_mask))
205
+ or (key_padding_mask is not None) and torch.is_floating_point(key_padding_mask)):
206
+ why_not_fast_path = "floating-point masks are not supported for fast path."
207
+
208
+ is_batched = query.dim() == 3
209
+
210
+ key_padding_mask = F._canonical_mask(
211
+ mask=key_padding_mask,
212
+ mask_name="key_padding_mask",
213
+ other_type=F._none_or_dtype(attn_mask),
214
+ other_name="attn_mask",
215
+ target_type=query.dtype
216
+ )
217
+ # _canonical_mask
218
+ attn_mask = F._canonical_mask(
219
+ mask=attn_mask,
220
+ mask_name="attn_mask",
221
+ other_type=None,
222
+ other_name="",
223
+ target_type=query.dtype,
224
+ check_other=False,
225
+ )
226
+
227
+
228
+ if not is_batched:
229
+ why_not_fast_path = f"input not batched; expected query.dim() of 3 but got {query.dim()}"
230
+ elif query is not key or key is not value:
231
+ # When lifting this restriction, don't forget to either
232
+ # enforce that the dtypes all match or test cases where
233
+ # they don't!
234
+ why_not_fast_path = "non-self attention was used (query, key, and value are not the same Tensor)"
235
+ elif self.in_proj_bias is not None and query.dtype != self.in_proj_bias.dtype:
236
+ why_not_fast_path = f"dtypes of query ({query.dtype}) and self.in_proj_bias ({self.in_proj_bias.dtype}) don't match"
237
+ elif self.in_proj_weight is None:
238
+ why_not_fast_path = "in_proj_weight was None"
239
+ elif query.dtype != self.in_proj_weight.dtype:
240
+ # this case will fail anyway, but at least they'll get a useful error message.
241
+ why_not_fast_path = f"dtypes of query ({query.dtype}) and self.in_proj_weight ({self.in_proj_weight.dtype}) don't match"
242
+ elif self.training:
243
+ why_not_fast_path = "training is enabled"
244
+ elif (self.num_heads % 2) != 0:
245
+ why_not_fast_path = "self.num_heads is not even"
246
+ elif not self.batch_first:
247
+ why_not_fast_path = "batch_first was not True"
248
+ elif self.bias_k is not None:
249
+ why_not_fast_path = "self.bias_k was not None"
250
+ elif self.bias_v is not None:
251
+ why_not_fast_path = "self.bias_v was not None"
252
+ elif self.add_zero_attn:
253
+ why_not_fast_path = "add_zero_attn was enabled"
254
+ elif not self._qkv_same_embed_dim:
255
+ why_not_fast_path = "_qkv_same_embed_dim was not True"
256
+ elif query.is_nested and (key_padding_mask is not None or attn_mask is not None):
257
+ why_not_fast_path = "supplying both src_key_padding_mask and src_mask at the same time \
258
+ is not supported with NestedTensor input"
259
+ elif torch.is_autocast_enabled():
260
+ why_not_fast_path = "autocast is enabled"
261
+
262
+ if not why_not_fast_path:
263
+ tensor_args = (
264
+ query,
265
+ key,
266
+ value,
267
+ self.in_proj_weight,
268
+ self.in_proj_bias,
269
+ self.out_proj.weight,
270
+ self.out_proj.bias,
271
+ )
272
+ # We have to use list comprehensions below because TorchScript does not support
273
+ # generator expressions.
274
+ if torch.overrides.has_torch_function(tensor_args):
275
+ why_not_fast_path = "some Tensor argument has_torch_function"
276
+ elif _is_make_fx_tracing():
277
+ why_not_fast_path = "we are running make_fx tracing"
278
+ elif not all(_check_arg_device(x) for x in tensor_args):
279
+ why_not_fast_path = ("some Tensor argument's device is neither one of "
280
+ f"cpu, cuda or {torch.utils.backend_registration._privateuse1_backend_name}")
281
+ elif torch.is_grad_enabled() and any(_arg_requires_grad(x) for x in tensor_args):
282
+ why_not_fast_path = ("grad is enabled and at least one of query or the "
283
+ "input/output projection weights or biases requires_grad")
284
+ if not why_not_fast_path:
285
+ merged_mask, mask_type = self.merge_masks(attn_mask, key_padding_mask, query)
286
+
287
+ if self.in_proj_bias is not None and self.in_proj_weight is not None:
288
+ return torch._native_multi_head_attention(
289
+ query,
290
+ key,
291
+ value,
292
+ self.embed_dim,
293
+ self.num_heads,
294
+ self.in_proj_weight,
295
+ self.in_proj_bias,
296
+ self.out_proj.weight,
297
+ self.out_proj.bias,
298
+ merged_mask,
299
+ need_weights,
300
+ average_attn_weights,
301
+ mask_type)
302
+
303
+ any_nested = query.is_nested or key.is_nested or value.is_nested
304
+ assert not any_nested, ("MultiheadAttention does not support NestedTensor outside of its fast path. " +
305
+ f"The fast path was not hit because {why_not_fast_path}")
306
+
307
+ if self.batch_first and is_batched:
308
+ # make sure that the transpose op does not affect the "is" property
309
+ if key is value:
310
+ if query is key:
311
+ query = key = value = query.transpose(1, 0)
312
+ else:
313
+ query, key = (x.transpose(1, 0) for x in (query, key))
314
+ value = key
315
+ else:
316
+ query, key, value = (x.transpose(1, 0) for x in (query, key, value))
317
+
318
+ if not self._qkv_same_embed_dim:
319
+ attn_output, attn_output_weights = self.multi_head_attention_forward(
320
+ query, key, value, self.embed_dim, self.num_heads,
321
+ self.in_proj_weight, self.in_proj_bias,
322
+ self.bias_k, self.bias_v, self.add_zero_attn,
323
+ self.dropout, self.out_proj.weight, self.out_proj.bias,
324
+ training=self.training,
325
+ key_padding_mask=key_padding_mask, need_weights=need_weights,
326
+ attn_mask=attn_mask,
327
+ use_separate_proj_weight=True,
328
+ q_proj_weight=self.q_proj_weight, k_proj_weight=self.k_proj_weight,
329
+ v_proj_weight=self.v_proj_weight,
330
+ average_attn_weights=average_attn_weights,
331
+ is_causal=is_causal)
332
+ else:
333
+ attn_output, attn_output_weights = self.multi_head_attention_forward(
334
+ query, key, value, self.embed_dim, self.num_heads,
335
+ self.in_proj_weight, self.in_proj_bias,
336
+ self.bias_k, self.bias_v, self.add_zero_attn,
337
+ self.dropout, self.out_proj.weight, self.out_proj.bias,
338
+ training=self.training,
339
+ key_padding_mask=key_padding_mask,
340
+ need_weights=need_weights,
341
+ attn_mask=attn_mask,
342
+ average_attn_weights=average_attn_weights,
343
+ is_causal=is_causal)
344
+ if self.batch_first and is_batched:
345
+ return attn_output.transpose(1, 0), attn_output_weights
346
+ else:
347
+ return attn_output, attn_output_weights
348
+
349
+ def multi_head_attention_forward(
350
+ self,
351
+ query: Tensor,
352
+ key: Tensor,
353
+ value: Tensor,
354
+ embed_dim_to_check: int,
355
+ num_heads: int,
356
+ in_proj_weight: Optional[Tensor],
357
+ in_proj_bias: Optional[Tensor],
358
+ bias_k: Optional[Tensor],
359
+ bias_v: Optional[Tensor],
360
+ add_zero_attn: bool,
361
+ dropout_p: float,
362
+ out_proj_weight: Tensor,
363
+ out_proj_bias: Optional[Tensor],
364
+ training: bool = True,
365
+ key_padding_mask: Optional[Tensor] = None,
366
+ need_weights: bool = True,
367
+ attn_mask: Optional[Tensor] = None,
368
+ use_separate_proj_weight: bool = False,
369
+ q_proj_weight: Optional[Tensor] = None,
370
+ k_proj_weight: Optional[Tensor] = None,
371
+ v_proj_weight: Optional[Tensor] = None,
372
+ static_k: Optional[Tensor] = None,
373
+ static_v: Optional[Tensor] = None,
374
+ average_attn_weights: bool = True,
375
+ is_causal: bool = False,
376
+ ) -> Tuple[Tensor, Optional[Tensor]]:
377
+ tens_ops = (query, key, value, in_proj_weight, in_proj_bias, bias_k, bias_v, out_proj_weight, out_proj_bias)
378
+ if has_torch_function(tens_ops):
379
+ return handle_torch_function(
380
+ multi_head_attention_forward,
381
+ tens_ops,
382
+ query,
383
+ key,
384
+ value,
385
+ embed_dim_to_check,
386
+ num_heads,
387
+ in_proj_weight,
388
+ in_proj_bias,
389
+ bias_k,
390
+ bias_v,
391
+ add_zero_attn,
392
+ dropout_p,
393
+ out_proj_weight,
394
+ out_proj_bias,
395
+ training=training,
396
+ key_padding_mask=key_padding_mask,
397
+ need_weights=need_weights,
398
+ attn_mask=attn_mask,
399
+ is_causal=is_causal,
400
+ use_separate_proj_weight=use_separate_proj_weight,
401
+ q_proj_weight=q_proj_weight,
402
+ k_proj_weight=k_proj_weight,
403
+ v_proj_weight=v_proj_weight,
404
+ static_k=static_k,
405
+ static_v=static_v,
406
+ average_attn_weights=average_attn_weights,
407
+ )
408
+
409
+ is_batched = _mha_shape_check(query, key, value, key_padding_mask, attn_mask, num_heads)
410
+
411
+ # For unbatched input, we unsqueeze at the expected batch-dim to pretend that the input
412
+ # is batched, run the computation and before returning squeeze the
413
+ # batch dimension so that the output doesn't carry this temporary batch dimension.
414
+ if not is_batched:
415
+ # unsqueeze if the input is unbatched
416
+ query = query.unsqueeze(1)
417
+ key = key.unsqueeze(1)
418
+ value = value.unsqueeze(1)
419
+ if key_padding_mask is not None:
420
+ key_padding_mask = key_padding_mask.unsqueeze(0)
421
+
422
+ # set up shape vars
423
+ tgt_len, bsz, embed_dim = query.shape
424
+ src_len, _, _ = key.shape
425
+
426
+ key_padding_mask = _canonical_mask(
427
+ mask=key_padding_mask,
428
+ mask_name="key_padding_mask",
429
+ other_type=_none_or_dtype(attn_mask),
430
+ other_name="attn_mask",
431
+ target_type=query.dtype
432
+ )
433
+
434
+ if is_causal and attn_mask is None:
435
+ raise RuntimeError(
436
+ "Need attn_mask if specifying the is_causal hint. "
437
+ "You may use the Transformer module method "
438
+ "`generate_square_subsequent_mask` to create this mask."
439
+ )
440
+
441
+ if is_causal and key_padding_mask is None and not need_weights:
442
+ # when we have a kpm or need weights, we need attn_mask
443
+ # Otherwise, we use the is_causal hint go as is_causal
444
+ # indicator to SDPA.
445
+ attn_mask = None
446
+ else:
447
+ attn_mask = _canonical_mask(
448
+ mask=attn_mask,
449
+ mask_name="attn_mask",
450
+ other_type=None,
451
+ other_name="",
452
+ target_type=query.dtype,
453
+ check_other=False,
454
+ )
455
+
456
+ if key_padding_mask is not None:
457
+ # We have the attn_mask, and use that to merge kpm into it.
458
+ # Turn off use of is_causal hint, as the merged mask is no
459
+ # longer causal.
460
+ is_causal = False
461
+
462
+ assert embed_dim == embed_dim_to_check, \
463
+ f"was expecting embedding dimension of {embed_dim_to_check}, but got {embed_dim}"
464
+ if isinstance(embed_dim, torch.Tensor):
465
+ # embed_dim can be a tensor when JIT tracing
466
+ head_dim = embed_dim.div(num_heads, rounding_mode='trunc')
467
+ else:
468
+ head_dim = embed_dim // num_heads
469
+ assert head_dim * num_heads == embed_dim, f"embed_dim {embed_dim} not divisible by num_heads {num_heads}"
470
+ if use_separate_proj_weight:
471
+ # allow MHA to have different embedding dimensions when separate projection weights are used
472
+ assert key.shape[:2] == value.shape[:2], \
473
+ f"key's sequence and batch dims {key.shape[:2]} do not match value's {value.shape[:2]}"
474
+ else:
475
+ assert key.shape == value.shape, f"key shape {key.shape} does not match value shape {value.shape}"
476
+
477
+ #
478
+ # compute in-projection
479
+ #
480
+
481
+ if not use_separate_proj_weight:
482
+ assert in_proj_weight is not None, "use_separate_proj_weight is False but in_proj_weight is None"
483
+ q, k, v = _in_projection_packed(query, key, value, in_proj_weight, in_proj_bias)
484
+ else:
485
+ assert q_proj_weight is not None, "use_separate_proj_weight is True but q_proj_weight is None"
486
+ assert k_proj_weight is not None, "use_separate_proj_weight is True but k_proj_weight is None"
487
+ assert v_proj_weight is not None, "use_separate_proj_weight is True but v_proj_weight is None"
488
+ if in_proj_bias is None:
489
+ b_q = b_k = b_v = None
490
+ else:
491
+ b_q, b_k, b_v = in_proj_bias.chunk(3)
492
+ q, k, v = _in_projection(query, key, value, q_proj_weight, k_proj_weight, v_proj_weight, b_q, b_k, b_v)
493
+
494
+ # prep attention mask
495
+
496
+ if attn_mask is not None:
497
+ # ensure attn_mask's dim is 3
498
+ if attn_mask.dim() == 2:
499
+ correct_2d_size = (tgt_len, src_len)
500
+ if attn_mask.shape != correct_2d_size:
501
+ raise RuntimeError(f"The shape of the 2D attn_mask is {attn_mask.shape}, but should be {correct_2d_size}.")
502
+ attn_mask = attn_mask.unsqueeze(0)
503
+ elif attn_mask.dim() == 3:
504
+ correct_3d_size = (bsz * num_heads, tgt_len, src_len)
505
+ if attn_mask.shape != correct_3d_size:
506
+ raise RuntimeError(f"The shape of the 3D attn_mask is {attn_mask.shape}, but should be {correct_3d_size}.")
507
+ else:
508
+ raise RuntimeError(f"attn_mask's dimension {attn_mask.dim()} is not supported")
509
+
510
+ # add bias along batch dimension (currently second)
511
+ if bias_k is not None and bias_v is not None:
512
+ assert static_k is None, "bias cannot be added to static key."
513
+ assert static_v is None, "bias cannot be added to static value."
514
+ k = torch.cat([k, bias_k.repeat(1, bsz, 1)])
515
+ v = torch.cat([v, bias_v.repeat(1, bsz, 1)])
516
+ if attn_mask is not None:
517
+ attn_mask = pad(attn_mask, (0, 1))
518
+ if key_padding_mask is not None:
519
+ key_padding_mask = pad(key_padding_mask, (0, 1))
520
+ else:
521
+ assert bias_k is None
522
+ assert bias_v is None
523
+
524
+ #
525
+ # reshape q, k, v for multihead attention and make em batch first
526
+ #
527
+ q = q.view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1)
528
+ if static_k is None:
529
+ k = k.view(k.shape[0], bsz * num_heads, head_dim).transpose(0, 1)
530
+ else:
531
+ # TODO finish disentangling control flow so we don't do in-projections when statics are passed
532
+ assert static_k.size(0) == bsz * num_heads, \
533
+ f"expecting static_k.size(0) of {bsz * num_heads}, but got {static_k.size(0)}"
534
+ assert static_k.size(2) == head_dim, \
535
+ f"expecting static_k.size(2) of {head_dim}, but got {static_k.size(2)}"
536
+ k = static_k
537
+ if static_v is None:
538
+ v = v.view(v.shape[0], bsz * num_heads, head_dim).transpose(0, 1)
539
+ else:
540
+ # TODO finish disentangling control flow so we don't do in-projections when statics are passed
541
+ assert static_v.size(0) == bsz * num_heads, \
542
+ f"expecting static_v.size(0) of {bsz * num_heads}, but got {static_v.size(0)}"
543
+ assert static_v.size(2) == head_dim, \
544
+ f"expecting static_v.size(2) of {head_dim}, but got {static_v.size(2)}"
545
+ v = static_v
546
+
547
+ # add zero attention along batch dimension (now first)
548
+ if add_zero_attn:
549
+ zero_attn_shape = (bsz * num_heads, 1, head_dim)
550
+ k = torch.cat([k, torch.zeros(zero_attn_shape, dtype=k.dtype, device=k.device)], dim=1)
551
+ v = torch.cat([v, torch.zeros(zero_attn_shape, dtype=v.dtype, device=v.device)], dim=1)
552
+ if attn_mask is not None:
553
+ attn_mask = pad(attn_mask, (0, 1))
554
+ if key_padding_mask is not None:
555
+ key_padding_mask = pad(key_padding_mask, (0, 1))
556
+
557
+ # update source sequence length after adjustments
558
+ src_len = k.size(1)
559
+
560
+ # merge key padding and attention masks
561
+ if key_padding_mask is not None:
562
+ assert key_padding_mask.shape == (bsz, src_len), \
563
+ f"expecting key_padding_mask shape of {(bsz, src_len)}, but got {key_padding_mask.shape}"
564
+ key_padding_mask = key_padding_mask.view(bsz, 1, 1, src_len). \
565
+ expand(-1, num_heads, -1, -1).reshape(bsz * num_heads, 1, src_len)
566
+ if attn_mask is None:
567
+ attn_mask = key_padding_mask
568
+ else:
569
+ attn_mask = attn_mask + key_padding_mask
570
+
571
+ # adjust dropout probability
572
+ if not training:
573
+ dropout_p = 0.0
574
+
575
+ #
576
+ # (deep breath) calculate attention and out projection
577
+ #
578
+
579
+ if need_weights:
580
+ B, Nt, E = q.shape
581
+ q_scaled = q / math.sqrt(E)
582
+
583
+ assert not (is_causal and attn_mask is None), "FIXME: is_causal not implemented for need_weights"
584
+
585
+ if attn_mask is not None:
586
+ attn_output_weights = torch.baddbmm(attn_mask, q_scaled, k.transpose(-2, -1))
587
+ else:
588
+ attn_output_weights = torch.bmm(q_scaled, k.transpose(-2, -1))
589
+ attn_output_weights = softmax(attn_output_weights, dim=-1)
590
+ if dropout_p > 0.0:
591
+ attn_output_weights = dropout(attn_output_weights, p=dropout_p)
592
+
593
+ attn_output = torch.bmm(attn_output_weights, v)
594
+
595
+ attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len * bsz, embed_dim)
596
+ attn_output = self.out_proj(attn_output)
597
+
598
+ attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))
599
+
600
+ # optionally average attention weights over heads
601
+ attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len)
602
+ if average_attn_weights:
603
+ attn_output_weights = attn_output_weights.mean(dim=1)
604
+
605
+ if not is_batched:
606
+ # squeeze the output if input was unbatched
607
+ attn_output = attn_output.squeeze(1)
608
+ attn_output_weights = attn_output_weights.squeeze(0)
609
+ return attn_output, attn_output_weights
610
+ else:
611
+ # attn_mask can be either (L,S) or (N*num_heads, L, S)
612
+ # if attn_mask's shape is (1, L, S) we need to unsqueeze to (1, 1, L, S)
613
+ # in order to match the input for SDPA of (N, num_heads, L, S)
614
+ if attn_mask is not None:
615
+ if attn_mask.size(0) == 1 and attn_mask.dim() == 3:
616
+ attn_mask = attn_mask.unsqueeze(0)
617
+ else:
618
+ attn_mask = attn_mask.view(bsz, num_heads, -1, src_len)
619
+
620
+ q = q.view(bsz, num_heads, tgt_len, head_dim)
621
+ k = k.view(bsz, num_heads, src_len, head_dim)
622
+ v = v.view(bsz, num_heads, src_len, head_dim)
623
+
624
+ attn_output = F.scaled_dot_product_attention(q, k, v, attn_mask, dropout_p, is_causal)
625
+ attn_output = attn_output.permute(2, 0, 1, 3).contiguous().view(bsz * tgt_len, embed_dim)
626
+
627
+ attn_output = self.out_proj(attn_output)
628
+
629
+ attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1))
630
+ if not is_batched:
631
+ # squeeze the output if input was unbatched
632
+ attn_output = attn_output.squeeze(1)
633
+ return attn_output, None
634
+
635
+
636
+ def _mha_shape_check(query: Tensor, key: Tensor, value: Tensor,
637
+ key_padding_mask: Optional[Tensor], attn_mask: Optional[Tensor], num_heads: int):
638
+ # Verifies the expected shape for `query, `key`, `value`, `key_padding_mask` and `attn_mask`
639
+ # and returns if the input is batched or not.
640
+ # Raises an error if `query` is not 2-D (unbatched) or 3-D (batched) tensor.
641
+
642
+ # Shape check.
643
+ if query.dim() == 3:
644
+ # Batched Inputs
645
+ is_batched = True
646
+ assert key.dim() == 3 and value.dim() == 3, \
647
+ ("For batched (3-D) `query`, expected `key` and `value` to be 3-D"
648
+ f" but found {key.dim()}-D and {value.dim()}-D tensors respectively")
649
+ if key_padding_mask is not None:
650
+ assert key_padding_mask.dim() == 2, \
651
+ ("For batched (3-D) `query`, expected `key_padding_mask` to be `None` or 2-D"
652
+ f" but found {key_padding_mask.dim()}-D tensor instead")
653
+ if attn_mask is not None:
654
+ assert attn_mask.dim() in (2, 3), \
655
+ ("For batched (3-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D"
656
+ f" but found {attn_mask.dim()}-D tensor instead")
657
+ elif query.dim() == 2:
658
+ # Unbatched Inputs
659
+ is_batched = False
660
+ assert key.dim() == 2 and value.dim() == 2, \
661
+ ("For unbatched (2-D) `query`, expected `key` and `value` to be 2-D"
662
+ f" but found {key.dim()}-D and {value.dim()}-D tensors respectively")
663
+
664
+ if key_padding_mask is not None:
665
+ assert key_padding_mask.dim() == 1, \
666
+ ("For unbatched (2-D) `query`, expected `key_padding_mask` to be `None` or 1-D"
667
+ f" but found {key_padding_mask.dim()}-D tensor instead")
668
+
669
+ if attn_mask is not None:
670
+ assert attn_mask.dim() in (2, 3), \
671
+ ("For unbatched (2-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D"
672
+ f" but found {attn_mask.dim()}-D tensor instead")
673
+ if attn_mask.dim() == 3:
674
+ expected_shape = (num_heads, query.shape[0], key.shape[0])
675
+ assert attn_mask.shape == expected_shape, \
676
+ (f"Expected `attn_mask` shape to be {expected_shape} but got {attn_mask.shape}")
677
+ else:
678
+ raise AssertionError(
679
+ f"query should be unbatched 2D or batched 3D tensor but received {query.dim()}-D query tensor")
680
+
681
+ return is_batched
682
+
683
+
684
+ def _canonical_mask(
685
+ mask: Optional[Tensor],
686
+ mask_name: str,
687
+ other_type: Optional[DType],
688
+ other_name: str,
689
+ target_type: DType,
690
+ check_other: bool = True,
691
+ ) -> Optional[Tensor]:
692
+
693
+ if mask is not None:
694
+ _mask_dtype = mask.dtype
695
+ _mask_is_float = torch.is_floating_point(mask)
696
+ if _mask_dtype != torch.bool and not _mask_is_float:
697
+ raise AssertionError(
698
+ f"only bool and floating types of {mask_name} are supported")
699
+ if check_other and other_type is not None:
700
+ if _mask_dtype != other_type:
701
+ warnings.warn(
702
+ f"Support for mismatched {mask_name} and {other_name} "
703
+ "is deprecated. Use same type for both instead."
704
+ )
705
+ if not _mask_is_float:
706
+ mask = (
707
+ torch.zeros_like(mask, dtype=target_type)
708
+ .masked_fill_(mask, float("-inf"))
709
+ )
710
+ return mask
711
+
712
+
713
+ def _none_or_dtype(input: Optional[Tensor]) -> Optional[DType]:
714
+ if input is None:
715
+ return None
716
+ elif isinstance(input, torch.Tensor):
717
+ return input.dtype
718
+ raise RuntimeError("input to _none_or_dtype() must be None or torch.Tensor")
719
+
720
+ def _in_projection_packed(
721
+ q: Tensor,
722
+ k: Tensor,
723
+ v: Tensor,
724
+ w: Tensor,
725
+ b: Optional[Tensor] = None,
726
+ ) -> List[Tensor]:
727
+ r"""
728
+ Performs the in-projection step of the attention operation, using packed weights.
729
+ Output is a triple containing projection tensors for query, key and value.
730
+ Args:
731
+ q, k, v: query, key and value tensors to be projected. For self-attention,
732
+ these are typically the same tensor; for encoder-decoder attention,
733
+ k and v are typically the same tensor. (We take advantage of these
734
+ identities for performance if they are present.) Regardless, q, k and v
735
+ must share a common embedding dimension; otherwise their shapes may vary.
736
+ w: projection weights for q, k and v, packed into a single tensor. Weights
737
+ are packed along dimension 0, in q, k, v order.
738
+ b: optional projection biases for q, k and v, packed into a single tensor
739
+ in q, k, v order.
740
+ Shape:
741
+ Inputs:
742
+ - q: :math:`(..., E)` where E is the embedding dimension
743
+ - k: :math:`(..., E)` where E is the embedding dimension
744
+ - v: :math:`(..., E)` where E is the embedding dimension
745
+ - w: :math:`(E * 3, E)` where E is the embedding dimension
746
+ - b: :math:`E * 3` where E is the embedding dimension
747
+ Output:
748
+ - in output list :math:`[q', k', v']`, each output tensor will have the
749
+ same shape as the corresponding input tensor.
750
+ """
751
+ E = q.size(-1)
752
+ if k is v:
753
+ if q is k:
754
+ # self-attention
755
+ proj = linear(q, w, b)
756
+ # reshape to 3, E and not E, 3 is deliberate for better memory coalescing and keeping same order as chunk()
757
+ proj = proj.unflatten(-1, (3, E)).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()
758
+ return proj[0], proj[1], proj[2]
759
+ else:
760
+ # encoder-decoder attention
761
+ w_q, w_kv = w.split([E, E * 2])
762
+ if b is None:
763
+ b_q = b_kv = None
764
+ else:
765
+ b_q, b_kv = b.split([E, E * 2])
766
+ q_proj = linear(q, w_q, b_q)
767
+ kv_proj = linear(k, w_kv, b_kv)
768
+ # reshape to 2, E and not E, 2 is deliberate for better memory coalescing and keeping same order as chunk()
769
+ kv_proj = kv_proj.unflatten(-1, (2, E)).unsqueeze(0).transpose(0, -2).squeeze(-2).contiguous()
770
+ return (q_proj, kv_proj[0], kv_proj[1])
771
+ else:
772
+ w_q, w_k, w_v = w.chunk(3)
773
+ if b is None:
774
+ b_q = b_k = b_v = None
775
+ else:
776
+ b_q, b_k, b_v = b.chunk(3)
777
+ return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v)
778
+
779
+
780
+ def _in_projection(
781
+ q: Tensor,
782
+ k: Tensor,
783
+ v: Tensor,
784
+ w_q: Tensor,
785
+ w_k: Tensor,
786
+ w_v: Tensor,
787
+ b_q: Optional[Tensor] = None,
788
+ b_k: Optional[Tensor] = None,
789
+ b_v: Optional[Tensor] = None,
790
+ ) -> Tuple[Tensor, Tensor, Tensor]:
791
+ r"""
792
+ Performs the in-projection step of the attention operation. This is simply
793
+ a triple of linear projections, with shape constraints on the weights which
794
+ ensure embedding dimension uniformity in the projected outputs.
795
+ Output is a triple containing projection tensors for query, key and value.
796
+ Args:
797
+ q, k, v: query, key and value tensors to be projected.
798
+ w_q, w_k, w_v: weights for q, k and v, respectively.
799
+ b_q, b_k, b_v: optional biases for q, k and v, respectively.
800
+ Shape:
801
+ Inputs:
802
+ - q: :math:`(Qdims..., Eq)` where Eq is the query embedding dimension and Qdims are any
803
+ number of leading dimensions.
804
+ - k: :math:`(Kdims..., Ek)` where Ek is the key embedding dimension and Kdims are any
805
+ number of leading dimensions.
806
+ - v: :math:`(Vdims..., Ev)` where Ev is the value embedding dimension and Vdims are any
807
+ number of leading dimensions.
808
+ - w_q: :math:`(Eq, Eq)`
809
+ - w_k: :math:`(Eq, Ek)`
810
+ - w_v: :math:`(Eq, Ev)`
811
+ - b_q: :math:`(Eq)`
812
+ - b_k: :math:`(Eq)`
813
+ - b_v: :math:`(Eq)`
814
+ Output: in output triple :math:`(q', k', v')`,
815
+ - q': :math:`[Qdims..., Eq]`
816
+ - k': :math:`[Kdims..., Eq]`
817
+ - v': :math:`[Vdims..., Eq]`
818
+ """
819
+ Eq, Ek, Ev = q.size(-1), k.size(-1), v.size(-1)
820
+ assert w_q.shape == (Eq, Eq), f"expecting query weights shape of {(Eq, Eq)}, but got {w_q.shape}"
821
+ assert w_k.shape == (Eq, Ek), f"expecting key weights shape of {(Eq, Ek)}, but got {w_k.shape}"
822
+ assert w_v.shape == (Eq, Ev), f"expecting value weights shape of {(Eq, Ev)}, but got {w_v.shape}"
823
+ assert b_q is None or b_q.shape == (Eq,), f"expecting query bias shape of {(Eq,)}, but got {b_q.shape}"
824
+ assert b_k is None or b_k.shape == (Eq,), f"expecting key bias shape of {(Eq,)}, but got {b_k.shape}"
825
+ assert b_v is None or b_v.shape == (Eq,), f"expecting value bias shape of {(Eq,)}, but got {b_v.shape}"
826
+ return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v)