Spaces:
Runtime error
Runtime error
| # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved | |
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| __all__ = [ | |
| "window_partition", | |
| "window_unpartition", | |
| "add_decomposed_rel_pos", | |
| "get_abs_pos", | |
| "PatchEmbed", | |
| ] | |
| def window_partition(x, window_size): | |
| """ | |
| Partition into non-overlapping windows with padding if needed. | |
| Args: | |
| x (tensor): input tokens with [B, H, W, C]. | |
| window_size (int): window size. | |
| Returns: | |
| windows: windows after partition with [B * num_windows, window_size, window_size, C]. | |
| (Hp, Wp): padded height and width before partition | |
| """ | |
| B, H, W, C = x.shape | |
| pad_h = (window_size - H % window_size) % window_size | |
| pad_w = (window_size - W % window_size) % window_size | |
| if pad_h > 0 or pad_w > 0: | |
| x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h)) | |
| Hp, Wp = H + pad_h, W + pad_w | |
| x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C) | |
| windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) | |
| return windows, (Hp, Wp) | |
| def window_unpartition(windows, window_size, pad_hw, hw): | |
| """ | |
| Window unpartition into original sequences and removing padding. | |
| Args: | |
| x (tensor): input tokens with [B * num_windows, window_size, window_size, C]. | |
| window_size (int): window size. | |
| pad_hw (Tuple): padded height and width (Hp, Wp). | |
| hw (Tuple): original height and width (H, W) before padding. | |
| Returns: | |
| x: unpartitioned sequences with [B, H, W, C]. | |
| """ | |
| Hp, Wp = pad_hw | |
| H, W = hw | |
| B = windows.shape[0] // (Hp * Wp // window_size // window_size) | |
| x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1) | |
| x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1) | |
| if Hp > H or Wp > W: | |
| x = x[:, :H, :W, :].contiguous() | |
| return x | |
| def get_rel_pos(q_size, k_size, rel_pos): | |
| """ | |
| Get relative positional embeddings according to the relative positions of | |
| query and key sizes. | |
| Args: | |
| q_size (int): size of query q. | |
| k_size (int): size of key k. | |
| rel_pos (Tensor): relative position embeddings (L, C). | |
| Returns: | |
| Extracted positional embeddings according to relative positions. | |
| """ | |
| max_rel_dist = int(2 * max(q_size, k_size) - 1) | |
| # Interpolate rel pos if needed. | |
| if rel_pos.shape[0] != max_rel_dist: | |
| # Interpolate rel pos. | |
| rel_pos_resized = F.interpolate( | |
| rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1), | |
| size=max_rel_dist, | |
| mode="linear", | |
| ) | |
| rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0) | |
| else: | |
| rel_pos_resized = rel_pos | |
| # Scale the coords with short length if shapes for q and k are different. | |
| q_coords = torch.arange(q_size)[:, None] * max(k_size / q_size, 1.0) | |
| k_coords = torch.arange(k_size)[None, :] * max(q_size / k_size, 1.0) | |
| relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0) | |
| return rel_pos_resized[relative_coords.long()] | |
| def add_decomposed_rel_pos(attn, q, rel_pos_h, rel_pos_w, q_size, k_size): | |
| """ | |
| Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`. | |
| https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py # noqa B950 | |
| Args: | |
| attn (Tensor): attention map. | |
| q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C). | |
| rel_pos_h (Tensor): relative position embeddings (Lh, C) for height axis. | |
| rel_pos_w (Tensor): relative position embeddings (Lw, C) for width axis. | |
| q_size (Tuple): spatial sequence size of query q with (q_h, q_w). | |
| k_size (Tuple): spatial sequence size of key k with (k_h, k_w). | |
| Returns: | |
| attn (Tensor): attention map with added relative positional embeddings. | |
| """ | |
| q_h, q_w = q_size | |
| k_h, k_w = k_size | |
| Rh = get_rel_pos(q_h, k_h, rel_pos_h) | |
| Rw = get_rel_pos(q_w, k_w, rel_pos_w) | |
| B, _, dim = q.shape | |
| r_q = q.reshape(B, q_h, q_w, dim) | |
| rel_h = torch.einsum("bhwc,hkc->bhwk", r_q, Rh) | |
| rel_w = torch.einsum("bhwc,wkc->bhwk", r_q, Rw) | |
| attn = ( | |
| attn.view(B, q_h, q_w, k_h, k_w) + rel_h[:, :, :, :, None] + rel_w[:, :, :, None, :] | |
| ).view(B, q_h * q_w, k_h * k_w) | |
| return attn | |
| def get_abs_pos(abs_pos, has_cls_token, hw): | |
| """ | |
| Calculate absolute positional embeddings. If needed, resize embeddings and remove cls_token | |
| dimension for the original embeddings. | |
| Args: | |
| abs_pos (Tensor): absolute positional embeddings with (1, num_position, C). | |
| has_cls_token (bool): If true, has 1 embedding in abs_pos for cls token. | |
| hw (Tuple): size of input image tokens. | |
| Returns: | |
| Absolute positional embeddings after processing with shape (1, H, W, C) | |
| """ | |
| h, w = hw | |
| if has_cls_token: | |
| abs_pos = abs_pos[:, 1:] | |
| xy_num = abs_pos.shape[1] | |
| size = int(math.sqrt(xy_num)) | |
| assert size * size == xy_num | |
| if size != h or size != w: | |
| new_abs_pos = F.interpolate( | |
| abs_pos.reshape(1, size, size, -1).permute(0, 3, 1, 2), | |
| size=(h, w), | |
| mode="bicubic", | |
| align_corners=False, | |
| ) | |
| return new_abs_pos.permute(0, 2, 3, 1) | |
| else: | |
| return abs_pos.reshape(1, h, w, -1) | |
| class PatchEmbed(nn.Module): | |
| """ | |
| Image to Patch Embedding. | |
| """ | |
| def __init__( | |
| self, kernel_size=(16, 16), stride=(16, 16), padding=(0, 0), in_chans=3, embed_dim=768 | |
| ): | |
| """ | |
| Args: | |
| kernel_size (Tuple): kernel size of the projection layer. | |
| stride (Tuple): stride of the projection layer. | |
| padding (Tuple): padding size of the projection layer. | |
| in_chans (int): Number of input image channels. | |
| embed_dim (int): embed_dim (int): Patch embedding dimension. | |
| """ | |
| super().__init__() | |
| self.proj = nn.Conv2d( | |
| in_chans, embed_dim, kernel_size=kernel_size, stride=stride, padding=padding | |
| ) | |
| def forward(self, x): | |
| x = self.proj(x) | |
| # B C H W -> B H W C | |
| x = x.permute(0, 2, 3, 1) | |
| return x | |