# Copyright (C) 2022-present Naver Corporation. All rights reserved. # Licensed under CC BY-NC-SA 4.0 (non-commercial use only). # -------------------------------------------------------- # Position embedding utils # -------------------------------------------------------- import numpy as np import torch # -------------------------------------------------------- # 2D sine-cosine position embedding # References: # MAE: https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py # Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py # MoCo v3: https://github.com/facebookresearch/moco-v3 # -------------------------------------------------------- def get_2d_sincos_pos_embed(embed_dim, grid_size, n_cls_token=0): """ grid_size: tuple (height, width) of the grid return: pos_embed: [grid_size[0]*grid_size[1], embed_dim] or [n_cls_token+grid_size[0]*grid_size[1], embed_dim] (w/ or w/o cls_token) """ grid_h = np.arange(grid_size[0], dtype=np.float32) grid_w = np.arange(grid_size[1], dtype=np.float32) grid = np.meshgrid(grid_w, grid_h) # here w goes first grid = np.stack(grid, axis=0) grid = grid.reshape([2, 1, grid_size[0], grid_size[1]]) pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) if n_cls_token>0: pos_embed = np.concatenate([np.zeros([n_cls_token, embed_dim]), pos_embed], axis=0) return pos_embed def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): assert embed_dim % 2 == 0 # use half of dimensions to encode grid_h emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) return emb def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): """ embed_dim: output dimension for each position pos: a list of positions to be encoded: size (M,) out: (M, D) """ assert embed_dim % 2 == 0 omega = np.arange(embed_dim // 2, dtype=float) omega /= embed_dim / 2. omega = 1. / 10000**omega # (D/2,) pos = pos.reshape(-1) # (M,) out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product emb_sin = np.sin(out) # (M, D/2) emb_cos = np.cos(out) # (M, D/2) emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) return emb # -------------------------------------------------------- # Interpolate position embeddings for high-resolution # References: # MAE: https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py # DeiT: https://github.com/facebookresearch/deit # -------------------------------------------------------- def interpolate_pos_embed(model, checkpoint_model): keys = ['enc_pos_embed']+(['dec_pos_embed'] if hasattr(model,'dec_blocks') else []) img_size = model.patch_embed.img_size if isinstance(img_size,int): img_size = (img_size,img_size) for k in keys: if not k in checkpoint_model: continue pos_embed_checkpoint = checkpoint_model[k] embedding_size = pos_embed_checkpoint.shape[-1] num_extra_tokens = 0 # no cls token # height (== width) for the checkpoint position embedding orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5) new_size = (img_size[0]//model.patch_embed.patch_size[0],img_size[1]//model.patch_embed.patch_size[1]) if orig_size != new_size[0] or orig_size != new_size[1]: print("Position interpolate %s from %dx%d to %dx%d" % (k, orig_size, orig_size, new_size[0], new_size[1])) extra_tokens = pos_embed_checkpoint[:num_extra_tokens,:] pos_tokens = pos_embed_checkpoint[num_extra_tokens:,:] pos_tokens = pos_tokens.reshape(1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2) pos_tokens = torch.nn.functional.interpolate(pos_tokens, size=(new_size[0], new_size[1]), mode='bicubic', align_corners=False) pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2).squeeze(0) new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=0) checkpoint_model[k] = new_pos_embed.squeeze(0) #---------------------------------------------------------- # RoPE2D: RoPE implementation in 2D #---------------------------------------------------------- try: from models.curope import cuRoPE2D RoPE2D = cuRoPE2D except ImportError: print('Warning, cannot find cuda-compiled version of RoPE2D, using a slow pytorch version instead') class RoPE2D(torch.nn.Module): def __init__(self, freq=100.0, F0=1.0): super().__init__() self.base = freq self.F0 = F0 self.cache = {} def get_cos_sin(self, D, seq_len, device, dtype): if (D,seq_len,device,dtype) not in self.cache: inv_freq = 1.0 / (self.base ** (torch.arange(0, D, 2).float().to(device) / D)) t = torch.arange(seq_len, device=device, dtype=inv_freq.dtype) freqs = torch.einsum("i,j->ij", t, inv_freq).to(dtype) freqs = torch.cat((freqs, freqs), dim=-1) cos = freqs.cos() # (Seq, Dim) sin = freqs.sin() self.cache[D,seq_len,device,dtype] = (cos,sin) return self.cache[D,seq_len,device,dtype] @staticmethod def rotate_half(x): x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :] return torch.cat((-x2, x1), dim=-1) def apply_rope1d(self, tokens, pos1d, cos, sin): assert pos1d.ndim==2 cos = torch.nn.functional.embedding(pos1d, cos)[:, None, :, :] sin = torch.nn.functional.embedding(pos1d, sin)[:, None, :, :] return (tokens * cos) + (self.rotate_half(tokens) * sin) def forward(self, tokens, positions): """ input: * tokens: batch_size x nheads x ntokens x dim * positions: batch_size x ntokens x 2 (y and x position of each token) output: * tokens after appplying RoPE2D (batch_size x nheads x ntokens x dim) """ assert tokens.size(3)%2==0, "number of dimensions should be a multiple of two" D = tokens.size(3) // 2 assert positions.ndim==3 and positions.shape[-1] == 2 # Batch, Seq, 2 cos, sin = self.get_cos_sin(D, int(positions.max())+1, tokens.device, tokens.dtype) # split features into two along the feature dimension, and apply rope1d on each half y, x = tokens.chunk(2, dim=-1) y = self.apply_rope1d(y, positions[:,:,0], cos, sin) x = self.apply_rope1d(x, positions[:,:,1], cos, sin) tokens = torch.cat((y, x), dim=-1) return tokens