Spaces:
Running
on
Zero
Running
on
Zero
| import numpy as np | |
| # -------------------------------------------------------- | |
| # 3D sine-cosine position embedding | |
| # References: | |
| # MVD: https://github.com/ruiwang2021/mvd/blob/main/modeling_finetune.py | |
| # -------------------------------------------------------- | |
| def get_3d_sincos_pos_embed(embed_dim, grid_size, t_size, cls_token=False): | |
| """ | |
| grid_size: int of the grid height and width | |
| t_size: int of the temporal size | |
| return: | |
| pos_embed: [t_size*grid_size*grid_size, embed_dim] or [1+t_size*grid_size*grid_size, embed_dim] (w/ or w/o cls_token) | |
| """ | |
| assert embed_dim % 4 == 0 | |
| embed_dim_spatial = embed_dim // 4 * 3 | |
| embed_dim_temporal = embed_dim // 4 | |
| # spatial | |
| grid_h = np.arange(grid_size, dtype=np.float32) | |
| grid_w = np.arange(grid_size, dtype=np.float32) | |
| grid = np.meshgrid(grid_w, grid_h) # here w goes first | |
| grid = np.stack(grid, axis=0) | |
| grid = grid.reshape([2, 1, grid_size, grid_size]) | |
| pos_embed_spatial = get_2d_sincos_pos_embed_from_grid( | |
| embed_dim_spatial, grid | |
| ) | |
| # temporal | |
| grid_t = np.arange(t_size, dtype=np.float32) | |
| pos_embed_temporal = get_1d_sincos_pos_embed_from_grid( | |
| embed_dim_temporal, grid_t | |
| ) | |
| # concate: [T, H, W] order | |
| pos_embed_temporal = pos_embed_temporal[:, np.newaxis, :] | |
| pos_embed_temporal = np.repeat( | |
| pos_embed_temporal, grid_size**2, axis=1 | |
| ) # [T, H*W, D // 4] | |
| pos_embed_spatial = pos_embed_spatial[np.newaxis, :, :] | |
| pos_embed_spatial = np.repeat( | |
| pos_embed_spatial, t_size, axis=0 | |
| ) # [T, H*W, D // 4 * 3] | |
| pos_embed = np.concatenate([pos_embed_temporal, pos_embed_spatial], axis=-1) | |
| pos_embed = pos_embed.reshape([-1, embed_dim]) # [T*H*W, D] | |
| if cls_token: | |
| pos_embed = np.concatenate( | |
| [np.zeros([1, embed_dim]), pos_embed], axis=0 | |
| ) | |
| return pos_embed | |
| # -------------------------------------------------------- | |
| # 2D sine-cosine position embedding | |
| # References: | |
| # Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py | |
| # MoCo v3: https://github.com/facebookresearch/moco-v3 | |
| # -------------------------------------------------------- | |
| def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False): | |
| """ | |
| grid_size: int of the grid height and width | |
| return: | |
| pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) | |
| """ | |
| grid_h = np.arange(grid_size, dtype=np.float32) | |
| grid_w = np.arange(grid_size, dtype=np.float32) | |
| grid = np.meshgrid(grid_w, grid_h) # here w goes first | |
| grid = np.stack(grid, axis=0) | |
| grid = grid.reshape([2, 1, grid_size, grid_size]) | |
| pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) | |
| if cls_token: | |
| pos_embed = np.concatenate( | |
| [np.zeros([1, embed_dim]), pos_embed], axis=0 | |
| ) | |
| return pos_embed | |
| def get_1d_sincos_pos_embed(embed_dim, t_size, cls_token=False): | |
| """ | |
| t_size: int of the temporal size | |
| return: | |
| pos_embed: [t_size, embed_dim] or [1+t_size, embed_dim] (w/ or w/o cls_token) | |
| """ | |
| grid_t = np.arange(t_size, dtype=np.float32) | |
| pos_embed = get_1d_sincos_pos_embed_from_grid(embed_dim, grid_t) | |
| if cls_token: | |
| pos_embed = np.concatenate( | |
| [np.zeros([1, embed_dim]), pos_embed], axis=0 | |
| ) | |
| return pos_embed | |
| def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): | |
| assert embed_dim % 2 == 0 | |
| # use half of dimensions to encode grid_h | |
| emb_h = get_1d_sincos_pos_embed_from_grid( | |
| embed_dim // 2, grid[0] | |
| ) # (H*W, D/2) | |
| emb_w = get_1d_sincos_pos_embed_from_grid( | |
| embed_dim // 2, grid[1] | |
| ) # (H*W, D/2) | |
| emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) | |
| return emb | |
| def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): | |
| """ | |
| embed_dim: output dimension for each position | |
| pos: a list of positions to be encoded: size (M,) | |
| out: (M, D) | |
| """ | |
| assert embed_dim % 2 == 0 | |
| omega = np.arange(embed_dim // 2, dtype=np.float32) | |
| omega /= embed_dim / 2.0 | |
| omega = 1.0 / 10000**omega # (D/2,) | |
| pos = pos.reshape(-1) # (M,) | |
| out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product | |
| emb_sin = np.sin(out) # (M, D/2) | |
| emb_cos = np.cos(out) # (M, D/2) | |
| emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) | |
| return emb |