import torch import torch.nn as nn import torch.nn.functional as F import numpy as np from typing import Tuple, Literal from functools import partial import itertools # LRM from .embedder import CameraEmbedder from .transformer import TransformerDecoder # from accelerate.logging import get_logger # logger = get_logger(__name__) class LRM_VSD_Mesh_Net(nn.Module): """ predict VSD using transformer """ def __init__(self, camera_embed_dim: int, transformer_dim: int, transformer_layers: int, transformer_heads: int, triplane_low_res: int, triplane_high_res: int, triplane_dim: int, encoder_freeze: bool = True, encoder_type: str = 'dino', encoder_model_name: str = 'facebook/dino-vitb16', encoder_feat_dim: int = 768, app_dim = 27, density_dim = 8, app_n_comp=24, density_n_comp=8): super().__init__() # attributes self.encoder_feat_dim = encoder_feat_dim self.camera_embed_dim = camera_embed_dim self.triplane_low_res = triplane_low_res self.triplane_high_res = triplane_high_res self.triplane_dim = triplane_dim self.transformer_dim=transformer_dim # modules self.encoder = self._encoder_fn(encoder_type)( model_name=encoder_model_name, modulation_dim=self.camera_embed_dim, #mod camera vector freeze=encoder_freeze, ) self.camera_embedder = CameraEmbedder( raw_dim=12+4, embed_dim=camera_embed_dim, ) self.n_comp=app_n_comp+density_n_comp self.app_dim=app_dim self.density_dim=density_dim self.app_n_comp=app_n_comp self.density_n_comp=density_n_comp self.pos_embed = nn.Parameter(torch.randn(1, 3*(triplane_low_res**2)+3*triplane_low_res, transformer_dim) * (1. / transformer_dim) ** 0.5) self.transformer = TransformerDecoder( block_type='cond', num_layers=transformer_layers, num_heads=transformer_heads, inner_dim=transformer_dim, cond_dim=encoder_feat_dim, mod_dim=None, ) # for plane self.upsampler = nn.ConvTranspose2d(transformer_dim, self.n_comp, kernel_size=2, stride=2, padding=0) self.dim_map = nn.Linear(transformer_dim,self.n_comp) self.up_line = nn.Linear(triplane_low_res,triplane_low_res*2) @staticmethod def _encoder_fn(encoder_type: str): encoder_type = encoder_type.lower() assert encoder_type in ['dino', 'dinov2'], "Unsupported encoder type" if encoder_type == 'dino': from .encoders.dino_wrapper import DinoWrapper #logger.info("Using DINO as the encoder") return DinoWrapper elif encoder_type == 'dinov2': from .encoders.dinov2_wrapper import Dinov2Wrapper #logger.info("Using DINOv2 as the encoder") return Dinov2Wrapper def forward_transformer(self, image_feats, camera_embeddings=None): N = image_feats.shape[0] x = self.pos_embed.repeat(N, 1, 1) # [N, L, D] x = self.transformer( x, cond=image_feats, mod=camera_embeddings, ) return x def reshape_upsample(self, tokens): #B,_,3*ncomp N = tokens.shape[0] H = W = self.triplane_low_res P=self.n_comp offset=3*H*W # planes plane_tokens= tokens[:,:3*H*W,:].view(N,H,W,3,self.transformer_dim) plane_tokens = torch.einsum('nhwip->inphw', plane_tokens) # [3, N, P, H, W] plane_tokens = plane_tokens.contiguous().view(3*N, -1, H, W) # [3*N, D, H, W] plane_tokens = self.upsampler(plane_tokens) # [3*N, P, H', W'] plane_tokens = plane_tokens.view(3, N, *plane_tokens.shape[-3:]) # [3, N, P, H', W'] plane_tokens = torch.einsum('inphw->niphw', plane_tokens) # [N, 3, P, H', W'] plane_tokens = plane_tokens.reshape(N, 3*P, *plane_tokens.shape[-2:]) # # [N, 3*P, H', W'] plane_tokens = plane_tokens.contiguous() #lines line_tokens= tokens[:,3*H*W:3*H*W+3*H,:].view(N,H,3,self.transformer_dim) line_tokens= self.dim_map(line_tokens) line_tokens = torch.einsum('nhip->npih', line_tokens) # [ N, P, 3, H] line_tokens=self.up_line(line_tokens) line_tokens = torch.einsum('npih->niph', line_tokens) # [ N, 3, P, H] line_tokens=line_tokens.reshape(N,3*P,line_tokens.shape[-1],1) line_tokens = line_tokens.contiguous() mat_tokens=None d_mat_tokens=None return plane_tokens[:,:self.app_n_comp*3,:,:],line_tokens[:,:self.app_n_comp*3,:,:],mat_tokens,d_mat_tokens,plane_tokens[:,self.app_n_comp*3:,:,:],line_tokens[:,self.app_n_comp*3:,:,:] def forward_planes(self, image, camera): # image: [N, V, C_img, H_img, W_img] # camera: [N,V, D_cam_raw] N,V,_,H,W = image.shape image=image.reshape(N*V,3,H,W) camera=camera.reshape(N*V,-1) # embed camera camera_embeddings = self.camera_embedder(camera) assert camera_embeddings.shape[-1] == self.camera_embed_dim, \ f"Feature dimension mismatch: {camera_embeddings.shape[-1]} vs {self.camera_embed_dim}" # encode image image_feats = self.encoder(image, camera_embeddings) assert image_feats.shape[-1] == self.encoder_feat_dim, \ f"Feature dimension mismatch: {image_feats.shape[-1]} vs {self.encoder_feat_dim}" image_feats=image_feats.reshape(N,V*image_feats.shape[-2],image_feats.shape[-1]) # transformer generating planes tokens = self.forward_transformer(image_feats) app_planes,app_lines,basis_mat,d_basis_mat,density_planes,density_lines = self.reshape_upsample(tokens) return app_planes,app_lines,basis_mat,d_basis_mat,density_planes,density_lines def forward(self, image,source_camera): # image: [N,V, C_img, H_img, W_img] # source_camera: [N, V, D_cam_raw] assert image.shape[0] == source_camera.shape[0], "Batch size mismatch for image and source_camera" planes = self.forward_planes(image, source_camera) #B,3,dim,H,W return planes