utils/eval_3D.py

import numpy as np
import torch
import threading
import mcubes
import trimesh
from utils.util_vis import show_att_on_image
from utils.camera import get_rotation_sphere

@torch.no_grad()
def get_dense_3D_grid(opt, var, N=None):
    batch_size = len(var.idx)
    N = N or opt.eval.vox_res
    # -0.6, 0.6
    range_min, range_max = opt.eval.range
    grid = torch.linspace(range_min, range_max, N+1, device=opt.device)
    points_3D = torch.stack(torch.meshgrid(grid, grid, grid, indexing='ij'), dim=-1) # [N, N, N, 3]
    # actually N+1 instead of N
    points_3D = points_3D.repeat(batch_size, 1, 1, 1, 1) # [B, N, N, N, 3]
    return points_3D

@torch.no_grad()
def compute_level_grid(opt, impl_network, latent_depth, latent_semantic, points_3D, images, vis_attn=False):
    # needed for amp
    latent_depth = latent_depth.to(torch.float32) if latent_depth is not None else None
    latent_semantic = latent_semantic.to(torch.float32) if latent_semantic is not None else None
    
    # process points in sliced way
    batch_size = points_3D.shape[0]
    N = points_3D.shape[1]
    assert N == points_3D.shape[2] == points_3D.shape[3]
    assert points_3D.shape[4] == 3
    
    points_3D = points_3D.view(batch_size, N, N*N, 3)
    occ = []
    attn = []
    for i in range(N):
        # [B, N*N, 3]
        points_slice = points_3D[:, i]
        # [B, N*N, 3] -> [B, N*N], [B, N*N, 1+feat_res**2]
        occ_slice, attn_slice = impl_network(latent_depth, latent_semantic, points_slice)
        occ.append(occ_slice)
        attn.append(attn_slice.detach())
    # [B, N, N*N] -> [B, N, N, N]
    occ = torch.stack(occ, dim=1).view(batch_size, N, N, N)
    occ = torch.sigmoid(occ)
    if vis_attn:
        N_global = 1
        feat_res = opt.H // opt.arch.win_size
        attn = torch.stack(attn, dim=1).view(batch_size, N, N, N, N_global+feat_res**2)
        # average along Z, [B, N, N, N_global+feat_res**2]
        attn = torch.mean(attn, dim=3)
        # [B, N, N, N_global] -> [B, N, N, 1]
        attn_global = attn[:, :, :, :N_global].sum(dim=-1, keepdim=True)
        # [B, N, N, feat_res, feat_res]
        attn_local = attn[:, :, :, N_global:].view(batch_size, N, N, feat_res, feat_res)
        # [B, N, N, feat_res, feat_res]
        attn_vis = attn_global.unsqueeze(-1) + attn_local
        # list of frame lists
        images_vis = []
        for b in range(batch_size):
            images_vis_sample = []
            for row in range(0, N, 8):
                if row % 16 == 0:
                    col_range = range(0, N//8*8+1, 8)
                else:
                    col_range = range(N//8*8, -1, -8)
                for col in col_range:
                    # [feat_res, feat_res], x is col
                    attn_curr = attn_vis[b, col, row]
                    attn_curr = torch.nn.functional.interpolate(
                        attn_curr.unsqueeze(0).unsqueeze(0), size=(opt.H, opt.W), 
                        mode='bilinear', align_corners=False
                    ).squeeze(0).squeeze(0).cpu().numpy()
                    attn_curr /= attn_curr.max()
                    # [feat_res, feat_res, 3]
                    image_curr = images[b].permute(1, 2, 0).cpu().numpy()
                    # merge the image and the attention
                    images_vis_sample.append(show_att_on_image(image_curr, attn_curr))
            images_vis.append(images_vis_sample)
    return occ, images_vis if vis_attn else None

@torch.no_grad()
def standardize_pc(pc):
    assert len(pc.shape) == 3
    pc_mean = pc.mean(dim=1, keepdim=True) 
    pc_zmean = pc - pc_mean
    origin_distance = (pc_zmean**2).sum(dim=2, keepdim=True).sqrt()
    scale = torch.sqrt(torch.sum(origin_distance**2, dim=1, keepdim=True) / pc.shape[1])
    pc_standardized = pc_zmean / (scale * 2)
    return pc_standardized

@torch.no_grad()
def normalize_pc(pc):
    assert len(pc.shape) == 3
    pc_mean = pc.mean(dim=1, keepdim=True) 
    pc_zmean = pc - pc_mean
    length_x = pc_zmean[:, :, 0].max(dim=-1)[0] - pc_zmean[:, :, 0].min(dim=-1)[0]
    length_y = pc_zmean[:, :, 1].max(dim=-1)[0] - pc_zmean[:, :, 1].min(dim=-1)[0]
    length_max = torch.stack([length_x, length_y], dim=-1).max(dim=-1)[0].unsqueeze(-1).unsqueeze(-1)
    pc_normalized = pc_zmean / (length_max + 1.e-7)
    return pc_normalized

def convert_to_explicit(opt, level_grids, isoval=0., to_pointcloud=False):
    N = len(level_grids)
    meshes = [None]*N
    pointclouds = [None]*N if to_pointcloud else None
    threads = [threading.Thread(target=convert_to_explicit_worker,
                                args=(opt, i, level_grids[i], isoval, meshes),
                                kwargs=dict(pointclouds=pointclouds),
                                daemon=False) for i in range(N)]
    for t in threads: t.start()
    for t in threads: t.join()
    if to_pointcloud:
        pointclouds = np.stack(pointclouds, axis=0)
        return meshes, pointclouds
    else: return meshes

def convert_to_explicit_worker(opt, i, level_vox_i, isoval, meshes, pointclouds=None):
    # use marching cubes to convert implicit surface to mesh
    vertices, faces = mcubes.marching_cubes(level_vox_i, isovalue=isoval)
    assert(level_vox_i.shape[0]==level_vox_i.shape[1]==level_vox_i.shape[2])
    S = level_vox_i.shape[0]
    range_min, range_max = opt.eval.range
    # marching cubes treat every cube as unit length
    vertices = vertices/S*(range_max-range_min)+range_min
    mesh = trimesh.Trimesh(vertices, faces)
    meshes[i] = mesh
    if pointclouds is not None:
        # randomly sample on mesh to get uniform dense point cloud
        if len(mesh.triangles)!=0:
            points = mesh.sample(opt.eval.num_points)
        else: points = np.zeros([opt.eval.num_points, 3])
        pointclouds[i] = points