lam/models/rendering/utils/uv_utils.py

import torch
import numpy as np
import math
import torch.nn as nn

from pytorch3d.structures       import Meshes
from pytorch3d.io               import load_obj
from pytorch3d.renderer.mesh    import rasterize_meshes
from pytorch3d.ops              import mesh_face_areas_normals

#-------------------------------------------------------------------------------#

def gen_tritex(vt: np.ndarray, vi: np.ndarray, vti: np.ndarray, texsize: int):
    """
    Copied from MVP
    Create 3 texture maps containing the vertex indices, texture vertex
    indices, and barycentric coordinates

    Parameters
    ----------
        vt: uv coordinates of texels
        vi: triangle list mapping into vertex positions
        vti: triangle list mapping into texel coordinates
        texsize: Size of the generated maps
    """
    # vt = ((vt + 1. ) / 2.)[:, :2]
    vt = vt[:, :2]

    vt = np.array(vt, dtype=np.float32)
    vi = np.array(vi, dtype=np.int32)
    vti = np.array(vti, dtype=np.int32)
    ntris = vi.shape[0]

    texu, texv = np.meshgrid(
        (np.arange(texsize) + 0.5) / texsize,
        (np.arange(texsize) + 0.5) / texsize)
    texuv = np.stack((texu, texv), axis=-1)

    vt = vt[vti]

    viim = np.zeros((texsize, texsize, 3), dtype=np.int32)
    vtiim = np.zeros((texsize, texsize, 3), dtype=np.int32)
    baryim = np.zeros((texsize, texsize, 3), dtype=np.float32)

    for i in list(range(ntris))[::-1]:
        bbox = (
            max(0, int(min(vt[i, 0, 0], min(vt[i, 1, 0], vt[i, 2, 0])) * texsize) - 1),
            min(texsize, int(max(vt[i, 0, 0], max(vt[i, 1, 0], vt[i, 2, 0])) * texsize) + 2),
            max(0, int(min(vt[i, 0, 1], min(vt[i, 1, 1], vt[i, 2, 1])) * texsize) - 1),
            min(texsize, int(max(vt[i, 0, 1], max(vt[i, 1, 1], vt[i, 2, 1])) * texsize) + 2))
        v0 = vt[None, None, i, 1, :] - vt[None, None, i, 0, :]
        v1 = vt[None, None, i, 2, :] - vt[None, None, i, 0, :]
        v2 = texuv[bbox[2]:bbox[3], bbox[0]:bbox[1], :] - vt[None, None, i, 0, :]
        d00 = np.sum(v0 * v0, axis=-1)
        d01 = np.sum(v0 * v1, axis=-1)
        d11 = np.sum(v1 * v1, axis=-1)
        d20 = np.sum(v2 * v0, axis=-1)
        d21 = np.sum(v2 * v1, axis=-1)
        denom = d00 * d11 - d01 * d01

        if denom != 0.:
            baryv = (d11 * d20 - d01 * d21) / denom
            baryw = (d00 * d21 - d01 * d20) / denom
            baryu = 1. - baryv - baryw

            baryim[bbox[2]:bbox[3], bbox[0]:bbox[1], :] = np.where(
                ((baryu >= 0.) & (baryv >= 0.) & (baryw >= 0.))[:, :, None],
                np.stack((baryu, baryv, baryw), axis=-1),
                baryim[bbox[2]:bbox[3], bbox[0]:bbox[1], :])
            viim[bbox[2]:bbox[3], bbox[0]:bbox[1], :] = np.where(
                ((baryu >= 0.) & (baryv >= 0.) & (baryw >= 0.))[:, :, None],
                np.stack((vi[i, 0], vi[i, 1], vi[i, 2]), axis=-1),
                viim[bbox[2]:bbox[3], bbox[0]:bbox[1], :])
            vtiim[bbox[2]:bbox[3], bbox[0]:bbox[1], :] = np.where(
                ((baryu >= 0.) & (baryv >= 0.) & (baryw >= 0.))[:, :, None],
                np.stack((vti[i, 0], vti[i, 1], vti[i, 2]), axis=-1),
                vtiim[bbox[2]:bbox[3], bbox[0]:bbox[1], :])

    return torch.LongTensor(viim), torch.Tensor(vtiim), torch.Tensor(baryim)


# modified from https://github.com/facebookresearch/pytorch3d
class Pytorch3dRasterizer(nn.Module):
    def __init__(self, image_size=224):
        """
        use fixed raster_settings for rendering faces
        """
        super().__init__()
        raster_settings = {
            'image_size': image_size,
            'blur_radius': 0.0,
            'faces_per_pixel': 1,
            'bin_size': None,
            'max_faces_per_bin':  None,
            'perspective_correct': False,
            'cull_backfaces': True
        }
        # raster_settings = dict2obj(raster_settings)
        self.raster_settings = raster_settings

    def forward(self, vertices, faces, h=None, w=None):
        fixed_vertices = vertices.clone()
        fixed_vertices[...,:2] = -fixed_vertices[...,:2]
        raster_settings = self.raster_settings
        if h is None and w is None:
            image_size = raster_settings['image_size']
        else:
            image_size = [h, w]
            if h>w:
                fixed_vertices[..., 1] = fixed_vertices[..., 1]*h/w
            else:
                fixed_vertices[..., 0] = fixed_vertices[..., 0]*w/h
            
        meshes_screen = Meshes(verts=fixed_vertices.float(), faces=faces.long())
        pix_to_face, zbuf, bary_coords, dists = rasterize_meshes(
            meshes_screen,
            image_size=image_size,
            blur_radius=raster_settings['blur_radius'],
            faces_per_pixel=raster_settings['faces_per_pixel'],
            bin_size=raster_settings['bin_size'],
            max_faces_per_bin=raster_settings['max_faces_per_bin'],
            perspective_correct=raster_settings['perspective_correct'],
            cull_backfaces=raster_settings['cull_backfaces']
        )

        return pix_to_face, bary_coords
    
#-------------------------------------------------------------------------------#

# borrowed from https://github.com/daniilidis-group/neural_renderer/blob/master/neural_renderer/vertices_to_faces.py
def face_vertices(vertices, faces):
    """ 
    Indexing the coordinates of the three vertices on each face.

    Args:
        vertices:   [bs, V, 3]
        faces:      [bs, F, 3]

    Return: 
        face_to_vertices: [bs, F, 3, 3]
    """
    assert (vertices.ndimension() == 3)
    assert (faces.ndimension() == 3)
    # assert (vertices.shape[0] == faces.shape[0])
    assert (vertices.shape[2] == 3)
    assert (faces.shape[2] == 3)

    bs, nv = vertices.shape[:2]
    bs, nf = faces.shape[:2]
    device = vertices.device
    faces = faces + (torch.arange(bs, dtype=torch.int32).to(device) * nv)[:, None, None]
    vertices = vertices.reshape((bs * nv, 3))
    # pytorch only supports long and byte tensors for indexing
    return vertices[faces.long()]

def uniform_sampling_barycoords(
        num_points:  int,
        tex_coord:   torch.Tensor,
        uv_faces:    torch.Tensor,
        d_size:      float=1.0,
        strict:      bool=False,
        use_mask:    bool=True,
    ):
    """
    Uniformly sampling barycentric coordinates using the rasterizer.

    Args:
        num_points:     int                 sampling points number
        tex_coord:      [5150, 2]           UV coords for each vert
        uv_faces:       [F,3]               UV faces to UV coords index
        d_size:         const               to control sampling points number
        use_mask:       use mask to mask valid points
    Returns:
        face_index      [num_points]        save which face each bary_coords belongs to
        bary_coords     [num_points, 3]
    """
    
    uv_size = int(math.sqrt(num_points) * d_size)
    uv_rasterizer = Pytorch3dRasterizer(uv_size)

    tex_coord   = tex_coord[None, ...]
    uv_faces    = uv_faces[None, ...]

    tex_coord_ = torch.cat([tex_coord, tex_coord[:,:,0:1]*0.+1.], -1)
    tex_coord_ = tex_coord_ * 2 - 1 
    tex_coord_[...,1] = - tex_coord_[...,1]

    pix_to_face, bary_coords = uv_rasterizer(tex_coord_.expand(1, -1, -1), uv_faces.expand(1, -1, -1))
    mask = (pix_to_face == -1)

    if use_mask:
        face_index = pix_to_face[~mask]
        bary_coords = bary_coords[~mask]
    else:
        return pix_to_face, bary_coords

    cur_n = face_index.shape[0]

    # fix sampling number to num_points
    if strict:
        if cur_n < num_points:
            pad_size        = num_points - cur_n
            new_face_index  = face_index[torch.randint(0, cur_n, (pad_size,))]
            new_bary_coords = torch.rand((pad_size, 3), device=bary_coords.device)
            new_bary_coords = new_bary_coords / new_bary_coords.sum(dim=-1, keepdim=True)
            face_index      = torch.cat([face_index, new_face_index], dim=0)
            bary_coords     = torch.cat([bary_coords, new_bary_coords], dim=0)
        elif cur_n > num_points:
            face_index  = face_index[:num_points]
            bary_coords = bary_coords[:num_points]

    return face_index, bary_coords

def random_sampling_barycoords(
        num_points:   int,
        vertices:     torch.Tensor,
        faces:        torch.Tensor
    ):
    """
    Randomly sampling barycentric coordinates using the rasterizer.

    Args:
        num_points:     int                 sampling points number
        vertices:       [V, 3]           
        faces:          [F,3]
    Returns:
        face_index      [num_points]        save which face each bary_coords belongs to
        bary_coords     [num_points, 3]
    """

    areas, _ = mesh_face_areas_normals(vertices.squeeze(0), faces)

    g1 = torch.Generator(device=vertices.device)
    g1.manual_seed(0)

    face_index = areas.multinomial(
            num_points, replacement=True, generator=g1
        )  # (N, num_samples)

    uvw = torch.rand((face_index.shape[0], 3), device=vertices.device)
    bary_coords = uvw / uvw.sum(dim=-1, keepdim=True)

    return face_index, bary_coords

def reweight_verts_by_barycoords(
        verts:       torch.Tensor,
        faces:       torch.Tensor,
        face_index:  torch.Tensor,
        bary_coords: torch.Tensor,
    ):
    """
    Reweights the vertices based on the barycentric coordinates for each face.

    Args:
        verts:          [bs, V, 3].
        faces:          [F, 3]
        face_index:     [N].
        bary_coords:    [N, 3].

    Returns:
        Reweighted vertex positions of shape [bs, N, 3].
    """
    
    # index attributes by face
    B               = verts.shape[0]

    face_verts      = face_vertices(verts,  faces.expand(B, -1, -1))   # [1, F, 3, 3]
    # gather idnex for every splat
    N               = face_index.shape[0]
    face_index_3    = face_index.view(1, N, 1, 1).expand(B, N, 3, 3)
    position_vals   = face_verts.gather(1, face_index_3)
    # reweight
    position_vals   = (bary_coords[..., None] * position_vals).sum(dim = -2)

    return position_vals

def reweight_uvcoords_by_barycoords(
        uvcoords:    torch.Tensor,
        uvfaces:     torch.Tensor,
        face_index:  torch.Tensor,
        bary_coords: torch.Tensor,
    ):
    """
    Reweights the UV coordinates based on the barycentric coordinates for each face.

    Args:
        uvcoords:       [bs, V', 2].
        uvfaces:        [F, 3].
        face_index:     [N].
        bary_coords:    [N, 3].

    Returns:
        Reweighted UV coordinates, shape [bs, N, 2].
    """

    # homogeneous coordinates
    num_v           = uvcoords.shape[0]
    uvcoords        = torch.cat([uvcoords, torch.ones((num_v, 1)).to(uvcoords.device)], dim=1)
    # index attributes by face
    uvcoords        = uvcoords[None, ...]
    face_verts      = face_vertices(uvcoords,  uvfaces.expand(1, -1, -1))   # [1, F, 3, 3]
    # gather idnex for every splat
    N               = face_index.shape[0]
    face_index_3    = face_index.view(1, N, 1, 1).expand(1, N, 3, 3)
    position_vals   = face_verts.gather(1, face_index_3)
    # reweight
    position_vals   = (bary_coords[..., None] * position_vals).sum(dim = -2)

    return position_vals

# modified from https://github.com/computational-imaging/GSM/blob/main/main/gsm/deformer/util.py
def get_shell_verts_from_base(
        template_verts: torch.Tensor,
        template_faces: torch.Tensor,
        offset_len: float,
        num_shells: int,
        deflat = False,
    ):
    """
    Generates shell vertices by offsetting the original mesh's vertices along their normals.

    Args:
        template_verts: [bs, V, 3].
        template_faces: [F, 3].
        offset_len:     Positive number specifying the offset length for generating shells.
        num_shells:     The number of shells to generate.
        deflat:         If True, performs a deflation process. Defaults to False.

    Returns:
        shell verts:    [bs, num_shells, n, 3]
    """
    out_offset_len = offset_len

    if deflat:
        in_offset_len = offset_len

    batch_size = template_verts.shape[0]
    mesh = Meshes(
        verts=template_verts, faces=template_faces[None].repeat(batch_size, 1, 1)
    )
    # bs, n, 3
    vertex_normal = mesh.verts_normals_padded()
    # only for inflating

    if deflat:
        n_inflated_shells = num_shells//2 + 1
    else:
        n_inflated_shells = num_shells
    
    linscale = torch.linspace(
        out_offset_len,
        0,
        n_inflated_shells,
        device=template_verts.device,
        dtype=template_verts.dtype,
    )
    offset = linscale.reshape(1,n_inflated_shells, 1, 1) * vertex_normal[:, None]
    
    if deflat:
        linscale = torch.linspace(0, -in_offset_len, num_shells - n_inflated_shells + 1, device=template_verts.device, dtype=template_verts.dtype)[1:]
        offset_in = linscale.reshape(1, -1, 1, 1) * vertex_normal[:, None]
        offset = torch.cat([offset, offset_in], dim=1)

    verts = template_verts[:, None] + offset
    assert verts.isfinite().all()
    return verts