lib/dataset/PIFuDataset.py

from lib.renderer.mesh import load_fit_body
from lib.dataset.hoppeMesh import HoppeMesh
from lib.dataset.body_model import TetraSMPLModel
from lib.common.render import Render
from lib.dataset.mesh_util import SMPLX, projection, cal_sdf_batch, get_visibility
from lib.pare.pare.utils.geometry import rotation_matrix_to_angle_axis
from termcolor import colored
import os.path as osp
import numpy as np
from PIL import Image
import random
import trimesh
import torch
import vedo
from kaolin.ops.mesh import check_sign
import torchvision.transforms as transforms
from ipdb import set_trace


class PIFuDataset():
    def __init__(self, cfg, split='train', vis=False):

        self.split = split
        self.root = cfg.root
        self.bsize = cfg.batch_size
        self.overfit = cfg.overfit

        # for debug, only used in visualize_sampling3D
        self.vis = vis

        self.opt = cfg.dataset
        self.datasets = self.opt.types
        self.input_size = self.opt.input_size
        self.scales = self.opt.scales
        self.workers = cfg.num_threads
        self.prior_type = cfg.net.prior_type

        self.noise_type = self.opt.noise_type
        self.noise_scale = self.opt.noise_scale

        noise_joints = [4, 5, 7, 8, 13, 14, 16, 17, 18, 19, 20, 21]

        self.noise_smpl_idx = []
        self.noise_smplx_idx = []

        for idx in noise_joints:
            self.noise_smpl_idx.append(idx * 3)
            self.noise_smpl_idx.append(idx * 3 + 1)
            self.noise_smpl_idx.append(idx * 3 + 2)

            self.noise_smplx_idx.append((idx-1) * 3)
            self.noise_smplx_idx.append((idx-1) * 3 + 1)
            self.noise_smplx_idx.append((idx-1) * 3 + 2)

        self.use_sdf = cfg.sdf
        self.sdf_clip = cfg.sdf_clip

        # [(feat_name, channel_num),...]
        self.in_geo = [item[0] for item in cfg.net.in_geo]
        self.in_nml = [item[0] for item in cfg.net.in_nml]

        self.in_geo_dim = [item[1] for item in cfg.net.in_geo]
        self.in_nml_dim = [item[1] for item in cfg.net.in_nml]

        self.in_total = self.in_geo + self.in_nml
        self.in_total_dim = self.in_geo_dim + self.in_nml_dim

        if self.split == 'train':
            self.rotations = np.arange(
                0, 360, 360 / self.opt.rotation_num).astype(np.int32)
        else:
            self.rotations = range(0, 360, 120)

        self.datasets_dict = {}

        for dataset_id, dataset in enumerate(self.datasets):

            mesh_dir = None
            smplx_dir = None

            dataset_dir = osp.join(self.root, dataset)

            if dataset in ['thuman2']:
                mesh_dir = osp.join(dataset_dir, "scans")
                smplx_dir = osp.join(dataset_dir, "fits")
                smpl_dir = osp.join(dataset_dir, "smpl")

            self.datasets_dict[dataset] = {
                "subjects": np.loadtxt(osp.join(dataset_dir, "all.txt"), dtype=str),
                "smplx_dir": smplx_dir,
                "smpl_dir": smpl_dir,
                "mesh_dir": mesh_dir,
                "scale": self.scales[dataset_id]
            }

        self.subject_list = self.get_subject_list(split)
        self.smplx = SMPLX()

        # PIL to tensor
        self.image_to_tensor = transforms.Compose([
            transforms.Resize(self.input_size),
            transforms.ToTensor(),
            transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
        ])

        # PIL to tensor
        self.mask_to_tensor = transforms.Compose([
            transforms.Resize(self.input_size),
            transforms.ToTensor(),
            transforms.Normalize((0.0, ), (1.0, ))
        ])

        self.device = torch.device(f"cuda:{cfg.gpus[0]}")
        self.render = Render(size=512, device=self.device)

    def render_normal(self, verts, faces):

        # render optimized mesh (normal, T_normal, image [-1,1])
        self.render.load_meshes(verts, faces)
        return self.render.get_rgb_image()

    def get_subject_list(self, split):

        subject_list = []

        for dataset in self.datasets:

            split_txt = osp.join(self.root, dataset, f'{split}.txt')

            if osp.exists(split_txt):
                print(f"load from {split_txt}")
                subject_list += np.loadtxt(split_txt, dtype=str).tolist()
            else:
                full_txt = osp.join(self.root, dataset, 'all.txt')
                print(f"split {full_txt} into train/val/test")

                full_lst = np.loadtxt(full_txt, dtype=str)
                full_lst = [dataset+"/"+item for item in full_lst]
                [train_lst, test_lst, val_lst] = np.split(
                    full_lst, [500, 500+5, ])

                np.savetxt(full_txt.replace(
                    "all", "train"), train_lst, fmt="%s")
                np.savetxt(full_txt.replace("all", "test"), test_lst, fmt="%s")
                np.savetxt(full_txt.replace("all", "val"), val_lst, fmt="%s")

                print(f"load from {split_txt}")
                subject_list += np.loadtxt(split_txt, dtype=str).tolist()

        if self.split != 'test':
            subject_list += subject_list[:self.bsize -
                                         len(subject_list) % self.bsize]
            print(colored(f"total: {len(subject_list)}", "yellow"))
            random.shuffle(subject_list)

        # subject_list = ["thuman2/0008"]
        return subject_list

    def __len__(self):
        return len(self.subject_list) * len(self.rotations)

    def __getitem__(self, index):

        # only pick the first data if overfitting
        if self.overfit:
            index = 0

        rid = index % len(self.rotations)
        mid = index // len(self.rotations)

        rotation = self.rotations[rid]
        subject = self.subject_list[mid].split("/")[1]
        dataset = self.subject_list[mid].split("/")[0]
        render_folder = "/".join([dataset +
                                 f"_{self.opt.rotation_num}views", subject])

        # setup paths
        data_dict = {
            'dataset': dataset,
            'subject': subject,
            'rotation': rotation,
            'scale': self.datasets_dict[dataset]["scale"],
            'mesh_path': osp.join(self.datasets_dict[dataset]["mesh_dir"], f"{subject}/{subject}.obj"),
            'smplx_path': osp.join(self.datasets_dict[dataset]["smplx_dir"], f"{subject}/smplx_param.pkl"),
            'smpl_path': osp.join(self.datasets_dict[dataset]["smpl_dir"], f"{subject}.pkl"),
            'calib_path': osp.join(self.root, render_folder, 'calib', f'{rotation:03d}.txt'),
            'vis_path': osp.join(self.root, render_folder, 'vis', f'{rotation:03d}.pt'),
            'image_path': osp.join(self.root, render_folder, 'render', f'{rotation:03d}.png')
        }

        # load training data
        data_dict.update(self.load_calib(data_dict))

        # image/normal/depth loader
        for name, channel in zip(self.in_total, self.in_total_dim):

            if f'{name}_path' not in data_dict.keys():
                data_dict.update({
                    f'{name}_path': osp.join(self.root, render_folder, name, f'{rotation:03d}.png')
                })

            # tensor update
            data_dict.update({
                name: self.imagepath2tensor(
                    data_dict[f'{name}_path'], channel, inv=False)
            })

        data_dict.update(self.load_mesh(data_dict))
        data_dict.update(self.get_sampling_geo(
            data_dict, is_valid=self.split == "val", is_sdf=self.use_sdf))
        data_dict.update(self.load_smpl(data_dict, self.vis))

        if self.prior_type == 'pamir':
            data_dict.update(self.load_smpl_voxel(data_dict))

        if (self.split != 'test') and (not self.vis):

            del data_dict['verts']
            del data_dict['faces']

        if not self.vis:
            del data_dict['mesh']

        path_keys = [
            key for key in data_dict.keys() if '_path' in key or '_dir' in key
        ]
        for key in path_keys:
            del data_dict[key]

        return data_dict

    def imagepath2tensor(self, path, channel=3, inv=False):

        rgba = Image.open(path).convert('RGBA')
        mask = rgba.split()[-1]
        image = rgba.convert('RGB')
        image = self.image_to_tensor(image)
        mask = self.mask_to_tensor(mask)
        image = (image * mask)[:channel]

        return (image * (0.5 - inv) * 2.0).float()

    def load_calib(self, data_dict):
        calib_data = np.loadtxt(data_dict['calib_path'], dtype=float)
        extrinsic = calib_data[:4, :4]
        intrinsic = calib_data[4:8, :4]
        calib_mat = np.matmul(intrinsic, extrinsic)
        calib_mat = torch.from_numpy(calib_mat).float()
        return {'calib': calib_mat}

    def load_mesh(self, data_dict):
        mesh_path = data_dict['mesh_path']
        scale = data_dict['scale']

        mesh_ori = trimesh.load(mesh_path,
                                skip_materials=True,
                                process=False,
                                maintain_order=True)
        verts = mesh_ori.vertices * scale
        faces = mesh_ori.faces

        vert_normals = np.array(mesh_ori.vertex_normals)
        face_normals = np.array(mesh_ori.face_normals)

        mesh = HoppeMesh(verts, faces, vert_normals, face_normals)

        return {
            'mesh': mesh,
            'verts': torch.as_tensor(mesh.verts).float(),
            'faces': torch.as_tensor(mesh.faces).long()
        }

    def add_noise(self,
                  beta_num,
                  smpl_pose,
                  smpl_betas,
                  noise_type,
                  noise_scale,
                  type,
                  hashcode):

        np.random.seed(hashcode)

        if type == 'smplx':
            noise_idx = self.noise_smplx_idx
        else:
            noise_idx = self.noise_smpl_idx

        if 'beta' in noise_type and noise_scale[noise_type.index("beta")] > 0.0:
            smpl_betas += (np.random.rand(beta_num) -
                           0.5) * 2.0 * noise_scale[noise_type.index("beta")]
            smpl_betas = smpl_betas.astype(np.float32)

        if 'pose' in noise_type and noise_scale[noise_type.index("pose")] > 0.0:
            smpl_pose[noise_idx] += (
                np.random.rand(len(noise_idx)) -
                0.5) * 2.0 * np.pi * noise_scale[noise_type.index("pose")]
            smpl_pose = smpl_pose.astype(np.float32)
        if type == 'smplx':
            return torch.as_tensor(smpl_pose[None, ...]), torch.as_tensor(smpl_betas[None, ...])
        else:
            return smpl_pose, smpl_betas

    def compute_smpl_verts(self, data_dict, noise_type=None, noise_scale=None):

        dataset = data_dict['dataset']
        smplx_dict = {}

        smplx_param = np.load(data_dict['smplx_path'], allow_pickle=True)
        smplx_pose = smplx_param["body_pose"]  # [1,63]
        smplx_betas = smplx_param["betas"]  # [1,10]
        smplx_pose, smplx_betas = self.add_noise(
            smplx_betas.shape[1],
            smplx_pose[0],
            smplx_betas[0],
            noise_type,
            noise_scale,
            type='smplx',
            hashcode=(hash(f"{data_dict['subject']}_{data_dict['rotation']}")) % (10**8))

        smplx_out, _ = load_fit_body(fitted_path=data_dict['smplx_path'],
                                     scale=self.datasets_dict[dataset]['scale'],
                                     smpl_type='smplx',
                                     smpl_gender='male',
                                     noise_dict=dict(betas=smplx_betas, body_pose=smplx_pose))

        smplx_dict.update({"type": "smplx",
                          "gender": 'male',
                           "body_pose": torch.as_tensor(smplx_pose),
                           "betas": torch.as_tensor(smplx_betas)})

        return smplx_out.vertices, smplx_dict

    def compute_voxel_verts(self,
                            data_dict,
                            noise_type=None,
                            noise_scale=None):

        smpl_param = np.load(data_dict['smpl_path'], allow_pickle=True)
        smplx_param = np.load(data_dict['smplx_path'], allow_pickle=True)

        smpl_pose = rotation_matrix_to_angle_axis(
            torch.as_tensor(smpl_param['full_pose'][0])).numpy()
        smpl_betas = smpl_param["betas"]

        smpl_path = osp.join(self.smplx.model_dir, "smpl/SMPL_MALE.pkl")
        tetra_path = osp.join(self.smplx.tedra_dir,
                              "tetra_male_adult_smpl.npz")

        smpl_model = TetraSMPLModel(smpl_path, tetra_path, 'adult')

        smpl_pose, smpl_betas = self.add_noise(
            smpl_model.beta_shape[0],
            smpl_pose.flatten(),
            smpl_betas[0],
            noise_type,
            noise_scale,
            type='smpl',
            hashcode=(hash(f"{data_dict['subject']}_{data_dict['rotation']}")) % (10**8))

        smpl_model.set_params(pose=smpl_pose.reshape(-1, 3),
                              beta=smpl_betas,
                              trans=smpl_param["transl"])
        
        verts = (np.concatenate([smpl_model.verts, smpl_model.verts_added],
                                axis=0) * smplx_param["scale"] + smplx_param["translation"]
                 ) * self.datasets_dict[data_dict['dataset']]['scale']
        faces = np.loadtxt(osp.join(self.smplx.tedra_dir, "tetrahedrons_male_adult.txt"),
                           dtype=np.int32) - 1

        pad_v_num = int(8000 - verts.shape[0])
        pad_f_num = int(25100 - faces.shape[0])

        verts = np.pad(verts, ((0, pad_v_num), (0, 0)),
                       mode='constant',
                       constant_values=0.0).astype(np.float32)
        faces = np.pad(faces, ((0, pad_f_num), (0, 0)),
                       mode='constant',
                       constant_values=0.0).astype(np.int32)
        

        return verts, faces, pad_v_num, pad_f_num

    def load_smpl(self, data_dict, vis=False):

        smplx_verts, smplx_dict = self.compute_smpl_verts(
            data_dict, self.noise_type,
            self.noise_scale)  # compute using smpl model

        smplx_verts = projection(smplx_verts, data_dict['calib']).float()
        smplx_faces = torch.as_tensor(self.smplx.faces).long()
        smplx_vis = torch.load(data_dict['vis_path']).float()
        smplx_cmap = torch.as_tensor(
            np.load(self.smplx.cmap_vert_path)).float()

        # get smpl_signs
        query_points = projection(data_dict['samples_geo'],
                                  data_dict['calib']).float()

        pts_signs = 2.0 * (check_sign(smplx_verts.unsqueeze(0),
                                      smplx_faces,
                                      query_points.unsqueeze(0)).float() - 0.5).squeeze(0)

        return_dict = {
            'smpl_verts': smplx_verts,
            'smpl_faces': smplx_faces,
            'smpl_vis': smplx_vis,
            'smpl_cmap': smplx_cmap,
            'pts_signs': pts_signs
        }
        if smplx_dict is not None:
            return_dict.update(smplx_dict)

        if vis:

            (xy, z) = torch.as_tensor(smplx_verts).to(
                self.device).split([2, 1], dim=1)
            smplx_vis = get_visibility(xy, z, torch.as_tensor(
                smplx_faces).to(self.device).long())

            T_normal_F, T_normal_B = self.render_normal(
                (smplx_verts*torch.tensor([1.0, -1.0, 1.0])).to(self.device),
                smplx_faces.to(self.device))

            return_dict.update({"T_normal_F": T_normal_F.squeeze(0),
                                "T_normal_B": T_normal_B.squeeze(0)})
            query_points = projection(data_dict['samples_geo'],
                                      data_dict['calib']).float()

            smplx_sdf, smplx_norm, smplx_cmap, smplx_vis = cal_sdf_batch(
                smplx_verts.unsqueeze(0).to(self.device),
                smplx_faces.unsqueeze(0).to(self.device),
                smplx_cmap.unsqueeze(0).to(self.device),
                smplx_vis.unsqueeze(0).to(self.device),
                query_points.unsqueeze(0).contiguous().to(self.device))

            return_dict.update({
                'smpl_feat':
                torch.cat(
                    (smplx_sdf[0].detach().cpu(),
                     smplx_cmap[0].detach().cpu(),
                     smplx_norm[0].detach().cpu(),
                     smplx_vis[0].detach().cpu()),
                    dim=1)
            })

        return return_dict

    def load_smpl_voxel(self, data_dict):

        smpl_verts, smpl_faces, pad_v_num, pad_f_num = self.compute_voxel_verts(
            data_dict, self.noise_type,
            self.noise_scale)  # compute using smpl model
        smpl_verts = projection(smpl_verts, data_dict['calib'])

        smpl_verts *= 0.5

        return {
            'voxel_verts': smpl_verts,
            'voxel_faces': smpl_faces,
            'pad_v_num': pad_v_num,
            'pad_f_num': pad_f_num
        }

    def get_sampling_geo(self, data_dict, is_valid=False, is_sdf=False):

        mesh = data_dict['mesh']
        calib = data_dict['calib']

        # Samples are around the true surface with an offset
        n_samples_surface = 4 * self.opt.num_sample_geo
        vert_ids = np.arange(mesh.verts.shape[0])
        thickness_sample_ratio = np.ones_like(vert_ids).astype(np.float32)

        thickness_sample_ratio /= thickness_sample_ratio.sum()

        samples_surface_ids = np.random.choice(vert_ids,
                                               n_samples_surface,
                                               replace=True,
                                               p=thickness_sample_ratio)

        samples_normal_ids = np.random.choice(vert_ids,
                                              self.opt.num_sample_geo // 2,
                                              replace=False,
                                              p=thickness_sample_ratio)

        surf_samples = mesh.verts[samples_normal_ids, :]
        surf_normals = mesh.vert_normals[samples_normal_ids, :]

        samples_surface = mesh.verts[samples_surface_ids, :]

        # Sampling offsets are random noise with constant scale (15cm - 20cm)
        offset = np.random.normal(scale=self.opt.sigma_geo,
                                  size=(n_samples_surface, 1))
        samples_surface += mesh.vert_normals[samples_surface_ids, :] * offset

        # Uniform samples in [-1, 1]
        calib_inv = np.linalg.inv(calib)
        n_samples_space = self.opt.num_sample_geo // 4
        samples_space_img = 2.0 * np.random.rand(n_samples_space, 3) - 1.0
        samples_space = projection(samples_space_img, calib_inv)

        # z-ray direction samples
        if self.opt.zray_type and not is_valid:
            n_samples_rayz = self.opt.ray_sample_num
            samples_surface_cube = projection(samples_surface, calib)
            samples_surface_cube_repeat = np.repeat(samples_surface_cube,
                                                    n_samples_rayz,
                                                    axis=0)

            thickness_repeat = np.repeat(0.5 *
                                         np.ones_like(samples_surface_ids),
                                         n_samples_rayz,
                                         axis=0)

            noise_repeat = np.random.normal(scale=0.40,
                                            size=(n_samples_surface *
                                                  n_samples_rayz, ))
            samples_surface_cube_repeat[:,
                                        -1] += thickness_repeat * noise_repeat
            samples_surface_rayz = projection(samples_surface_cube_repeat,
                                              calib_inv)

            samples = np.concatenate(
                [samples_surface, samples_space, samples_surface_rayz], 0)
        else:
            samples = np.concatenate([samples_surface, samples_space], 0)

        np.random.shuffle(samples)

        # labels: in->1.0; out->0.0.
        if is_sdf:
            sdfs = mesh.get_sdf(samples)
            inside_samples = samples[sdfs < 0]
            outside_samples = samples[sdfs >= 0]

            inside_sdfs = sdfs[sdfs < 0]
            outside_sdfs = sdfs[sdfs >= 0]
        else:
            inside = mesh.contains(samples)
            inside_samples = samples[inside >= 0.5]
            outside_samples = samples[inside < 0.5]

        nin = inside_samples.shape[0]

        if nin > self.opt.num_sample_geo // 2:
            inside_samples = inside_samples[:self.opt.num_sample_geo // 2]
            outside_samples = outside_samples[:self.opt.num_sample_geo // 2]
            if is_sdf:
                inside_sdfs = inside_sdfs[:self.opt.num_sample_geo // 2]
                outside_sdfs = outside_sdfs[:self.opt.num_sample_geo // 2]
        else:
            outside_samples = outside_samples[:(self.opt.num_sample_geo - nin)]
            if is_sdf:
                outside_sdfs = outside_sdfs[:(self.opt.num_sample_geo - nin)]

        if is_sdf:
            samples = np.concatenate(
                [inside_samples, outside_samples, surf_samples], 0)

            labels = np.concatenate([
                inside_sdfs, outside_sdfs, 0.0 * np.ones(surf_samples.shape[0])
            ])

            normals = np.zeros_like(samples)
            normals[-self.opt.num_sample_geo // 2:, :] = surf_normals

            # convert sdf from [-14, 130] to [0, 1]
            # outside: 0, inside: 1
            # Note: Marching cubes is defined on occupancy space (inside=1.0, outside=0.0)

            labels = -labels.clip(min=-self.sdf_clip, max=self.sdf_clip)
            labels += self.sdf_clip
            labels /= (self.sdf_clip * 2)

        else:
            samples = np.concatenate([inside_samples, outside_samples])
            labels = np.concatenate([
                np.ones(inside_samples.shape[0]),
                np.zeros(outside_samples.shape[0])
            ])

            normals = np.zeros_like(samples)

        samples = torch.from_numpy(samples).float()
        labels = torch.from_numpy(labels).float()
        normals = torch.from_numpy(normals).float()

        return {'samples_geo': samples, 'labels_geo': labels}

    def visualize_sampling3D(self, data_dict, mode='vis'):

        # create plot
        vp = vedo.Plotter(title="", size=(1500, 1500), axes=0, bg='white')
        vis_list = []

        assert mode in ['vis', 'sdf', 'normal', 'cmap', 'occ']

        # sdf-1 cmap-3 norm-3 vis-1
        if mode == 'vis':
            labels = data_dict[f'smpl_feat'][:, [-1]]  # visibility
            colors = np.concatenate([labels, labels, labels], axis=1)
        elif mode == 'occ':
            labels = data_dict[f'labels_geo'][..., None]  # occupancy
            colors = np.concatenate([labels, labels, labels], axis=1)
        elif mode == 'sdf':
            labels = data_dict[f'smpl_feat'][:, [0]]  # sdf
            labels -= labels.min()
            labels /= labels.max()
            colors = np.concatenate([labels, labels, labels], axis=1)
        elif mode == 'normal':
            labels = data_dict[f'smpl_feat'][:, -4:-1]  # normal
            colors = (labels + 1.0) * 0.5
        elif mode == 'cmap':
            labels = data_dict[f'smpl_feat'][:, -7:-4]  # colormap
            colors = np.array(labels)

        points = projection(data_dict['samples_geo'], data_dict['calib'])
        verts = projection(data_dict['verts'], data_dict['calib'])
        points[:, 1] *= -1
        verts[:, 1] *= -1

        # create a mesh
        mesh = trimesh.Trimesh(verts, data_dict['faces'], process=True)
        mesh.visual.vertex_colors = [128.0, 128.0, 128.0, 255.0]
        vis_list.append(mesh)

        if 'voxel_verts' in data_dict.keys():
            print(colored("voxel verts", "green"))
            voxel_verts = data_dict['voxel_verts'] * 2.0
            voxel_faces = data_dict['voxel_faces']
            voxel_verts[:, 1] *= -1
            voxel = trimesh.Trimesh(
                voxel_verts, voxel_faces[:, [0, 2, 1]], process=False, maintain_order=True)
            voxel.visual.vertex_colors = [0.0, 128.0, 0.0, 255.0]
            vis_list.append(voxel)

        if 'smpl_verts' in data_dict.keys():
            print(colored("smpl verts", "green"))
            smplx_verts = data_dict['smpl_verts']
            smplx_faces = data_dict['smpl_faces']
            smplx_verts[:, 1] *= -1
            smplx = trimesh.Trimesh(
                smplx_verts, smplx_faces[:, [0, 2, 1]], process=False, maintain_order=True)
            smplx.visual.vertex_colors = [128.0, 128.0, 0.0, 255.0]
            vis_list.append(smplx)

        # create a picure
        img_pos = [1.0, 0.0, -1.0]
        for img_id, img_key in enumerate(['normal_F', 'image', 'T_normal_B']):
            image_arr = (data_dict[img_key].detach().cpu().permute(
                1, 2, 0).numpy() + 1.0) * 0.5 * 255.0
            image_dim = image_arr.shape[0]
            image = vedo.Picture(image_arr).scale(
                2.0 / image_dim).pos(-1.0, -1.0, img_pos[img_id])
            vis_list.append(image)

        # create a pointcloud
        pc = vedo.Points(points, r=15, c=np.float32(colors))
        vis_list.append(pc)

        vp.show(*vis_list, bg="white", axes=1.0, interactive=True)