clean unnecessary libs

README.md

@@ -14,7 +14,7 @@
     ·
     <a href="https://ps.is.tuebingen.mpg.de/person/black"><strong>Michael J. Black</strong></a>
   </p>
-  <h2 align="center">CVPR 2023</h2>
+  <h2 align="center">CVPR 2023 (Highlight)</h2>
   <div align="center">
     <img src="./assets/teaser.gif" alt="Logo" width="100%">
   </div>
@@ -48,7 +48,7 @@ ECON is designed for "Human digitization from a color image", which combines the
 
 ## News :triangular_flag_on_post:
 
-- [2023/02/27] ECON got accepted by CVPR 2023!
+- [2023/02/27] ECON got accepted by CVPR 2023 as Highlight (top 10%)!
 - [2023/01/12] [Carlos Barreto](https://twitter.com/carlosedubarret/status/1613252471035494403) creates a Blender Addon ([Download](https://carlosedubarreto.gumroad.com/l/CEB_ECON), [Tutorial](https://youtu.be/sbWZbTf6ZYk)).
 - [2023/01/08] [Teddy Huang](https://github.com/Teddy12155555) creates [install-with-docker](docs/installation-docker.md) for ECON .
 - [2023/01/06] [Justin John](https://github.com/justinjohn0306) and [Carlos Barreto](https://github.com/carlosedubarreto) creates [install-on-windows](docs/installation-windows.md) for ECON .
@@ -60,6 +60,28 @@ ECON is designed for "Human digitization from a color image", which combines the
 - [ ] Blender add-on for FBX export
 - [ ] Full RGB texture generation
 
+## Key idea: d-BiNI
+
+d-BiNI jointly optimizes front-back 2.5D surfaces such that: (1) high-frequency surface details agree with normal maps, (2) low-frequency surface variations, including discontinuities, align with SMPL-X surfaces, and (3) front-back 2.5D surface silhouettes are coherent with each other.
+
+|Front-view|Back-view|Side-view|
+|:--:|:--:|:---:|
+|![](assets/front-45.gif)|![](assets/back-45.gif)|![](assets/double-90.gif)||
+
+<details><summary>Please consider cite <strong>BiNI</strong> if it also helps on your project</summary>
+
+```bibtex
+@inproceedings{cao2022bilateral,
+  title={Bilateral normal integration},
+  author={Cao, Xu and Santo, Hiroaki and Shi, Boxin and Okura, Fumio and Matsushita, Yasuyuki},
+  booktitle={Computer Vision--ECCV 2022: 17th European Conference, Tel Aviv, Israel, October 23--27, 2022, Proceedings, Part I},
+  pages={552--567},
+  year={2022},
+  organization={Springer}
+}
+```
+</details>
+
 <br>
 
 <!-- TABLE OF CONTENTS -->

configs/econ.yaml

@@ -35,5 +35,5 @@ bni:
   face_thres: 6e-2
   thickness: 0.02
   hps_type: "pixie"
-  texture_src: "SD"
+  texture_src: "image"
   cut_intersection: True

docs/installation-ubuntu.md

@@ -10,11 +10,11 @@ cd ECON
 ## Environment
 
 - Ubuntu 20 / 18, (Windows as well, see [issue#7](https://github.com/YuliangXiu/ECON/issues/7))
-- **CUDA=11.4, GPU Memory > 12GB**
+- **CUDA=11.6, GPU Memory > 12GB**
 - Python = 3.8
 - PyTorch >= 1.13.0 (official [Get Started](https://pytorch.org/get-started/locally/))
 - Cupy >= 11.3.0 (offcial [Installation](https://docs.cupy.dev/en/stable/install.html#installing-cupy-from-pypi))
-- PyTorch3D (official [INSTALL.md](https://github.com/facebookresearch/pytorch3d/blob/main/INSTALL.md), recommend [install-from-local-clone](https://github.com/facebookresearch/pytorch3d/blob/main/INSTALL.md#2-install-from-a-local-clone))
+- PyTorch3D = 0.7.1 (official [INSTALL.md](https://github.com/facebookresearch/pytorch3d/blob/main/INSTALL.md), recommend [install-from-local-clone](https://github.com/facebookresearch/pytorch3d/blob/main/INSTALL.md#2-install-from-a-local-clone))
 
 ```bash
 
@@ -26,6 +26,9 @@ conda env create -f environment.yaml
 conda activate econ
 pip install -r requirements.txt
 
+# the installation(incl. compilation) of PyTorch3D will take ~20min
+pip install git+https://github.com/facebookresearch/pytorch3d.git@v0.7.1
+
 # install libmesh & libvoxelize
 cd lib/common/libmesh
 python setup.py build_ext --inplace

docs/installation-windows.md

@@ -31,11 +31,11 @@ cd ECON
 ## Environment
 
 - Windows 10 / 11
-- **CUDA=11.4**
+- **CUDA=11.3**
 - Python = 3.8
 - PyTorch >= 1.12.1 (official [Get Started](https://pytorch.org/get-started/locally/))
 - Cupy >= 11.3.0 (offcial [Installation](https://docs.cupy.dev/en/stable/install.html#installing-cupy-from-pypi))
-- PyTorch3D (official [INSTALL.md](https://github.com/facebookresearch/pytorch3d/blob/main/INSTALL.md), recommend [install-from-local-clone](https://github.com/facebookresearch/pytorch3d/blob/main/INSTALL.md#2-install-from-a-local-clone))
+- PyTorch3D = 0.7.1 (official [INSTALL.md](https://github.com/facebookresearch/pytorch3d/blob/main/INSTALL.md), recommend [install-from-local-clone](https://github.com/facebookresearch/pytorch3d/blob/main/INSTALL.md#2-install-from-a-local-clone))
 
 ```bash
 # install required packages
@@ -44,18 +44,10 @@ conda env create -f environment-windows.yaml
 conda activate econ
 
 # install pytorch and cupy
-pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1 --extra-index-url https://download.pytorch.org/whl/cu113
-pip install -r requirements-win.txt
+pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 --extra-index-url https://download.pytorch.org/whl/cu113
+pip install -r requirements.txt
 pip install cupy-cuda11x
-
-## If you have a RTX 30 series GPU then run this cmd below for installing neural_voxelization_layer
-pip install git+https://github.com/YuliangXiu/neural_voxelization_layer.git
-## If you have GPU below RTX 30 series then you gotta build neural_voxelization_layer (steps below)
-git clone https://github.com/justinjohn0306/neural_voxelization_layer.git
-
-cd neural_voxelization_layer
-python setup install
-cd..
+pip install git+https://github.com/facebookresearch/pytorch3d.git@v0.7.1
 
 # install libmesh & libvoxelize
 cd lib/common/libmesh

environment-windows.yaml

@@ -1,6 +1,7 @@
 name: econ
 channels:
   - nvidia
+  - pytorch
   - conda-forge
   - fvcore
   - iopath
@@ -8,6 +9,7 @@ channels:
   - defaults
 dependencies:
   - python=3.8
+  - pytorch-cuda=11.3
   - fvcore
   - iopath
   - cupy

environment.yaml

@@ -9,11 +9,12 @@ channels:
   - defaults
 dependencies:
   - python=3.8
-  - pytorch
+  - pytorch-cuda=11.7
+  - pytorch=1.13.0
+  - nvidiacub
   - torchvision
   - fvcore
   - iopath
-  - nvidiacub
   - pyembree
   - cupy
   - cython

lib/dataset/EvalDataset.py

@@ -239,13 +239,10 @@ class EvalDataset:
         mesh_path = data_dict["mesh_path"]
         scale = data_dict["scale"]
 
-        # tinyobjloader has wired bug
         scan_mesh = trimesh.load(mesh_path)
         verts = scan_mesh.vertices
         faces = scan_mesh.faces
 
-        # verts, faces = obj_loader(mesh_path, with_uv=False)
-
         mesh = HoppeMesh(verts * scale, faces)
 
         return {

lib/dataset/TestDataset.py

@@ -144,47 +144,6 @@ class TestDataset:
             ]).unsqueeze(0).to(self.device),
         }
 
-    def compute_voxel_verts(self, body_pose, global_orient, betas, trans, scale):
-
-        smpl_path = osp.join(self.smpl_data.model_dir, "smpl/SMPL_NEUTRAL.pkl")
-        tetra_path = osp.join(self.smpl_data.tedra_dir, "tetra_neutral_adult_smpl.npz")
-        smpl_model = TetraSMPLModel(smpl_path, tetra_path, "adult")
-
-        pose = torch.cat([global_orient[0], body_pose[0]], dim=0)
-        smpl_model.set_params(rotation_matrix_to_angle_axis(rot6d_to_rotmat(pose)), beta=betas[0])
-
-        verts = (
-            np.concatenate([smpl_model.verts, smpl_model.verts_added], axis=0) * scale.item() +
-            trans.detach().cpu().numpy()
-        )
-        faces = (
-            np.loadtxt(
-                osp.join(self.smpl_data.tedra_dir, "tetrahedrons_neutral_adult.txt"),
-                dtype=np.int32,
-            ) - 1
-        )
-
-        pad_v_num = int(8000 - verts.shape[0])
-        pad_f_num = int(25100 - faces.shape[0])
-
-        verts = (
-            np.pad(verts, ((0, pad_v_num),
-                           (0, 0)), mode="constant", constant_values=0.0).astype(np.float32) * 0.5
-        )
-        faces = np.pad(faces, ((0, pad_f_num), (0, 0)), mode="constant",
-                       constant_values=0.0).astype(np.int32)
-
-        verts[:, 2] *= -1.0
-
-        voxel_dict = {
-            "voxel_verts": torch.from_numpy(verts).to(self.device).unsqueeze(0).float(),
-            "voxel_faces": torch.from_numpy(faces).to(self.device).unsqueeze(0).long(),
-            "pad_v_num": torch.tensor(pad_v_num).to(self.device).unsqueeze(0).long(),
-            "pad_f_num": torch.tensor(pad_f_num).to(self.device).unsqueeze(0).long(),
-        }
-
-        return voxel_dict
-
     def __getitem__(self, index):
 
         img_path = self.subject_list[index]

lib/dataset/mesh_util.py

@@ -21,7 +21,6 @@ import torchvision
 import trimesh
 import json
 import open3d as o3d
-import tinyobjloader
 import os.path as osp
 import _pickle as cPickle
 from termcolor import colored
@@ -110,7 +109,6 @@ class SMPLX:
         self.smplx_to_smpl = cPickle.load(open(self.smplx_to_smplx_path, "rb"))
 
         self.model_dir = osp.join(self.current_dir, "models")
-        self.tedra_dir = osp.join(self.current_dir, "../tedra_data")
 
         self.ghum_smpl_pairs = torch.tensor(
             [
@@ -288,33 +286,6 @@ def part_removal(full_mesh, part_mesh, thres, device, smpl_obj, region, clean=Tr
     return full_mesh
 
 
-def obj_loader(path, with_uv=True):
-    # Create reader.
-    reader = tinyobjloader.ObjReader()
-
-    # Load .obj(and .mtl) using default configuration
-    ret = reader.ParseFromFile(path)
-
-    # note here for wavefront obj, #v might not equal to #vt, same as #vn.
-    attrib = reader.GetAttrib()
-    v = np.array(attrib.vertices).reshape(-1, 3)
-    vt = np.array(attrib.texcoords).reshape(-1, 2)
-
-    shapes = reader.GetShapes()
-    tri = shapes[0].mesh.numpy_indices().reshape(-1, 9)
-    f_v = tri[:, [0, 3, 6]]
-    f_vt = tri[:, [2, 5, 8]]
-
-    if with_uv:
-        face_uvs = vt[f_vt].mean(axis=1)    #[m, 2]
-        vert_uvs = np.zeros((v.shape[0], 2), dtype=np.float32)    #[n, 2]
-        vert_uvs[f_v.reshape(-1)] = vt[f_vt.reshape(-1)]
-
-        return v, f_v, vert_uvs, face_uvs
-    else:
-        return v, f_v
-
-
 class HoppeMesh:
     def __init__(self, verts, faces, uvs=None, texture=None):
         """

lib/net/IFGeoNet.py

@@ -2,9 +2,7 @@ from pickle import TRUE
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-from lib.net.voxelize import Voxelization
 from lib.net.geometry import orthogonal
-from lib.dataset.mesh_util import read_smpl_constants, SMPLX
 
 
 class SelfAttention(torch.nn.Module):
@@ -88,31 +86,11 @@ class IFGeoNet(nn.Module):
         self.conv3_1_bn = nn.InstanceNorm3d(128)
         self.conv4_1_bn = nn.InstanceNorm3d(128)
 
-        self.smplx = SMPLX()
-        voxel_param = read_smpl_constants(self.smplx.tedra_dir)
-
-        self.voxelization = Voxelization(
-            torch.ones_like(voxel_param["smpl_vertex_code"]),
-            torch.ones_like(voxel_param["smpl_face_code"]),
-            voxel_param["smpl_faces"],
-            voxel_param["smpl_tetras"],
-            volume_res=cfg.dataset.voxel_res,
-            sigma=0.05,
-            smooth_kernel_size=7,
-            batch_size=cfg.batch_size,
-        )
-
         self.l1_loss = nn.SmoothL1Loss()
 
     def forward(self, batch):
 
-        if "body_voxels" in batch.keys():
-            x_smpl = batch["body_voxels"]
-        else:
-            with torch.no_grad():
-                self.voxelization.update_param(batch["voxel_faces"])
-                x_smpl = self.voxelization(batch["voxel_verts"])[:, 0]    #[B, 128, 128, 128]
-
+        x_smpl = batch["body_voxels"]
         p = orthogonal(batch["samples_geo"].permute(0, 2, 1),
                        batch["calib"]).permute(0, 2, 1)    #[2, 60000, 3]
         x = batch["depth_voxels"]    #[B, 128, 128, 128]

lib/net/IFGeoNet_nobody.py

@@ -2,9 +2,7 @@ from pickle import TRUE
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-from lib.net.voxelize import Voxelization
 from lib.net.geometry import orthogonal
-from lib.dataset.mesh_util import read_smpl_constants, SMPLX
 
 
 class SelfAttention(torch.nn.Module):
@@ -82,20 +80,6 @@ class IFGeoNet(nn.Module):
         self.conv3_1_bn = nn.InstanceNorm3d(128)
         self.conv4_1_bn = nn.InstanceNorm3d(128)
 
-        self.smplx = SMPLX()
-        voxel_param = read_smpl_constants(self.smplx.tedra_dir)
-
-        self.voxelization = Voxelization(
-            torch.ones_like(voxel_param["smpl_vertex_code"]),
-            torch.ones_like(voxel_param["smpl_face_code"]),
-            voxel_param["smpl_faces"],
-            voxel_param["smpl_tetras"],
-            volume_res=cfg.dataset.voxel_res,
-            sigma=0.05,
-            smooth_kernel_size=7,
-            batch_size=cfg.batch_size,
-        )
-
         self.l1_loss = nn.SmoothL1Loss()
 
     def forward(self, batch):

requirements-win.txt

@@ -1,19 +0,0 @@
-matplotlib
-scikit-image
-trimesh
-rtree
-pytorch_lightning
-kornia>0.4.0
-chumpy
-opencv-python
-opencv_contrib_python
-scikit-learn
-protobuf
-dataclasses
-mediapipe
-einops
-boto3
-open3d
-tinyobjloader==2.0.0rc7
-git+https://github.com/facebookresearch/pytorch3d.git
-git+https://github.com/YuliangXiu/rembg.git

requirements.txt

@@ -14,7 +14,4 @@ mediapipe
 einops
 boto3
 open3d
-tinyobjloader==2.0.0rc7
-git+https://github.com/facebookresearch/pytorch3d.git
-git+https://github.com/YuliangXiu/neural_voxelization_layer.git
 git+https://github.com/YuliangXiu/rembg.git