Spaces:
Running
Running
File size: 3,911 Bytes
10dcc2e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 |
'''
COTR two view reconstruction with known extrinsic/intrinsic demo
'''
import argparse
import os
import time
import numpy as np
import torch
import imageio
import open3d as o3d
from COTR.utils import utils, debug_utils
from COTR.models import build_model
from COTR.options.options import *
from COTR.options.options_utils import *
from COTR.inference.sparse_engine import SparseEngine, FasterSparseEngine
from COTR.projector import pcd_projector
utils.fix_randomness(0)
torch.set_grad_enabled(False)
def triangulate_rays_to_pcd(center_a, dir_a, center_b, dir_b):
A = center_a
a = dir_a / np.linalg.norm(dir_a, axis=1, keepdims=True)
B = center_b
b = dir_b / np.linalg.norm(dir_b, axis=1, keepdims=True)
c = B - A
D = A + a * ((-np.sum(a * b, axis=1) * np.sum(b * c, axis=1) + np.sum(a * c, axis=1) * np.sum(b * b, axis=1)) / (np.sum(a * a, axis=1) * np.sum(b * b, axis=1) - np.sum(a * b, axis=1) * np.sum(a * b, axis=1)))[..., None]
return D
def main(opt):
model = build_model(opt)
model = model.cuda()
weights = torch.load(opt.load_weights_path, map_location='cpu')['model_state_dict']
utils.safe_load_weights(model, weights)
model = model.eval()
img_a = imageio.imread('./sample_data/imgs/img_0.jpg', pilmode='RGB')
img_b = imageio.imread('./sample_data/imgs/img_1.jpg', pilmode='RGB')
if opt.faster_infer:
engine = FasterSparseEngine(model, 32, mode='tile')
else:
engine = SparseEngine(model, 32, mode='tile')
t0 = time.time()
corrs = engine.cotr_corr_multiscale_with_cycle_consistency(img_a, img_b, np.linspace(0.5, 0.0625, 4), 1, max_corrs=opt.max_corrs, queries_a=None)
t1 = time.time()
print(f'spent {t1-t0} seconds for {opt.max_corrs} correspondences.')
camera_a = np.load('./sample_data/camera_0.npy', allow_pickle=True).item()
camera_b = np.load('./sample_data/camera_1.npy', allow_pickle=True).item()
center_a = camera_a['cam_center']
center_b = camera_b['cam_center']
rays_a = pcd_projector.PointCloudProjector.pcd_2d_to_pcd_3d_np(corrs[:, :2], np.ones([corrs.shape[0], 1]) * 2, camera_a['intrinsic'], motion=camera_a['c2w'])
rays_b = pcd_projector.PointCloudProjector.pcd_2d_to_pcd_3d_np(corrs[:, 2:], np.ones([corrs.shape[0], 1]) * 2, camera_b['intrinsic'], motion=camera_b['c2w'])
dir_a = rays_a - center_a
dir_b = rays_b - center_b
center_a = np.array([center_a] * corrs.shape[0])
center_b = np.array([center_b] * corrs.shape[0])
points = triangulate_rays_to_pcd(center_a, dir_a, center_b, dir_b)
colors = (img_a[tuple(np.floor(corrs[:, :2]).astype(int)[:, ::-1].T)] / 255 + img_b[tuple(np.floor(corrs[:, 2:]).astype(int)[:, ::-1].T)] / 255) / 2
colors = np.array(colors)
pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(points)
pcd.colors = o3d.utility.Vector3dVector(colors)
o3d.visualization.draw_geometries([pcd])
if __name__ == "__main__":
parser = argparse.ArgumentParser()
set_COTR_arguments(parser)
parser.add_argument('--out_dir', type=str, default=general_config['out'], help='out directory')
parser.add_argument('--load_weights', type=str, default=None, help='load a pretrained set of weights, you need to provide the model id')
parser.add_argument('--max_corrs', type=int, default=2048, help='number of correspondences')
parser.add_argument('--faster_infer', type=str2bool, default=False, help='use fatser inference')
opt = parser.parse_args()
opt.command = ' '.join(sys.argv)
layer_2_channels = {'layer1': 256,
'layer2': 512,
'layer3': 1024,
'layer4': 2048, }
opt.dim_feedforward = layer_2_channels[opt.layer]
if opt.load_weights:
opt.load_weights_path = os.path.join(opt.out_dir, opt.load_weights, 'checkpoint.pth.tar')
print_opt(opt)
main(opt)
|