dist_coeff=0.1

handcrafted_solution.py

@@ -13,7 +13,6 @@ from hoho.color_mappings import gestalt_color_mapping
 from hoho.read_write_colmap import read_cameras_binary, read_images_binary, read_points3D_binary
 from scipy.spatial import KDTree
 from scipy.spatial.distance import cdist
-import scipy.cluster.hierarchy as shc
 from sklearn.cluster import DBSCAN
 
 apex_color = gestalt_color_mapping["apex"]
@@ -257,7 +256,7 @@ def get_vertices_and_edges_from_segmentation(gest_seg_np, *, point_radius=30, ma
     for edge_class in ['eave',
                        'step_flashing',
                        'flashing',
-                       'post',
+                       # 'post',
                        'valley',
                        'hip',
                        'transition_line']:
@@ -358,15 +357,40 @@ def merge_vertices_3d(vert_edge_per_image, merge_th=0.1, **kwargs):
     '''Merge vertices that are close to each other in 3D space and are of same types'''
     all_3d_vertices = []
     connections_3d = []
-    all_indexes = []
     cur_start = 0
     types = []
+
     for cimg_idx, (vertices, connections, vertices_3d) in vert_edge_per_image.items():
+        # remove nan values and remap the connections
+        connections = [[a, b]
+                       for (a, b) in connections
+                       if
+                       not np.any(np.isnan(vertices_3d[a]))
+                       and
+                       not np.any(np.isnan(vertices_3d[b]))
+                       ]
+        left_vertex_indices = np.where(np.all(~np.isnan(vertices_3d), axis=1))[0]
+
+        new_indices = np.arange(len(left_vertex_indices))
+
+        new_vertex_mapping = dict(zip(left_vertex_indices, new_indices))
+
+        vertices = [v for i, v in enumerate(vertices) if i in new_vertex_mapping]
         types += [int(v['type'] == 'apex') for v in vertices]
+        vertices_3d = vertices_3d[left_vertex_indices]
+        connections = [[new_vertex_mapping[a] + cur_start, new_vertex_mapping[b] + cur_start] for a, b in connections]
+
+
+
         all_3d_vertices.append(vertices_3d)
-        connections_3d += [(x + cur_start, y + cur_start) for (x, y) in connections]
+        connections_3d += connections
         cur_start += len(vertices_3d)
+
     all_3d_vertices = np.concatenate(all_3d_vertices, axis=0)
+
+    # dbscan = DBSCAN(eps=merge_th, min_samples=1).fit(all_3d_vertices)
+    # print(dbscan.core_sample_indices_)
+    # print(dbscan.labels_[dbscan.core_sample_indices_])
     # print (connections_3d)
     distmat = cdist(all_3d_vertices, all_3d_vertices)
     types = np.array(types).reshape(-1, 1)
@@ -430,7 +454,8 @@ def prune_not_connected(all_3d_vertices, connections_3d):
     return np.array(new_verts), connected_out
 
 
-def predict(entry, visualize=False, scale_estimation_coefficient=2.5, clustering_eps=100, **kwargs) -> Tuple[np.ndarray, List[int]]:
+def predict(entry, visualize=False, scale_estimation_coefficient=2.5, clustering_eps=100, dist_coeff=0, **kwargs) -> Tuple[
+    np.ndarray, List[int]]:
     if 'gestalt' not in entry or 'depthcm' not in entry or 'K' not in entry or 'R' not in entry or 't' not in entry:
         print('Missing required fields in the entry')
         return (entry['__key__'], *empty_solution())
@@ -474,23 +499,23 @@ def predict(entry, visualize=False, scale_estimation_coefficient=2.5, clustering
                                                                )):
 
         try:
-            gest_seg = gest.resize(depthcm.size)
-            gest_seg_np = np.array(gest_seg).astype(np.uint8)
+            # gest_seg = gest.resize(depthcm.size)
+            gest_seg_np = np.array(gest).astype(np.uint8)
             vertices, connections = get_vertices_and_edges_from_segmentation(gest_seg_np, **kwargs)
             if (len(vertices) < 2) or (len(connections) < 1):
                 print(f'Not enough vertices or connections in image {i}')
                 vert_edge_per_image[i] = np.empty((0, 2)), [], np.empty((0, 3))
                 continue
             belonging_points = []
-            for i in image_dict[imagekey].point3D_ids[np.where(image_dict[imagekey].point3D_ids != -1)]:
-                if i in biggest_cluster_keys:
-                    belonging_points.append(entry["points3d"][i])
+            for point_id in image_dict[imagekey].point3D_ids[np.where(image_dict[imagekey].point3D_ids != -1)]:
+                if point_id in biggest_cluster_keys:
+                    belonging_points.append(entry["points3d"][point_id])
 
             if len(belonging_points) < 1:
                 print(f'No 3D points in image {i}')
                 vert_edge_per_image[i] = np.empty((0, 2)), [], np.empty((0, 3))
                 raise KeyError
-            projected2d, _ = cv2.projectPoints(np.array([i.xyz for i in belonging_points]), R, t, K, 0)
+            projected2d, _ = cv2.projectPoints(np.array([i.xyz for i in belonging_points]), R, t, K, dist_coeff)
             important = np.where(np.all(projected2d >= 0, axis=2))
             # Normalize the uv to the camera intrinsics
             world_to_cam = np.eye(4)
@@ -499,7 +524,7 @@ def predict(entry, visualize=False, scale_estimation_coefficient=2.5, clustering
 
             homo_belonging_points = cv2.convertPointsToHomogeneous(np.array([i.xyz for i in belonging_points]))
             depth = cv2.convertPointsFromHomogeneous(cv2.transform(homo_belonging_points, world_to_cam))
-            depth = np.array([i[0][2] for i in depth])
+            depth = depth[:, 0, 2]
             depth = depth[important[0]]
             projected2d = projected2d[important]
             if len(depth) < 1:
@@ -508,7 +533,8 @@ def predict(entry, visualize=False, scale_estimation_coefficient=2.5, clustering
                 raise KeyError
             # print(projected2d.shape, depth.shape)
 
-            interpolator = si.NearestNDInterpolator(projected2d, depth)
+            interpolator = si.NearestNDInterpolator(projected2d, depth, rescale=True)
+            # interpolator = si.CloughTocher2DInterpolator(projected2d, depth, np.nan)
 
             vertex_coordinates = np.array([v['xy'] for v in vertices])
             xi, yi = vertex_coordinates[:, 0], vertex_coordinates[:, 1]
@@ -530,6 +556,9 @@ def predict(entry, visualize=False, scale_estimation_coefficient=2.5, clustering
             gest_seg_np = np.array(gest_seg).astype(np.uint8)
             # Metric3D
             depth_np = np.array(depthcm) / scale_estimation_coefficient
+            cv2.GaussianBlur(depth_np, (21, 21), 1, depth_np)
+            # cv2.medianBlur(depth_np, 5)
+            # depth_np = np.zeros_like(depth_np)
             vertices, connections = get_vertices_and_edges_from_segmentation(gest_seg_np, **kwargs)
             if (len(vertices) < 2) or (len(connections) < 1):
                 print(f'Not enough vertices or connections in image {i}')

script.py

@@ -138,6 +138,7 @@ if __name__ == "__main__":
                                        min_missing_distance=350.0,
                                        scale_estimation_coefficient=2.54,
                                        clustering_eps=100,
+                                       dist_coeff=0.1,
                                        ))
         
         for i, result in enumerate(tqdm(results)):