added notebooks

.gitattributes

@@ -34,3 +34,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 *.whl filter=lfs diff=lfs merge=lfs -text
+*.ipynb filter=lfs diff=lfs merge=lfs -text

examine_results.ipynb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a301378de22ca2d5d2798aa02f8380ef75fb75a039d8bcfd7ec64a8f5efe6c0b
+size 3250980

explore.ipynb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8e5020524a5e7fea1d1d97a4a0ed5d2da4fff7ddfd6870a418510018285da0e
+size 86613

handcrafted_solution.py

@@ -146,7 +146,12 @@ def get_vertices(image_gestalt, *, color_range=3.5, dialations=2, erosions=1, ke
     *_, apex_stats, apex_centroids = cv2.connectedComponentsWithStats(apex_mask, connectivity=4, stats=cv2.CV_32S)
     *_, other_stats, other_centroids = cv2.connectedComponentsWithStats(eave_end_point_mask, connectivity=4, stats=cv2.CV_32S)
 
-    return apex_centroids[1:], other_centroids[1:], apex_mask, eave_end_point_mask, apex_stats[1:, cv2.CC_STAT_WIDTH]/2, other_stats[1:, cv2.CC_STAT_WIDTH]/2
+    return (apex_centroids[1:],
+            other_centroids[1:],
+            apex_mask,
+            eave_end_point_mask,
+            np.maximum(apex_stats[1:, cv2.CC_STAT_WIDTH], apex_stats[1:, cv2.CC_STAT_HEIGHT])/2,
+            np.maximum(other_stats[1:, cv2.CC_STAT_WIDTH], other_stats[1:, cv2.CC_STAT_HEIGHT])/2)
 
 
 def infer_vertices(image_gestalt, *, color_range=4.):
@@ -180,14 +185,14 @@ def get_missed_vertices(vertices, inferred_centroids, *, min_missing_distance=20
 
 
 def get_lines_and_directions(gest_seg_np, edge_class, *, color_range=4., rho, theta, threshold, min_line_length,
-                             max_line_gap, extend, **kwargs):
+                             max_line_gap, extend=30, kernel_size=3, dilation_iterations=1, **kwargs):
     edge_color = np.array(gestalt_color_mapping[edge_class])
 
     mask = cv2.inRange(gest_seg_np,
                        edge_color - color_range,
                        edge_color + color_range)
     mask = cv2.morphologyEx(mask,
-                            cv2.MORPH_DILATE, np.ones((3, 3)), iterations=1)
+                            cv2.MORPH_DILATE, np.ones((kernel_size, kernel_size)), iterations=dilation_iterations)
 
     if not np.any(mask):
         return [], []
@@ -203,20 +208,22 @@ def get_lines_and_directions(gest_seg_np, edge_class, *, color_range=4., rho, th
 
     line_directions = []
     edges = []
+
     for line_idx, line in enumerate(lines):
         for x1, y1, x2, y2 in line:
             if x1 < x2:
                 x1, y1, x2, y2 = x2, y2, x1, y1
             direction = (np.array([x2 - x1, y2 - y1]))
             direction = direction / np.linalg.norm(direction)
-            line_directions.append(direction)
 
-            direction = extend * direction
+            for extend_value in range(0, int(extend), 5):
+                new_direction = extend_value * direction
 
-            x1, y1 = -direction + (x1, y1)
-            x2, y2 = + direction + (x2, y2)
+                x1, y1 = -new_direction + (x1, y1)
+                x2, y2 = + new_direction + (x2, y2)
 
-            edges.append((x1, y1, x2, y2))
+                line_directions.append(direction)
+                edges.append((x1, y1, x2, y2))
     return edges, line_directions
 
 
@@ -259,7 +266,7 @@ def get_vertices_and_edges_from_segmentation(gest_seg_np, *,
 
     rho = 1  # distance resolution in pixels of the Hough grid
     theta = np.pi / 180  # angular resolution in radians of the Hough grid
-    threshold = 20  # minimum number of votes (intersections in Hough grid cell)
+    threshold = 40  # minimum number of votes (intersections in Hough grid cell)
     min_line_length = 60  # minimum number of pixels making up a line
     max_line_gap = 40  # maximum gap in pixels between connectable line segments
     ridge_edges, ridge_directions = get_lines_and_directions(gest_seg_np, "ridge",
@@ -268,6 +275,8 @@ def get_vertices_and_edges_from_segmentation(gest_seg_np, *,
                                                              threshold=threshold,
                                                              min_line_length=min_line_length,
                                                              max_line_gap=max_line_gap,
+                                                             kernel_size=3,
+                                                             dilation_iterations=3,
                                                              **kwargs)
 
     rake_edges, rake_directions = get_lines_and_directions(gest_seg_np, "rake",
@@ -295,12 +304,14 @@ def get_vertices_and_edges_from_segmentation(gest_seg_np, *,
         return [], []
 
     vertex_size = np.full(len(vertices), point_radius/2)
-    apex_radii *= point_radius_scale
-    eave_radii *= point_radius_scale
-    apex_radii = np.clip(apex_radii, 10, point_radius)
-    eave_radii = np.clip(eave_radii, 10, point_radius)
-    vertex_size[:len(apex_radii)] = apex_radii
-    vertex_size[len(apex_radii):len(apex_radii) + len(eave_radii)] = eave_radii
+    if len(apex_radii) > 0 and len(eave_radii) > 0:
+        apex_radii *= point_radius_scale
+        eave_radii *= point_radius_scale
+        apex_radii = np.maximum(apex_radii, 10)
+        eave_radii = np.maximum(eave_radii, 10)
+        point_radius = np.max([np.max(apex_radii), np.max(eave_radii)])
+        vertex_size[:len(apex_radii)] = apex_radii
+        vertex_size[len(apex_radii):len(apex_radii) + len(eave_radii)] = eave_radii
 
     # for i, coords in enumerate(vertices):
         # coords = np.round(coords).astype(np.uint32)
@@ -553,7 +564,7 @@ def clean_points3d(entry, clustering_eps):
     
     return points3d_kdtree, biggest_cluster_keys, image_dict
     
-def get_depthmap_from_pointcloud(image, pointcloud, biggest_cluster_keys, R, t):
+def get_depth_from_pointcloud(image, pointcloud, biggest_cluster_keys, R, t):
     belonging_points3d = []
     belonging_points2d = []
     point_indices = np.where(image.point3D_ids != -1)[0]
@@ -582,6 +593,57 @@ def get_depthmap_from_pointcloud(image, pointcloud, biggest_cluster_keys, R, t):
     # projected2d = projected2d[important[0]]
     projected2d = belonging_points2d[important[0]]
     return projected2d, depth
+
+def get_depthmap_from_pointcloud(pointcloud, biggest_cluster_keys, K, R, t, depthmap):
+    belonging_points3d = []
+    for point_id in biggest_cluster_keys:
+        belonging_points3d.append(pointcloud[point_id].xyz)
+    belonging_points3d = np.array(belonging_points3d)
+
+    projected2d, _ = cv2.projectPoints(belonging_points3d, R, t, K, 0)
+    projected2d = projected2d.reshape(-1, 2)
+    important = np.where(np.all(projected2d >= 0, axis=1))[0]
+    # Normalize the uv to the camera intrinsics
+    world_to_cam = np.eye(4)
+    world_to_cam[:3, :3] = R
+    world_to_cam[:3, 3] = t
+
+    homo_belonging_points = cv2.convertPointsToHomogeneous(belonging_points3d)
+    depth = cv2.convertPointsFromHomogeneous(cv2.transform(homo_belonging_points, world_to_cam))
+    depth = depth[:, 0, 2]
+    # projected2d = projected2d[:, 0, :]
+    depth = depth[important]
+    projected2d = projected2d[important]
+
+    projected2d = projected2d.astype(np.int32)
+    projected2d[:, 1] = np.clip(projected2d[:, 1], 0, depthmap.shape[1]-1)
+    projected2d[:, 0] = np.clip(projected2d[:, 0], 0, depthmap.shape[0]-1)
+    pointcloud_depthmap = np.full_like(depthmap, 4000)
+
+    sorted_indices = np.argsort(depth)[::-1]
+    projected2d, depth = projected2d[sorted_indices], depth[sorted_indices]
+    idx = np.searchsorted(depth, 1000)
+    projected2d, depth = projected2d[:idx], depth[:idx]
+    for point, depth_value in zip(projected2d, depth):
+        cv2.circle(pointcloud_depthmap, point, 20, depth_value, -1)
+    return pointcloud_depthmap
+
+def get_mesh_depthmap(mesh, K, R, t, depthmap, fill_value=4000):
+
+    world_to_cam = np.eye(4)
+    world_to_cam[:3, :3] = R
+    world_to_cam[:3, 3] = t
+    camera_transform = np.linalg.inv(world_to_cam)
+    import trimesh
+    camera = trimesh.scene.Camera(focal=(K[0, 0], K[1, 1]), resolution=depthmap.shape[::-1])
+    scene = trimesh.scene.scene.Scene(camera=camera, camera_transform=camera_transform)
+    ray_origins, ray_directions, corresponding_pixels = scene.camera_rays()
+    locations, index_ray, index_tri = mesh.ray.intersects_location(ray_origins, -ray_directions, multiple_hits=False)
+    mesh_depths = np.linalg.norm(ray_origins[index_ray] - locations, axis=1)
+    mesh_depthmap = np.full(camera.resolution, fill_value)
+    mesh_depthmap[corresponding_pixels[index_ray, 0], corresponding_pixels[index_ray, 1]] = mesh_depths
+    mesh_depthmap = cv2.flip(cv2.rotate(mesh_depthmap, cv2.ROTATE_90_COUNTERCLOCKWISE), 1)
+    return mesh_depthmap
 def predict(entry, visualize=False,
             scale_estimation_coefficient=2.5,
             clustering_eps=100,
@@ -597,6 +659,8 @@ def predict(entry, visualize=False,
     vert_edge_per_image = {}
 
     points3d_kdtree, biggest_cluster_keys, image_dict = clean_points3d(entry, clustering_eps)
+    # import trimesh
+    # mesh = trimesh.Trimesh(vertices=entry["mesh_vertices"], faces=entry["mesh_faces"][:,1:], use_embree=True)
 
 
     for i, (gest, depthcm, K, R, t, imagekey) in enumerate(zip(entry['gestalt'],
@@ -615,14 +679,18 @@ def predict(entry, visualize=False,
             print(f'Not enough vertices or connections in image {i}')
             vert_edge_per_image[i] = np.empty((0, 2)), [], np.empty((0, 3))
             continue
+
         depth_np = np.array(depthcm) / scale_estimation_coefficient
+        # depth_np = get_mesh_depthmap(mesh, K, R, t, depth_np, 4000).astype(np.float32)
         # kernel = np.array([[-1,-1,-1], [-1,9,-1], [-1,-1,-1]])
         # depth_np = cv2.filter2D(depth_np, -1, kernel)
         uv, depth_vert_from_depth_map = get_uv_depth(vertices, depth_np)
+
         try:
+            # raise KeyError
             image = image_dict[imagekey]
 
-            projected2d, depth = get_depthmap_from_pointcloud(image, entry["points3d"], biggest_cluster_keys, R, t)
+            projected2d, depth = get_depth_from_pointcloud(image, entry["points3d"], biggest_cluster_keys, R, t)
             if len(depth) < 1:
                 print(f'No 3D points in image {i}')
                 # vert_edge_per_image[i] = np.empty((0, 2)), [], np.empty((0, 3))
@@ -639,6 +707,9 @@ def predict(entry, visualize=False,
 
         except KeyError:
             #Revert to the depthmap
+            # if len(biggest_cluster_keys) > 0:
+                # depth_map = get_depthmap_from_pointcloud(entry["points3d"], biggest_cluster_keys, K, R, t, depth_np)
+                # uv, depth_vert_from_depth_map = get_uv_depth(vertices, depth_map)
             depthmap_used = True
 
         # Normalize the uv to the camera intrinsics
@@ -672,7 +743,7 @@ def predict(entry, visualize=False,
             vertices_3d_local /= norm_factor_max
         else:
             vertices_3d_local[depth_vert_nan_idxs] /= norm_factor_max
-            vertices_3d_local[~np.isin(np.arange(len(vertices_3d_local)), depth_vert_nan_idxs)] /= norm_factor_min
+            vertices_3d_local[~np.isin(np.arange(len(vertices_3d_local)), depth_vert_nan_idxs)] /= norm_factor_max
 
         world_to_cam = np.eye(4)
         world_to_cam[:3, :3] = R
@@ -682,11 +753,11 @@ def predict(entry, visualize=False,
         vertices_3d = cv2.transform(cv2.convertPointsToHomogeneous(vertices_3d_local), cam_to_world)
         vertices_3d = cv2.convertPointsFromHomogeneous(vertices_3d).reshape(-1, 3)
 
-        if not depthmap_used:
-            not_nan_items = np.all(~np.isnan(vertices_3d), axis=1)
-            _, closest_fitted = points3d_kdtree.query(vertices_3d[not_nan_items])
-
-            vertices_3d[not_nan_items] = points3d_kdtree.data[closest_fitted]
+        # if not depthmap_used:
+        #     not_nan_items = np.all(~np.isnan(vertices_3d), axis=1)
+        #     _, closest_fitted = points3d_kdtree.query(vertices_3d[not_nan_items])
+        #
+        #     vertices_3d[not_nan_items] = points3d_kdtree.data[closest_fitted]
 
         vert_edge_per_image[i] = vertices, connections, vertices_3d
     all_3d_vertices, connections_3d = merge_vertices_3d(vert_edge_per_image, **kwargs)

pointclouds.ipynb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:965401b68d1da3152141a478322bc4ebfc4d4b4b7a16b3cffefcd7c0b5e0e915
+size 151533905

seek_top_level_roofs.ipynb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e0d1b2dd8a7fcb8683520c0aa8890ec6b9788e669627468efe8a45a7a3ec160
+size 8948456