import einops import matplotlib.cm as cm import numba import numpy as np import plotly.graph_objects as go import torch import torch.nn.functional as F classes = { 0: "unlabeled", 1: "car", 2: "bicycle", 3: "motorcycle", 4: "truck", 5: "other-vehicle", 6: "person", 7: "bicyclist", 8: "motorcyclist", 9: "road", 10: "parking", 11: "sidewalk", 12: "other-ground", 13: "building", 14: "fence", 15: "vegetation", 16: "trunk", 17: "terrain", 18: "pole", 19: "traffic-sign", } def get_hdl64e_linear_ray_angles( H: int = 64, W: int = 2048, device: torch.device = "cpu" ): h_up, h_down = 3, -25 w_left, w_right = 180, -180 elevation = 1 - torch.arange(H, device=device) / H # [0, 1] elevation = elevation * (h_up - h_down) + h_down # [-25, 3] azimuth = 1 - torch.arange(W, device=device) / W # [0, 1] azimuth = azimuth * (w_left - w_right) + w_right # [-180, 180] [elevation, azimuth] = torch.meshgrid([elevation, azimuth], indexing="ij") angles = torch.stack([elevation, azimuth])[None].deg2rad() return angles def make_semantickitti_cmap(): from matplotlib.colors import ListedColormap label_colors = { 0: [0, 0, 0], 1: [245, 150, 100], 2: [245, 230, 100], 3: [150, 60, 30], 4: [180, 30, 80], 5: [255, 0, 0], 6: [30, 30, 255], 7: [200, 40, 255], 8: [90, 30, 150], 9: [255, 0, 255], 10: [255, 150, 255], 11: [75, 0, 75], 12: [75, 0, 175], 13: [0, 200, 255], 14: [50, 120, 255], 15: [0, 175, 0], 16: [0, 60, 135], 17: [80, 240, 150], 18: [150, 240, 255], 19: [0, 0, 255], } num_classes = max(label_colors.keys()) + 1 label_colormap = np.zeros((num_classes, 3), dtype=np.uint8) for label_id, color in label_colors.items(): label_colormap[label_id] = color[::-1] # BGR -> RGB cmap = ListedColormap(label_colormap / 255.0) return cmap def colorize(tensor, cmap_fn=cm.turbo): colors = cmap_fn(np.linspace(0, 1, 256))[:, :3] colors = torch.from_numpy(colors).to(tensor) tensor = tensor.squeeze(1) if tensor.ndim == 4 else tensor ids = (tensor * 256).clamp(0, 255).long() tensor = F.embedding(ids, colors).permute(0, 3, 1, 2) tensor = tensor.mul(255).clamp(0, 255).byte() return tensor @numba.jit(nopython=True, parallel=False) def scatter(array, index, value): for (h, w), v in zip(index, value): array[h, w] = v return array def load_points_as_images( point_cloud_obj, scan_unfolding: bool = True, H: int = 64, W: int = 2048, min_depth: float = 1.45, max_depth: float = 80.0, ): # load xyz & intensity and add depth & mask points = np.frombuffer(point_cloud_obj, dtype=np.float32).reshape((-1, 4)) xyz = points[:, :3] # xyz x = xyz[:, [0]] y = xyz[:, [1]] z = xyz[:, [2]] depth = np.linalg.norm(xyz, ord=2, axis=1, keepdims=True) mask = (depth >= min_depth) & (depth <= max_depth) points = np.concatenate([points, depth, mask], axis=1) if scan_unfolding: # the i-th quadrant # suppose the points are ordered counterclockwise quads = np.zeros_like(x, dtype=np.int32) quads[(x >= 0) & (y >= 0)] = 0 # 1st quads[(x < 0) & (y >= 0)] = 1 # 2nd quads[(x < 0) & (y < 0)] = 2 # 3rd quads[(x >= 0) & (y < 0)] = 3 # 4th # split between the 3rd and 1st quadrants diff = np.roll(quads, shift=1, axis=0) - quads delim_inds, _ = np.where(diff == 3) # number of lines inds = list(delim_inds) + [len(points)] # add the last index # vertical grid grid_h = np.zeros_like(x, dtype=np.int32) cur_ring_idx = H - 1 # ...0 for i in reversed(range(len(delim_inds))): grid_h[inds[i] : inds[i + 1]] = cur_ring_idx if cur_ring_idx >= 0: cur_ring_idx -= 1 else: break else: h_up, h_down = np.deg2rad(3), np.deg2rad(-25) elevation = np.arcsin(z / depth) + abs(h_down) grid_h = 1 - elevation / (h_up - h_down) grid_h = np.floor(grid_h * H).clip(0, H - 1).astype(np.int32) # horizontal grid azimuth = -np.arctan2(y, x) # [-pi,pi] grid_w = (azimuth / np.pi + 1) / 2 % 1 # [0,1] grid_w = np.floor(grid_w * W).clip(0, W - 1).astype(np.int32) grid = np.concatenate((grid_h, grid_w), axis=1) # projection order = np.argsort(-depth.squeeze(1)) proj_points = np.zeros((H, W, 4 + 2), dtype=points.dtype) proj_points = scatter(proj_points, grid[order], points[order]) return proj_points.astype(np.float32) def to_xyz(depth, angles, min_depth, max_depth): phi = angles[:, [0]] theta = angles[:, [1]] grid_x = depth * phi.cos() * theta.cos() grid_y = depth * phi.cos() * theta.sin() grid_z = depth * phi.sin() xyz = torch.cat((grid_x, grid_y, grid_z), dim=1) xyz = xyz * (depth > 0).float() return xyz def render_point_cloud(depth, colors=None, labels=None): _, _, H, W = depth.shape angles = get_hdl64e_linear_ray_angles(H, W, device=depth.device) points = to_xyz(depth, angles, min_depth=1.45, max_depth=80.0) points = einops.rearrange(points, "1 c h w -> c (h w)") if colors is None: marker = dict( size=1, color=points[2], colorscale="viridis", autocolorscale=False, cauto=False, cmin=-2, cmax=0.5, ) else: colors = einops.rearrange(colors, "1 c h w -> (h w) c") marker = dict(size=1, color=colors) fig = go.Figure( data=[ go.Scatter3d( x=-points[0], y=-points[1], z=points[2], mode="markers", marker=marker, text=labels, hoverinfo="text", ) ], layout=dict( scene=dict( xaxis=dict(showticklabels=False, visible=False), yaxis=dict(showticklabels=False, visible=False), zaxis=dict(showticklabels=False, visible=False), aspectmode="data", ), margin=dict(l=0, r=0, b=0, t=0), paper_bgcolor="rgba(0,0,0,0)", plot_bgcolor="rgba(0,0,0,0)", ), ) return fig