# Copyright 2024 Anton Obukhov, ETH Zurich. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # -------------------------------------------------------------------------- # If you find this code useful, we kindly ask you to cite our paper in your work. # Please find bibtex at: https://github.com/prs-eth/Marigold#-citation # More information about the method can be found at https://marigoldmonodepth.github.io # -------------------------------------------------------------------------- import math import os import zipfile import numpy as np import pygltflib import trimesh from PIL import Image from scipy.ndimage import median_filter def quaternion_multiply(q1, q2): x1, y1, z1, w1 = q1 x2, y2, z2, w2 = q2 return [ w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2, w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2, w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2, w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2, ] def glb_add_lights(path_input, path_output): """ Adds directional lights in the horizontal plane to the glb file. :param path_input: path to input glb :param path_output: path to output glb :return: None """ glb = pygltflib.GLTF2().load(path_input) N = 3 # default max num lights in Babylon.js is 4 angle_step = 2 * math.pi / N elevation_angle = math.radians(75) light_colors = [ [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], ] lights_extension = { "lights": [ {"type": "directional", "color": light_colors[i], "intensity": 2.0} for i in range(N) ] } if "KHR_lights_punctual" not in glb.extensionsUsed: glb.extensionsUsed.append("KHR_lights_punctual") glb.extensions["KHR_lights_punctual"] = lights_extension light_nodes = [] for i in range(N): angle = i * angle_step pos_rot = [0.0, 0.0, math.sin(angle / 2), math.cos(angle / 2)] elev_rot = [ math.sin(elevation_angle / 2), 0.0, 0.0, math.cos(elevation_angle / 2), ] rotation = quaternion_multiply(pos_rot, elev_rot) node = { "rotation": rotation, "extensions": {"KHR_lights_punctual": {"light": i}}, } light_nodes.append(node) light_node_indices = list(range(len(glb.nodes), len(glb.nodes) + N)) glb.nodes.extend(light_nodes) root_node_index = glb.scenes[glb.scene].nodes[0] root_node = glb.nodes[root_node_index] if hasattr(root_node, "children"): root_node.children.extend(light_node_indices) else: root_node.children = light_node_indices glb.save(path_output) def extrude_depth_3d( path_rgb, path_depth, path_out_base=None, output_model_scale=100, filter_size=3, coef_near=0.0, coef_far=1.0, emboss=0.3, f_thic=0.05, f_near=-0.15, f_back=0.01, vertex_colors=True, scene_lights=True, prepare_for_3d_printing=False, zip_outputs=False, ): f_far_inner = -emboss f_far_outer = f_far_inner - f_back f_near = max(f_near, f_far_inner) depth_image = Image.open(path_depth) w, h = depth_image.size d_max = max(w, h) depth_image = np.array(depth_image).astype(np.double) depth_image = median_filter(depth_image, size=filter_size) z_min, z_max = np.min(depth_image), np.max(depth_image) depth_image = (depth_image.astype(np.double) - z_min) / (z_max - z_min) depth_image[depth_image < coef_near] = coef_near depth_image[depth_image > coef_far] = coef_far depth_image = emboss * (depth_image - coef_near) / (coef_far - coef_near) rgb_image = np.array( Image.open(path_rgb).convert("RGB").resize((w, h), Image.Resampling.LANCZOS) ) w_norm = w / float(d_max - 1) h_norm = h / float(d_max - 1) w_half = w_norm / 2 h_half = h_norm / 2 x, y = np.meshgrid(np.arange(w), np.arange(h)) x = x / float(d_max - 1) - w_half # [-w_half, w_half] y = -y / float(d_max - 1) + h_half # [-h_half, h_half] z = -depth_image # -depth_emboss (far) - 0 (near) vertices_2d = np.stack((x, y, z), axis=-1) vertices = vertices_2d.reshape(-1, 3) colors = rgb_image[:, :, :3].reshape(-1, 3) / 255.0 faces = [] for y in range(h - 1): for x in range(w - 1): idx = y * w + x faces.append([idx, idx + w, idx + 1]) faces.append([idx + 1, idx + w, idx + 1 + w]) # OUTER frame nv = len(vertices) vertices = np.append( vertices, [ [-w_half - f_thic, -h_half - f_thic, f_near], # 00 [-w_half - f_thic, -h_half - f_thic, f_far_outer], # 01 [w_half + f_thic, -h_half - f_thic, f_near], # 02 [w_half + f_thic, -h_half - f_thic, f_far_outer], # 03 [w_half + f_thic, h_half + f_thic, f_near], # 04 [w_half + f_thic, h_half + f_thic, f_far_outer], # 05 [-w_half - f_thic, h_half + f_thic, f_near], # 06 [-w_half - f_thic, h_half + f_thic, f_far_outer], # 07 ], axis=0, ) faces.extend( [ [nv + 0, nv + 1, nv + 2], [nv + 2, nv + 1, nv + 3], [nv + 2, nv + 3, nv + 4], [nv + 4, nv + 3, nv + 5], [nv + 4, nv + 5, nv + 6], [nv + 6, nv + 5, nv + 7], [nv + 6, nv + 7, nv + 0], [nv + 0, nv + 7, nv + 1], ] ) colors = np.append(colors, [[0.5, 0.5, 0.5]] * 8, axis=0) # INNER frame nv = len(vertices) vertices_left_data = vertices_2d[:, 0] # H x 3 vertices_left_frame = vertices_2d[:, 0].copy() # H x 3 vertices_left_frame[:, 2] = f_near vertices = np.append(vertices, vertices_left_data, axis=0) vertices = np.append(vertices, vertices_left_frame, axis=0) colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 * h), axis=0) for i in range(h - 1): nvi_d = nv + i nvi_f = nvi_d + h faces.append([nvi_d, nvi_f, nvi_d + 1]) faces.append([nvi_d + 1, nvi_f, nvi_f + 1]) nv = len(vertices) vertices_right_data = vertices_2d[:, -1] # H x 3 vertices_right_frame = vertices_2d[:, -1].copy() # H x 3 vertices_right_frame[:, 2] = f_near vertices = np.append(vertices, vertices_right_data, axis=0) vertices = np.append(vertices, vertices_right_frame, axis=0) colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 * h), axis=0) for i in range(h - 1): nvi_d = nv + i nvi_f = nvi_d + h faces.append([nvi_d, nvi_d + 1, nvi_f]) faces.append([nvi_d + 1, nvi_f + 1, nvi_f]) nv = len(vertices) vertices_top_data = vertices_2d[0, :] # H x 3 vertices_top_frame = vertices_2d[0, :].copy() # H x 3 vertices_top_frame[:, 2] = f_near vertices = np.append(vertices, vertices_top_data, axis=0) vertices = np.append(vertices, vertices_top_frame, axis=0) colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 * w), axis=0) for i in range(w - 1): nvi_d = nv + i nvi_f = nvi_d + w faces.append([nvi_d, nvi_d + 1, nvi_f]) faces.append([nvi_d + 1, nvi_f + 1, nvi_f]) nv = len(vertices) vertices_bottom_data = vertices_2d[-1, :] # H x 3 vertices_bottom_frame = vertices_2d[-1, :].copy() # H x 3 vertices_bottom_frame[:, 2] = f_near vertices = np.append(vertices, vertices_bottom_data, axis=0) vertices = np.append(vertices, vertices_bottom_frame, axis=0) colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 * w), axis=0) for i in range(w - 1): nvi_d = nv + i nvi_f = nvi_d + w faces.append([nvi_d, nvi_f, nvi_d + 1]) faces.append([nvi_d + 1, nvi_f, nvi_f + 1]) # FRONT frame nv = len(vertices) vertices = np.append( vertices, [ [-w_half - f_thic, -h_half - f_thic, f_near], [-w_half - f_thic, h_half + f_thic, f_near], ], axis=0, ) vertices = np.append(vertices, vertices_left_frame, axis=0) colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 + h), axis=0) for i in range(h - 1): faces.append([nv, nv + 2 + i + 1, nv + 2 + i]) faces.append([nv, nv + 2, nv + 1]) nv = len(vertices) vertices = np.append( vertices, [ [w_half + f_thic, h_half + f_thic, f_near], [w_half + f_thic, -h_half - f_thic, f_near], ], axis=0, ) vertices = np.append(vertices, vertices_right_frame, axis=0) colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 + h), axis=0) for i in range(h - 1): faces.append([nv, nv + 2 + i, nv + 2 + i + 1]) faces.append([nv, nv + h + 1, nv + 1]) nv = len(vertices) vertices = np.append( vertices, [ [w_half + f_thic, h_half + f_thic, f_near], [-w_half - f_thic, h_half + f_thic, f_near], ], axis=0, ) vertices = np.append(vertices, vertices_top_frame, axis=0) colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 + w), axis=0) for i in range(w - 1): faces.append([nv, nv + 2 + i, nv + 2 + i + 1]) faces.append([nv, nv + 1, nv + 2]) nv = len(vertices) vertices = np.append( vertices, [ [-w_half - f_thic, -h_half - f_thic, f_near], [w_half + f_thic, -h_half - f_thic, f_near], ], axis=0, ) vertices = np.append(vertices, vertices_bottom_frame, axis=0) colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 + w), axis=0) for i in range(w - 1): faces.append([nv, nv + 2 + i + 1, nv + 2 + i]) faces.append([nv, nv + 1, nv + w + 1]) # BACK frame nv = len(vertices) vertices = np.append( vertices, [ [-w_half - f_thic, -h_half - f_thic, f_far_outer], # 00 [w_half + f_thic, -h_half - f_thic, f_far_outer], # 01 [w_half + f_thic, h_half + f_thic, f_far_outer], # 02 [-w_half - f_thic, h_half + f_thic, f_far_outer], # 03 ], axis=0, ) faces.extend( [ [nv + 0, nv + 2, nv + 1], [nv + 2, nv + 0, nv + 3], ] ) colors = np.append(colors, [[0.5, 0.5, 0.5]] * 4, axis=0) trimesh_kwargs = {} if vertex_colors: trimesh_kwargs["vertex_colors"] = colors mesh = trimesh.Trimesh(vertices=vertices, faces=faces, **trimesh_kwargs) mesh.merge_vertices() current_max_dimension = max(mesh.extents) scaling_factor = output_model_scale / current_max_dimension mesh.apply_scale(scaling_factor) if prepare_for_3d_printing: rotation_mat = trimesh.transformations.rotation_matrix( np.radians(90), [-1, 0, 0] ) mesh.apply_transform(rotation_mat) if path_out_base is None: path_out_base = os.path.splitext(path_depth)[0].replace("_16bit", "") path_out_glb = path_out_base + ".glb" path_out_stl = path_out_base + ".stl" path_out_obj = path_out_base + ".obj" mesh.export(path_out_glb, file_type="glb") if scene_lights: glb_add_lights(path_out_glb, path_out_glb) mesh.export(path_out_stl, file_type="stl") mesh.export(path_out_obj, file_type="obj") if zip_outputs: with zipfile.ZipFile(path_out_glb + ".zip", "w", zipfile.ZIP_DEFLATED) as zipf: arcname = os.path.basename(os.path.splitext(path_out_glb)[0]) + ".glb" zipf.write(path_out_glb, arcname=arcname) path_out_glb = path_out_glb + ".zip" with zipfile.ZipFile(path_out_stl + ".zip", "w", zipfile.ZIP_DEFLATED) as zipf: arcname = os.path.basename(os.path.splitext(path_out_stl)[0]) + ".stl" zipf.write(path_out_stl, arcname=arcname) path_out_stl = path_out_stl + ".zip" with zipfile.ZipFile(path_out_obj + ".zip", "w", zipfile.ZIP_DEFLATED) as zipf: arcname = os.path.basename(os.path.splitext(path_out_obj)[0]) + ".obj" zipf.write(path_out_obj, arcname=arcname) path_out_obj = path_out_obj + ".zip" return path_out_glb, path_out_stl, path_out_obj if __name__ == "__main__": img_rgb = "files/basrelief/einstein.jpg" img_depth = "gradio_cached_examples/examples_image/Depth outputs/54d74157894322bdc77c/einstein_depth_16bit.png" Image.open(img_rgb).resize((512, 512), Image.LANCZOS).save( "einstein_3d_tex_512.jpg" ) Image.open(img_depth).convert(mode="F").resize((512, 512), Image.BILINEAR).convert( "I" ).save("einstein_3d_depth_512.png") extrude_depth_3d( "einstein_3d_tex_512.jpg", "einstein_3d_depth_512.png", path_out_base="einstein_3d_out", output_model_scale=100, filter_size=3, coef_near=0.0, coef_far=0.5, emboss=0.5, f_thic=0.05, f_near=-0.25, f_back=0.01, vertex_colors=True, scene_lights=True, prepare_for_3d_printing=True, )