Spaces:
Runtime error
Runtime error
"""PBR renderer for Python. | |
Author: Matthew Matl | |
""" | |
import sys | |
import numpy as np | |
import PIL | |
from .constants import (RenderFlags, TextAlign, GLTF, BufFlags, TexFlags, | |
ProgramFlags, DEFAULT_Z_FAR, DEFAULT_Z_NEAR, | |
SHADOW_TEX_SZ, MAX_N_LIGHTS) | |
from .shader_program import ShaderProgramCache | |
from .material import MetallicRoughnessMaterial, SpecularGlossinessMaterial | |
from .light import PointLight, SpotLight, DirectionalLight | |
from .font import FontCache | |
from .utils import format_color_vector | |
from OpenGL.GL import * | |
class Renderer(object): | |
"""Class for handling all rendering operations on a scene. | |
Note | |
---- | |
This renderer relies on the existence of an OpenGL context and | |
does not create one on its own. | |
Parameters | |
---------- | |
viewport_width : int | |
Width of the viewport in pixels. | |
viewport_height : int | |
Width of the viewport height in pixels. | |
point_size : float, optional | |
Size of points in pixels. Defaults to 1.0. | |
""" | |
def __init__(self, viewport_width, viewport_height, point_size=1.0): | |
self.dpscale = 1 | |
# Scaling needed on retina displays | |
if sys.platform == 'darwin': | |
self.dpscale = 2 | |
self.viewport_width = viewport_width | |
self.viewport_height = viewport_height | |
self.point_size = point_size | |
# Optional framebuffer for offscreen renders | |
self._main_fb = None | |
self._main_cb = None | |
self._main_db = None | |
self._main_fb_ms = None | |
self._main_cb_ms = None | |
self._main_db_ms = None | |
self._main_fb_dims = (None, None) | |
self._shadow_fb = None | |
self._latest_znear = DEFAULT_Z_NEAR | |
self._latest_zfar = DEFAULT_Z_FAR | |
# Shader Program Cache | |
self._program_cache = ShaderProgramCache() | |
self._font_cache = FontCache() | |
self._meshes = set() | |
self._mesh_textures = set() | |
self._shadow_textures = set() | |
self._texture_alloc_idx = 0 | |
def viewport_width(self): | |
"""int : The width of the main viewport, in pixels. | |
""" | |
return self._viewport_width | |
def viewport_width(self, value): | |
self._viewport_width = self.dpscale * value | |
def viewport_height(self): | |
"""int : The height of the main viewport, in pixels. | |
""" | |
return self._viewport_height | |
def viewport_height(self, value): | |
self._viewport_height = self.dpscale * value | |
def point_size(self): | |
"""float : The size of screen-space points, in pixels. | |
""" | |
return self._point_size | |
def point_size(self, value): | |
self._point_size = float(value) | |
def render(self, scene, flags, seg_node_map=None): | |
"""Render a scene with the given set of flags. | |
Parameters | |
---------- | |
scene : :class:`Scene` | |
A scene to render. | |
flags : int | |
A specification from :class:`.RenderFlags`. | |
seg_node_map : dict | |
A map from :class:`.Node` objects to (3,) colors for each. | |
If specified along with flags set to :attr:`.RenderFlags.SEG`, | |
the color image will be a segmentation image. | |
Returns | |
------- | |
color_im : (h, w, 3) uint8 or (h, w, 4) uint8 | |
If :attr:`RenderFlags.OFFSCREEN` is set, the color buffer. This is | |
normally an RGB buffer, but if :attr:`.RenderFlags.RGBA` is set, | |
the buffer will be a full RGBA buffer. | |
depth_im : (h, w) float32 | |
If :attr:`RenderFlags.OFFSCREEN` is set, the depth buffer | |
in linear units. | |
""" | |
# Update context with meshes and textures | |
self._update_context(scene, flags) | |
# Render necessary shadow maps | |
if not bool(flags & RenderFlags.DEPTH_ONLY or flags & RenderFlags.SEG): | |
for ln in scene.light_nodes: | |
take_pass = False | |
if (isinstance(ln.light, DirectionalLight) and | |
bool(flags & RenderFlags.SHADOWS_DIRECTIONAL)): | |
take_pass = True | |
elif (isinstance(ln.light, SpotLight) and | |
bool(flags & RenderFlags.SHADOWS_SPOT)): | |
take_pass = True | |
elif (isinstance(ln.light, PointLight) and | |
bool(flags & RenderFlags.SHADOWS_POINT)): | |
take_pass = True | |
if take_pass: | |
self._shadow_mapping_pass(scene, ln, flags) | |
# Make forward pass | |
retval = self._forward_pass(scene, flags, seg_node_map=seg_node_map) | |
# If necessary, make normals pass | |
if flags & (RenderFlags.VERTEX_NORMALS | RenderFlags.FACE_NORMALS): | |
self._normals_pass(scene, flags) | |
# Update camera settings for retrieving depth buffers | |
self._latest_znear = scene.main_camera_node.camera.znear | |
self._latest_zfar = scene.main_camera_node.camera.zfar | |
return retval | |
def render_text(self, text, x, y, font_name='OpenSans-Regular', | |
font_pt=40, color=None, scale=1.0, | |
align=TextAlign.BOTTOM_LEFT): | |
"""Render text into the current viewport. | |
Note | |
---- | |
This cannot be done into an offscreen buffer. | |
Parameters | |
---------- | |
text : str | |
The text to render. | |
x : int | |
Horizontal pixel location of text. | |
y : int | |
Vertical pixel location of text. | |
font_name : str | |
Name of font, from the ``pyrender/fonts`` folder, or | |
a path to a ``.ttf`` file. | |
font_pt : int | |
Height of the text, in font points. | |
color : (4,) float | |
The color of the text. Default is black. | |
scale : int | |
Scaling factor for text. | |
align : int | |
One of the :class:`TextAlign` options which specifies where the | |
``x`` and ``y`` parameters lie on the text. For example, | |
:attr:`TextAlign.BOTTOM_LEFT` means that ``x`` and ``y`` indicate | |
the position of the bottom-left corner of the textbox. | |
""" | |
x *= self.dpscale | |
y *= self.dpscale | |
font_pt *= self.dpscale | |
if color is None: | |
color = np.array([0.0, 0.0, 0.0, 1.0]) | |
else: | |
color = format_color_vector(color, 4) | |
# Set up viewport for render | |
self._configure_forward_pass_viewport(0) | |
# Load font | |
font = self._font_cache.get_font(font_name, font_pt) | |
if not font._in_context(): | |
font._add_to_context() | |
# Load program | |
program = self._get_text_program() | |
program._bind() | |
# Set uniforms | |
p = np.eye(4) | |
p[0,0] = 2.0 / self.viewport_width | |
p[0,3] = -1.0 | |
p[1,1] = 2.0 / self.viewport_height | |
p[1,3] = -1.0 | |
program.set_uniform('projection', p) | |
program.set_uniform('text_color', color) | |
# Draw text | |
font.render_string(text, x, y, scale, align) | |
def read_color_buf(self): | |
"""Read and return the current viewport's color buffer. | |
Alpha cannot be computed for an on-screen buffer. | |
Returns | |
------- | |
color_im : (h, w, 3) uint8 | |
The color buffer in RGB byte format. | |
""" | |
# Extract color image from frame buffer | |
width, height = self.viewport_width, self.viewport_height | |
glBindFramebuffer(GL_READ_FRAMEBUFFER, 0) | |
glReadBuffer(GL_FRONT) | |
color_buf = glReadPixels(0, 0, width, height, GL_RGB, GL_UNSIGNED_BYTE) | |
# Re-format them into numpy arrays | |
color_im = np.frombuffer(color_buf, dtype=np.uint8) | |
color_im = color_im.reshape((height, width, 3)) | |
color_im = np.flip(color_im, axis=0) | |
# Resize for macos if needed | |
if sys.platform == 'darwin': | |
color_im = self._resize_image(color_im, True) | |
return color_im | |
def read_depth_buf(self): | |
"""Read and return the current viewport's color buffer. | |
Returns | |
------- | |
depth_im : (h, w) float32 | |
The depth buffer in linear units. | |
""" | |
width, height = self.viewport_width, self.viewport_height | |
glBindFramebuffer(GL_READ_FRAMEBUFFER, 0) | |
glReadBuffer(GL_FRONT) | |
depth_buf = glReadPixels( | |
0, 0, width, height, GL_DEPTH_COMPONENT, GL_FLOAT | |
) | |
depth_im = np.frombuffer(depth_buf, dtype=np.float32) | |
depth_im = depth_im.reshape((height, width)) | |
depth_im = np.flip(depth_im, axis=0) | |
inf_inds = (depth_im == 1.0) | |
depth_im = 2.0 * depth_im - 1.0 | |
z_near, z_far = self._latest_znear, self._latest_zfar | |
noninf = np.logical_not(inf_inds) | |
if z_far is None: | |
depth_im[noninf] = 2 * z_near / (1.0 - depth_im[noninf]) | |
else: | |
depth_im[noninf] = ((2.0 * z_near * z_far) / | |
(z_far + z_near - depth_im[noninf] * | |
(z_far - z_near))) | |
depth_im[inf_inds] = 0.0 | |
# Resize for macos if needed | |
if sys.platform == 'darwin': | |
depth_im = self._resize_image(depth_im) | |
return depth_im | |
def delete(self): | |
"""Free all allocated OpenGL resources. | |
""" | |
# Free shaders | |
self._program_cache.clear() | |
# Free fonts | |
self._font_cache.clear() | |
# Free meshes | |
for mesh in self._meshes: | |
for p in mesh.primitives: | |
p.delete() | |
# Free textures | |
for mesh_texture in self._mesh_textures: | |
mesh_texture.delete() | |
for shadow_texture in self._shadow_textures: | |
shadow_texture.delete() | |
self._meshes = set() | |
self._mesh_textures = set() | |
self._shadow_textures = set() | |
self._texture_alloc_idx = 0 | |
self._delete_main_framebuffer() | |
self._delete_shadow_framebuffer() | |
def __del__(self): | |
try: | |
self.delete() | |
except Exception: | |
pass | |
########################################################################### | |
# Rendering passes | |
########################################################################### | |
def _forward_pass(self, scene, flags, seg_node_map=None): | |
# Set up viewport for render | |
self._configure_forward_pass_viewport(flags) | |
# Clear it | |
if bool(flags & RenderFlags.SEG): | |
glClearColor(0.0, 0.0, 0.0, 1.0) | |
if seg_node_map is None: | |
seg_node_map = {} | |
else: | |
glClearColor(*scene.bg_color) | |
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) | |
if not bool(flags & RenderFlags.SEG): | |
glEnable(GL_MULTISAMPLE) | |
else: | |
glDisable(GL_MULTISAMPLE) | |
# Set up camera matrices | |
V, P = self._get_camera_matrices(scene) | |
program = None | |
# Now, render each object in sorted order | |
for node in self._sorted_mesh_nodes(scene): | |
mesh = node.mesh | |
# Skip the mesh if it's not visible | |
if not mesh.is_visible: | |
continue | |
# If SEG, set color | |
if bool(flags & RenderFlags.SEG): | |
if node not in seg_node_map: | |
continue | |
color = seg_node_map[node] | |
if not isinstance(color, (list, tuple, np.ndarray)): | |
color = np.repeat(color, 3) | |
else: | |
color = np.asanyarray(color) | |
color = color / 255.0 | |
for primitive in mesh.primitives: | |
# First, get and bind the appropriate program | |
program = self._get_primitive_program( | |
primitive, flags, ProgramFlags.USE_MATERIAL | |
) | |
program._bind() | |
# Set the camera uniforms | |
program.set_uniform('V', V) | |
program.set_uniform('P', P) | |
program.set_uniform( | |
'cam_pos', scene.get_pose(scene.main_camera_node)[:3,3] | |
) | |
if bool(flags & RenderFlags.SEG): | |
program.set_uniform('color', color) | |
# Next, bind the lighting | |
if not (flags & RenderFlags.DEPTH_ONLY or flags & RenderFlags.FLAT or | |
flags & RenderFlags.SEG): | |
self._bind_lighting(scene, program, node, flags) | |
# Finally, bind and draw the primitive | |
self._bind_and_draw_primitive( | |
primitive=primitive, | |
pose=scene.get_pose(node), | |
program=program, | |
flags=flags | |
) | |
self._reset_active_textures() | |
# Unbind the shader and flush the output | |
if program is not None: | |
program._unbind() | |
glFlush() | |
# If doing offscreen render, copy result from framebuffer and return | |
if flags & RenderFlags.OFFSCREEN: | |
return self._read_main_framebuffer(scene, flags) | |
else: | |
return | |
def _shadow_mapping_pass(self, scene, light_node, flags): | |
light = light_node.light | |
# Set up viewport for render | |
self._configure_shadow_mapping_viewport(light, flags) | |
# Set up camera matrices | |
V, P = self._get_light_cam_matrices(scene, light_node, flags) | |
# Now, render each object in sorted order | |
for node in self._sorted_mesh_nodes(scene): | |
mesh = node.mesh | |
# Skip the mesh if it's not visible | |
if not mesh.is_visible: | |
continue | |
for primitive in mesh.primitives: | |
# First, get and bind the appropriate program | |
program = self._get_primitive_program( | |
primitive, flags, ProgramFlags.NONE | |
) | |
program._bind() | |
# Set the camera uniforms | |
program.set_uniform('V', V) | |
program.set_uniform('P', P) | |
program.set_uniform( | |
'cam_pos', scene.get_pose(scene.main_camera_node)[:3,3] | |
) | |
# Finally, bind and draw the primitive | |
self._bind_and_draw_primitive( | |
primitive=primitive, | |
pose=scene.get_pose(node), | |
program=program, | |
flags=RenderFlags.DEPTH_ONLY | |
) | |
self._reset_active_textures() | |
# Unbind the shader and flush the output | |
if program is not None: | |
program._unbind() | |
glFlush() | |
def _normals_pass(self, scene, flags): | |
# Set up viewport for render | |
self._configure_forward_pass_viewport(flags) | |
program = None | |
# Set up camera matrices | |
V, P = self._get_camera_matrices(scene) | |
# Now, render each object in sorted order | |
for node in self._sorted_mesh_nodes(scene): | |
mesh = node.mesh | |
# Skip the mesh if it's not visible | |
if not mesh.is_visible: | |
continue | |
for primitive in mesh.primitives: | |
# Skip objects that don't have normals | |
if not primitive.buf_flags & BufFlags.NORMAL: | |
continue | |
# First, get and bind the appropriate program | |
pf = ProgramFlags.NONE | |
if flags & RenderFlags.VERTEX_NORMALS: | |
pf = pf | ProgramFlags.VERTEX_NORMALS | |
if flags & RenderFlags.FACE_NORMALS: | |
pf = pf | ProgramFlags.FACE_NORMALS | |
program = self._get_primitive_program(primitive, flags, pf) | |
program._bind() | |
# Set the camera uniforms | |
program.set_uniform('V', V) | |
program.set_uniform('P', P) | |
program.set_uniform('normal_magnitude', 0.05 * primitive.scale) | |
program.set_uniform( | |
'normal_color', np.array([0.1, 0.1, 1.0, 1.0]) | |
) | |
# Finally, bind and draw the primitive | |
self._bind_and_draw_primitive( | |
primitive=primitive, | |
pose=scene.get_pose(node), | |
program=program, | |
flags=RenderFlags.DEPTH_ONLY | |
) | |
self._reset_active_textures() | |
# Unbind the shader and flush the output | |
if program is not None: | |
program._unbind() | |
glFlush() | |
########################################################################### | |
# Handlers for binding uniforms and drawing primitives | |
########################################################################### | |
def _bind_and_draw_primitive(self, primitive, pose, program, flags): | |
# Set model pose matrix | |
program.set_uniform('M', pose) | |
# Bind mesh buffers | |
primitive._bind() | |
# Bind mesh material | |
if not (flags & RenderFlags.DEPTH_ONLY or flags & RenderFlags.SEG): | |
material = primitive.material | |
# Bind textures | |
tf = material.tex_flags | |
if tf & TexFlags.NORMAL: | |
self._bind_texture(material.normalTexture, | |
'material.normal_texture', program) | |
if tf & TexFlags.OCCLUSION: | |
self._bind_texture(material.occlusionTexture, | |
'material.occlusion_texture', program) | |
if tf & TexFlags.EMISSIVE: | |
self._bind_texture(material.emissiveTexture, | |
'material.emissive_texture', program) | |
if tf & TexFlags.BASE_COLOR: | |
self._bind_texture(material.baseColorTexture, | |
'material.base_color_texture', program) | |
if tf & TexFlags.METALLIC_ROUGHNESS: | |
self._bind_texture(material.metallicRoughnessTexture, | |
'material.metallic_roughness_texture', | |
program) | |
if tf & TexFlags.DIFFUSE: | |
self._bind_texture(material.diffuseTexture, | |
'material.diffuse_texture', program) | |
if tf & TexFlags.SPECULAR_GLOSSINESS: | |
self._bind_texture(material.specularGlossinessTexture, | |
'material.specular_glossiness_texture', | |
program) | |
# Bind other uniforms | |
b = 'material.{}' | |
program.set_uniform(b.format('emissive_factor'), | |
material.emissiveFactor) | |
if isinstance(material, MetallicRoughnessMaterial): | |
program.set_uniform(b.format('base_color_factor'), | |
material.baseColorFactor) | |
program.set_uniform(b.format('metallic_factor'), | |
material.metallicFactor) | |
program.set_uniform(b.format('roughness_factor'), | |
material.roughnessFactor) | |
elif isinstance(material, SpecularGlossinessMaterial): | |
program.set_uniform(b.format('diffuse_factor'), | |
material.diffuseFactor) | |
program.set_uniform(b.format('specular_factor'), | |
material.specularFactor) | |
program.set_uniform(b.format('glossiness_factor'), | |
material.glossinessFactor) | |
# Set blending options | |
if material.alphaMode == 'BLEND': | |
glEnable(GL_BLEND) | |
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA) | |
else: | |
glEnable(GL_BLEND) | |
glBlendFunc(GL_ONE, GL_ZERO) | |
# Set wireframe mode | |
wf = material.wireframe | |
if flags & RenderFlags.FLIP_WIREFRAME: | |
wf = not wf | |
if (flags & RenderFlags.ALL_WIREFRAME) or wf: | |
glPolygonMode(GL_FRONT_AND_BACK, GL_LINE) | |
else: | |
glPolygonMode(GL_FRONT_AND_BACK, GL_FILL) | |
# Set culling mode | |
if material.doubleSided or flags & RenderFlags.SKIP_CULL_FACES: | |
glDisable(GL_CULL_FACE) | |
else: | |
glEnable(GL_CULL_FACE) | |
glCullFace(GL_BACK) | |
else: | |
glEnable(GL_CULL_FACE) | |
glEnable(GL_BLEND) | |
glCullFace(GL_BACK) | |
glBlendFunc(GL_ONE, GL_ZERO) | |
glPolygonMode(GL_FRONT_AND_BACK, GL_FILL) | |
# Set point size if needed | |
glDisable(GL_PROGRAM_POINT_SIZE) | |
if primitive.mode == GLTF.POINTS: | |
glEnable(GL_PROGRAM_POINT_SIZE) | |
glPointSize(self.point_size) | |
# Render mesh | |
n_instances = 1 | |
if primitive.poses is not None: | |
n_instances = len(primitive.poses) | |
if primitive.indices is not None: | |
glDrawElementsInstanced( | |
primitive.mode, primitive.indices.size, GL_UNSIGNED_INT, | |
ctypes.c_void_p(0), n_instances | |
) | |
else: | |
glDrawArraysInstanced( | |
primitive.mode, 0, len(primitive.positions), n_instances | |
) | |
# Unbind mesh buffers | |
primitive._unbind() | |
def _bind_lighting(self, scene, program, node, flags): | |
"""Bind all lighting uniform values for a scene. | |
""" | |
max_n_lights = self._compute_max_n_lights(flags) | |
n_d = min(len(scene.directional_light_nodes), max_n_lights[0]) | |
n_s = min(len(scene.spot_light_nodes), max_n_lights[1]) | |
n_p = min(len(scene.point_light_nodes), max_n_lights[2]) | |
program.set_uniform('ambient_light', scene.ambient_light) | |
program.set_uniform('n_directional_lights', n_d) | |
program.set_uniform('n_spot_lights', n_s) | |
program.set_uniform('n_point_lights', n_p) | |
plc = 0 | |
slc = 0 | |
dlc = 0 | |
light_nodes = scene.light_nodes | |
if (len(scene.directional_light_nodes) > max_n_lights[0] or | |
len(scene.spot_light_nodes) > max_n_lights[1] or | |
len(scene.point_light_nodes) > max_n_lights[2]): | |
light_nodes = self._sorted_nodes_by_distance( | |
scene, scene.light_nodes, node | |
) | |
for n in light_nodes: | |
light = n.light | |
pose = scene.get_pose(n) | |
position = pose[:3,3] | |
direction = -pose[:3,2] | |
if isinstance(light, PointLight): | |
if plc == max_n_lights[2]: | |
continue | |
b = 'point_lights[{}].'.format(plc) | |
plc += 1 | |
shadow = bool(flags & RenderFlags.SHADOWS_POINT) | |
program.set_uniform(b + 'position', position) | |
elif isinstance(light, SpotLight): | |
if slc == max_n_lights[1]: | |
continue | |
b = 'spot_lights[{}].'.format(slc) | |
slc += 1 | |
shadow = bool(flags & RenderFlags.SHADOWS_SPOT) | |
las = 1.0 / max(0.001, np.cos(light.innerConeAngle) - | |
np.cos(light.outerConeAngle)) | |
lao = -np.cos(light.outerConeAngle) * las | |
program.set_uniform(b + 'direction', direction) | |
program.set_uniform(b + 'position', position) | |
program.set_uniform(b + 'light_angle_scale', las) | |
program.set_uniform(b + 'light_angle_offset', lao) | |
else: | |
if dlc == max_n_lights[0]: | |
continue | |
b = 'directional_lights[{}].'.format(dlc) | |
dlc += 1 | |
shadow = bool(flags & RenderFlags.SHADOWS_DIRECTIONAL) | |
program.set_uniform(b + 'direction', direction) | |
program.set_uniform(b + 'color', light.color) | |
program.set_uniform(b + 'intensity', light.intensity) | |
# if light.range is not None: | |
# program.set_uniform(b + 'range', light.range) | |
# else: | |
# program.set_uniform(b + 'range', 0) | |
if shadow: | |
self._bind_texture(light.shadow_texture, | |
b + 'shadow_map', program) | |
if not isinstance(light, PointLight): | |
V, P = self._get_light_cam_matrices(scene, n, flags) | |
program.set_uniform(b + 'light_matrix', P.dot(V)) | |
else: | |
raise NotImplementedError( | |
'Point light shadows not implemented' | |
) | |
def _sorted_mesh_nodes(self, scene): | |
cam_loc = scene.get_pose(scene.main_camera_node)[:3,3] | |
solid_nodes = [] | |
trans_nodes = [] | |
for node in scene.mesh_nodes: | |
mesh = node.mesh | |
if mesh.is_transparent: | |
trans_nodes.append(node) | |
else: | |
solid_nodes.append(node) | |
# TODO BETTER SORTING METHOD | |
trans_nodes.sort( | |
key=lambda n: -np.linalg.norm(scene.get_pose(n)[:3,3] - cam_loc) | |
) | |
solid_nodes.sort( | |
key=lambda n: -np.linalg.norm(scene.get_pose(n)[:3,3] - cam_loc) | |
) | |
return solid_nodes + trans_nodes | |
def _sorted_nodes_by_distance(self, scene, nodes, compare_node): | |
nodes = list(nodes) | |
compare_posn = scene.get_pose(compare_node)[:3,3] | |
nodes.sort(key=lambda n: np.linalg.norm( | |
scene.get_pose(n)[:3,3] - compare_posn) | |
) | |
return nodes | |
########################################################################### | |
# Context Management | |
########################################################################### | |
def _update_context(self, scene, flags): | |
# Update meshes | |
scene_meshes = scene.meshes | |
# Add new meshes to context | |
for mesh in scene_meshes - self._meshes: | |
for p in mesh.primitives: | |
p._add_to_context() | |
# Remove old meshes from context | |
for mesh in self._meshes - scene_meshes: | |
for p in mesh.primitives: | |
p.delete() | |
self._meshes = scene_meshes.copy() | |
# Update mesh textures | |
mesh_textures = set() | |
for m in scene_meshes: | |
for p in m.primitives: | |
mesh_textures |= p.material.textures | |
# Add new textures to context | |
for texture in mesh_textures - self._mesh_textures: | |
texture._add_to_context() | |
# Remove old textures from context | |
for texture in self._mesh_textures - mesh_textures: | |
texture.delete() | |
self._mesh_textures = mesh_textures.copy() | |
shadow_textures = set() | |
for l in scene.lights: | |
# Create if needed | |
active = False | |
if (isinstance(l, DirectionalLight) and | |
flags & RenderFlags.SHADOWS_DIRECTIONAL): | |
active = True | |
elif (isinstance(l, PointLight) and | |
flags & RenderFlags.SHADOWS_POINT): | |
active = True | |
elif isinstance(l, SpotLight) and flags & RenderFlags.SHADOWS_SPOT: | |
active = True | |
if active and l.shadow_texture is None: | |
l._generate_shadow_texture() | |
if l.shadow_texture is not None: | |
shadow_textures.add(l.shadow_texture) | |
# Add new textures to context | |
for texture in shadow_textures - self._shadow_textures: | |
texture._add_to_context() | |
# Remove old textures from context | |
for texture in self._shadow_textures - shadow_textures: | |
texture.delete() | |
self._shadow_textures = shadow_textures.copy() | |
########################################################################### | |
# Texture Management | |
########################################################################### | |
def _bind_texture(self, texture, uniform_name, program): | |
"""Bind a texture to an active texture unit and return | |
the texture unit index that was used. | |
""" | |
tex_id = self._get_next_active_texture() | |
glActiveTexture(GL_TEXTURE0 + tex_id) | |
texture._bind() | |
program.set_uniform(uniform_name, tex_id) | |
def _get_next_active_texture(self): | |
val = self._texture_alloc_idx | |
self._texture_alloc_idx += 1 | |
return val | |
def _reset_active_textures(self): | |
self._texture_alloc_idx = 0 | |
########################################################################### | |
# Camera Matrix Management | |
########################################################################### | |
def _get_camera_matrices(self, scene): | |
main_camera_node = scene.main_camera_node | |
if main_camera_node is None: | |
raise ValueError('Cannot render scene without a camera') | |
P = main_camera_node.camera.get_projection_matrix( | |
width=self.viewport_width, height=self.viewport_height | |
) | |
pose = scene.get_pose(main_camera_node) | |
V = np.linalg.inv(pose) # V maps from world to camera | |
return V, P | |
def _get_light_cam_matrices(self, scene, light_node, flags): | |
light = light_node.light | |
pose = scene.get_pose(light_node).copy() | |
s = scene.scale | |
camera = light._get_shadow_camera(s) | |
P = camera.get_projection_matrix() | |
if isinstance(light, DirectionalLight): | |
direction = -pose[:3,2] | |
c = scene.centroid | |
loc = c - direction * s | |
pose[:3,3] = loc | |
V = np.linalg.inv(pose) # V maps from world to camera | |
return V, P | |
########################################################################### | |
# Shader Program Management | |
########################################################################### | |
def _get_text_program(self): | |
program = self._program_cache.get_program( | |
vertex_shader='text.vert', | |
fragment_shader='text.frag' | |
) | |
if not program._in_context(): | |
program._add_to_context() | |
return program | |
def _compute_max_n_lights(self, flags): | |
max_n_lights = [MAX_N_LIGHTS, MAX_N_LIGHTS, MAX_N_LIGHTS] | |
n_tex_units = glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS) | |
# Reserved texture units: 6 | |
# Normal Map | |
# Occlusion Map | |
# Emissive Map | |
# Base Color or Diffuse Map | |
# MR or SG Map | |
# Environment cubemap | |
n_reserved_textures = 6 | |
n_available_textures = n_tex_units - n_reserved_textures | |
# Distribute textures evenly among lights with shadows, with | |
# a preference for directional lights | |
n_shadow_types = 0 | |
if flags & RenderFlags.SHADOWS_DIRECTIONAL: | |
n_shadow_types += 1 | |
if flags & RenderFlags.SHADOWS_SPOT: | |
n_shadow_types += 1 | |
if flags & RenderFlags.SHADOWS_POINT: | |
n_shadow_types += 1 | |
if n_shadow_types > 0: | |
tex_per_light = n_available_textures // n_shadow_types | |
if flags & RenderFlags.SHADOWS_DIRECTIONAL: | |
max_n_lights[0] = ( | |
tex_per_light + | |
(n_available_textures - tex_per_light * n_shadow_types) | |
) | |
if flags & RenderFlags.SHADOWS_SPOT: | |
max_n_lights[1] = tex_per_light | |
if flags & RenderFlags.SHADOWS_POINT: | |
max_n_lights[2] = tex_per_light | |
return max_n_lights | |
def _get_primitive_program(self, primitive, flags, program_flags): | |
vertex_shader = None | |
fragment_shader = None | |
geometry_shader = None | |
defines = {} | |
if (bool(program_flags & ProgramFlags.USE_MATERIAL) and | |
not flags & RenderFlags.DEPTH_ONLY and | |
not flags & RenderFlags.FLAT and | |
not flags & RenderFlags.SEG): | |
vertex_shader = 'mesh.vert' | |
fragment_shader = 'mesh.frag' | |
elif bool(program_flags & (ProgramFlags.VERTEX_NORMALS | | |
ProgramFlags.FACE_NORMALS)): | |
vertex_shader = 'vertex_normals.vert' | |
if primitive.mode == GLTF.POINTS: | |
geometry_shader = 'vertex_normals_pc.geom' | |
else: | |
geometry_shader = 'vertex_normals.geom' | |
fragment_shader = 'vertex_normals.frag' | |
elif flags & RenderFlags.FLAT: | |
vertex_shader = 'flat.vert' | |
fragment_shader = 'flat.frag' | |
elif flags & RenderFlags.SEG: | |
vertex_shader = 'segmentation.vert' | |
fragment_shader = 'segmentation.frag' | |
else: | |
vertex_shader = 'mesh_depth.vert' | |
fragment_shader = 'mesh_depth.frag' | |
# Set up vertex buffer DEFINES | |
bf = primitive.buf_flags | |
buf_idx = 1 | |
if bf & BufFlags.NORMAL: | |
defines['NORMAL_LOC'] = buf_idx | |
buf_idx += 1 | |
if bf & BufFlags.TANGENT: | |
defines['TANGENT_LOC'] = buf_idx | |
buf_idx += 1 | |
if bf & BufFlags.TEXCOORD_0: | |
defines['TEXCOORD_0_LOC'] = buf_idx | |
buf_idx += 1 | |
if bf & BufFlags.TEXCOORD_1: | |
defines['TEXCOORD_1_LOC'] = buf_idx | |
buf_idx += 1 | |
if bf & BufFlags.COLOR_0: | |
defines['COLOR_0_LOC'] = buf_idx | |
buf_idx += 1 | |
if bf & BufFlags.JOINTS_0: | |
defines['JOINTS_0_LOC'] = buf_idx | |
buf_idx += 1 | |
if bf & BufFlags.WEIGHTS_0: | |
defines['WEIGHTS_0_LOC'] = buf_idx | |
buf_idx += 1 | |
defines['INST_M_LOC'] = buf_idx | |
# Set up shadow mapping defines | |
if flags & RenderFlags.SHADOWS_DIRECTIONAL: | |
defines['DIRECTIONAL_LIGHT_SHADOWS'] = 1 | |
if flags & RenderFlags.SHADOWS_SPOT: | |
defines['SPOT_LIGHT_SHADOWS'] = 1 | |
if flags & RenderFlags.SHADOWS_POINT: | |
defines['POINT_LIGHT_SHADOWS'] = 1 | |
max_n_lights = self._compute_max_n_lights(flags) | |
defines['MAX_DIRECTIONAL_LIGHTS'] = max_n_lights[0] | |
defines['MAX_SPOT_LIGHTS'] = max_n_lights[1] | |
defines['MAX_POINT_LIGHTS'] = max_n_lights[2] | |
# Set up vertex normal defines | |
if program_flags & ProgramFlags.VERTEX_NORMALS: | |
defines['VERTEX_NORMALS'] = 1 | |
if program_flags & ProgramFlags.FACE_NORMALS: | |
defines['FACE_NORMALS'] = 1 | |
# Set up material texture defines | |
if bool(program_flags & ProgramFlags.USE_MATERIAL): | |
tf = primitive.material.tex_flags | |
if tf & TexFlags.NORMAL: | |
defines['HAS_NORMAL_TEX'] = 1 | |
if tf & TexFlags.OCCLUSION: | |
defines['HAS_OCCLUSION_TEX'] = 1 | |
if tf & TexFlags.EMISSIVE: | |
defines['HAS_EMISSIVE_TEX'] = 1 | |
if tf & TexFlags.BASE_COLOR: | |
defines['HAS_BASE_COLOR_TEX'] = 1 | |
if tf & TexFlags.METALLIC_ROUGHNESS: | |
defines['HAS_METALLIC_ROUGHNESS_TEX'] = 1 | |
if tf & TexFlags.DIFFUSE: | |
defines['HAS_DIFFUSE_TEX'] = 1 | |
if tf & TexFlags.SPECULAR_GLOSSINESS: | |
defines['HAS_SPECULAR_GLOSSINESS_TEX'] = 1 | |
if isinstance(primitive.material, MetallicRoughnessMaterial): | |
defines['USE_METALLIC_MATERIAL'] = 1 | |
elif isinstance(primitive.material, SpecularGlossinessMaterial): | |
defines['USE_GLOSSY_MATERIAL'] = 1 | |
program = self._program_cache.get_program( | |
vertex_shader=vertex_shader, | |
fragment_shader=fragment_shader, | |
geometry_shader=geometry_shader, | |
defines=defines | |
) | |
if not program._in_context(): | |
program._add_to_context() | |
return program | |
########################################################################### | |
# Viewport Management | |
########################################################################### | |
def _configure_forward_pass_viewport(self, flags): | |
# If using offscreen render, bind main framebuffer | |
if flags & RenderFlags.OFFSCREEN: | |
self._configure_main_framebuffer() | |
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb_ms) | |
else: | |
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0) | |
glViewport(0, 0, self.viewport_width, self.viewport_height) | |
glEnable(GL_DEPTH_TEST) | |
glDepthMask(GL_TRUE) | |
glDepthFunc(GL_LESS) | |
glDepthRange(0.0, 1.0) | |
def _configure_shadow_mapping_viewport(self, light, flags): | |
self._configure_shadow_framebuffer() | |
glBindFramebuffer(GL_FRAMEBUFFER, self._shadow_fb) | |
light.shadow_texture._bind() | |
light.shadow_texture._bind_as_depth_attachment() | |
glActiveTexture(GL_TEXTURE0) | |
light.shadow_texture._bind() | |
glDrawBuffer(GL_NONE) | |
glReadBuffer(GL_NONE) | |
glClear(GL_DEPTH_BUFFER_BIT) | |
glViewport(0, 0, SHADOW_TEX_SZ, SHADOW_TEX_SZ) | |
glEnable(GL_DEPTH_TEST) | |
glDepthMask(GL_TRUE) | |
glDepthFunc(GL_LESS) | |
glDepthRange(0.0, 1.0) | |
glDisable(GL_CULL_FACE) | |
glDisable(GL_BLEND) | |
########################################################################### | |
# Framebuffer Management | |
########################################################################### | |
def _configure_shadow_framebuffer(self): | |
if self._shadow_fb is None: | |
self._shadow_fb = glGenFramebuffers(1) | |
def _delete_shadow_framebuffer(self): | |
if self._shadow_fb is not None: | |
glDeleteFramebuffers(1, [self._shadow_fb]) | |
def _configure_main_framebuffer(self): | |
# If mismatch with prior framebuffer, delete it | |
if (self._main_fb is not None and | |
self.viewport_width != self._main_fb_dims[0] or | |
self.viewport_height != self._main_fb_dims[1]): | |
self._delete_main_framebuffer() | |
# If framebuffer doesn't exist, create it | |
if self._main_fb is None: | |
# Generate standard buffer | |
self._main_cb, self._main_db = glGenRenderbuffers(2) | |
glBindRenderbuffer(GL_RENDERBUFFER, self._main_cb) | |
glRenderbufferStorage( | |
GL_RENDERBUFFER, GL_RGBA, | |
self.viewport_width, self.viewport_height | |
) | |
glBindRenderbuffer(GL_RENDERBUFFER, self._main_db) | |
glRenderbufferStorage( | |
GL_RENDERBUFFER, GL_DEPTH_COMPONENT24, | |
self.viewport_width, self.viewport_height | |
) | |
self._main_fb = glGenFramebuffers(1) | |
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb) | |
glFramebufferRenderbuffer( | |
GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, | |
GL_RENDERBUFFER, self._main_cb | |
) | |
glFramebufferRenderbuffer( | |
GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, | |
GL_RENDERBUFFER, self._main_db | |
) | |
# Generate multisample buffer | |
self._main_cb_ms, self._main_db_ms = glGenRenderbuffers(2) | |
glBindRenderbuffer(GL_RENDERBUFFER, self._main_cb_ms) | |
# glRenderbufferStorageMultisample( | |
# GL_RENDERBUFFER, 4, GL_RGBA, | |
# self.viewport_width, self.viewport_height | |
# ) | |
# glBindRenderbuffer(GL_RENDERBUFFER, self._main_db_ms) | |
# glRenderbufferStorageMultisample( | |
# GL_RENDERBUFFER, 4, GL_DEPTH_COMPONENT24, | |
# self.viewport_width, self.viewport_height | |
# ) | |
# 增加这一行 | |
num_samples = min(glGetIntegerv(GL_MAX_SAMPLES), 4) # No more than GL_MAX_SAMPLES | |
# 其实就是把 4 替换成 num_samples,其余不变 | |
glRenderbufferStorageMultisample(GL_RENDERBUFFER, num_samples, GL_RGBA, self.viewport_width, self.viewport_height) | |
glBindRenderbuffer(GL_RENDERBUFFER, self._main_db_ms) # 这行不变 | |
# 这一行也是将 4 替换成 num_samples | |
glRenderbufferStorageMultisample(GL_RENDERBUFFER, num_samples, GL_DEPTH_COMPONENT24, self.viewport_width, self.viewport_height) | |
self._main_fb_ms = glGenFramebuffers(1) | |
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb_ms) | |
glFramebufferRenderbuffer( | |
GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, | |
GL_RENDERBUFFER, self._main_cb_ms | |
) | |
glFramebufferRenderbuffer( | |
GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, | |
GL_RENDERBUFFER, self._main_db_ms | |
) | |
self._main_fb_dims = (self.viewport_width, self.viewport_height) | |
def _delete_main_framebuffer(self): | |
if self._main_fb is not None: | |
glDeleteFramebuffers(2, [self._main_fb, self._main_fb_ms]) | |
if self._main_cb is not None: | |
glDeleteRenderbuffers(2, [self._main_cb, self._main_cb_ms]) | |
if self._main_db is not None: | |
glDeleteRenderbuffers(2, [self._main_db, self._main_db_ms]) | |
self._main_fb = None | |
self._main_cb = None | |
self._main_db = None | |
self._main_fb_ms = None | |
self._main_cb_ms = None | |
self._main_db_ms = None | |
self._main_fb_dims = (None, None) | |
def _read_main_framebuffer(self, scene, flags): | |
width, height = self._main_fb_dims[0], self._main_fb_dims[1] | |
# Bind framebuffer and blit buffers | |
glBindFramebuffer(GL_READ_FRAMEBUFFER, self._main_fb_ms) | |
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb) | |
glBlitFramebuffer( | |
0, 0, width, height, 0, 0, width, height, | |
GL_COLOR_BUFFER_BIT, GL_LINEAR | |
) | |
glBlitFramebuffer( | |
0, 0, width, height, 0, 0, width, height, | |
GL_DEPTH_BUFFER_BIT, GL_NEAREST | |
) | |
glBindFramebuffer(GL_READ_FRAMEBUFFER, self._main_fb) | |
# Read depth | |
depth_buf = glReadPixels( | |
0, 0, width, height, GL_DEPTH_COMPONENT, GL_FLOAT | |
) | |
depth_im = np.frombuffer(depth_buf, dtype=np.float32) | |
depth_im = depth_im.reshape((height, width)) | |
depth_im = np.flip(depth_im, axis=0) | |
inf_inds = (depth_im == 1.0) | |
depth_im = 2.0 * depth_im - 1.0 | |
z_near = scene.main_camera_node.camera.znear | |
z_far = scene.main_camera_node.camera.zfar | |
noninf = np.logical_not(inf_inds) | |
if z_far is None: | |
depth_im[noninf] = 2 * z_near / (1.0 - depth_im[noninf]) | |
else: | |
depth_im[noninf] = ((2.0 * z_near * z_far) / | |
(z_far + z_near - depth_im[noninf] * | |
(z_far - z_near))) | |
depth_im[inf_inds] = 0.0 | |
# Resize for macos if needed | |
if sys.platform == 'darwin': | |
depth_im = self._resize_image(depth_im) | |
if flags & RenderFlags.DEPTH_ONLY: | |
return depth_im | |
# Read color | |
if flags & RenderFlags.RGBA: | |
color_buf = glReadPixels( | |
0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE | |
) | |
color_im = np.frombuffer(color_buf, dtype=np.uint8) | |
color_im = color_im.reshape((height, width, 4)) | |
else: | |
color_buf = glReadPixels( | |
0, 0, width, height, GL_RGB, GL_UNSIGNED_BYTE | |
) | |
color_im = np.frombuffer(color_buf, dtype=np.uint8) | |
color_im = color_im.reshape((height, width, 3)) | |
color_im = np.flip(color_im, axis=0) | |
# Resize for macos if needed | |
if sys.platform == 'darwin': | |
color_im = self._resize_image(color_im, True) | |
return color_im, depth_im | |
def _resize_image(self, value, antialias=False): | |
"""If needed, rescale the render for MacOS.""" | |
img = PIL.Image.fromarray(value) | |
resample = PIL.Image.NEAREST | |
if antialias: | |
resample = PIL.Image.BILINEAR | |
size = (self.viewport_width // self.dpscale, | |
self.viewport_height // self.dpscale) | |
img = img.resize(size, resample=resample) | |
return np.array(img) | |
########################################################################### | |
# Shadowmap Debugging | |
########################################################################### | |
def _forward_pass_no_reset(self, scene, flags): | |
# Set up camera matrices | |
V, P = self._get_camera_matrices(scene) | |
# Now, render each object in sorted order | |
for node in self._sorted_mesh_nodes(scene): | |
mesh = node.mesh | |
# Skip the mesh if it's not visible | |
if not mesh.is_visible: | |
continue | |
for primitive in mesh.primitives: | |
# First, get and bind the appropriate program | |
program = self._get_primitive_program( | |
primitive, flags, ProgramFlags.USE_MATERIAL | |
) | |
program._bind() | |
# Set the camera uniforms | |
program.set_uniform('V', V) | |
program.set_uniform('P', P) | |
program.set_uniform( | |
'cam_pos', scene.get_pose(scene.main_camera_node)[:3,3] | |
) | |
# Next, bind the lighting | |
if not flags & RenderFlags.DEPTH_ONLY and not flags & RenderFlags.FLAT: | |
self._bind_lighting(scene, program, node, flags) | |
# Finally, bind and draw the primitive | |
self._bind_and_draw_primitive( | |
primitive=primitive, | |
pose=scene.get_pose(node), | |
program=program, | |
flags=flags | |
) | |
self._reset_active_textures() | |
# Unbind the shader and flush the output | |
if program is not None: | |
program._unbind() | |
glFlush() | |
def _render_light_shadowmaps(self, scene, light_nodes, flags, tile=False): | |
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0) | |
glClearColor(*scene.bg_color) | |
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) | |
glEnable(GL_DEPTH_TEST) | |
glDepthMask(GL_TRUE) | |
glDepthFunc(GL_LESS) | |
glDepthRange(0.0, 1.0) | |
w = self.viewport_width | |
h = self.viewport_height | |
num_nodes = len(light_nodes) | |
viewport_dims = { | |
(0, 2): [0, h // 2, w // 2, h], | |
(1, 2): [w // 2, h // 2, w, h], | |
(0, 3): [0, h // 2, w // 2, h], | |
(1, 3): [w // 2, h // 2, w, h], | |
(2, 3): [0, 0, w // 2, h // 2], | |
(0, 4): [0, h // 2, w // 2, h], | |
(1, 4): [w // 2, h // 2, w, h], | |
(2, 4): [0, 0, w // 2, h // 2], | |
(3, 4): [w // 2, 0, w, h // 2] | |
} | |
if tile: | |
for i, ln in enumerate(light_nodes): | |
light = ln.light | |
if light.shadow_texture is None: | |
raise ValueError('Light does not have a shadow texture') | |
glViewport(*viewport_dims[(i, num_nodes + 1)]) | |
program = self._get_debug_quad_program() | |
program._bind() | |
self._bind_texture(light.shadow_texture, 'depthMap', program) | |
self._render_debug_quad() | |
self._reset_active_textures() | |
glFlush() | |
i += 1 | |
glViewport(*viewport_dims[(i, num_nodes + 1)]) | |
self._forward_pass_no_reset(scene, flags) | |
else: | |
for i, ln in enumerate(light_nodes): | |
light = ln.light | |
if light.shadow_texture is None: | |
raise ValueError('Light does not have a shadow texture') | |
glViewport(0, 0, self.viewport_width, self.viewport_height) | |
program = self._get_debug_quad_program() | |
program._bind() | |
self._bind_texture(light.shadow_texture, 'depthMap', program) | |
self._render_debug_quad() | |
self._reset_active_textures() | |
glFlush() | |
return | |
def _get_debug_quad_program(self): | |
program = self._program_cache.get_program( | |
vertex_shader='debug_quad.vert', | |
fragment_shader='debug_quad.frag' | |
) | |
if not program._in_context(): | |
program._add_to_context() | |
return program | |
def _render_debug_quad(self): | |
x = glGenVertexArrays(1) | |
glBindVertexArray(x) | |
glDrawArrays(GL_TRIANGLES, 0, 6) | |
glBindVertexArray(0) | |
glDeleteVertexArrays(1, [x]) | |