vumichien's picture
First commit
b4c8bc3
raw history blame
No virus
49.1 kB
"""PBR renderer for Python.
Author: Matthew Matl
"""
import sys
import numpy as np
import PIL
from .constants import (RenderFlags, TextAlign, GLTF, BufFlags, TexFlags,
ProgramFlags, DEFAULT_Z_FAR, DEFAULT_Z_NEAR,
SHADOW_TEX_SZ, MAX_N_LIGHTS)
from .shader_program import ShaderProgramCache
from .material import MetallicRoughnessMaterial, SpecularGlossinessMaterial
from .light import PointLight, SpotLight, DirectionalLight
from .font import FontCache
from .utils import format_color_vector
from OpenGL.GL import *
class Renderer(object):
"""Class for handling all rendering operations on a scene.
Note
----
This renderer relies on the existence of an OpenGL context and
does not create one on its own.
Parameters
----------
viewport_width : int
Width of the viewport in pixels.
viewport_height : int
Width of the viewport height in pixels.
point_size : float, optional
Size of points in pixels. Defaults to 1.0.
"""
def __init__(self, viewport_width, viewport_height, point_size=1.0):
self.dpscale = 1
# Scaling needed on retina displays
if sys.platform == 'darwin':
self.dpscale = 2
self.viewport_width = viewport_width
self.viewport_height = viewport_height
self.point_size = point_size
# Optional framebuffer for offscreen renders
self._main_fb = None
self._main_cb = None
self._main_db = None
self._main_fb_ms = None
self._main_cb_ms = None
self._main_db_ms = None
self._main_fb_dims = (None, None)
self._shadow_fb = None
self._latest_znear = DEFAULT_Z_NEAR
self._latest_zfar = DEFAULT_Z_FAR
# Shader Program Cache
self._program_cache = ShaderProgramCache()
self._font_cache = FontCache()
self._meshes = set()
self._mesh_textures = set()
self._shadow_textures = set()
self._texture_alloc_idx = 0
@property
def viewport_width(self):
"""int : The width of the main viewport, in pixels.
"""
return self._viewport_width
@viewport_width.setter
def viewport_width(self, value):
self._viewport_width = self.dpscale * value
@property
def viewport_height(self):
"""int : The height of the main viewport, in pixels.
"""
return self._viewport_height
@viewport_height.setter
def viewport_height(self, value):
self._viewport_height = self.dpscale * value
@property
def point_size(self):
"""float : The size of screen-space points, in pixels.
"""
return self._point_size
@point_size.setter
def point_size(self, value):
self._point_size = float(value)
def render(self, scene, flags, seg_node_map=None):
"""Render a scene with the given set of flags.
Parameters
----------
scene : :class:`Scene`
A scene to render.
flags : int
A specification from :class:`.RenderFlags`.
seg_node_map : dict
A map from :class:`.Node` objects to (3,) colors for each.
If specified along with flags set to :attr:`.RenderFlags.SEG`,
the color image will be a segmentation image.
Returns
-------
color_im : (h, w, 3) uint8 or (h, w, 4) uint8
If :attr:`RenderFlags.OFFSCREEN` is set, the color buffer. This is
normally an RGB buffer, but if :attr:`.RenderFlags.RGBA` is set,
the buffer will be a full RGBA buffer.
depth_im : (h, w) float32
If :attr:`RenderFlags.OFFSCREEN` is set, the depth buffer
in linear units.
"""
# Update context with meshes and textures
self._update_context(scene, flags)
# Render necessary shadow maps
if not bool(flags & RenderFlags.DEPTH_ONLY or flags & RenderFlags.SEG):
for ln in scene.light_nodes:
take_pass = False
if (isinstance(ln.light, DirectionalLight) and
bool(flags & RenderFlags.SHADOWS_DIRECTIONAL)):
take_pass = True
elif (isinstance(ln.light, SpotLight) and
bool(flags & RenderFlags.SHADOWS_SPOT)):
take_pass = True
elif (isinstance(ln.light, PointLight) and
bool(flags & RenderFlags.SHADOWS_POINT)):
take_pass = True
if take_pass:
self._shadow_mapping_pass(scene, ln, flags)
# Make forward pass
retval = self._forward_pass(scene, flags, seg_node_map=seg_node_map)
# If necessary, make normals pass
if flags & (RenderFlags.VERTEX_NORMALS | RenderFlags.FACE_NORMALS):
self._normals_pass(scene, flags)
# Update camera settings for retrieving depth buffers
self._latest_znear = scene.main_camera_node.camera.znear
self._latest_zfar = scene.main_camera_node.camera.zfar
return retval
def render_text(self, text, x, y, font_name='OpenSans-Regular',
font_pt=40, color=None, scale=1.0,
align=TextAlign.BOTTOM_LEFT):
"""Render text into the current viewport.
Note
----
This cannot be done into an offscreen buffer.
Parameters
----------
text : str
The text to render.
x : int
Horizontal pixel location of text.
y : int
Vertical pixel location of text.
font_name : str
Name of font, from the ``pyrender/fonts`` folder, or
a path to a ``.ttf`` file.
font_pt : int
Height of the text, in font points.
color : (4,) float
The color of the text. Default is black.
scale : int
Scaling factor for text.
align : int
One of the :class:`TextAlign` options which specifies where the
``x`` and ``y`` parameters lie on the text. For example,
:attr:`TextAlign.BOTTOM_LEFT` means that ``x`` and ``y`` indicate
the position of the bottom-left corner of the textbox.
"""
x *= self.dpscale
y *= self.dpscale
font_pt *= self.dpscale
if color is None:
color = np.array([0.0, 0.0, 0.0, 1.0])
else:
color = format_color_vector(color, 4)
# Set up viewport for render
self._configure_forward_pass_viewport(0)
# Load font
font = self._font_cache.get_font(font_name, font_pt)
if not font._in_context():
font._add_to_context()
# Load program
program = self._get_text_program()
program._bind()
# Set uniforms
p = np.eye(4)
p[0,0] = 2.0 / self.viewport_width
p[0,3] = -1.0
p[1,1] = 2.0 / self.viewport_height
p[1,3] = -1.0
program.set_uniform('projection', p)
program.set_uniform('text_color', color)
# Draw text
font.render_string(text, x, y, scale, align)
def read_color_buf(self):
"""Read and return the current viewport's color buffer.
Alpha cannot be computed for an on-screen buffer.
Returns
-------
color_im : (h, w, 3) uint8
The color buffer in RGB byte format.
"""
# Extract color image from frame buffer
width, height = self.viewport_width, self.viewport_height
glBindFramebuffer(GL_READ_FRAMEBUFFER, 0)
glReadBuffer(GL_FRONT)
color_buf = glReadPixels(0, 0, width, height, GL_RGB, GL_UNSIGNED_BYTE)
# Re-format them into numpy arrays
color_im = np.frombuffer(color_buf, dtype=np.uint8)
color_im = color_im.reshape((height, width, 3))
color_im = np.flip(color_im, axis=0)
# Resize for macos if needed
if sys.platform == 'darwin':
color_im = self._resize_image(color_im, True)
return color_im
def read_depth_buf(self):
"""Read and return the current viewport's color buffer.
Returns
-------
depth_im : (h, w) float32
The depth buffer in linear units.
"""
width, height = self.viewport_width, self.viewport_height
glBindFramebuffer(GL_READ_FRAMEBUFFER, 0)
glReadBuffer(GL_FRONT)
depth_buf = glReadPixels(
0, 0, width, height, GL_DEPTH_COMPONENT, GL_FLOAT
)
depth_im = np.frombuffer(depth_buf, dtype=np.float32)
depth_im = depth_im.reshape((height, width))
depth_im = np.flip(depth_im, axis=0)
inf_inds = (depth_im == 1.0)
depth_im = 2.0 * depth_im - 1.0
z_near, z_far = self._latest_znear, self._latest_zfar
noninf = np.logical_not(inf_inds)
if z_far is None:
depth_im[noninf] = 2 * z_near / (1.0 - depth_im[noninf])
else:
depth_im[noninf] = ((2.0 * z_near * z_far) /
(z_far + z_near - depth_im[noninf] *
(z_far - z_near)))
depth_im[inf_inds] = 0.0
# Resize for macos if needed
if sys.platform == 'darwin':
depth_im = self._resize_image(depth_im)
return depth_im
def delete(self):
"""Free all allocated OpenGL resources.
"""
# Free shaders
self._program_cache.clear()
# Free fonts
self._font_cache.clear()
# Free meshes
for mesh in self._meshes:
for p in mesh.primitives:
p.delete()
# Free textures
for mesh_texture in self._mesh_textures:
mesh_texture.delete()
for shadow_texture in self._shadow_textures:
shadow_texture.delete()
self._meshes = set()
self._mesh_textures = set()
self._shadow_textures = set()
self._texture_alloc_idx = 0
self._delete_main_framebuffer()
self._delete_shadow_framebuffer()
def __del__(self):
try:
self.delete()
except Exception:
pass
###########################################################################
# Rendering passes
###########################################################################
def _forward_pass(self, scene, flags, seg_node_map=None):
# Set up viewport for render
self._configure_forward_pass_viewport(flags)
# Clear it
if bool(flags & RenderFlags.SEG):
glClearColor(0.0, 0.0, 0.0, 1.0)
if seg_node_map is None:
seg_node_map = {}
else:
glClearColor(*scene.bg_color)
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
if not bool(flags & RenderFlags.SEG):
glEnable(GL_MULTISAMPLE)
else:
glDisable(GL_MULTISAMPLE)
# Set up camera matrices
V, P = self._get_camera_matrices(scene)
program = None
# Now, render each object in sorted order
for node in self._sorted_mesh_nodes(scene):
mesh = node.mesh
# Skip the mesh if it's not visible
if not mesh.is_visible:
continue
# If SEG, set color
if bool(flags & RenderFlags.SEG):
if node not in seg_node_map:
continue
color = seg_node_map[node]
if not isinstance(color, (list, tuple, np.ndarray)):
color = np.repeat(color, 3)
else:
color = np.asanyarray(color)
color = color / 255.0
for primitive in mesh.primitives:
# First, get and bind the appropriate program
program = self._get_primitive_program(
primitive, flags, ProgramFlags.USE_MATERIAL
)
program._bind()
# Set the camera uniforms
program.set_uniform('V', V)
program.set_uniform('P', P)
program.set_uniform(
'cam_pos', scene.get_pose(scene.main_camera_node)[:3,3]
)
if bool(flags & RenderFlags.SEG):
program.set_uniform('color', color)
# Next, bind the lighting
if not (flags & RenderFlags.DEPTH_ONLY or flags & RenderFlags.FLAT or
flags & RenderFlags.SEG):
self._bind_lighting(scene, program, node, flags)
# Finally, bind and draw the primitive
self._bind_and_draw_primitive(
primitive=primitive,
pose=scene.get_pose(node),
program=program,
flags=flags
)
self._reset_active_textures()
# Unbind the shader and flush the output
if program is not None:
program._unbind()
glFlush()
# If doing offscreen render, copy result from framebuffer and return
if flags & RenderFlags.OFFSCREEN:
return self._read_main_framebuffer(scene, flags)
else:
return
def _shadow_mapping_pass(self, scene, light_node, flags):
light = light_node.light
# Set up viewport for render
self._configure_shadow_mapping_viewport(light, flags)
# Set up camera matrices
V, P = self._get_light_cam_matrices(scene, light_node, flags)
# Now, render each object in sorted order
for node in self._sorted_mesh_nodes(scene):
mesh = node.mesh
# Skip the mesh if it's not visible
if not mesh.is_visible:
continue
for primitive in mesh.primitives:
# First, get and bind the appropriate program
program = self._get_primitive_program(
primitive, flags, ProgramFlags.NONE
)
program._bind()
# Set the camera uniforms
program.set_uniform('V', V)
program.set_uniform('P', P)
program.set_uniform(
'cam_pos', scene.get_pose(scene.main_camera_node)[:3,3]
)
# Finally, bind and draw the primitive
self._bind_and_draw_primitive(
primitive=primitive,
pose=scene.get_pose(node),
program=program,
flags=RenderFlags.DEPTH_ONLY
)
self._reset_active_textures()
# Unbind the shader and flush the output
if program is not None:
program._unbind()
glFlush()
def _normals_pass(self, scene, flags):
# Set up viewport for render
self._configure_forward_pass_viewport(flags)
program = None
# Set up camera matrices
V, P = self._get_camera_matrices(scene)
# Now, render each object in sorted order
for node in self._sorted_mesh_nodes(scene):
mesh = node.mesh
# Skip the mesh if it's not visible
if not mesh.is_visible:
continue
for primitive in mesh.primitives:
# Skip objects that don't have normals
if not primitive.buf_flags & BufFlags.NORMAL:
continue
# First, get and bind the appropriate program
pf = ProgramFlags.NONE
if flags & RenderFlags.VERTEX_NORMALS:
pf = pf | ProgramFlags.VERTEX_NORMALS
if flags & RenderFlags.FACE_NORMALS:
pf = pf | ProgramFlags.FACE_NORMALS
program = self._get_primitive_program(primitive, flags, pf)
program._bind()
# Set the camera uniforms
program.set_uniform('V', V)
program.set_uniform('P', P)
program.set_uniform('normal_magnitude', 0.05 * primitive.scale)
program.set_uniform(
'normal_color', np.array([0.1, 0.1, 1.0, 1.0])
)
# Finally, bind and draw the primitive
self._bind_and_draw_primitive(
primitive=primitive,
pose=scene.get_pose(node),
program=program,
flags=RenderFlags.DEPTH_ONLY
)
self._reset_active_textures()
# Unbind the shader and flush the output
if program is not None:
program._unbind()
glFlush()
###########################################################################
# Handlers for binding uniforms and drawing primitives
###########################################################################
def _bind_and_draw_primitive(self, primitive, pose, program, flags):
# Set model pose matrix
program.set_uniform('M', pose)
# Bind mesh buffers
primitive._bind()
# Bind mesh material
if not (flags & RenderFlags.DEPTH_ONLY or flags & RenderFlags.SEG):
material = primitive.material
# Bind textures
tf = material.tex_flags
if tf & TexFlags.NORMAL:
self._bind_texture(material.normalTexture,
'material.normal_texture', program)
if tf & TexFlags.OCCLUSION:
self._bind_texture(material.occlusionTexture,
'material.occlusion_texture', program)
if tf & TexFlags.EMISSIVE:
self._bind_texture(material.emissiveTexture,
'material.emissive_texture', program)
if tf & TexFlags.BASE_COLOR:
self._bind_texture(material.baseColorTexture,
'material.base_color_texture', program)
if tf & TexFlags.METALLIC_ROUGHNESS:
self._bind_texture(material.metallicRoughnessTexture,
'material.metallic_roughness_texture',
program)
if tf & TexFlags.DIFFUSE:
self._bind_texture(material.diffuseTexture,
'material.diffuse_texture', program)
if tf & TexFlags.SPECULAR_GLOSSINESS:
self._bind_texture(material.specularGlossinessTexture,
'material.specular_glossiness_texture',
program)
# Bind other uniforms
b = 'material.{}'
program.set_uniform(b.format('emissive_factor'),
material.emissiveFactor)
if isinstance(material, MetallicRoughnessMaterial):
program.set_uniform(b.format('base_color_factor'),
material.baseColorFactor)
program.set_uniform(b.format('metallic_factor'),
material.metallicFactor)
program.set_uniform(b.format('roughness_factor'),
material.roughnessFactor)
elif isinstance(material, SpecularGlossinessMaterial):
program.set_uniform(b.format('diffuse_factor'),
material.diffuseFactor)
program.set_uniform(b.format('specular_factor'),
material.specularFactor)
program.set_uniform(b.format('glossiness_factor'),
material.glossinessFactor)
# Set blending options
if material.alphaMode == 'BLEND':
glEnable(GL_BLEND)
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA)
else:
glEnable(GL_BLEND)
glBlendFunc(GL_ONE, GL_ZERO)
# Set wireframe mode
wf = material.wireframe
if flags & RenderFlags.FLIP_WIREFRAME:
wf = not wf
if (flags & RenderFlags.ALL_WIREFRAME) or wf:
glPolygonMode(GL_FRONT_AND_BACK, GL_LINE)
else:
glPolygonMode(GL_FRONT_AND_BACK, GL_FILL)
# Set culling mode
if material.doubleSided or flags & RenderFlags.SKIP_CULL_FACES:
glDisable(GL_CULL_FACE)
else:
glEnable(GL_CULL_FACE)
glCullFace(GL_BACK)
else:
glEnable(GL_CULL_FACE)
glEnable(GL_BLEND)
glCullFace(GL_BACK)
glBlendFunc(GL_ONE, GL_ZERO)
glPolygonMode(GL_FRONT_AND_BACK, GL_FILL)
# Set point size if needed
glDisable(GL_PROGRAM_POINT_SIZE)
if primitive.mode == GLTF.POINTS:
glEnable(GL_PROGRAM_POINT_SIZE)
glPointSize(self.point_size)
# Render mesh
n_instances = 1
if primitive.poses is not None:
n_instances = len(primitive.poses)
if primitive.indices is not None:
glDrawElementsInstanced(
primitive.mode, primitive.indices.size, GL_UNSIGNED_INT,
ctypes.c_void_p(0), n_instances
)
else:
glDrawArraysInstanced(
primitive.mode, 0, len(primitive.positions), n_instances
)
# Unbind mesh buffers
primitive._unbind()
def _bind_lighting(self, scene, program, node, flags):
"""Bind all lighting uniform values for a scene.
"""
max_n_lights = self._compute_max_n_lights(flags)
n_d = min(len(scene.directional_light_nodes), max_n_lights[0])
n_s = min(len(scene.spot_light_nodes), max_n_lights[1])
n_p = min(len(scene.point_light_nodes), max_n_lights[2])
program.set_uniform('ambient_light', scene.ambient_light)
program.set_uniform('n_directional_lights', n_d)
program.set_uniform('n_spot_lights', n_s)
program.set_uniform('n_point_lights', n_p)
plc = 0
slc = 0
dlc = 0
light_nodes = scene.light_nodes
if (len(scene.directional_light_nodes) > max_n_lights[0] or
len(scene.spot_light_nodes) > max_n_lights[1] or
len(scene.point_light_nodes) > max_n_lights[2]):
light_nodes = self._sorted_nodes_by_distance(
scene, scene.light_nodes, node
)
for n in light_nodes:
light = n.light
pose = scene.get_pose(n)
position = pose[:3,3]
direction = -pose[:3,2]
if isinstance(light, PointLight):
if plc == max_n_lights[2]:
continue
b = 'point_lights[{}].'.format(plc)
plc += 1
shadow = bool(flags & RenderFlags.SHADOWS_POINT)
program.set_uniform(b + 'position', position)
elif isinstance(light, SpotLight):
if slc == max_n_lights[1]:
continue
b = 'spot_lights[{}].'.format(slc)
slc += 1
shadow = bool(flags & RenderFlags.SHADOWS_SPOT)
las = 1.0 / max(0.001, np.cos(light.innerConeAngle) -
np.cos(light.outerConeAngle))
lao = -np.cos(light.outerConeAngle) * las
program.set_uniform(b + 'direction', direction)
program.set_uniform(b + 'position', position)
program.set_uniform(b + 'light_angle_scale', las)
program.set_uniform(b + 'light_angle_offset', lao)
else:
if dlc == max_n_lights[0]:
continue
b = 'directional_lights[{}].'.format(dlc)
dlc += 1
shadow = bool(flags & RenderFlags.SHADOWS_DIRECTIONAL)
program.set_uniform(b + 'direction', direction)
program.set_uniform(b + 'color', light.color)
program.set_uniform(b + 'intensity', light.intensity)
# if light.range is not None:
# program.set_uniform(b + 'range', light.range)
# else:
# program.set_uniform(b + 'range', 0)
if shadow:
self._bind_texture(light.shadow_texture,
b + 'shadow_map', program)
if not isinstance(light, PointLight):
V, P = self._get_light_cam_matrices(scene, n, flags)
program.set_uniform(b + 'light_matrix', P.dot(V))
else:
raise NotImplementedError(
'Point light shadows not implemented'
)
def _sorted_mesh_nodes(self, scene):
cam_loc = scene.get_pose(scene.main_camera_node)[:3,3]
solid_nodes = []
trans_nodes = []
for node in scene.mesh_nodes:
mesh = node.mesh
if mesh.is_transparent:
trans_nodes.append(node)
else:
solid_nodes.append(node)
# TODO BETTER SORTING METHOD
trans_nodes.sort(
key=lambda n: -np.linalg.norm(scene.get_pose(n)[:3,3] - cam_loc)
)
solid_nodes.sort(
key=lambda n: -np.linalg.norm(scene.get_pose(n)[:3,3] - cam_loc)
)
return solid_nodes + trans_nodes
def _sorted_nodes_by_distance(self, scene, nodes, compare_node):
nodes = list(nodes)
compare_posn = scene.get_pose(compare_node)[:3,3]
nodes.sort(key=lambda n: np.linalg.norm(
scene.get_pose(n)[:3,3] - compare_posn)
)
return nodes
###########################################################################
# Context Management
###########################################################################
def _update_context(self, scene, flags):
# Update meshes
scene_meshes = scene.meshes
# Add new meshes to context
for mesh in scene_meshes - self._meshes:
for p in mesh.primitives:
p._add_to_context()
# Remove old meshes from context
for mesh in self._meshes - scene_meshes:
for p in mesh.primitives:
p.delete()
self._meshes = scene_meshes.copy()
# Update mesh textures
mesh_textures = set()
for m in scene_meshes:
for p in m.primitives:
mesh_textures |= p.material.textures
# Add new textures to context
for texture in mesh_textures - self._mesh_textures:
texture._add_to_context()
# Remove old textures from context
for texture in self._mesh_textures - mesh_textures:
texture.delete()
self._mesh_textures = mesh_textures.copy()
shadow_textures = set()
for l in scene.lights:
# Create if needed
active = False
if (isinstance(l, DirectionalLight) and
flags & RenderFlags.SHADOWS_DIRECTIONAL):
active = True
elif (isinstance(l, PointLight) and
flags & RenderFlags.SHADOWS_POINT):
active = True
elif isinstance(l, SpotLight) and flags & RenderFlags.SHADOWS_SPOT:
active = True
if active and l.shadow_texture is None:
l._generate_shadow_texture()
if l.shadow_texture is not None:
shadow_textures.add(l.shadow_texture)
# Add new textures to context
for texture in shadow_textures - self._shadow_textures:
texture._add_to_context()
# Remove old textures from context
for texture in self._shadow_textures - shadow_textures:
texture.delete()
self._shadow_textures = shadow_textures.copy()
###########################################################################
# Texture Management
###########################################################################
def _bind_texture(self, texture, uniform_name, program):
"""Bind a texture to an active texture unit and return
the texture unit index that was used.
"""
tex_id = self._get_next_active_texture()
glActiveTexture(GL_TEXTURE0 + tex_id)
texture._bind()
program.set_uniform(uniform_name, tex_id)
def _get_next_active_texture(self):
val = self._texture_alloc_idx
self._texture_alloc_idx += 1
return val
def _reset_active_textures(self):
self._texture_alloc_idx = 0
###########################################################################
# Camera Matrix Management
###########################################################################
def _get_camera_matrices(self, scene):
main_camera_node = scene.main_camera_node
if main_camera_node is None:
raise ValueError('Cannot render scene without a camera')
P = main_camera_node.camera.get_projection_matrix(
width=self.viewport_width, height=self.viewport_height
)
pose = scene.get_pose(main_camera_node)
V = np.linalg.inv(pose) # V maps from world to camera
return V, P
def _get_light_cam_matrices(self, scene, light_node, flags):
light = light_node.light
pose = scene.get_pose(light_node).copy()
s = scene.scale
camera = light._get_shadow_camera(s)
P = camera.get_projection_matrix()
if isinstance(light, DirectionalLight):
direction = -pose[:3,2]
c = scene.centroid
loc = c - direction * s
pose[:3,3] = loc
V = np.linalg.inv(pose) # V maps from world to camera
return V, P
###########################################################################
# Shader Program Management
###########################################################################
def _get_text_program(self):
program = self._program_cache.get_program(
vertex_shader='text.vert',
fragment_shader='text.frag'
)
if not program._in_context():
program._add_to_context()
return program
def _compute_max_n_lights(self, flags):
max_n_lights = [MAX_N_LIGHTS, MAX_N_LIGHTS, MAX_N_LIGHTS]
n_tex_units = glGetIntegerv(GL_MAX_TEXTURE_IMAGE_UNITS)
# Reserved texture units: 6
# Normal Map
# Occlusion Map
# Emissive Map
# Base Color or Diffuse Map
# MR or SG Map
# Environment cubemap
n_reserved_textures = 6
n_available_textures = n_tex_units - n_reserved_textures
# Distribute textures evenly among lights with shadows, with
# a preference for directional lights
n_shadow_types = 0
if flags & RenderFlags.SHADOWS_DIRECTIONAL:
n_shadow_types += 1
if flags & RenderFlags.SHADOWS_SPOT:
n_shadow_types += 1
if flags & RenderFlags.SHADOWS_POINT:
n_shadow_types += 1
if n_shadow_types > 0:
tex_per_light = n_available_textures // n_shadow_types
if flags & RenderFlags.SHADOWS_DIRECTIONAL:
max_n_lights[0] = (
tex_per_light +
(n_available_textures - tex_per_light * n_shadow_types)
)
if flags & RenderFlags.SHADOWS_SPOT:
max_n_lights[1] = tex_per_light
if flags & RenderFlags.SHADOWS_POINT:
max_n_lights[2] = tex_per_light
return max_n_lights
def _get_primitive_program(self, primitive, flags, program_flags):
vertex_shader = None
fragment_shader = None
geometry_shader = None
defines = {}
if (bool(program_flags & ProgramFlags.USE_MATERIAL) and
not flags & RenderFlags.DEPTH_ONLY and
not flags & RenderFlags.FLAT and
not flags & RenderFlags.SEG):
vertex_shader = 'mesh.vert'
fragment_shader = 'mesh.frag'
elif bool(program_flags & (ProgramFlags.VERTEX_NORMALS |
ProgramFlags.FACE_NORMALS)):
vertex_shader = 'vertex_normals.vert'
if primitive.mode == GLTF.POINTS:
geometry_shader = 'vertex_normals_pc.geom'
else:
geometry_shader = 'vertex_normals.geom'
fragment_shader = 'vertex_normals.frag'
elif flags & RenderFlags.FLAT:
vertex_shader = 'flat.vert'
fragment_shader = 'flat.frag'
elif flags & RenderFlags.SEG:
vertex_shader = 'segmentation.vert'
fragment_shader = 'segmentation.frag'
else:
vertex_shader = 'mesh_depth.vert'
fragment_shader = 'mesh_depth.frag'
# Set up vertex buffer DEFINES
bf = primitive.buf_flags
buf_idx = 1
if bf & BufFlags.NORMAL:
defines['NORMAL_LOC'] = buf_idx
buf_idx += 1
if bf & BufFlags.TANGENT:
defines['TANGENT_LOC'] = buf_idx
buf_idx += 1
if bf & BufFlags.TEXCOORD_0:
defines['TEXCOORD_0_LOC'] = buf_idx
buf_idx += 1
if bf & BufFlags.TEXCOORD_1:
defines['TEXCOORD_1_LOC'] = buf_idx
buf_idx += 1
if bf & BufFlags.COLOR_0:
defines['COLOR_0_LOC'] = buf_idx
buf_idx += 1
if bf & BufFlags.JOINTS_0:
defines['JOINTS_0_LOC'] = buf_idx
buf_idx += 1
if bf & BufFlags.WEIGHTS_0:
defines['WEIGHTS_0_LOC'] = buf_idx
buf_idx += 1
defines['INST_M_LOC'] = buf_idx
# Set up shadow mapping defines
if flags & RenderFlags.SHADOWS_DIRECTIONAL:
defines['DIRECTIONAL_LIGHT_SHADOWS'] = 1
if flags & RenderFlags.SHADOWS_SPOT:
defines['SPOT_LIGHT_SHADOWS'] = 1
if flags & RenderFlags.SHADOWS_POINT:
defines['POINT_LIGHT_SHADOWS'] = 1
max_n_lights = self._compute_max_n_lights(flags)
defines['MAX_DIRECTIONAL_LIGHTS'] = max_n_lights[0]
defines['MAX_SPOT_LIGHTS'] = max_n_lights[1]
defines['MAX_POINT_LIGHTS'] = max_n_lights[2]
# Set up vertex normal defines
if program_flags & ProgramFlags.VERTEX_NORMALS:
defines['VERTEX_NORMALS'] = 1
if program_flags & ProgramFlags.FACE_NORMALS:
defines['FACE_NORMALS'] = 1
# Set up material texture defines
if bool(program_flags & ProgramFlags.USE_MATERIAL):
tf = primitive.material.tex_flags
if tf & TexFlags.NORMAL:
defines['HAS_NORMAL_TEX'] = 1
if tf & TexFlags.OCCLUSION:
defines['HAS_OCCLUSION_TEX'] = 1
if tf & TexFlags.EMISSIVE:
defines['HAS_EMISSIVE_TEX'] = 1
if tf & TexFlags.BASE_COLOR:
defines['HAS_BASE_COLOR_TEX'] = 1
if tf & TexFlags.METALLIC_ROUGHNESS:
defines['HAS_METALLIC_ROUGHNESS_TEX'] = 1
if tf & TexFlags.DIFFUSE:
defines['HAS_DIFFUSE_TEX'] = 1
if tf & TexFlags.SPECULAR_GLOSSINESS:
defines['HAS_SPECULAR_GLOSSINESS_TEX'] = 1
if isinstance(primitive.material, MetallicRoughnessMaterial):
defines['USE_METALLIC_MATERIAL'] = 1
elif isinstance(primitive.material, SpecularGlossinessMaterial):
defines['USE_GLOSSY_MATERIAL'] = 1
program = self._program_cache.get_program(
vertex_shader=vertex_shader,
fragment_shader=fragment_shader,
geometry_shader=geometry_shader,
defines=defines
)
if not program._in_context():
program._add_to_context()
return program
###########################################################################
# Viewport Management
###########################################################################
def _configure_forward_pass_viewport(self, flags):
# If using offscreen render, bind main framebuffer
if flags & RenderFlags.OFFSCREEN:
self._configure_main_framebuffer()
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb_ms)
else:
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0)
glViewport(0, 0, self.viewport_width, self.viewport_height)
glEnable(GL_DEPTH_TEST)
glDepthMask(GL_TRUE)
glDepthFunc(GL_LESS)
glDepthRange(0.0, 1.0)
def _configure_shadow_mapping_viewport(self, light, flags):
self._configure_shadow_framebuffer()
glBindFramebuffer(GL_FRAMEBUFFER, self._shadow_fb)
light.shadow_texture._bind()
light.shadow_texture._bind_as_depth_attachment()
glActiveTexture(GL_TEXTURE0)
light.shadow_texture._bind()
glDrawBuffer(GL_NONE)
glReadBuffer(GL_NONE)
glClear(GL_DEPTH_BUFFER_BIT)
glViewport(0, 0, SHADOW_TEX_SZ, SHADOW_TEX_SZ)
glEnable(GL_DEPTH_TEST)
glDepthMask(GL_TRUE)
glDepthFunc(GL_LESS)
glDepthRange(0.0, 1.0)
glDisable(GL_CULL_FACE)
glDisable(GL_BLEND)
###########################################################################
# Framebuffer Management
###########################################################################
def _configure_shadow_framebuffer(self):
if self._shadow_fb is None:
self._shadow_fb = glGenFramebuffers(1)
def _delete_shadow_framebuffer(self):
if self._shadow_fb is not None:
glDeleteFramebuffers(1, [self._shadow_fb])
def _configure_main_framebuffer(self):
# If mismatch with prior framebuffer, delete it
if (self._main_fb is not None and
self.viewport_width != self._main_fb_dims[0] or
self.viewport_height != self._main_fb_dims[1]):
self._delete_main_framebuffer()
# If framebuffer doesn't exist, create it
if self._main_fb is None:
# Generate standard buffer
self._main_cb, self._main_db = glGenRenderbuffers(2)
glBindRenderbuffer(GL_RENDERBUFFER, self._main_cb)
glRenderbufferStorage(
GL_RENDERBUFFER, GL_RGBA,
self.viewport_width, self.viewport_height
)
glBindRenderbuffer(GL_RENDERBUFFER, self._main_db)
glRenderbufferStorage(
GL_RENDERBUFFER, GL_DEPTH_COMPONENT24,
self.viewport_width, self.viewport_height
)
self._main_fb = glGenFramebuffers(1)
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb)
glFramebufferRenderbuffer(
GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_RENDERBUFFER, self._main_cb
)
glFramebufferRenderbuffer(
GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,
GL_RENDERBUFFER, self._main_db
)
# Generate multisample buffer
self._main_cb_ms, self._main_db_ms = glGenRenderbuffers(2)
glBindRenderbuffer(GL_RENDERBUFFER, self._main_cb_ms)
# glRenderbufferStorageMultisample(
# GL_RENDERBUFFER, 4, GL_RGBA,
# self.viewport_width, self.viewport_height
# )
# glBindRenderbuffer(GL_RENDERBUFFER, self._main_db_ms)
# glRenderbufferStorageMultisample(
# GL_RENDERBUFFER, 4, GL_DEPTH_COMPONENT24,
# self.viewport_width, self.viewport_height
# )
# 增加这一行
num_samples = min(glGetIntegerv(GL_MAX_SAMPLES), 4) # No more than GL_MAX_SAMPLES
# 其实就是把 4 替换成 num_samples,其余不变
glRenderbufferStorageMultisample(GL_RENDERBUFFER, num_samples, GL_RGBA, self.viewport_width, self.viewport_height)
glBindRenderbuffer(GL_RENDERBUFFER, self._main_db_ms) # 这行不变
# 这一行也是将 4 替换成 num_samples
glRenderbufferStorageMultisample(GL_RENDERBUFFER, num_samples, GL_DEPTH_COMPONENT24, self.viewport_width, self.viewport_height)
self._main_fb_ms = glGenFramebuffers(1)
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb_ms)
glFramebufferRenderbuffer(
GL_DRAW_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
GL_RENDERBUFFER, self._main_cb_ms
)
glFramebufferRenderbuffer(
GL_DRAW_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,
GL_RENDERBUFFER, self._main_db_ms
)
self._main_fb_dims = (self.viewport_width, self.viewport_height)
def _delete_main_framebuffer(self):
if self._main_fb is not None:
glDeleteFramebuffers(2, [self._main_fb, self._main_fb_ms])
if self._main_cb is not None:
glDeleteRenderbuffers(2, [self._main_cb, self._main_cb_ms])
if self._main_db is not None:
glDeleteRenderbuffers(2, [self._main_db, self._main_db_ms])
self._main_fb = None
self._main_cb = None
self._main_db = None
self._main_fb_ms = None
self._main_cb_ms = None
self._main_db_ms = None
self._main_fb_dims = (None, None)
def _read_main_framebuffer(self, scene, flags):
width, height = self._main_fb_dims[0], self._main_fb_dims[1]
# Bind framebuffer and blit buffers
glBindFramebuffer(GL_READ_FRAMEBUFFER, self._main_fb_ms)
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, self._main_fb)
glBlitFramebuffer(
0, 0, width, height, 0, 0, width, height,
GL_COLOR_BUFFER_BIT, GL_LINEAR
)
glBlitFramebuffer(
0, 0, width, height, 0, 0, width, height,
GL_DEPTH_BUFFER_BIT, GL_NEAREST
)
glBindFramebuffer(GL_READ_FRAMEBUFFER, self._main_fb)
# Read depth
depth_buf = glReadPixels(
0, 0, width, height, GL_DEPTH_COMPONENT, GL_FLOAT
)
depth_im = np.frombuffer(depth_buf, dtype=np.float32)
depth_im = depth_im.reshape((height, width))
depth_im = np.flip(depth_im, axis=0)
inf_inds = (depth_im == 1.0)
depth_im = 2.0 * depth_im - 1.0
z_near = scene.main_camera_node.camera.znear
z_far = scene.main_camera_node.camera.zfar
noninf = np.logical_not(inf_inds)
if z_far is None:
depth_im[noninf] = 2 * z_near / (1.0 - depth_im[noninf])
else:
depth_im[noninf] = ((2.0 * z_near * z_far) /
(z_far + z_near - depth_im[noninf] *
(z_far - z_near)))
depth_im[inf_inds] = 0.0
# Resize for macos if needed
if sys.platform == 'darwin':
depth_im = self._resize_image(depth_im)
if flags & RenderFlags.DEPTH_ONLY:
return depth_im
# Read color
if flags & RenderFlags.RGBA:
color_buf = glReadPixels(
0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE
)
color_im = np.frombuffer(color_buf, dtype=np.uint8)
color_im = color_im.reshape((height, width, 4))
else:
color_buf = glReadPixels(
0, 0, width, height, GL_RGB, GL_UNSIGNED_BYTE
)
color_im = np.frombuffer(color_buf, dtype=np.uint8)
color_im = color_im.reshape((height, width, 3))
color_im = np.flip(color_im, axis=0)
# Resize for macos if needed
if sys.platform == 'darwin':
color_im = self._resize_image(color_im, True)
return color_im, depth_im
def _resize_image(self, value, antialias=False):
"""If needed, rescale the render for MacOS."""
img = PIL.Image.fromarray(value)
resample = PIL.Image.NEAREST
if antialias:
resample = PIL.Image.BILINEAR
size = (self.viewport_width // self.dpscale,
self.viewport_height // self.dpscale)
img = img.resize(size, resample=resample)
return np.array(img)
###########################################################################
# Shadowmap Debugging
###########################################################################
def _forward_pass_no_reset(self, scene, flags):
# Set up camera matrices
V, P = self._get_camera_matrices(scene)
# Now, render each object in sorted order
for node in self._sorted_mesh_nodes(scene):
mesh = node.mesh
# Skip the mesh if it's not visible
if not mesh.is_visible:
continue
for primitive in mesh.primitives:
# First, get and bind the appropriate program
program = self._get_primitive_program(
primitive, flags, ProgramFlags.USE_MATERIAL
)
program._bind()
# Set the camera uniforms
program.set_uniform('V', V)
program.set_uniform('P', P)
program.set_uniform(
'cam_pos', scene.get_pose(scene.main_camera_node)[:3,3]
)
# Next, bind the lighting
if not flags & RenderFlags.DEPTH_ONLY and not flags & RenderFlags.FLAT:
self._bind_lighting(scene, program, node, flags)
# Finally, bind and draw the primitive
self._bind_and_draw_primitive(
primitive=primitive,
pose=scene.get_pose(node),
program=program,
flags=flags
)
self._reset_active_textures()
# Unbind the shader and flush the output
if program is not None:
program._unbind()
glFlush()
def _render_light_shadowmaps(self, scene, light_nodes, flags, tile=False):
glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0)
glClearColor(*scene.bg_color)
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glEnable(GL_DEPTH_TEST)
glDepthMask(GL_TRUE)
glDepthFunc(GL_LESS)
glDepthRange(0.0, 1.0)
w = self.viewport_width
h = self.viewport_height
num_nodes = len(light_nodes)
viewport_dims = {
(0, 2): [0, h // 2, w // 2, h],
(1, 2): [w // 2, h // 2, w, h],
(0, 3): [0, h // 2, w // 2, h],
(1, 3): [w // 2, h // 2, w, h],
(2, 3): [0, 0, w // 2, h // 2],
(0, 4): [0, h // 2, w // 2, h],
(1, 4): [w // 2, h // 2, w, h],
(2, 4): [0, 0, w // 2, h // 2],
(3, 4): [w // 2, 0, w, h // 2]
}
if tile:
for i, ln in enumerate(light_nodes):
light = ln.light
if light.shadow_texture is None:
raise ValueError('Light does not have a shadow texture')
glViewport(*viewport_dims[(i, num_nodes + 1)])
program = self._get_debug_quad_program()
program._bind()
self._bind_texture(light.shadow_texture, 'depthMap', program)
self._render_debug_quad()
self._reset_active_textures()
glFlush()
i += 1
glViewport(*viewport_dims[(i, num_nodes + 1)])
self._forward_pass_no_reset(scene, flags)
else:
for i, ln in enumerate(light_nodes):
light = ln.light
if light.shadow_texture is None:
raise ValueError('Light does not have a shadow texture')
glViewport(0, 0, self.viewport_width, self.viewport_height)
program = self._get_debug_quad_program()
program._bind()
self._bind_texture(light.shadow_texture, 'depthMap', program)
self._render_debug_quad()
self._reset_active_textures()
glFlush()
return
def _get_debug_quad_program(self):
program = self._program_cache.get_program(
vertex_shader='debug_quad.vert',
fragment_shader='debug_quad.frag'
)
if not program._in_context():
program._add_to_context()
return program
def _render_debug_quad(self):
x = glGenVertexArrays(1)
glBindVertexArray(x)
glDrawArrays(GL_TRIANGLES, 0, 6)
glBindVertexArray(0)
glDeleteVertexArrays(1, [x])