Repository: Genesis-Embodied-AI/Genesis
File: genesis/ext/pyrender/interaction/viewer_interaction.py (Python)

Code before:
        
            # ray_hit = self._raycast_against_ground(self.get_camera_ray())
            ray_hit = self._raycast_against_ground(self.screen_position_to_ray(*self.prev_mouse_pos))
            if ray_hit.is_hit:
                self.scene.draw_debug_sphere(ray_hit.position.v, 0.01, (0, 1, 0, 1))
                self._draw_arrow(ray_hit.position, ray_hit.normal, (0, 1, 0, 1))

    def screen_position_to_ray(self, x: float, y: float) -> Ray:
        # convert screen position to ray
        if True:
            x = x - 0.5 * self.viewport_size[0]
            y = y - 0.5 * self.viewport_size[1]
            x = 2.0 * x / self.viewport_size[1] * self.tan_half_fov
            y = 2.0 * y / self.viewport_size[1] * self.tan_half_fov
        else:
            # alternative way
            projection_matrix = self.camera.camera.get_projection_matrix(*self.viewport_size)
            x = x - 0.5 * self.viewport_size[0]
            y = y - 0.5 * self.viewport_size[1]
            x = 2.0 * x / self.viewport_size[0] / projection_matrix[0, 0]
            y = 2.0 * y / self.viewport_size[1] / projection_matrix[1, 1]

        # Note: ignoring pixel aspect ratio

        mtx = self.camera.matrix
        position = Vec3(mtx[:3, 3])
        forward = Vec3(-mtx[:3, 2])
        right = Vec3(mtx[:3, 0])
        up = Vec3(mtx[:3, 1])

        direction = forward + right * x + up * y
        return Ray(position, direction)

    def get_camera_ray(self) -> Ray:
        mtx = self.camera.matrix
        position = Vec3(mtx[:3, 3])
        forward = Vec3(-mtx[:3, 2])
        return Ray(position, forward)

    def _raycast_against_ground(self, ray: Ray) -> RayHit:
        ground_plane = Plane(Vec3.from_xyz(0, 0, 1), Vec3.zero())
        return ground_plane.raycast(ray)

    def _draw_arrow(self, pos: Vec3, dir: Vec3, color: tuple[float, float, float, float] = (1, 1, 1, 1)) -> None:
        # Only draws arrowhead:
        self.scene.draw_debug_arrow(
            pos=pos.v,
            vec=dir.v,
            color=color,
        )
        self.scene.draw_debug_line(

Review comment (bug):
@YilingQiao @duburcqa On the topic of strong typing.

I just realized, that here, in code, we still use an incorrect dtype, cos camera.matrix returns an array of float64's -- And while the expected type is linted in Vec3 constructor, it was not asserted at runtime.

So, just having an alias for types is not sufficient for guarding data type correctness at this moment. Possibly we could run some strong linter on our code as a required commit check. But currently, I can only rely on runtime checks, afaiu. 

I'll add another classmethod to Vec3 to convert explicitly from an array[np.float64]

Code after:
        mouse_ray = self.screen_position_to_ray(x, y)
        # print(f"mouse_ray: {mouse_ray}")

    @override
    def on_draw(self) -> None:
        super().on_draw()
        if self.scene._visualizer is not None and self.scene._visualizer.viewer_lock is not None:
            self.scene.clear_debug_objects()
        
            ray_hit = self._raycast_against_ground(self.screen_position_to_ray(*self.prev_mouse_pos))
            if ray_hit.is_hit:
                self.scene.draw_debug_sphere(ray_hit.position.v, 0.01, (0, 1, 0, 1))
                self._draw_arrow(ray_hit.position, ray_hit.normal, (0, 1, 0, 1))

    def screen_position_to_ray(self, x: float, y: float) -> Ray:
        # convert screen position to ray
        if True:
            x = x - 0.5 * self.viewport_size[0]
            y = y - 0.5 * self.viewport_size[1]
            x = 2.0 * x / self.viewport_size[1] * self.tan_half_fov
            y = 2.0 * y / self.viewport_size[1] * self.tan_half_fov
        else:
            # alternative way
            projection_matrix = self.camera.camera.get_projection_matrix(*self.viewport_size)
            x = x - 0.5 * self.viewport_size[0]
            y = y - 0.5 * self.viewport_size[1]
            x = 2.0 * x / self.viewport_size[0] / projection_matrix[0, 0]
            y = 2.0 * y / self.viewport_size[1] / projection_matrix[1, 1]

        # Note: ignoring pixel aspect ratio

        mtx = self.camera.matrix
        position = Vec3.from_float64(mtx[:3, 3])
        forward = Vec3.from_float64(-mtx[:3, 2])
        right = Vec3.from_float64(mtx[:3, 0])
        up = Vec3.from_float64(mtx[:3, 1])

        direction = forward + right * x + up * y
        return Ray(position, direction)

    def get_camera_ray(self) -> Ray:
        mtx = self.camera.matrix
        position = Vec3.from_float64(mtx[:3, 3])
        forward = Vec3.from_float64(-mtx[:3, 2])
        return Ray(position, forward)

    def _raycast_against_ground(self, ray: Ray) -> RayHit:
        ground_plane = Plane(Vec3.from_xyz(0, 0, 1), Vec3.zero())
        return ground_plane.raycast(ray)

    def _draw_arrow(