Repository: vispy/vispy

Issue:
Impossible to read the depth buffer
According to the document http://vispy.org/gloo.html, `vispy.gloo.FrameBuffer.read` supports read the depth buffer.  However in practice it always returns an numpy array whose shape is either `(w, h, 3)` or `(w, h, 4)`, which is not the depth buffer.



Discussion:
May be the read pixels function in https://github.com/vispy/vispy/blob/master/vispy/gloo/framebuffer.py is wrong.

The code for `vispy.gloo.FrameBuffer.read` is

```
def read(self, mode='color', alpha=True):
    _check_valid('mode', mode, ['color', 'depth', 'stencil'])
    buffer = getattr(self, mode+'_buffer')
    h, w = buffer.shape[:2]

    # todo: this is ostensibly required, but not available in gloo.gl
    #gl.glReadBuffer(buffer._target)

    return read_pixels((0, 0, w, h), alpha=alpha)
```

which does not uses the `mode` parameter at all.

Do either of you have example/test code that I could use to test this and know that I'm testing it correctly?
It seems that as the comment said, the blocker is `gl.glReadBuffer`. I will try to provide a working test code.
@zhou13 Any updates on this?
```python
from vispy import gloo
from vispy import app
import numpy as np
import matplotlib.pyplot as plt

# Create vetices
vPosition = np.array([[-0.8, -0.8, 0.0], [+0.7, -0.7, 0.0],
                      [-0.7, +0.7, 0.0], [+0.8, +0.8, 0.0, ]], np.float32)
vPosition_full = np.array([[-1.0, -1.0, 0.0], [+1.0, -1.0, 0.0],
                           [-1.0, +1.0, 0.0], [+1.0, +1.0, 0.0, ]], np.float32)
vTexcoord = np.array([[0.0, 0.0], [0.0, 1.0],
                      [1.0, 0.0], [1.0, 1.0]], np.float32)

# For initial quad
VERT_SHADER1 = """
attribute vec3 a_position;
void main (void) {
    gl_Position = vec4(a_position, 1.0);
}
"""

FRAG_SHADER1 = """
uniform vec4 u_color;
void main()
{
    gl_FragColor = u_color;
}
"""

# To render the result of the FBO
VERT_SHADER2 = """
attribute vec3 a_position;
attribute vec2 a_texcoord;
varying vec2 v_texcoord;
void main (void) {
    // Pass tex coords
    v_texcoord = a_texcoord;
    // Calculate position
    gl_Position = vec4(a_position.x, a_position.y, a_position.z, 1.0);
}
"""

FRAG_SHADER2 = """
uniform sampler2D u_texture1;
varying vec2 v_texcoord;
const float c_zero = 0.0;
const int c_sze = 5;
void main()
{
    float scalefactor = 1.0 / float(c_sze * c_sze * 4 + 1);
    gl_FragColor = vec4(c_zero, c_zero, c_zero, 1.0);
    for (int y=-c_sze; y<=c_sze; y++) {
        for (int x=-c_sze; x<=c_sze; x++) {
            vec2 step = vec2(x,y) * 0.01;
            vec3 color = texture2D(u_texture1, v_texcoord.st+step).rgb;
            gl_FragColor.rgb += color * scalefactor;
        }
    }
}
"""


SIZE = 50


class Canvas(app.Canvas):

    def __init__(self):
        app.Canvas.__init__(self, keys='interactive', size=(560, 420))

        # Create texture to render to
        shape = self.physical_size[1], self.physical_size[0]
        self._rendertex = gloo.Texture2D((shape + (3,)))

        # Create FBO, attach the color buffer and depth buffer
        self._fbo = gloo.FrameBuffer(self._rendertex, gloo.RenderBuffer(shape))

        # Create program to render a shape
        self._program1 = gloo.Program(VERT_SHADER1, FRAG_SHADER1)
        self._program1['u_color'] = 0.9, 1.0, 0.4, 1
        self._program1['a_position'] = gloo.VertexBuffer(vPosition)

        # Create program to render FBO result
        self._program2 = gloo.Program(VERT_SHADER2, FRAG_SHADER2)
        self._program2['a_position'] = gloo.VertexBuffer(vPosition)
        self._program2['a_texcoord'] = gloo.VertexBuffer(vTexcoord)
        self._program2['u_texture1'] = self._rendertex

        self.show()

    def on_resize(self, event):
        width, height = event.physical_size
        gloo.set_viewport(0, 0, width, height)

    def on_draw(self, event):
        # Draw the same scene as as in hello_quad.py, but draw it to the FBO
        with self._fbo:
            gloo.set_clear_color((0.0, 0.0, 0.5, 1))
            gloo.clear(color=True, depth=True)
            gloo.set_viewport(0, 0, *self.physical_size)
            self._program1.draw('triangle_strip')

            image = self._fbo.read(mode="color")
            depth = self._fbo.read(mode="depth")
            plt.figure()
            plt.imshow(image)
            plt.figure()
            plt.imshow(depth)
            plt.show()

            app.quit()

        # Now draw result to a full-screen quad
        # Init
        gloo.set_clear_color('white')
        gloo.clear(color=True, depth=True)
        self._program2.draw('triangle_strip')


if __name__ == '__main__':
    c = Canvas()
    app.run()
```
This is definitely bug and I think I have a fix. For the record, here is the version of your code that I've adjusted to work properly. Mainly I enabled depth_test which wasn't being done before. Otherwise I just rearranged the plotting stuff to make it easier to analyze.

<details>

```
from vispy import gloo
from vispy import app
import numpy as np
import matplotlib.pyplot as plt

# Create vetices
vPosition = np.array([[-0.8, -0.8, 0.8], [+0.7, -0.7, 0.2],
                      [-0.7, +0.7, 0.5], [+0.8, +0.8, 0.9, ]], np.float32)
vPosition_full = np.array([[-1.0, -1.0, 0.8], [+1.0, -1.0, 0.2],
                           [-1.0, +1.0, 0.5], [+1.0, +1.0, 0.9, ]], np.float32)
vTexcoord = np.array([[0.0, 0.0], [0.0, 1.0],
                      [1.0, 0.0], [1.0, 1.0]], np.float32)

# For initial quad
VERT_SHADER1 = """
attribute vec3 a_position;
void main (void) {
    gl_Position = vec4(a_position, 1.0);
}
"""

FRAG_SHADER1 = """
uniform vec4 u_color;
void main()
{
    gl_FragColor = u_color;
}
"""

# To render the result of the FBO
VERT_SHADER2 = """
attribute vec3 a_position;
attribute vec2 a_texcoord;
varying vec2 v_texcoord;
void main (void) {
    // Pass tex coords
    v_texcoord = a_texcoord;
    // Calculate position
    gl_Position = vec4(a_position.x, a_position.y, a_position.z, 1.0);
}
"""

FRAG_SHADER2 = """
uniform sampler2D u_texture1;
varying vec2 v_texcoord;
const float c_zero = 0.0;
const int c_sze = 5;
void main()
{
    float scalefactor = 1.0 / float(c_sze * c_sze * 4 + 1);
    gl_FragColor = vec4(c_zero, c_zero, c_zero, 1.0);
    for (int y=-c_sze; y<=c_sze; y++) {
        for (int x=-c_sze; x<=c_sze; x++) {
            vec2 step = vec2(x,y) * 0.01;
            vec3 color = texture2D(u_texture1, v_texcoord.st+step).rgb;
            gl_FragColor.rgb += color * scalefactor;
        }
    }
}
"""


SIZE = 50


class Canvas(app.Canvas):

    def __init__(self):
        app.Canvas.__init__(self, keys='interactive', size=(560, 420))

        # Create texture to render to
        shape = self.physical_size[1], self.physical_size[0]
        self._rendertex = gloo.Texture2D((shape + (3,)))

        # Create FBO, attach the color buffer and depth buffer
        self._fbo = gloo.FrameBuffer(self._rendertex, gloo.RenderBuffer(shape))

        # Create program to render a shape
        self._program1 = gloo.Program(VERT_SHADER1, FRAG_SHADER1)
        self._program1['u_color'] = 0.9, 1.0, 0.4, 1
        self._program1['a_position'] = gloo.VertexBuffer(vPosition)

        # Create program to render FBO result
        self._program2 = gloo.Program(VERT_SHADER2, FRAG_SHADER2)
        self._program2['a_position'] = gloo.VertexBuffer(vPosition)
        self._program2['a_texcoord'] = gloo.VertexBuffer(vTexcoord)
        self._program2['u_texture1'] = self._rendertex

        gloo.set_state(depth_test=True)
        self.show()

    def on_resize(self, event):
        width, height = event.physical_size
        gloo.set_viewport(0, 0, width, height)

    def on_draw(self, event):
        # Draw the same scene as as in hello_quad.py, but draw it to the FBO
        with self._fbo:
            gloo.set_clear_color((0.0, 0.0, 0.5, 1))
            gloo.clear(color=True, depth=True)
            gloo.set_viewport(0, 0, *self.physical_size)
            self._program1.draw('triangle_strip')

            image = self._fbo.read(mode="color")
            depth = self._fbo.read(mode="depth")[:, :, 0]
            print(image.shape, depth.shape)
            fig = plt.figure()
            ax = plt.subplot(1, 2, 1)
            ax.imshow(image)
            ax.set_title('Image')
            ax = plt.subplot(1, 2, 2)
            img = ax.imshow(depth)
            ax.set_title('Depth')
            fig.colorbar(img, ax=ax)
            plt.show()

            app.quit()

        # Now draw result to a full-screen quad
        # Init
        gloo.set_clear_color('white')
        gloo.clear(color=True, depth=True)
        self._program2.draw('triangle_strip')


if __name__ == '__main__':
    c = Canvas()
    app.run()


```

</details>

Produces (with my fixes):

![image](https://user-images.githubusercontent.com/1828519/60928723-14c6f500-a274-11e9-9c8c-6d3b38ecf01a.png)


Resolving patch:
diff --git a/vispy/gloo/framebuffer.py b/vispy/gloo/framebuffer.py
--- a/vispy/gloo/framebuffer.py
+++ b/vispy/gloo/framebuffer.py
@@ -250,12 +250,14 @@ def read(self, mode='color', alpha=True, crop=None):
         
         """
         _check_valid('mode', mode, ['color', 'depth', 'stencil'])
+        buffer = getattr(self, mode + '_buffer')
+        if buffer is None:
+            raise ValueError("Can't read pixels for buffer {}, "
+                             "buffer does not exist.".format(mode))
         if crop is None:
-            buffer = getattr(self, mode+'_buffer')
             h, w = buffer.shape[:2]
             crop = (0, 0, w, h)
         
         # todo: this is ostensibly required, but not available in gloo.gl
         #gl.glReadBuffer(buffer._target)
-        
-        return read_pixels(crop, alpha=alpha)
+        return read_pixels(crop, alpha=alpha, mode=mode)
diff --git a/vispy/gloo/wrappers.py b/vispy/gloo/wrappers.py
--- a/vispy/gloo/wrappers.py
+++ b/vispy/gloo/wrappers.py
@@ -602,7 +602,7 @@ def glir(self):
 ## Functions that do not use the glir queue
 
 
-def read_pixels(viewport=None, alpha=True, out_type='unsigned_byte'):
+def read_pixels(viewport=None, alpha=True, mode='color', out_type='unsigned_byte'):
     """Read pixels from the currently selected buffer.
 
     Under most circumstances, this function reads from the front buffer.
@@ -616,7 +616,10 @@ def read_pixels(viewport=None, alpha=True, out_type='unsigned_byte'):
         the current GL viewport will be queried and used.
     alpha : bool
         If True (default), the returned array has 4 elements (RGBA).
-        If False, it has 3 (RGB).
+        If False, it has 3 (RGB). This only effects the color mode.
+    mode : str
+        Type of buffer data to read. Can be one of 'colors', 'depth',
+        or 'stencil'. See returns for more information.
     out_type : str | dtype
         Can be 'unsigned_byte' or 'float'. Note that this does not
         use casting, but instead determines how values are read from
@@ -627,9 +630,13 @@ def read_pixels(viewport=None, alpha=True, out_type='unsigned_byte'):
     -------
     pixels : array
         3D array of pixels in np.uint8 or np.float32 format.
-        The array shape is (h, w, 3) or (h, w, 4), with the top-left corner
-        of the framebuffer at index [0, 0] in the returned array.
+        The array shape is (h, w, 3) or (h, w, 4) for colors mode,
+        with the top-left corner of the framebuffer at index [0, 0] in the
+        returned array. If 'mode' is depth or stencil then the last dimension
+        is 1.
     """
+    _check_valid('mode', mode, ['color', 'depth', 'stencil'])
+
     # Check whether the GL context is direct or remote
     context = get_current_canvas().context
     if context.shared.parser.is_remote():
@@ -649,7 +656,18 @@ def read_pixels(viewport=None, alpha=True, out_type='unsigned_byte'):
                          % (viewport,))
     x, y, w, h = viewport
     gl.glPixelStorei(gl.GL_PACK_ALIGNMENT, 1)  # PACK, not UNPACK
-    fmt = gl.GL_RGBA if alpha else gl.GL_RGB
+    if mode == 'depth':
+        fmt = gl.GL_DEPTH_COMPONENT
+        shape = (h, w, 1)
+    elif mode == 'stencil':
+        fmt = gl.GL_STENCIL_INDEX8
+        shape = (h, w, 1)
+    elif alpha:
+        fmt = gl.GL_RGBA
+        shape = (h, w, 4)
+    else:
+        fmt = gl.GL_RGB
+        shape = (h, w, 3)
     im = gl.glReadPixels(x, y, w, h, fmt, type_)
     gl.glPixelStorei(gl.GL_PACK_ALIGNMENT, 4)
     # reshape, flip, and return
@@ -657,8 +675,8 @@ def read_pixels(viewport=None, alpha=True, out_type='unsigned_byte'):
         np_dtype = np.uint8 if type_ == gl.GL_UNSIGNED_BYTE else np.float32
         im = np.frombuffer(im, np_dtype)
 
-    im.shape = h, w, (4 if alpha else 3)  # RGBA vs RGB
-    im = im[::-1, :, :]  # flip the image
+    im.shape = shape
+    im = im[::-1, ...]  # flip the image
     return im
 
 


Test patch:
diff --git a/vispy/gloo/tests/test_wrappers.py b/vispy/gloo/tests/test_wrappers.py
--- a/vispy/gloo/tests/test_wrappers.py
+++ b/vispy/gloo/tests/test_wrappers.py
@@ -211,13 +211,14 @@ def test_wrappers():
 def test_read_pixels():
     """Test read_pixels to ensure that the image is not flipped"""
     # Create vertices
-    vPosition = np.array([[-1, 1], [0, 1],  # For drawing a square to top left
-                          [-1, 0], [0, 0]], np.float32)
+    vPosition = np.array(
+        [[-1, 1, 0.0], [0, 1, 0.5],  # For drawing a square to top left
+         [-1, 0, 0.0], [0, 0, 0.5]], np.float32)
 
     VERT_SHADER = """ // simple vertex shader
-    attribute vec2 a_position;
+    attribute vec3 a_position;
     void main (void) {
-        gl_Position = vec4(a_position, 0., 1.0);
+        gl_Position = vec4(a_position, 1.0);
     }
     """
 
@@ -231,6 +232,7 @@ def test_read_pixels():
     with Canvas() as c:
         c.set_current()
         gloo.set_viewport(0, 0, *c.size)
+        gloo.set_state(depth_test=True)
         c._program = gloo.Program(VERT_SHADER, FRAG_SHADER)
         c._program['a_position'] = gloo.VertexBuffer(vPosition)
         gloo.clear(color='black')
@@ -247,5 +249,15 @@ def test_read_pixels():
         gloo.flush()
         gloo.finish()
 
+        # Check that we can read the depth buffer
+        img = read_pixels(mode='depth')
+        assert_equal(img.shape[:2], c.size[::-1])
+        assert_equal(img.shape[2], 1)
+        unique_img = np.unique(img)
+        # we should have quite a few different depth values
+        assert unique_img.shape[0] > 50
+        assert unique_img.max() == 255
+        assert unique_img.min() > 0
+
 
 run_tests_if_main()