Repository: napari/napari

Issue:
2d->3d->2d points z-ordering bug
## 🐛 Bug

2d->3d->2d z-ordering bug using `examples/add_points.py`

## To Reproduce

Steps to reproduce the behavior:

1. Run `examples/add_points.py`
2. Click 3D rendering button
3. Click 2D rendering button
4. Observe points that were in front of image are now behind image.

## Expected behavior

Points should retain correct z-order

## Environment

Last commit 0179c0dad1d93326388d3ab0f768e4ef028b3f09

## Additional context

Identified in https://github.com/napari/napari/pull/1374#issuecomment-659233743
Layer order not always remaining synchronized
## 🐛 Bug
See https://forum.image.sc/t/certain-layers-mysteriously-invisible-when-removing-then-re-adding-layers/40484, where user naten7k shared this script:
```python
import napari
import numpy as np

with napari.gui_qt():
    viewer = napari.Viewer()
    image = np.random.rand(1000,1000)
    points = np.random.rand(100, 2) * 1000
    shapes = [np.random.rand(10,2) * 1000 for x in range(10)]
    
    viewer.add_image(image)
    viewer.add_shapes(shapes, shape_type='polygon')
    viewer.add_points(points)

    # remove and re-add image    
    viewer.layers.remove('image')
    viewer.add_image(image, name='image')
    
    # remove non-image data
    viewer.layers.remove('shapes')
    viewer.layers.remove('points')
    # add it back
    viewer.add_shapes(shapes, shape_type='polygon')
    viewer.add_points(points)
   # and mysteriously, shapes is not visible
```

at the end of that script, the shapes are not visible even though they are in front of the image:
```
[viewer.window.qt_viewer.layer_to_visual[x].order for x in viewer.layers]
# output is [3,2,3] ... so two layers erroneously have the same value
```
there's probably a quick fix, but I will also mention a conversation that @sofroniewn and I had recently about this: this is a good example of the multiple-sources-of-truth problem.  Ideally, the visual.node.order would be a dynamic property that pulled the index of the layer in the LayerList when needed, rather than something that was help independently on the node.  That's a bit tricky because this is a vispy class, but we could create a subclass that knew how to retrieve the proper order from a provided layerlist.


Discussion:
Note this was working in `0.3.5` I will investigate when fail started. We'll want to fix before `0.3.6` release - I don't think it will take long to identify/ add test for
Note this was broken in #1445 I will look into fix now


Resolving patch:
diff --git a/napari/_qt/qt_viewer.py b/napari/_qt/qt_viewer.py
--- a/napari/_qt/qt_viewer.py
+++ b/napari/_qt/qt_viewer.py
@@ -248,7 +248,7 @@ def _add_layer(self, event):
         layer = event.item
         vispy_layer = create_vispy_visual(layer)
         vispy_layer.node.parent = self.view.scene
-        vispy_layer.order = len(layers)
+        vispy_layer.order = len(layers) - 1
         self.canvas.connect(vispy_layer.on_draw)
         self.layer_to_visual[layer] = vispy_layer
 
@@ -266,6 +266,7 @@ def _remove_layer(self, event):
         vispy_layer.node.transforms = ChainTransform()
         vispy_layer.node.parent = None
         del vispy_layer
+        self._reorder_layers(None)
 
     def _reorder_layers(self, event):
         """When the list is reordered, propagate changes to draw order.
diff --git a/napari/_vispy/vispy_image_layer.py b/napari/_vispy/vispy_image_layer.py
--- a/napari/_vispy/vispy_image_layer.py
+++ b/napari/_vispy/vispy_image_layer.py
@@ -47,6 +47,7 @@ def _on_display_change(self, data=None):
 
         self.node.set_data(data)
         self.node.parent = parent
+        self.node.order = self.order
         self.reset()
 
     def _on_data_change(self, event=None):
diff --git a/napari/_vispy/vispy_points_layer.py b/napari/_vispy/vispy_points_layer.py
--- a/napari/_vispy/vispy_points_layer.py
+++ b/napari/_vispy/vispy_points_layer.py
@@ -1,7 +1,6 @@
 import numpy as np
 from vispy.scene.visuals import Line, Compound, Text
 from .markers import Markers
-from vispy.visuals.transforms import ChainTransform
 
 from .vispy_base_layer import VispyBaseLayer
 from ..utils.colormaps.standardize_color import transform_color
@@ -17,7 +16,7 @@ def __init__(self, layer):
         # Lines: The lines of the interaction box used for highlights.
         # Markers: The the outlines for each point used for highlights.
         # Markers: The actual markers of each point.
-        node = Compound([Markers(), Markers(), Line()])
+        node = Compound([Markers(), Markers(), Line(), Text()])
 
         super().__init__(layer, node)
 
@@ -29,31 +28,9 @@ def __init__(self, layer):
             self._on_text_change, self._on_blending_change
         )
         self.layer.events.highlight.connect(self._on_highlight_change)
-        self._on_display_change()
         self._on_data_change()
 
-    def _on_display_change(self):
-        parent = self.node.parent
-        self.node.transforms = ChainTransform()
-        self.node.parent = None
-
-        if self.layer.dims.ndisplay == 2:
-            self.node = Compound([Markers(), Markers(), Line(), Text()])
-        else:
-            self.node = Compound([Markers(), Markers(), Text()])
-        self.node.parent = parent
-        self._reset_base()
-
     def _on_data_change(self, event=None):
-        # Check if ndisplay has changed current node type needs updating
-        if (
-            self.layer.dims.ndisplay == 3 and len(self.node._subvisuals) != 3
-        ) or (
-            self.layer.dims.ndisplay == 2 and len(self.node._subvisuals) != 4
-        ):
-            self._on_display_change()
-            self._on_highlight_change()
-
         if len(self.layer._indices_view) > 0:
             edge_color = self.layer._view_edge_color
             face_color = self.layer._view_face_color
@@ -128,6 +105,10 @@ def _on_highlight_change(self, event=None):
                 color=self._highlight_color,
                 width=width,
             )
+        else:
+            self.node._subvisuals[2].set_data(
+                pos=np.zeros((1, self.layer.dims.ndisplay)), width=0,
+            )
 
         self.node.update()
 


Test patch:
diff --git a/napari/_qt/_tests/test_qt_viewer.py b/napari/_qt/_tests/test_qt_viewer.py
--- a/napari/_qt/_tests/test_qt_viewer.py
+++ b/napari/_qt/_tests/test_qt_viewer.py
@@ -158,6 +158,39 @@ def test_new_shapes_empty_viewer(make_test_viewer):
     assert np.sum(view.dims._displayed_sliders) == 0
 
 
+def test_z_order_adding_removing_images(make_test_viewer):
+    """Test z order is correct after adding/ removing images."""
+    data = np.ones((10, 10))
+
+    viewer = make_test_viewer()
+    vis = viewer.window.qt_viewer.layer_to_visual
+    viewer.add_image(data, colormap='red', name='red')
+    viewer.add_image(data, colormap='green', name='green')
+    viewer.add_image(data, colormap='blue', name='blue')
+    order = [vis[x].order for x in viewer.layers]
+    np.testing.assert_almost_equal(order, list(range(len(viewer.layers))))
+
+    # Remove and re-add image
+    viewer.layers.remove('red')
+    order = [vis[x].order for x in viewer.layers]
+    np.testing.assert_almost_equal(order, list(range(len(viewer.layers))))
+    viewer.add_image(data, colormap='red', name='red')
+    order = [vis[x].order for x in viewer.layers]
+    np.testing.assert_almost_equal(order, list(range(len(viewer.layers))))
+
+    # Remove two other images
+    viewer.layers.remove('green')
+    viewer.layers.remove('blue')
+    order = [vis[x].order for x in viewer.layers]
+    np.testing.assert_almost_equal(order, list(range(len(viewer.layers))))
+
+    # Add two other layers back
+    viewer.add_image(data, colormap='green', name='green')
+    viewer.add_image(data, colormap='blue', name='blue')
+    order = [vis[x].order for x in viewer.layers]
+    np.testing.assert_almost_equal(order, list(range(len(viewer.layers))))
+
+
 def test_screenshot(make_test_viewer):
     "Test taking a screenshot"
     viewer = make_test_viewer()
diff --git a/napari/_tests/test_with_screenshot.py b/napari/_tests/test_with_screenshot.py
--- a/napari/_tests/test_with_screenshot.py
+++ b/napari/_tests/test_with_screenshot.py
@@ -4,6 +4,160 @@
 import pytest
 
 
+@pytest.mark.skipif(
+    sys.platform.startswith('win') or not os.getenv("CI"),
+    reason='Screenshot tests are not supported on napari windows CI.',
+)
+def test_z_order_adding_removing_images(make_test_viewer):
+    """Test z order is correct after adding/ removing images."""
+    data = np.ones((10, 10))
+
+    viewer = make_test_viewer(show=True)
+    viewer.add_image(data, colormap='red', name='red')
+    viewer.add_image(data, colormap='green', name='green')
+    viewer.add_image(data, colormap='blue', name='blue')
+
+    # Check that blue is visible
+    screenshot = viewer.screenshot(canvas_only=True)
+    center = tuple(np.round(np.divide(screenshot.shape[:2], 2)).astype(int))
+    np.testing.assert_almost_equal(screenshot[center], [0, 0, 255, 255])
+
+    # Remove and re-add image
+    viewer.layers.remove('red')
+    viewer.add_image(data, colormap='red', name='red')
+    # Check that red is visible
+    screenshot = viewer.screenshot(canvas_only=True)
+    np.testing.assert_almost_equal(screenshot[center], [255, 0, 0, 255])
+
+    # Remove two other images
+    viewer.layers.remove('green')
+    viewer.layers.remove('blue')
+    # Check that red is still visible
+    screenshot = viewer.screenshot(canvas_only=True)
+    np.testing.assert_almost_equal(screenshot[center], [255, 0, 0, 255])
+
+    # Add two other layers back
+    viewer.add_image(data, colormap='green', name='green')
+    viewer.add_image(data, colormap='blue', name='blue')
+    # Check that blue is visible
+    screenshot = viewer.screenshot(canvas_only=True)
+    np.testing.assert_almost_equal(screenshot[center], [0, 0, 255, 255])
+
+    # Hide blue
+    viewer.layers['blue'].visible = False
+    # Check that green is visible. Note this assert was failing before #1463
+    screenshot = viewer.screenshot(canvas_only=True)
+    np.testing.assert_almost_equal(screenshot[center], [0, 255, 0, 255])
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith('win') or not os.getenv("CI"),
+    reason='Screenshot tests are not supported on napari windows CI.',
+)
+def test_z_order_images(make_test_viewer):
+    """Test changing order of images changes z order in display."""
+    data = np.ones((10, 10))
+
+    viewer = make_test_viewer(show=True)
+    viewer.add_image(data, colormap='red')
+    viewer.add_image(data, colormap='blue')
+    screenshot = viewer.screenshot(canvas_only=True)
+    center = tuple(np.round(np.divide(screenshot.shape[:2], 2)).astype(int))
+    # Check that blue is visible
+    np.testing.assert_almost_equal(screenshot[center], [0, 0, 255, 255])
+
+    viewer.layers[0, 1] = viewer.layers[1, 0]
+    screenshot = viewer.screenshot(canvas_only=True)
+    center = tuple(np.round(np.divide(screenshot.shape[:2], 2)).astype(int))
+    # Check that red is now visible
+    np.testing.assert_almost_equal(screenshot[center], [255, 0, 0, 255])
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith('win') or not os.getenv("CI"),
+    reason='Screenshot tests are not supported on napari windows CI.',
+)
+def test_z_order_image_points(make_test_viewer):
+    """Test changing order of image and points changes z order in display."""
+    data = np.ones((10, 10))
+
+    viewer = make_test_viewer(show=True)
+    viewer.add_image(data, colormap='red')
+    viewer.add_points([5, 5], face_color='blue', size=10)
+    screenshot = viewer.screenshot(canvas_only=True)
+    center = tuple(np.round(np.divide(screenshot.shape[:2], 2)).astype(int))
+    # Check that blue is visible
+    np.testing.assert_almost_equal(screenshot[center], [0, 0, 255, 255])
+
+    viewer.layers[0, 1] = viewer.layers[1, 0]
+    screenshot = viewer.screenshot(canvas_only=True)
+    center = tuple(np.round(np.divide(screenshot.shape[:2], 2)).astype(int))
+    # Check that red is now visible
+    np.testing.assert_almost_equal(screenshot[center], [255, 0, 0, 255])
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith('win') or not os.getenv("CI"),
+    reason='Screenshot tests are not supported on napari windows CI.',
+)
+def test_z_order_images_after_ndisplay(make_test_viewer):
+    """Test z order of images remanins constant after chaning ndisplay."""
+    data = np.ones((10, 10))
+
+    viewer = make_test_viewer(show=True)
+    viewer.add_image(data, colormap='red')
+    viewer.add_image(data, colormap='blue')
+    screenshot = viewer.screenshot(canvas_only=True)
+    center = tuple(np.round(np.divide(screenshot.shape[:2], 2)).astype(int))
+    # Check that blue is visible
+    np.testing.assert_almost_equal(screenshot[center], [0, 0, 255, 255])
+
+    # Switch to 3D rendering
+    viewer.dims.ndisplay = 3
+    screenshot = viewer.screenshot(canvas_only=True)
+    center = tuple(np.round(np.divide(screenshot.shape[:2], 2)).astype(int))
+    # Check that blue is still visible
+    np.testing.assert_almost_equal(screenshot[center], [0, 0, 255, 255])
+
+    # Switch back to 2D rendering
+    viewer.dims.ndisplay = 2
+    screenshot = viewer.screenshot(canvas_only=True)
+    center = tuple(np.round(np.divide(screenshot.shape[:2], 2)).astype(int))
+    # Check that blue is still visible
+    np.testing.assert_almost_equal(screenshot[center], [0, 0, 255, 255])
+
+
+@pytest.mark.skipif(
+    sys.platform.startswith('win') or not os.getenv("CI"),
+    reason='Screenshot tests are not supported on napari windows CI.',
+)
+def test_z_order_image_points_after_ndisplay(make_test_viewer):
+    """Test z order of image and points remanins constant after chaning ndisplay."""
+    data = np.ones((10, 10))
+
+    viewer = make_test_viewer(show=True)
+    viewer.add_image(data, colormap='red')
+    viewer.add_points([5, 5], face_color='blue', size=10)
+    screenshot = viewer.screenshot(canvas_only=True)
+    center = tuple(np.round(np.divide(screenshot.shape[:2], 2)).astype(int))
+    # Check that blue is visible
+    np.testing.assert_almost_equal(screenshot[center], [0, 0, 255, 255])
+
+    # Switch to 3D rendering
+    viewer.dims.ndisplay = 3
+    screenshot = viewer.screenshot(canvas_only=True)
+    center = tuple(np.round(np.divide(screenshot.shape[:2], 2)).astype(int))
+    # Check that blue is still visible
+    np.testing.assert_almost_equal(screenshot[center], [0, 0, 255, 255])
+
+    # Switch back to 2D rendering
+    viewer.dims.ndisplay = 2
+    screenshot = viewer.screenshot(canvas_only=True)
+    center = tuple(np.round(np.divide(screenshot.shape[:2], 2)).astype(int))
+    # Check that blue is still visible
+    np.testing.assert_almost_equal(screenshot[center], [0, 0, 255, 255])
+
+
 @pytest.mark.skipif(
     sys.platform.startswith('win') or not os.getenv("CI"),
     reason='Screenshot tests are not supported on napari windows CI.',