Repository: pyxem/kikuchipy (language: python)

Issue:
Fix merging of maps with non-indexed points and release v0.8.5
#### Description of the change
<!-- Remember to branch off the develop branch for new features and the main branch for patches. -->
Fixes #637.

@Erlendos12, can you test this branch in EBSP Indexer against your problem reported in #637 for me?

#### Progress of the PR
- [n/a] [Docstrings for all functions](https://github.com/numpy/numpy/blob/master/doc/example.py)
- [x] Unit tests with pytest for all lines
- [x] Clean code style by [running black via pre-commit](https://kikuchipy.org/en/latest/contributing.html#code-style)

#### Minimal example of the bug fix or new feature
Create two crystal maps with some points considered not-indexed, only some of them not-indexed in both maps
```python
import kikuchipy as kp
import numpy as np
from orix.crystal_map import CrystalMap
from orix.quaternion import Rotation


xmap_a = CrystalMap.empty((4, 3))
xmap_a.phases[0].name = "a"
is_indexed_a = np.array([[1, 1, 0], [1, 0, 1], [0, 1, 1], [0, 1, 1]], dtype=bool).ravel()
xmap_a.phases.add_not_indexed()
xmap_a[~is_indexed_a].phase_id = -1
xmap_a.prop["scores"] = np.array([[2, 2, 0], [3, 0, 4], [0, 4, 3], [0, 2, 1]], dtype=float).ravel()

xmap_b = CrystalMap.empty((4, 3))
xmap_b.phases[0].name = "b"
is_indexed_b = np.array([[1, 1, 0], [1, 1, 1], [0, 1, 1], [0, 1, 0]], dtype=bool).ravel()
xmap_b.phases.add_not_indexed()
xmap_b[~is_indexed_b].phase_id = -1
xmap_b.prop["scores"] = np.array([[3, 1, 0], [2, 1, 5], [0, 2, 4], [0, 1, 0]], dtype=float).ravel()

xmap_ab = kp.indexing.merge_crystal_maps([xmap_a, xmap_b])

print(xmap_ab)
# Before:
# Phase  Orientations         Name  Space group  Point group  Proper point group       Color
#     0   12 (100.0%)  not_indexed         None         None                None      w
# Properties: scores, merged_scores
# Scan unit: px

# Now:
# Phase  Orientations         Name  Space group  Point group  Proper point group       Color
#    -1     3 (25.0%)  not_indexed         None         None                None           w
#     0     5 (41.7%)            a         None         None                None    tab:blue
#     1     4 (33.3%)            b         None         None                None  tab:orange
# Properties: scores, merged_scores
# Scan unit: px
```

#### For reviewers
<!-- Don't remove the checklist below. -->
- [ ] The PR title is short, concise, and will make sense 1 year later.
- [ ] New functions are imported in corresponding `__init__.py`.
- [ ] New features, API changes, and deprecations are mentioned in the unreleased
      section in `CHANGELOG.rst`.
- [ ] New contributors are added to `release.py`, `.zenodo.json` and
      `.all-contributorsrc` with the table regenerated.



Discussion / hints:

Fix patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 1af0a279..909723d4 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -36,6 +36,14 @@ Fixed
 Security
 --------
 
+0.8.5 (2023-05-21)
+==================
+
+Fixed
+-----
+- Not-indexed points in crystal maps are handled correctly when merging.
+  (`#639 <https://github.com/pyxem/kikuchipy/pull/639>`_)
+
 0.8.4 (2023-04-07)
 ==================
 
diff --git a/kikuchipy/indexing/_merge_crystal_maps.py b/kikuchipy/indexing/_merge_crystal_maps.py
index 13cb35b1..1a81988e 100644
--- a/kikuchipy/indexing/_merge_crystal_maps.py
+++ b/kikuchipy/indexing/_merge_crystal_maps.py
@@ -194,7 +194,9 @@ def merge_crystal_maps(
 
     # Combined (unsorted) scores array of shape (M, N, K) or (M, K)
     scores_dtype = crystal_maps[0].prop[scores_prop].dtype
-    combined_scores = np.full(comb_shape, np.nan, dtype=scores_dtype)
+    combined_scores = np.full(
+        comb_shape, np.nan, dtype=np.dtype(f"f{scores_dtype.itemsize}")
+    )
     for i, (mask, xmap) in enumerate(zip(navigation_masks1d, crystal_maps)):
         if mask is not None:
             combined_scores[mask, ..., i] = xmap.prop[scores_prop]
@@ -212,6 +214,18 @@ def merge_crystal_maps(
     # Phase of best score in each map point
     phase_id = np.nanargmax(sign * best_scores, axis=1)
 
+    # Set the phase ID of points marked as not-indexed in all maps to -1
+    not_indexed = np.zeros((n_maps, map_size), dtype=bool)
+    for i in range(n_maps):
+        mask = navigation_masks1d[i]
+        xmap = crystal_maps[i]
+        if mask is not None:
+            not_indexed[i, mask][xmap.phase_id == -1] = True
+        else:
+            not_indexed[i, xmap.phase_id == -1] = True
+    not_indexed = np.logical_and.reduce(not_indexed)
+    phase_id[not_indexed] = -1
+
     # Get the new crystal map's rotations, scores and indices,
     # restricted to one phase per point (uncombined)
     new_rotations = np.zeros(comb_shape[:-1] + (4,), dtype="float")
@@ -221,12 +235,16 @@ def merge_crystal_maps(
         new_indices = np.zeros(comb_shape[:-1], dtype="int32")
 
     phase_list = PhaseList()
+    if -1 in phase_id:
+        phase_list.add_not_indexed()
     for i, (nav_mask1d, xmap) in enumerate(zip(navigation_masks1d, crystal_maps)):
         phase_mask = phase_id == i
 
         if phase_mask.any():
-            current_id = xmap.phases_in_data.ids[0]
-            phase = xmap.phases_in_data[current_id].deepcopy()
+            phase_ids = xmap.phases_in_data.ids
+            if -1 in phase_ids:
+                phase_ids.remove(-1)
+            phase = xmap.phases_in_data[phase_ids[0]].deepcopy()
             if phase.name in phase_list.names:
                 # If they are equal, do not duplicate it in the phase
                 # list but update the phase ID
@@ -235,12 +253,12 @@ def merge_crystal_maps(
                     phase_id[phase_mask] = phase_list.id_from_name(phase.name)
                 else:
                     name = phase.name
+                    phase.name = name + str(i)
                     warnings.warn(
                         f"There are duplicates of phase '{name}' but the phases have "
                         f"different {different}, will therefore rename this phase's "
-                        f"name to '{name + str(i)}' in the merged PhaseList",
+                        f"name to '{phase.name}' in the merged PhaseList",
                     )
-                    phase.name = name + str(i)
                     phase_list.add(phase)
             else:
                 phase_list.add(phase)


Test patch:
diff --git a/kikuchipy/indexing/tests/test_merge_crystal_maps.py b/kikuchipy/indexing/tests/test_merge_crystal_maps.py
index 8ebda8c1..fea54027 100644
--- a/kikuchipy/indexing/tests/test_merge_crystal_maps.py
+++ b/kikuchipy/indexing/tests/test_merge_crystal_maps.py
@@ -559,7 +559,7 @@ def test_merging_with_navigation_masks(self):
         )
         # fmt: on
 
-        # All points in a map should be used, but not in another one:
+        # All points in one map should be used, but not in another:
         # Only consider xmap1 in the first row and first column (mask it
         # out everywhere else)
         xmap6 = merge_crystal_maps(
@@ -644,3 +644,41 @@ def test_merging_with_navigation_masks_raises(self):
                 [xmap1[~nav_mask1.ravel()], xmap2[~nav_mask2.ravel()]],
                 navigation_masks=[nav_mask1, list(nav_mask2)],
             )
+
+    def test_not_indexed(self):
+        xmap_a = CrystalMap.empty((4, 3))
+        is_indexed_a = np.array(
+            [[1, 1, 0], [1, 0, 1], [0, 1, 1], [0, 1, 1]], dtype=bool
+        ).ravel()
+        xmap_a.phases.add_not_indexed()
+        xmap_a.phases[0].name = "a"
+        xmap_a[~is_indexed_a].phase_id = -1
+        xmap_a.prop["scores"] = np.array(
+            [[2, 2, 0], [3, 0, 4], [0, 4, 3], [0, 2, 1]], dtype=float
+        ).ravel()
+        xmap_a._rotations = xmap_a.rotations * Rotation.from_axes_angles(
+            [0, 0, 1], 30, degrees=True
+        )
+
+        xmap_b = CrystalMap.empty((4, 3))
+        is_indexed_b = np.array(
+            [[1, 1, 0], [1, 1, 1], [0, 1, 1], [0, 1, 0]], dtype=bool
+        ).ravel()
+        xmap_b.phases.add_not_indexed()
+        xmap_b.phases[0].name = "b"
+        xmap_b[~is_indexed_b].phase_id = -1
+        xmap_b.prop["scores"] = np.array(
+            [[3, 1, 0], [2, 1, 5], [0, 2, 4], [0, 1, 0]], dtype=float
+        ).ravel()
+        xmap_b._rotations = xmap_b.rotations * Rotation.from_axes_angles(
+            [0, 0, 1], 60, degrees=True
+        )
+
+        xmap_ab = merge_crystal_maps([xmap_a, xmap_b])
+
+        assert np.allclose(xmap_ab.phase_id, [1, 0, -1, 0, 1, 1, -1, 0, 1, -1, 0, 0])
+        assert np.allclose(
+            xmap_ab["indexed"].rotations.angle,
+            np.deg2rad([60, 30, 30, 60, 60, 30, 60, 30, 30]),
+        )
+        assert np.allclose(xmap_ab["indexed"].scores, [3, 2, 3, 1, 5, 4, 4, 2, 1])
diff --git a/kikuchipy/pattern/tests/test_pattern.py b/kikuchipy/pattern/tests/test_pattern.py
index 4f195aa8..c0ab3d27 100644
--- a/kikuchipy/pattern/tests/test_pattern.py
+++ b/kikuchipy/pattern/tests/test_pattern.py
@@ -181,6 +181,7 @@ def test_remove_static_background_subtract(self, dummy_signal, dummy_background)
         assert np.allclose(p0, p)
         assert p0.dtype == p.dtype
 
+    @pytest.mark.filterwarnings("ignore:invalid value")
     def test_remove_static_background_divide(self, dummy_signal, dummy_background):
         p = dummy_signal.inav[0, 0].data
         dtype_out = p.dtype
diff --git a/kikuchipy/signals/tests/test_ebsd.py b/kikuchipy/signals/tests/test_ebsd.py
index 2033ed18..45bd2e09 100644
--- a/kikuchipy/signals/tests/test_ebsd.py
+++ b/kikuchipy/signals/tests/test_ebsd.py
@@ -557,6 +557,7 @@ def test_remove_dynamic_background_raises(self, dummy_signal):
         with pytest.raises(ValueError, match=f"{filter_domain} must be "):
             dummy_signal.remove_dynamic_background(filter_domain=filter_domain)
 
+    @pytest.mark.filterwarnings("ignore:invalid value")
     def test_inplace(self, dummy_signal):
         # Current signal is unaffected
         s = dummy_signal.deepcopy()
@@ -580,6 +581,7 @@ def test_inplace(self, dummy_signal):
         s4.compute()
         assert np.allclose(s4.data, dummy_signal.data)
 
+    @pytest.mark.filterwarnings("ignore:invalid value")
     def test_lazy_output(self, dummy_signal):
         with pytest.raises(
             ValueError, match="`lazy_output=True` requires `inplace=False`"