Repository: BlueBrain/atlas-splitter

Issue:
`atlas-splitter split-isocortex-layer-23` outputs an annotation volume with region IDs not present in the output hierarchy
Occurred with
```
cd /gpfs/bbp.cscs.ch/data/project/proj84/atlas_pipeline_runs/2024-03-12T15:01:39

atlas-splitter --version
atlas-splitter, version 0.1.3

atlas-splitter split-isocortex-layer-23  \
  --hierarchy-path hierarchy.json  \
  --annotation-path annotation_ccfv3.nrrd  \
  --direction-vectors-path direction_vectors/direction_vectors_isocortex_ccfv3.nrrd  \
  --output-hierarchy-path hierarchy_ccfv3_l23split.json  \
  --output-annotation-path annotation_ccfv3_l23split.nrrd
```

49 regions in the output annotation volume are not found in the output hierarchy:
[74352560, 96323749, 117943286, 144508775, 146617060, 156558204, 215093570, 259388834, 373656076, 410200320, 586135081, 740752775, 775557082, 940760025, 972952746, 976342702, 990831169, 1160972157, 1174775454, 1209880404, 1324692064, 1361664772, 1622995831, 1635953398, 1636706498, 1735586154, 1776372845, 1797874319, 2278275862, 2357063861, 2655117045, 2927565312, 3204811968, 3232181544, 3373402385, 3402245361, 3447123322, 3484927447, 3561909895, 3670850531, 3743715038, 3840653638, 3903157542, 3965686099, 4108857031, 4207091109, 4219725843, 4256935451, 4261721241]


Discussion:

Patch:
diff --git a/atlas_splitter/utils.py b/atlas_splitter/utils.py
index 66186d0..95407d8 100644
--- a/atlas_splitter/utils.py
+++ b/atlas_splitter/utils.py
@@ -9,6 +9,9 @@ from atlas_splitter.exceptions import AtlasSplitterError
 
 HierarchyDict = Dict[str, Any]
 
+MIN_CUSTOM_ID = 1_000_000_000
+MAX_CUSTOM_ID = 4_000_000_000
+
 
 def get_isocortex_hierarchy(allen_hierachy: HierarchyDict):
     """
@@ -61,13 +64,23 @@ def id_from_acronym(region_map: RegionMap, acronym: str) -> int:
     if region_id_set:
         [region_id] = region_id_set
     else:  # acronym not present in hierarchy, generating a corresponding id
-        sha = hashlib.sha256()
-        sha.update(acronym.encode("utf-8"))
-        region_id = int(str(int(sha.hexdigest(), 16))[0:10])
-
+        region_id = _hash_derived_id(acronym)
     return region_id
 
 
+def _hash_derived_id(acronym: str) -> int:
+    """Create an id from the acronym's sha256 digest.
+
+    Notes:
+        The id is generated in the [MIN_CUSTOM_ID, MAX_CUSTOM_ID] interval for two reasons:
+            - Be outside the current ids range
+            - Fit within the range of uint32 annotation dtype
+    """
+    sha = hashlib.sha256(acronym.encode("utf-8"))
+    integer = int.from_bytes(sha.digest(), "big")
+    return MIN_CUSTOM_ID + integer % (MAX_CUSTOM_ID - MIN_CUSTOM_ID)
+
+
 def _assert_is_leaf_node(node) -> None:
     """
     Raises an AtalasSplitterError if `node` is not a leaf node.


Test patch:
diff --git a/tests/barrel_splitter/test_somatosensory_barrels.py b/tests/barrel_splitter/test_somatosensory_barrels.py
index 4f5ee74..e8d2a93 100644
--- a/tests/barrel_splitter/test_somatosensory_barrels.py
+++ b/tests/barrel_splitter/test_somatosensory_barrels.py
@@ -46,10 +46,11 @@ def test_layer_ids():
     names = ["region1", "region2", "region3"]
     layers = ["layer1", "layer2"]
     result = tested.layer_ids(region_map, names, layers)
+
     expected = {
-        "region1": {"region1": 5293416420, "layer1": 1111509459, "layer2": 8291842637},
-        "region2": {"region2": 9197100151, "layer1": 1048759989, "layer2": 8562892645},
-        "region3": {"region3": 2807168083, "layer1": 2207162267, "layer2": 3619321798},
+        "region1": {"region1": 3631290122, "layer1": 2267454475, "layer2": 1012379119},
+        "region2": {"region2": 1722831506, "layer1": 3787876483, "layer2": 1416363748},
+        "region3": {"region3": 1193141608, "layer1": 3031486657, "layer2": 3890489924},
     }
     assert result == expected
 
diff --git a/tests/layer_splitter/test_isocortex_layer_23.py b/tests/layer_splitter/test_isocortex_layer_23.py
index fd89f49..f4a1c04 100644
--- a/tests/layer_splitter/test_isocortex_layer_23.py
+++ b/tests/layer_splitter/test_isocortex_layer_23.py
@@ -91,7 +91,7 @@ def test_edit_hierarchy():
             "parent_structure_id": 219,
         },
         {
-            "id": 2438771567,
+            "id": 2906756445,
             "acronym": "MO3",
             "name": "Somatomotor areas, Layer 3",
             "children": [],
@@ -109,7 +109,6 @@ def test_edit_hierarchy():
         ids_to_reuse,
         region_map,
     )
-
     assert isocortex_hierarchy == expected_hierarchy
 
 
diff --git a/tests/test_app_barrel_splitter.py b/tests/test_app_barrel_splitter.py
index eccb9b1..ec913c7 100644
--- a/tests/test_app_barrel_splitter.py
+++ b/tests/test_app_barrel_splitter.py
@@ -19,24 +19,24 @@ output_bfd = {
     1070,
     1998,
     1999,
-    6808354304,
-    4625922081,
-    9632197987,
-    6740949287,
-    4025629096,
-    1028864429,
-    1132763284,
-    1135236927,
-    1569938381,
-    4159694229,
-    5965588094,
-    7178447226,
-    7758245998,
-    7766041724,
-    7882359060,
-    8701962317,
-    8773083888,
-    9288923888,
+    1050271831,
+    1419828837,
+    1541598958,
+    1561898146,
+    1669999094,
+    1742227897,
+    1779524838,
+    1914058509,
+    1941812702,
+    2061835172,
+    2112620658,
+    2190770412,
+    2321675771,
+    2491030215,
+    3335143301,
+    3364575989,
+    3625413550,
+    3951617891,
 }
 
 
@@ -74,4 +74,4 @@ def test_split_barrels():
 
         barrel_cortex_ids = output_region_map.find("SSp-bfd", attr="acronym", with_descendants=True)
 
-        assert barrel_cortex_ids == set(output_bfd)
+        assert barrel_cortex_ids == output_bfd
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 8921641..b385e65 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -2,6 +2,7 @@
 import json
 from pathlib import Path
 
+import numpy as np
 import numpy.testing as npt
 import pytest
 from voxcell import RegionMap
@@ -59,14 +60,25 @@ def test_id_from_acronym(region_map):
 
     # existing region -> existing id
     res1 = tested.id_from_acronym(region_map, "VISp1")
+    _assert_within_integer_type_range(res1, np.uint32)
     assert region_map.get(res1, attr="acronym") == "VISp1"
 
-    # new regeion -> non-existing id
-    res2 = tested.id_from_acronym(region_map, "MontyPython")
+    # new region -> non-existing id
+    res2 = tested.id_from_acronym(region_map, "VISPXXX")
+    _assert_within_integer_type_range(res2, np.uint32)
     with pytest.raises(VoxcellError, match="Region ID not found"):
         region_map.get(res2, attr="acronym")
 
 
+def _assert_within_integer_type_range(value, int_dtype):
+    info = np.iinfo(int_dtype)
+    check = info.min <= value < info.max
+    assert check, (
+        f"Value not within dtype '{int_dtype.__name__}' range: "
+        f"{info.min} <= {value} < {info.max}"
+    )
+
+
 def test_create_id_generator(region_map):
     id_generator = tested.create_id_generator(region_map)