Repository: eEcoLiDAR/laserchicken

Issue:
calculate normalized height
This module should have the digital surface model DTM and a point cloud as input and add the normalized height attribute to the point cloud by subtracting the local height of the DTM from each point.

Comments:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d2e3ef4..e6ac92a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,12 +6,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Added
+- Normalization module
 - General tests that all current and future feature extractors will be checked against.
+- Possibility to have a randomly subsampled (fixed) number of neighbors (eg for faster feature calculation) 
 
 ## Changed
+- Many feature calculations are done in a vectorized way
 
 ## Fixed
-- Fixed many feature extractors for corner cases (e.g. zero points)
+- Corner cases for many feature extractors (e.g. zero points)
 
 ## Removed
 
diff --git a/laserchicken/feature_extractor/__init__.py b/laserchicken/feature_extractor/__init__.py
index f909613..f2106f9 100644
--- a/laserchicken/feature_extractor/__init__.py
+++ b/laserchicken/feature_extractor/__init__.py
@@ -95,26 +95,6 @@ def compute_features(env_point_cloud, neighborhoods, target_idx_base, target_poi
             if provided_feature in features_to_do:
                 features_to_do.remove(provided_feature)
 
-    # for feature in extended_features:
-    #     if (target_idx_base == 0) and (not overwrite) and (feature in target_point_cloud[keys.point]):
-    #         continue  # Skip feature calc if it is already there and we do not overwrite
-    #
-    #     if verbose:
-    #         sys.stdout.write('Feature "{}"\n'.format(feature))
-    #         sys.stdout.flush()
-    #         start = time.time()
-    #
-    #     extractor = FEATURES[feature]()
-    #     _add_or_update_feature(env_point_cloud, neighborhoods, target_idx_base,
-    #                            target_point_cloud, extractor, volume, overwrite, kwargs)
-    #     utils.add_metadata(target_point_cloud, type(
-    #         extractor).__module__, extractor.get_params())
-    #
-    #     if verbose:
-    #         elapsed = time.time() - start
-    #         sys.stdout.write(' took {:.2f} seconds\n'.format(elapsed))
-    #         sys.stdout.flush()
-
     _keep_only_wanted_features(target_point_cloud, wanted_feature_names)
 
 
diff --git a/laserchicken/keys.py b/laserchicken/keys.py
index 761dfa8..3c2f9f7 100644
--- a/laserchicken/keys.py
+++ b/laserchicken/keys.py
@@ -1,3 +1,11 @@
+# Name of point data section in point cloud structure
 point = 'vertex'
+
+# Name of the normalized height point attribute
+normalized_height = 'normalized_height'
+
+#
 point_cloud = 'pointcloud'
+
+#
 provenance = 'log'
diff --git a/laserchicken/normalization.py b/laserchicken/normalization.py
new file mode 100644
index 0000000..f793faf
--- /dev/null
+++ b/laserchicken/normalization.py
@@ -0,0 +1,56 @@
+from laserchicken.compute_neighbors import compute_neighborhoods
+from laserchicken import keys
+from laserchicken.feature_extractor.range_z_feature_extractor import RangeZFeatureExtractor as range_extractor
+from laserchicken.keys import normalized_height
+import numpy as np
+
+from laserchicken.test_tools import create_point_cloud
+from laserchicken.volume_specification import Cell
+
+
+def normalize(point_cloud, cell_size=None):
+    z = point_cloud[keys.point]['z']['data']
+    point_cloud[keys.point][normalized_height] = {"type": 'float64', "data": np.array(z)}
+    if cell_size is None:
+        n_points = point_cloud[keys.point][normalized_height]['data'].size
+        _, min_z, _ = range_extractor().extract(point_cloud, range(n_points), None, None, None)
+        point_cloud[keys.point][normalized_height]['data'] = z - min_z
+    else:
+        targets = create_spanning_grid(point_cloud, cell_size)
+
+        neighborhood_sets = compute_neighborhoods(point_cloud, targets, Cell(cell_size), sample_size=None)
+
+        for neighborhood_set in neighborhood_sets:
+            for neighborhood in neighborhood_set:
+                _, min_z, _ = range_extractor().extract(point_cloud, neighborhood, None, None, None)
+                point_cloud[keys.point][normalized_height]['data'][neighborhood] = z[neighborhood] - min_z
+
+    return point_cloud
+
+
+def create_spanning_grid(point_cloud, cell_size):
+    x = point_cloud[keys.point]['x']['data']
+    y = point_cloud[keys.point]['y']['data']
+    min_x = np.min(x)
+    max_x = np.max(x)
+    min_y = np.min(y)
+    max_y = np.max(y)
+
+    cell_x_lengths, n_grid_points = _count_steps_and_points(cell_size, max_x, max_y, min_x, min_y)
+
+    xs = [min_x + cell_size * (0.5 + (i % cell_x_lengths)) for i in range(n_grid_points)]
+    ys = [min_y + cell_size * (0.5 + np.floor(i / cell_x_lengths)) for i in range(n_grid_points)]
+    zs = np.zeros_like(xs)
+    return create_point_cloud(xs, ys, zs)
+
+
+def _count_steps_and_points(cell_size, max_x, max_y, min_x, min_y):
+    cell_x_lengths = _count_steps(min_x, max_x, cell_size)
+    cell_y_lengths = _count_steps(min_y, max_y, cell_size)
+    n_grid_points = cell_x_lengths * cell_y_lengths
+    return cell_x_lengths, n_grid_points
+
+
+def _count_steps(min_x, max_x, cell_size):
+    """Count the number of steps in a grid in a single direction."""
+    return max(int(np.ceil((max_x - min_x) / float(cell_size))), 1)
diff --git a/laserchicken/utils.py b/laserchicken/utils.py
index aa923d7..c660eac 100644
--- a/laserchicken/utils.py
+++ b/laserchicken/utils.py
@@ -67,6 +67,23 @@ def get_features(point_cloud, attribute_names, index=None):
     return (point_cloud[keys.point][f]["data"][index] for f in attribute_names)
 
 
+def create_point_cloud(x, y, z):
+    """
+    Create a point cloud object given only the x y z values.
+
+    :param x: x attribute values
+    :param y: y attribute values
+    :param z: z attribute values
+    :return: point cloud object
+    """
+    return {keys.point: {'x': {'type': 'float', 'data': np.array(x)},
+                         'y': {'type': 'float', 'data': np.array(y)},
+                         'z': {'type': 'float', 'data': np.array(z)}},
+            keys.point_cloud: {},
+            keys.provenance: []
+            }
+
+
 def copy_point_cloud(source_point_cloud, array_mask=None):
     """
     Makes a deep copy of a point cloud dict using the array mask when copying the points.


Test patch:
diff --git a/laserchicken/test_normalize.py b/laserchicken/test_normalize.py
new file mode 100644
index 0000000..bfe02a1
--- /dev/null
+++ b/laserchicken/test_normalize.py
@@ -0,0 +1,41 @@
+import os
+import shutil
+import unittest
+import pytest
+
+import numpy as np
+import pandas as pd
+from numpy.testing import assert_almost_equal, assert_equal
+
+from laserchicken.keys import point, normalized_height
+from laserchicken.normalization import normalize
+from laserchicken.spatial_selections import points_in_polygon_wkt, points_in_polygon_wkt_file, \
+    points_in_polygon_shp_file
+from laserchicken.test_tools import create_point_cloud
+from laserchicken.utils import get_attribute_value
+
+
+class TestNormalize(unittest.TestCase):
+    def test_normalize_empty_point_cloud(self):
+        point_cloud = create_point_cloud([], [], [])
+        normalized_point_cloud = normalize(point_cloud)
+        self.assertTrue(normalized_height in normalized_point_cloud[point])
+
+    def test_normalize_tiny_equal_point_cloud(self):
+        point_cloud = create_point_cloud([0, 0, 0], [0, 0, 0], [0, 0, 0])
+        normalized_point_cloud = normalize(point_cloud)
+        normalized_values = get_attribute_value(normalized_point_cloud, range(3), normalized_height)
+        np.testing.assert_allclose(normalized_values, np.array([0, 0, 0]), atol=1e-7)
+
+    def test_normalize_tiny_unequal_point_cloud(self):
+        point_cloud = create_point_cloud([0, 0, 0], [0, 0, 0], [1, 2, 3])
+        normalized_point_cloud = normalize(point_cloud)
+        normalized_values = get_attribute_value(normalized_point_cloud, range(3), normalized_height)
+        np.testing.assert_allclose(normalized_values, np.array([0, 1, 2]), atol=1e-7)
+
+    def test_normalize_tiny_unequal_point_cloud_multiple_cells(self):
+        """Last of the 3 points is not in the neighborhood of the others."""
+        point_cloud = create_point_cloud([0, 0, 5], [0, 0, 0], [1, 2, 3])
+        normalized_point_cloud = normalize(point_cloud, cell_size=2)
+        normalized_values = get_attribute_value(normalized_point_cloud, range(3), normalized_height)
+        np.testing.assert_allclose(normalized_values, np.array([0, 1, 0]), atol=1e-7)
diff --git a/laserchicken/test_tools.py b/laserchicken/test_tools.py
index 9ed84b2..993099a 100644
--- a/laserchicken/test_tools.py
+++ b/laserchicken/test_tools.py
@@ -6,7 +6,7 @@ import numpy as np
 from laserchicken import keys
 
 
-def generate_test_point_cloud():
+def generate_tiny_test_point_cloud():
     """Generate a simple but valid point cloud with 3 points."""
     pc = {keys.point: {'x': {'type': 'double', 'data': np.array([1, 2, 3], dtype=np.float64)},
                        'y': {'type': 'double', 'data': np.array([2, 3, 4], dtype=np.float64)},
diff --git a/laserchicken/test_utils.py b/laserchicken/test_utils.py
index b028d1e..b691ef8 100644
--- a/laserchicken/test_utils.py
+++ b/laserchicken/test_utils.py
@@ -8,7 +8,7 @@ from time import time
 class TestUtils(unittest.TestCase):
     def test_GetPointCloudPoint(self):
         """ Should not raise exception. """
-        pc = test_tools.generate_test_point_cloud()
+        pc = test_tools.generate_tiny_test_point_cloud()
         x, y, z = utils.get_point(pc, 1)
         self.assertEqual(2, x)
         self.assertEqual(3, y)
@@ -16,7 +16,7 @@ class TestUtils(unittest.TestCase):
 
     def test_GetPointCloudPointFeature(self):
         """ Should not raise exception. """
-        pc = test_tools.generate_test_point_cloud()
+        pc = test_tools.generate_tiny_test_point_cloud()
         cols = 0.5 * (pc[keys.point]["x"]["data"] + pc[keys.point]["y"]["data"])
         pc[keys.point]["color"] = {"type": "double", "data": cols}
         x, y, z = utils.get_point(pc, 1)
@@ -25,7 +25,7 @@ class TestUtils(unittest.TestCase):
 
     def test_GetPointCloudPointFeatures(self):
         """ Should not raise exception. """
-        pc = test_tools.generate_test_point_cloud()
+        pc = test_tools.generate_tiny_test_point_cloud()
         cols = 0.5 * (pc[keys.point]["x"]["data"] + pc[keys.point]["y"]["data"])
         flavs = 0.5 * (pc[keys.point]["x"]["data"] - pc[keys.point]["y"]["data"])
         pc[keys.point]["color"] = {"type": "double", "data": cols}
@@ -37,7 +37,7 @@ class TestUtils(unittest.TestCase):
 
     def test_CopyEmptyPointCloud(self):
         """ Should not raise exception. """
-        pc = test_tools.generate_test_point_cloud()
+        pc = test_tools.generate_tiny_test_point_cloud()
         pc[keys.point]["x"]["data"] = np.array([])
         pc[keys.point]["y"]["data"] = np.array([])
         pc[keys.point]["z"]["data"] = np.array([])
@@ -47,7 +47,7 @@ class TestUtils(unittest.TestCase):
 
     def test_CopyNonEmptyPointCloud(self):
         """ Test whether coordinates are copied """
-        pc = test_tools.generate_test_point_cloud()
+        pc = test_tools.generate_tiny_test_point_cloud()
         x = pc[keys.point]["x"]["data"]
         y = pc[keys.point]["y"]["data"]
         z = pc[keys.point]["z"]["data"]
@@ -59,7 +59,7 @@ class TestUtils(unittest.TestCase):
 
     def test_CopyPointCloudMetaData(self):
         """ Test whether metadata are copied """
-        pc = test_tools.generate_test_point_cloud()
+        pc = test_tools.generate_tiny_test_point_cloud()
         pc["log"] = [
             {"time": datetime.datetime(2018, 1, 23, 12, 15, 59), "module": "filter", "parameters": [("z", "gt", 0.5)]}]
 
@@ -70,7 +70,7 @@ class TestUtils(unittest.TestCase):
 
     def test_CopyNonEmptyPointCloudBoolMask(self):
         """ Test whether coordinates are copied with boolean mask """
-        pc = test_tools.generate_test_point_cloud()
+        pc = test_tools.generate_tiny_test_point_cloud()
         x = pc[keys.point]["x"]["data"][2]
         y = pc[keys.point]["y"]["data"][2]
         z = pc[keys.point]["z"]["data"][2]
@@ -82,7 +82,7 @@ class TestUtils(unittest.TestCase):
 
     def test_CopyNonEmptyPointCloudIntMask(self):
         """ Test whether coordinates are copied with array indexing """
-        pc = test_tools.generate_test_point_cloud()
+        pc = test_tools.generate_tiny_test_point_cloud()
         x0, x1 = pc[keys.point]["x"]["data"][0], pc[keys.point]["x"]["data"][1]
         y0, y1 = pc[keys.point]["y"]["data"][0], pc[keys.point]["y"]["data"][1]
         z0, z1 = pc[keys.point]["z"]["data"][0], pc[keys.point]["z"]["data"][1]
@@ -94,7 +94,7 @@ class TestUtils(unittest.TestCase):
 
     def test_AddMetaDataToPointCloud(self):
         """ Test adding info to the point cloud for test module """
-        pc = test_tools.generate_test_point_cloud()
+        pc = test_tools.generate_tiny_test_point_cloud()
         from laserchicken import select as somemodule
         utils.add_metadata(pc,somemodule,params = (0.5,"cylinder",4))
         self.assertEqual(len(pc[keys.provenance]),1)