Repository: great-expectations/great_expectations

Issue:
Use cleaner solution for non-truncating division in python 2
Prefer `from __future__ import division` to `1.*x/y`


Discussion:
Done inline during improve_partitions push.

Resolving patch:
diff --git a/great_expectations/rule_based_profiler/helpers/util.py b/great_expectations/rule_based_profiler/helpers/util.py
--- a/great_expectations/rule_based_profiler/helpers/util.py
+++ b/great_expectations/rule_based_profiler/helpers/util.py
@@ -484,9 +484,11 @@ def compute_quantiles(
         axis=0,
         method=quantile_statistic_interpolation_method,
     )
-    return NumericRangeEstimationResult(
-        estimation_histogram=np.histogram(a=metric_values, bins=NUM_HISTOGRAM_BINS)[0],
-        value_range=np.asarray([lower_quantile, upper_quantile]),
+
+    return build_numeric_range_estimation_result(
+        metric_values=metric_values,
+        min_value=lower_quantile,
+        max_value=upper_quantile,
     )
 
 
@@ -510,6 +512,7 @@ def compute_kde_quantiles_point_estimate(
     Bandwidth Method: https://stats.stackexchange.com/questions/90656/kernel-bandwidth-scotts-vs-silvermans-rules
 
     Args:
+        metric_values: "numpy.ndarray" of "dtype.float" values with elements corresponding to "Batch" data samples.
         false_positive_rate: user-configured fraction between 0 and 1 expressing desired false positive rate for
             identifying unexpected values as judged by the upper- and lower- quantiles of the observed metric data.
         n_resamples: A positive integer indicating the sample size resulting from the sampling with replacement
@@ -549,14 +552,10 @@ def compute_kde_quantiles_point_estimate(
         method=quantile_statistic_interpolation_method,
     )
 
-    return NumericRangeEstimationResult(
-        estimation_histogram=np.histogram(
-            a=metric_values_gaussian_sample, bins=NUM_HISTOGRAM_BINS
-        )[0],
-        value_range=[
-            lower_quantile_point_estimate,
-            upper_quantile_point_estimate,
-        ],
+    return build_numeric_range_estimation_result(
+        metric_values=metric_values,
+        min_value=lower_quantile_point_estimate,
+        max_value=upper_quantile_point_estimate,
     )
 
 
@@ -665,14 +664,10 @@ def compute_bootstrap_quantiles_point_estimate(
         sample_quantile=sample_upper_quantile,
     )
 
-    return NumericRangeEstimationResult(
-        estimation_histogram=np.histogram(
-            a=bootstraps.flatten(), bins=NUM_HISTOGRAM_BINS
-        )[0],
-        value_range=[
-            lower_quantile_bias_corrected_point_estimate,
-            upper_quantile_bias_corrected_point_estimate,
-        ],
+    return build_numeric_range_estimation_result(
+        metric_values=metric_values,
+        min_value=lower_quantile_bias_corrected_point_estimate,
+        max_value=upper_quantile_bias_corrected_point_estimate,
     )
 
 
@@ -713,6 +708,41 @@ def _determine_quantile_bias_corrected_point_estimate(
     return quantile_bias_corrected_point_estimate
 
 
+def build_numeric_range_estimation_result(
+    metric_values: np.ndarray,
+    min_value: Number,
+    max_value: Number,
+) -> NumericRangeEstimationResult:
+    """
+    Computes histogram of 1-dimensional set of data points and packages it together with value range as returned output.
+
+    Args:
+        metric_values: "numpy.ndarray" of "dtype.float" values with elements corresponding to "Batch" data samples.
+        min_value: pre-computed supremum of "metric_values" (properly conditioned for output).
+        max_value: pre-computed infimum of "metric_values" (properly conditioned for output).
+
+    Returns:
+        Structured "NumericRangeEstimationResult" object, containing histogram and value_range attributes.
+    """
+    histogram: Tuple[np.ndarray, np.ndarray] = np.histogram(
+        a=metric_values, bins=NUM_HISTOGRAM_BINS
+    )
+    return NumericRangeEstimationResult(
+        estimation_histogram=np.vstack(
+            (
+                np.pad(
+                    array=histogram[0],
+                    pad_width=(0, 1),
+                    mode="constant",
+                    constant_values=0,
+                ),
+                histogram[1],
+            )
+        ),
+        value_range=np.asarray([min_value, max_value]),
+    )
+
+
 def get_validator_with_expectation_suite(
     data_context: "BaseDataContext",  # noqa: F821
     batch_list: Optional[List[Batch]] = None,
diff --git a/great_expectations/rule_based_profiler/parameter_builder/numeric_metric_range_multi_batch_parameter_builder.py b/great_expectations/rule_based_profiler/parameter_builder/numeric_metric_range_multi_batch_parameter_builder.py
--- a/great_expectations/rule_based_profiler/parameter_builder/numeric_metric_range_multi_batch_parameter_builder.py
+++ b/great_expectations/rule_based_profiler/parameter_builder/numeric_metric_range_multi_batch_parameter_builder.py
@@ -10,6 +10,7 @@
 from great_expectations.rule_based_profiler.config import ParameterBuilderConfig
 from great_expectations.rule_based_profiler.helpers.util import (
     NP_EPSILON,
+    build_numeric_range_estimation_result,
     compute_bootstrap_quantiles_point_estimate,
     compute_kde_quantiles_point_estimate,
     compute_quantiles,
@@ -533,13 +534,17 @@ def _estimate_metric_value_range(
         metric_value_range_shape: tuple = metric_value_shape + (2,)
         metric_value_range: np.ndarray = np.zeros(shape=metric_value_range_shape)
         # Initialize observed_values for multi-dimensional metric to all trivial values (to be updated in situ).
-        # Since "numpy.histogram()" uses 10 bins by default, histogram contains 10-element least-significant dimension.
-        estimation_histogram_shape: tuple = metric_value_shape + (10,)
-        estimation_histogram: np.ndarray = np.zeros(shape=estimation_histogram_shape)
+        # Return values of "numpy.histogram()", histogram and bin edges, are packaged in least-significant dimensions.
+        estimation_histogram_shape: tuple = metric_value_shape + (
+            2,
+            NUM_HISTOGRAM_BINS + 1,
+        )
+        estimation_histogram: np.ndarray = np.empty(shape=estimation_histogram_shape)
 
         metric_value_vector: np.ndarray
         metric_value_range_min_idx: tuple
         metric_value_range_max_idx: tuple
+        metric_value_estimation_histogram_idx: tuple
         numeric_range_estimation_result: NumericRangeEstimationResult
         # Traverse indices of sample vectors corresponding to every element of multi-dimensional metric.
         for metric_value_idx in metric_value_vector_indices:
@@ -547,13 +552,10 @@ def _estimate_metric_value_range(
             metric_value_vector = metric_values[metric_value_idx]
             if np.all(np.isclose(metric_value_vector, metric_value_vector[0])):
                 # Computation is unnecessary if distribution is degenerate.
-                numeric_range_estimation_result = NumericRangeEstimationResult(
-                    estimation_histogram=np.histogram(
-                        a=metric_value_vector, bins=NUM_HISTOGRAM_BINS
-                    )[0],
-                    value_range=np.asarray(
-                        [metric_value_vector[0], metric_value_vector[0]]
-                    ),
+                numeric_range_estimation_result = build_numeric_range_estimation_result(
+                    metric_values=metric_value_vector,
+                    min_value=metric_value_vector[0],
+                    max_value=metric_value_vector[0],
                 )
             else:
                 # Compute low and high estimates for vector of samples for given element of multi-dimensional metric.
@@ -576,7 +578,7 @@ def _estimate_metric_value_range(
             # Obtain index of metric element (by discarding "N"-element samples dimension).
             metric_value_idx = metric_value_idx[1:]
 
-            # Compute indices for min and max value range estimates.
+            # Compute indices for metric value range min and max estimates.
             metric_value_range_min_idx = metric_value_idx + (
                 slice(0, 1, None),
             )  # appends "[0]" element
@@ -584,10 +586,9 @@ def _estimate_metric_value_range(
                 slice(1, 2, None),
             )  # appends "[1]" element
 
-            # Store computed estimation_histogram into allocated range estimate for multi-dimensional metric.
-            estimation_histogram[
-                metric_value_idx
-            ] = numeric_range_estimation_result.estimation_histogram
+            # Compute index for metric value estimation histogram.
+            metric_value_estimation_histogram_idx = metric_value_idx
+
             # Store computed min and max value estimates into allocated range estimate for multi-dimensional metric.
             metric_value_range[metric_value_range_min_idx] = round(
                 cast(float, min_value), round_decimals
@@ -596,10 +597,15 @@ def _estimate_metric_value_range(
                 cast(float, max_value), round_decimals
             )
 
+            # Store computed estimation_histogram into allocated range estimate for multi-dimensional metric.
+            estimation_histogram[
+                metric_value_estimation_histogram_idx
+            ] = numeric_range_estimation_result.estimation_histogram
+
         # As a simplification, apply reduction to scalar in case of one-dimensional metric (for convenience).
         if metric_value_range.shape[0] == 1:
-            estimation_histogram = estimation_histogram[0]
             metric_value_range = metric_value_range[0]
+            estimation_histogram = estimation_histogram[0]
 
         if round_decimals == 0:
             metric_value_range = metric_value_range.astype(np.int64)
@@ -828,16 +834,14 @@ def _get_oneshot_estimate(
             quantile_statistic_interpolation_method=quantile_statistic_interpolation_method,
         )
 
+    # noinspection PyUnusedLocal
     @staticmethod
     def _get_exact_estimate(
         metric_values: np.ndarray,
         **kwargs,
     ) -> NumericRangeEstimationResult:
-        return NumericRangeEstimationResult(
-            estimation_histogram=np.histogram(a=metric_values, bins=NUM_HISTOGRAM_BINS)[
-                0
-            ],
-            value_range=np.asarray(
-                [np.amin(a=metric_values), np.amax(a=metric_values)]
-            ),
+        return build_numeric_range_estimation_result(
+            metric_values=metric_values,
+            min_value=np.amin(a=metric_values),
+            max_value=np.amax(a=metric_values),
         )
diff --git a/great_expectations/rule_based_profiler/types/numeric_range_estimation_result.py b/great_expectations/rule_based_profiler/types/numeric_range_estimation_result.py
--- a/great_expectations/rule_based_profiler/types/numeric_range_estimation_result.py
+++ b/great_expectations/rule_based_profiler/types/numeric_range_estimation_result.py
@@ -3,6 +3,7 @@
 
 import numpy as np
 
+from great_expectations.core.util import convert_to_json_serializable
 from great_expectations.types import DictDot
 
 NUM_HISTOGRAM_BINS: int = 10  # Equal to "numpy.histogram()" default (can be turned into configurable argument).
@@ -13,6 +14,10 @@ class NumericRangeEstimationResult(DictDot):
     """
     NumericRangeEstimationResult is a "dataclass" object, designed to hold results of executing numeric range estimator
     for multidimensional datasets, which consist of "estimation_histogram" and "value_range" for each numeric dimension.
+
+    In particular, "estimation_histogram" is "numpy.ndarray" of shape [2, NUM_HISTOGRAM_BINS + 1], containing
+    [0] "histogram": (integer array of dimension [NUM_HISTOGRAM_BINS + 1] padded with 0 at right edge) histogram values;
+    [1] "bin_edges": (float array of dimension [NUM_HISTOGRAM_BINS + 1]) binning edges.
     """
 
     estimation_histogram: np.ndarray
@@ -21,3 +26,7 @@ class NumericRangeEstimationResult(DictDot):
     def to_dict(self) -> dict:
         """Returns: this NumericRangeEstimationResult as a dictionary"""
         return asdict(self)
+
+    def to_json_dict(self) -> dict:
+        """Returns: this NumericRangeEstimationResult as a JSON dictionary"""
+        return convert_to_json_serializable(data=self.to_dict())


Test patch:
diff --git a/tests/rule_based_profiler/parameter_builder/test_numeric_metric_range_multi_batch_parameter_builder.py b/tests/rule_based_profiler/parameter_builder/test_numeric_metric_range_multi_batch_parameter_builder.py
--- a/tests/rule_based_profiler/parameter_builder/test_numeric_metric_range_multi_batch_parameter_builder.py
+++ b/tests/rule_based_profiler/parameter_builder/test_numeric_metric_range_multi_batch_parameter_builder.py
@@ -109,7 +109,7 @@ def test_bootstrap_numeric_metric_range_multi_batch_parameter_builder_bobby(
 
     assert parameter_node == expected_value_dict
 
-    expected_value: np.ndarray = np.array([7510, 8806])
+    expected_value: np.ndarray = np.asarray([7510, 8806])
 
     # Measure of "closeness" between "actual" and "desired" is computed as: atol + rtol * abs(desired)
     # (see "https://numpy.org/doc/stable/reference/generated/numpy.testing.assert_allclose.html" for details).
@@ -125,24 +125,25 @@ def test_bootstrap_numeric_metric_range_multi_batch_parameter_builder_bobby(
         err_msg=f"Actual value of {actual_value} differs from expected value of {expected_value} by more than {atol + rtol * abs(expected_value)} tolerance.",
     )
 
-    expected_estimation_histogram: np.ndarray = np.array(
+    expected_estimation_histogram: np.ndarray = np.asarray(
         [
-            10100.0,
+            1.0,
+            0.0,
             0.0,
             0.0,
             0.0,
             0.0,
+            1.0,
             0.0,
-            9889.0,
             0.0,
+            1.0,
             0.0,
-            10008.0,
         ]
     )
 
     # Assert no significant difference between expected (null hypothesis) and actual estimation histograms.
     ks_result: tuple = stats.ks_2samp(
-        data1=actual_estimation_histogram, data2=expected_estimation_histogram
+        data1=actual_estimation_histogram[0], data2=expected_estimation_histogram
     )
     p_value: float = ks_result[1]
     assert p_value > 9.5e-1
@@ -245,7 +246,7 @@ def test_oneshot_numeric_metric_range_multi_batch_parameter_builder_bobby(
     assert actual_value_01_lower == expected_value_01_lower
     assert actual_value_01_upper == expected_value_01_upper
 
-    expected_estimation_histogram: np.ndarray = np.array(
+    expected_estimation_histogram: np.ndarray = np.asarray(
         [
             1.0,
             0.0,
@@ -257,12 +258,13 @@ def test_oneshot_numeric_metric_range_multi_batch_parameter_builder_bobby(
             0.0,
             0.0,
             2.0,
+            0.0,
         ]
     )
 
     # Assert no significant difference between expected (null hypothesis) and actual estimation histograms.
     ks_result: tuple = stats.ks_2samp(
-        data1=actual_estimation_histogram, data2=expected_estimation_histogram
+        data1=actual_estimation_histogram[0], data2=expected_estimation_histogram
     )
     p_value: float = ks_result[1]
     assert p_value > 9.5e-1
@@ -320,7 +322,7 @@ def test_oneshot_numeric_metric_range_multi_batch_parameter_builder_bobby(
     assert actual_value_01_lower < actual_value_05_lower
     assert actual_value_01_upper > actual_value_05_upper
 
-    expected_estimation_histogram: np.ndarray = np.array(
+    expected_estimation_histogram: np.ndarray = np.asarray(
         [
             1.0,
             0.0,
@@ -332,12 +334,13 @@ def test_oneshot_numeric_metric_range_multi_batch_parameter_builder_bobby(
             0.0,
             0.0,
             2.0,
+            0.0,
         ]
     )
 
     # Assert no significant difference between expected (null hypothesis) and actual estimation histograms.
     ks_result = stats.ks_2samp(
-        data1=actual_estimation_histogram, data2=expected_estimation_histogram
+        data1=actual_estimation_histogram[0], data2=expected_estimation_histogram
     )
     p_value: float = ks_result[1]
     assert p_value > 9.5e-1
@@ -440,7 +443,7 @@ def test_exact_numeric_metric_range_multi_batch_parameter_builder_bobby(
     assert actual_value_01_lower == expected_value_01_lower
     assert actual_value_01_upper == expected_value_01_upper
 
-    expected_estimation_histogram: np.ndarray = np.array(
+    expected_estimation_histogram: np.ndarray = np.asarray(
         [
             1.0,
             0.0,
@@ -452,12 +455,13 @@ def test_exact_numeric_metric_range_multi_batch_parameter_builder_bobby(
             0.0,
             0.0,
             2.0,
-        ]
+            0.0,
+        ],
     )
 
     # Assert no significant difference between expected (null hypothesis) and actual estimation histograms.
     ks_result: tuple = stats.ks_2samp(
-        data1=actual_estimation_histogram, data2=expected_estimation_histogram
+        data1=actual_estimation_histogram[0], data2=expected_estimation_histogram
     )
     p_value: float = ks_result[1]
     assert p_value > 9.5e-1
@@ -575,7 +579,7 @@ def test_oneshot_numeric_metric_range_multi_batch_parameter_builder_with_evaluat
     assert actual_value_01_lower == expected_value_01_lower
     assert actual_value_01_upper == expected_value_01_upper
 
-    expected_estimation_histogram: np.ndarray = np.array(
+    expected_estimation_histogram: np.ndarray = np.asarray(
         [
             1.0,
             0.0,
@@ -587,12 +591,13 @@ def test_oneshot_numeric_metric_range_multi_batch_parameter_builder_with_evaluat
             0.0,
             0.0,
             2.0,
+            0.0,
         ]
     )
 
     # Assert no significant difference between expected (null hypothesis) and actual estimation histograms.
     ks_result: tuple = stats.ks_2samp(
-        data1=actual_estimation_histogram, data2=expected_estimation_histogram
+        data1=actual_estimation_histogram[0], data2=expected_estimation_histogram
     )
     p_value: float = ks_result[1]
     assert p_value > 9.5e-1
@@ -650,7 +655,7 @@ def test_oneshot_numeric_metric_range_multi_batch_parameter_builder_with_evaluat
     assert actual_value_01_lower < actual_value_05_lower
     assert actual_value_01_upper > actual_value_05_upper
 
-    expected_estimation_histogram: np.ndarray = np.array(
+    expected_estimation_histogram: np.ndarray = np.asarray(
         [
             1.0,
             0.0,
@@ -662,12 +667,13 @@ def test_oneshot_numeric_metric_range_multi_batch_parameter_builder_with_evaluat
             0.0,
             0.0,
             2.0,
+            0.0,
         ]
     )
 
     # Assert no significant difference between expected (null hypothesis) and actual estimation histograms.
     ks_result = stats.ks_2samp(
-        data1=actual_estimation_histogram, data2=expected_estimation_histogram
+        data1=actual_estimation_histogram[0], data2=expected_estimation_histogram
     )
     p_value: float = ks_result[1]
     assert p_value > 9.5e-1
@@ -990,7 +996,7 @@ def test_kde_numeric_metric_range_multi_batch_parameter_builder_bobby(
 
     assert parameter_node == expected_value_dict
 
-    expected_value: np.ndarray = np.array([6180, 10277])
+    expected_value: np.ndarray = np.asarray([6180, 10277])
 
     # Measure of "closeness" between "actual" and "desired" is computed as: atol + rtol * abs(desired)
     # (see "https://numpy.org/doc/stable/reference/generated/numpy.testing.assert_allclose.html" for details).
@@ -1006,24 +1012,25 @@ def test_kde_numeric_metric_range_multi_batch_parameter_builder_bobby(
         err_msg=f"Actual value of {actual_value} differs from expected value of {expected_value} by more than {atol + rtol * abs(expected_value)} tolerance.",
     )
 
-    expected_estimation_histogram: np.ndarray = np.array(
+    expected_estimation_histogram: np.ndarray = np.asarray(
         [
-            13.0,
-            155.0,
-            719.0,
-            1546.0,
-            2221.0,
-            2570.0,
-            1946.0,
-            683.0,
-            137.0,
-            9.0,
+            1.0,
+            0.0,
+            0.0,
+            0.0,
+            0.0,
+            0.0,
+            1.0,
+            0.0,
+            0.0,
+            1.0,
+            0.0,
         ]
     )
 
     # Assert no significant difference between expected (null hypothesis) and actual estimation histograms.
     ks_result: tuple = stats.ks_2samp(
-        data1=actual_estimation_histogram, data2=expected_estimation_histogram
+        data1=actual_estimation_histogram[0], data2=expected_estimation_histogram
     )
     p_value: float = ks_result[1]
     assert p_value > 9.5e-1