Repository: open-telemetry/opentelemetry-python

Issue:
ExplicitBucketHistogram resets counts to zero even if aggregation temporality is cumulative
              While this might help for deltas, the SDK should be returning metrics every single collection for readers that are CUMULATIVE (the default). However that seems to not be working corrctly https://github.com/open-telemetry/opentelemetry-python/issues/3277#issuecomment-1684521139

Yes, noticed that now that I added another test case. I think this is a problem for the [explicit bucket histogram aggregation](https://github.com/open-telemetry/opentelemetry-python/blob/main/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/aggregation.py#L293) who is resetting to zero all bucket counts after every collection even in the aggregation temporality is cumulative. I say we review this PR and address that issue separately.

_Originally posted by @aabmass in https://github.com/open-telemetry/opentelemetry-python/pull/3335#pullrequestreview-1585296752_
            


Discussion:
I looked into this a bit and found the issue https://github.com/open-telemetry/opentelemetry-python/issues/3277#issuecomment-1684545842

> I think I found the issue https://github.com/open-telemetry/opentelemetry-python/blob/d8490c5f557dd7005badeb800095cb51b553c98c/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/aggregation.py#L292-L293
> 
> This short circuiting logic is incorrect, it just assumes if no new counts were recorded since the last collection that we can return None. We should return the previous point values instead.

Resolving patch:
diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/aggregation.py b/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/aggregation.py
--- a/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/aggregation.py
+++ b/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/aggregation.py
@@ -377,6 +377,7 @@ class _ExplicitBucketHistogramAggregation(_Aggregation[HistogramPoint]):
     def __init__(
         self,
         attributes: Attributes,
+        instrument_aggregation_temporality: AggregationTemporality,
         start_time_unix_nano: int,
         boundaries: Sequence[float] = (
             0.0,
@@ -398,33 +399,43 @@ def __init__(
         record_min_max: bool = True,
     ):
         super().__init__(attributes)
+
         self._boundaries = tuple(boundaries)
-        self._bucket_counts = self._get_empty_bucket_counts()
+        self._record_min_max = record_min_max
         self._min = inf
         self._max = -inf
         self._sum = 0
-        self._record_min_max = record_min_max
+
         self._start_time_unix_nano = start_time_unix_nano
-        # It is assumed that the "natural" aggregation temporality for a
-        # Histogram instrument is DELTA, like the "natural" aggregation
-        # temporality for a Counter is DELTA and the "natural" aggregation
-        # temporality for an ObservableCounter is CUMULATIVE.
-        self._instrument_aggregation_temporality = AggregationTemporality.DELTA
+        self._instrument_aggregation_temporality = (
+            instrument_aggregation_temporality
+        )
+
+        self._current_value = None
+
+        self._previous_collection_start_nano = self._start_time_unix_nano
+        self._previous_cumulative_value = self._get_empty_bucket_counts()
+        self._previous_min = inf
+        self._previous_max = -inf
+        self._previous_sum = 0
 
     def _get_empty_bucket_counts(self) -> List[int]:
         return [0] * (len(self._boundaries) + 1)
 
     def aggregate(self, measurement: Measurement) -> None:
+        with self._lock:
+            if self._current_value is None:
+                self._current_value = self._get_empty_bucket_counts()
 
-        value = measurement.value
+            value = measurement.value
 
-        if self._record_min_max:
-            self._min = min(self._min, value)
-            self._max = max(self._max, value)
+            self._sum += value
 
-        self._sum += value
+            if self._record_min_max:
+                self._min = min(self._min, value)
+                self._max = max(self._max, value)
 
-        self._bucket_counts[bisect_left(self._boundaries, value)] += 1
+            self._current_value[bisect_left(self._boundaries, value)] += 1
 
     def collect(
         self,
@@ -434,84 +445,78 @@ def collect(
         """
         Atomically return a point for the current value of the metric.
         """
-        with self._lock:
-            if not any(self._bucket_counts):
-                return None
 
-            bucket_counts = self._bucket_counts
-            start_time_unix_nano = self._start_time_unix_nano
+        with self._lock:
+            current_value = self._current_value
             sum_ = self._sum
-            max_ = self._max
             min_ = self._min
+            max_ = self._max
 
-            self._bucket_counts = self._get_empty_bucket_counts()
-            self._start_time_unix_nano = collection_start_nano
+            self._current_value = None
             self._sum = 0
             self._min = inf
             self._max = -inf
 
-        current_point = HistogramDataPoint(
-            attributes=self._attributes,
-            start_time_unix_nano=start_time_unix_nano,
-            time_unix_nano=collection_start_nano,
-            count=sum(bucket_counts),
-            sum=sum_,
-            bucket_counts=tuple(bucket_counts),
-            explicit_bounds=self._boundaries,
-            min=min_,
-            max=max_,
-        )
+            if (
+                self._instrument_aggregation_temporality
+                is AggregationTemporality.DELTA
+            ):
+                # This happens when the corresponding instrument for this
+                # aggregation is synchronous.
+                if (
+                    collection_aggregation_temporality
+                    is AggregationTemporality.DELTA
+                ):
 
-        if self._previous_point is None or (
-            self._instrument_aggregation_temporality
-            is collection_aggregation_temporality
-        ):
-            self._previous_point = current_point
-            return current_point
+                    if current_value is None:
+                        return None
 
-        max_ = current_point.max
-        min_ = current_point.min
+                    previous_collection_start_nano = (
+                        self._previous_collection_start_nano
+                    )
+                    self._previous_collection_start_nano = (
+                        collection_start_nano
+                    )
 
-        if (
-            collection_aggregation_temporality
-            is AggregationTemporality.CUMULATIVE
-        ):
-            start_time_unix_nano = self._previous_point.start_time_unix_nano
-            sum_ = current_point.sum + self._previous_point.sum
-            # Only update min/max on delta -> cumulative
-            max_ = max(current_point.max, self._previous_point.max)
-            min_ = min(current_point.min, self._previous_point.min)
-            bucket_counts = [
-                curr_count + prev_count
-                for curr_count, prev_count in zip(
-                    current_point.bucket_counts,
-                    self._previous_point.bucket_counts,
-                )
-            ]
-        else:
-            start_time_unix_nano = self._previous_point.time_unix_nano
-            sum_ = current_point.sum - self._previous_point.sum
-            bucket_counts = [
-                curr_count - prev_count
-                for curr_count, prev_count in zip(
-                    current_point.bucket_counts,
-                    self._previous_point.bucket_counts,
+                    return HistogramDataPoint(
+                        attributes=self._attributes,
+                        start_time_unix_nano=previous_collection_start_nano,
+                        time_unix_nano=collection_start_nano,
+                        count=sum(current_value),
+                        sum=sum_,
+                        bucket_counts=tuple(current_value),
+                        explicit_bounds=self._boundaries,
+                        min=min_,
+                        max=max_,
+                    )
+
+                if current_value is None:
+                    current_value = self._get_empty_bucket_counts()
+
+                self._previous_cumulative_value = [
+                    current_value_element + previous_cumulative_value_element
+                    for (
+                        current_value_element,
+                        previous_cumulative_value_element,
+                    ) in zip(current_value, self._previous_cumulative_value)
+                ]
+                self._previous_min = min(min_, self._previous_min)
+                self._previous_max = max(max_, self._previous_max)
+                self._previous_sum = sum_ + self._previous_sum
+
+                return HistogramDataPoint(
+                    attributes=self._attributes,
+                    start_time_unix_nano=self._start_time_unix_nano,
+                    time_unix_nano=collection_start_nano,
+                    count=sum(self._previous_cumulative_value),
+                    sum=self._previous_sum,
+                    bucket_counts=tuple(self._previous_cumulative_value),
+                    explicit_bounds=self._boundaries,
+                    min=self._previous_min,
+                    max=self._previous_max,
                 )
-            ]
 
-        current_point = HistogramDataPoint(
-            attributes=self._attributes,
-            start_time_unix_nano=start_time_unix_nano,
-            time_unix_nano=current_point.time_unix_nano,
-            count=sum(bucket_counts),
-            sum=sum_,
-            bucket_counts=tuple(bucket_counts),
-            explicit_bounds=current_point.explicit_bounds,
-            min=min_,
-            max=max_,
-        )
-        self._previous_point = current_point
-        return current_point
+            return None
 
 
 # pylint: disable=protected-access
@@ -1100,7 +1105,11 @@ def _create_aggregation(
 
         if isinstance(instrument, Histogram):
             return _ExplicitBucketHistogramAggregation(
-                attributes, start_time_unix_nano
+                attributes,
+                instrument_aggregation_temporality=(
+                    AggregationTemporality.DELTA
+                ),
+                start_time_unix_nano=start_time_unix_nano,
             )
 
         if isinstance(instrument, ObservableGauge):
@@ -1179,8 +1188,18 @@ def _create_aggregation(
         attributes: Attributes,
         start_time_unix_nano: int,
     ) -> _Aggregation:
+
+        instrument_aggregation_temporality = AggregationTemporality.UNSPECIFIED
+        if isinstance(instrument, Synchronous):
+            instrument_aggregation_temporality = AggregationTemporality.DELTA
+        elif isinstance(instrument, Asynchronous):
+            instrument_aggregation_temporality = (
+                AggregationTemporality.CUMULATIVE
+            )
+
         return _ExplicitBucketHistogramAggregation(
             attributes,
+            instrument_aggregation_temporality,
             start_time_unix_nano,
             self._boundaries,
             self._record_min_max,
@@ -1200,16 +1219,18 @@ def _create_aggregation(
         start_time_unix_nano: int,
     ) -> _Aggregation:
 
-        temporality = AggregationTemporality.UNSPECIFIED
+        instrument_aggregation_temporality = AggregationTemporality.UNSPECIFIED
         if isinstance(instrument, Synchronous):
-            temporality = AggregationTemporality.DELTA
+            instrument_aggregation_temporality = AggregationTemporality.DELTA
         elif isinstance(instrument, Asynchronous):
-            temporality = AggregationTemporality.CUMULATIVE
+            instrument_aggregation_temporality = (
+                AggregationTemporality.CUMULATIVE
+            )
 
         return _SumAggregation(
             attributes,
             isinstance(instrument, (Counter, ObservableCounter)),
-            temporality,
+            instrument_aggregation_temporality,
             start_time_unix_nano,
         )
 


Test patch:
diff --git a/opentelemetry-sdk/tests/metrics/integration_test/test_explicit_bucket_histogram_aggregation.py b/opentelemetry-sdk/tests/metrics/integration_test/test_explicit_bucket_histogram_aggregation.py
new file mode 100644
--- /dev/null
+++ b/opentelemetry-sdk/tests/metrics/integration_test/test_explicit_bucket_histogram_aggregation.py
@@ -0,0 +1,244 @@
+# Copyright The OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from platform import system
+from unittest import TestCase
+
+from pytest import mark
+
+from opentelemetry.sdk.metrics import Histogram, MeterProvider
+from opentelemetry.sdk.metrics.export import (
+    AggregationTemporality,
+    InMemoryMetricReader,
+)
+from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation
+
+
+class TestExplicitBucketHistogramAggregation(TestCase):
+
+    test_values = [1, 6, 11, 26, 51, 76, 101, 251, 501, 751]
+
+    @mark.skipif(
+        system() == "Windows",
+        reason=(
+            "Tests fail because Windows time_ns resolution is too low so "
+            "two different time measurements may end up having the exact same"
+            "value."
+        ),
+    )
+    def test_synchronous_delta_temporality(self):
+
+        aggregation = ExplicitBucketHistogramAggregation()
+
+        reader = InMemoryMetricReader(
+            preferred_aggregation={Histogram: aggregation},
+            preferred_temporality={Histogram: AggregationTemporality.DELTA},
+        )
+
+        provider = MeterProvider(metric_readers=[reader])
+        meter = provider.get_meter("name", "version")
+
+        histogram = meter.create_histogram("histogram")
+
+        results = []
+
+        for _ in range(10):
+
+            results.append(reader.get_metrics_data())
+
+        for metrics_data in results:
+            self.assertIsNone(metrics_data)
+
+        results = []
+
+        for test_value in self.test_values:
+            histogram.record(test_value)
+            results.append(reader.get_metrics_data())
+
+        metric_data = (
+            results[0]
+            .resource_metrics[0]
+            .scope_metrics[0]
+            .metrics[0]
+            .data.data_points[0]
+        )
+
+        previous_time_unix_nano = metric_data.time_unix_nano
+
+        self.assertEqual(
+            metric_data.bucket_counts,
+            (0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
+        )
+
+        self.assertLess(
+            metric_data.start_time_unix_nano,
+            previous_time_unix_nano,
+        )
+        self.assertEqual(metric_data.min, self.test_values[0])
+        self.assertEqual(metric_data.max, self.test_values[0])
+        self.assertEqual(metric_data.sum, self.test_values[0])
+
+        for index, metrics_data in enumerate(results[1:]):
+            metric_data = (
+                metrics_data.resource_metrics[0]
+                .scope_metrics[0]
+                .metrics[0]
+                .data.data_points[0]
+            )
+
+            self.assertEqual(
+                previous_time_unix_nano, metric_data.start_time_unix_nano
+            )
+            previous_time_unix_nano = metric_data.time_unix_nano
+            self.assertEqual(
+                metric_data.bucket_counts,
+                tuple(
+                    [
+                        1 if internal_index == index + 2 else 0
+                        for internal_index in range(16)
+                    ]
+                ),
+            )
+            self.assertLess(
+                metric_data.start_time_unix_nano, metric_data.time_unix_nano
+            )
+            self.assertEqual(metric_data.min, self.test_values[index + 1])
+            self.assertEqual(metric_data.max, self.test_values[index + 1])
+            self.assertEqual(metric_data.sum, self.test_values[index + 1])
+
+        results = []
+
+        for _ in range(10):
+
+            results.append(reader.get_metrics_data())
+
+        provider.shutdown()
+
+        for metrics_data in results:
+            self.assertIsNone(metrics_data)
+
+    @mark.skipif(
+        system() != "Linux",
+        reason=(
+            "Tests fail because Windows time_ns resolution is too low so "
+            "two different time measurements may end up having the exact same"
+            "value."
+        ),
+    )
+    def test_synchronous_cumulative_temporality(self):
+
+        aggregation = ExplicitBucketHistogramAggregation()
+
+        reader = InMemoryMetricReader(
+            preferred_aggregation={Histogram: aggregation},
+            preferred_temporality={
+                Histogram: AggregationTemporality.CUMULATIVE
+            },
+        )
+
+        provider = MeterProvider(metric_readers=[reader])
+        meter = provider.get_meter("name", "version")
+
+        histogram = meter.create_histogram("histogram")
+
+        results = []
+
+        for _ in range(10):
+
+            results.append(reader.get_metrics_data())
+
+        for metrics_data in results:
+            self.assertIsNone(metrics_data)
+
+        results = []
+
+        for test_value in self.test_values:
+
+            histogram.record(test_value)
+            results.append(reader.get_metrics_data())
+
+        start_time_unix_nano = (
+            results[0]
+            .resource_metrics[0]
+            .scope_metrics[0]
+            .metrics[0]
+            .data.data_points[0]
+            .start_time_unix_nano
+        )
+
+        for index, metrics_data in enumerate(results):
+
+            metric_data = (
+                metrics_data.resource_metrics[0]
+                .scope_metrics[0]
+                .metrics[0]
+                .data.data_points[0]
+            )
+
+            self.assertEqual(
+                start_time_unix_nano, metric_data.start_time_unix_nano
+            )
+            self.assertEqual(
+                metric_data.bucket_counts,
+                tuple(
+                    [
+                        0
+                        if internal_index < 1 or internal_index > index + 1
+                        else 1
+                        for internal_index in range(16)
+                    ]
+                ),
+            )
+            self.assertEqual(metric_data.min, self.test_values[0])
+            self.assertEqual(metric_data.max, self.test_values[index])
+            self.assertEqual(
+                metric_data.sum, sum(self.test_values[: index + 1])
+            )
+
+        results = []
+
+        for _ in range(10):
+
+            results.append(reader.get_metrics_data())
+
+        provider.shutdown()
+
+        start_time_unix_nano = (
+            results[0]
+            .resource_metrics[0]
+            .scope_metrics[0]
+            .metrics[0]
+            .data.data_points[0]
+            .start_time_unix_nano
+        )
+
+        for metrics_data in results:
+
+            metric_data = (
+                metrics_data.resource_metrics[0]
+                .scope_metrics[0]
+                .metrics[0]
+                .data.data_points[0]
+            )
+
+            self.assertEqual(
+                start_time_unix_nano, metric_data.start_time_unix_nano
+            )
+            self.assertEqual(
+                metric_data.bucket_counts,
+                (0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0),
+            )
+            self.assertEqual(metric_data.min, self.test_values[0])
+            self.assertEqual(metric_data.max, self.test_values[-1])
+            self.assertEqual(metric_data.sum, sum(self.test_values))
diff --git a/opentelemetry-sdk/tests/metrics/integration_test/test_histogram_export.py b/opentelemetry-sdk/tests/metrics/integration_test/test_histogram_export.py
--- a/opentelemetry-sdk/tests/metrics/integration_test/test_histogram_export.py
+++ b/opentelemetry-sdk/tests/metrics/integration_test/test_histogram_export.py
@@ -65,10 +65,8 @@ def test_histogram_counter_collection(self):
 
         metric_data = in_memory_metric_reader.get_metrics_data()
 
-        # FIXME ExplicitBucketHistogramAggregation is resetting counts to zero
-        # even if aggregation temporality is cumulative.
         self.assertEqual(
-            len(metric_data.resource_metrics[0].scope_metrics[0].metrics), 1
+            len(metric_data.resource_metrics[0].scope_metrics[0].metrics), 2
         )
         self.assertEqual(
             (
@@ -76,6 +74,16 @@ def test_histogram_counter_collection(self):
                 .scope_metrics[0]
                 .metrics[0]
                 .data.data_points[0]
+                .bucket_counts
+            ),
+            (0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0),
+        )
+        self.assertEqual(
+            (
+                metric_data.resource_metrics[0]
+                .scope_metrics[0]
+                .metrics[1]
+                .data.data_points[0]
                 .value
             ),
             1,
diff --git a/opentelemetry-sdk/tests/metrics/test_aggregation.py b/opentelemetry-sdk/tests/metrics/test_aggregation.py
--- a/opentelemetry-sdk/tests/metrics/test_aggregation.py
+++ b/opentelemetry-sdk/tests/metrics/test_aggregation.py
@@ -270,7 +270,10 @@ def test_aggregate(self):
 
         explicit_bucket_histogram_aggregation = (
             _ExplicitBucketHistogramAggregation(
-                Mock(), 0, boundaries=[0, 2, 4]
+                Mock(),
+                AggregationTemporality.DELTA,
+                0,
+                boundaries=[0, 2, 4],
             )
         )
 
@@ -284,22 +287,22 @@ def test_aggregate(self):
 
         # The first bucket keeps count of values between (-inf, 0] (-1 and 0)
         self.assertEqual(
-            explicit_bucket_histogram_aggregation._bucket_counts[0], 2
+            explicit_bucket_histogram_aggregation._current_value[0], 2
         )
 
         # The second bucket keeps count of values between (0, 2] (1 and 2)
         self.assertEqual(
-            explicit_bucket_histogram_aggregation._bucket_counts[1], 2
+            explicit_bucket_histogram_aggregation._current_value[1], 2
         )
 
         # The third bucket keeps count of values between (2, 4] (3 and 4)
         self.assertEqual(
-            explicit_bucket_histogram_aggregation._bucket_counts[2], 2
+            explicit_bucket_histogram_aggregation._current_value[2], 2
         )
 
         # The fourth bucket keeps count of values between (4, inf) (3 and 4)
         self.assertEqual(
-            explicit_bucket_histogram_aggregation._bucket_counts[3], 1
+            explicit_bucket_histogram_aggregation._current_value[3], 1
         )
 
         histo = explicit_bucket_histogram_aggregation.collect(
@@ -314,7 +317,9 @@ def test_min_max(self):
         """
 
         explicit_bucket_histogram_aggregation = (
-            _ExplicitBucketHistogramAggregation(Mock(), 0)
+            _ExplicitBucketHistogramAggregation(
+                Mock(), AggregationTemporality.CUMULATIVE, 0
+            )
         )
 
         explicit_bucket_histogram_aggregation.aggregate(measurement(-1))
@@ -328,7 +333,10 @@ def test_min_max(self):
 
         explicit_bucket_histogram_aggregation = (
             _ExplicitBucketHistogramAggregation(
-                Mock(), 0, record_min_max=False
+                Mock(),
+                AggregationTemporality.CUMULATIVE,
+                0,
+                record_min_max=False,
             )
         )
 
@@ -348,7 +356,10 @@ def test_collect(self):
 
         explicit_bucket_histogram_aggregation = (
             _ExplicitBucketHistogramAggregation(
-                Mock(), 0, boundaries=[0, 1, 2]
+                Mock(),
+                AggregationTemporality.DELTA,
+                0,
+                boundaries=[0, 1, 2],
             )
         )
 
@@ -368,6 +379,7 @@ def test_collect(self):
         explicit_bucket_histogram_aggregation.aggregate(measurement(1))
         # 2 is used here directly to simulate the instant the second
         # collection process starts.
+
         second_histogram = explicit_bucket_histogram_aggregation.collect(
             AggregationTemporality.CUMULATIVE, 2
         )
@@ -381,7 +393,9 @@ def test_collect(self):
 
     def test_boundaries(self):
         self.assertEqual(
-            _ExplicitBucketHistogramAggregation(Mock(), 0)._boundaries,
+            _ExplicitBucketHistogramAggregation(
+                Mock(), AggregationTemporality.CUMULATIVE, 0
+            )._boundaries,
             (
                 0.0,
                 5.0,