Repository: open-telemetry/opentelemetry-python

Issue:
PeriodicExportingMetricReader thread dies when metrics collection times out
**Describe your environment** 
Python 3.10.

**Steps to reproduce**
1. Configure metrics with the `PeriodicExportingMetricReader`
2. Implement an observable metric that takes longer than 30 seconds to return (a simple thread sleep would work)

**What is the expected behavior?**
An error log regarding the collection timing out, but the next collection interval to begin on schedule and attempt another collection.

**What is the actual behavior?**
The `OtelPeriodicExportingMetricReader` thread dies, and the process stops emitting metrics altogether. 

**Additional context**
The `MetricsTimeoutError` raised by the `MeasurementConsumer` is not caught at any point and handled.
As far as I can see, there's no way for the application to identify this timeout has occurred and compensate by triggering a thread restart, or re-configuration.



Discussion:

Resolving patch:
diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/export/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/export/__init__.py
--- a/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/export/__init__.py
+++ b/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/export/__init__.py
@@ -41,6 +41,7 @@
     AggregationTemporality,
     DefaultAggregation,
 )
+from opentelemetry.sdk.metrics._internal.exceptions import MetricsTimeoutError
 from opentelemetry.sdk.metrics._internal.instrument import (
     Counter,
     Histogram,
@@ -497,7 +498,14 @@ def _at_fork_reinit(self):
     def _ticker(self) -> None:
         interval_secs = self._export_interval_millis / 1e3
         while not self._shutdown_event.wait(interval_secs):
-            self.collect(timeout_millis=self._export_timeout_millis)
+            try:
+                self.collect(timeout_millis=self._export_timeout_millis)
+            except MetricsTimeoutError:
+                _logger.warning(
+                    "Metric collection timed out. Will try again after %s seconds",
+                    interval_secs,
+                    exc_info=True,
+                )
         # one last collection below before shutting down completely
         self.collect(timeout_millis=self._export_interval_millis)
 


Test patch:
diff --git a/opentelemetry-sdk/tests/metrics/test_periodic_exporting_metric_reader.py b/opentelemetry-sdk/tests/metrics/test_periodic_exporting_metric_reader.py
--- a/opentelemetry-sdk/tests/metrics/test_periodic_exporting_metric_reader.py
+++ b/opentelemetry-sdk/tests/metrics/test_periodic_exporting_metric_reader.py
@@ -14,12 +14,12 @@
 
 import math
 from time import sleep, time_ns
-from typing import Sequence
+from typing import Optional, Sequence
 from unittest.mock import Mock
 
 from flaky import flaky
 
-from opentelemetry.sdk.metrics import Counter
+from opentelemetry.sdk.metrics import Counter, MetricsTimeoutError
 from opentelemetry.sdk.metrics._internal import _Counter
 from opentelemetry.sdk.metrics.export import (
     AggregationTemporality,
@@ -67,6 +67,25 @@ def force_flush(self, timeout_millis: float = 10_000) -> bool:
         return True
 
 
+class ExceptionAtCollectionPeriodicExportingMetricReader(
+    PeriodicExportingMetricReader
+):
+    def __init__(
+        self,
+        exporter: MetricExporter,
+        exception: Exception,
+        export_interval_millis: Optional[float] = None,
+        export_timeout_millis: Optional[float] = None,
+    ) -> None:
+        super().__init__(
+            exporter, export_interval_millis, export_timeout_millis
+        )
+        self._collect_exception = exception
+
+    def collect(self, timeout_millis: float = 10_000) -> None:
+        raise self._collect_exception
+
+
 metrics_list = [
     Metric(
         name="sum_name",
@@ -111,11 +130,13 @@ def test_defaults(self):
         pmr.shutdown()
 
     def _create_periodic_reader(
-        self, metrics, exporter, collect_wait=0, interval=60000
+        self, metrics, exporter, collect_wait=0, interval=60000, timeout=30000
     ):
 
         pmr = PeriodicExportingMetricReader(
-            exporter, export_interval_millis=interval
+            exporter,
+            export_interval_millis=interval,
+            export_timeout_millis=timeout,
         )
 
         def _collect(reader, timeout_millis):
@@ -219,3 +240,15 @@ def test_exporter_aggregation_preference(self):
                 self.assertTrue(isinstance(value, DefaultAggregation))
             else:
                 self.assertTrue(isinstance(value, LastValueAggregation))
+
+    def test_metric_timeout_does_not_kill_worker_thread(self):
+        exporter = FakeMetricsExporter()
+        pmr = ExceptionAtCollectionPeriodicExportingMetricReader(
+            exporter,
+            MetricsTimeoutError("test timeout"),
+            export_timeout_millis=1,
+        )
+
+        sleep(0.1)
+        self.assertTrue(pmr._daemon_thread.is_alive())
+        pmr.shutdown()