Repository: open-telemetry/opentelemetry-python

Issue:
Add OT collector metrics exporter
Exporter to export metrics to the OT collector: https://github.com/open-telemetry/opentelemetry-collector. Note that the release may be blocked on the release of https://github.com/open-telemetry/opentelemetry-proto.

This may be blocked by ongoing metrics export work in #341.

See also #343.


Discussion:
Releasing assignment to focus on other issues.

Resolving patch:
diff --git a/examples/metrics/collector.py b/examples/metrics/collector.py
new file mode 100644
--- /dev/null
+++ b/examples/metrics/collector.py
@@ -0,0 +1,53 @@
+# Copyright 2020, OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+This module serves as an example for a simple application using metrics
+exporting to Collector
+"""
+
+from opentelemetry import metrics
+from opentelemetry.ext.otcollector.metrics_exporter import (
+    CollectorMetricsExporter,
+)
+from opentelemetry.sdk.metrics import Counter, MeterProvider
+from opentelemetry.sdk.metrics.export.controller import PushController
+
+# Meter is responsible for creating and recording metrics
+metrics.set_preferred_meter_provider_implementation(lambda _: MeterProvider())
+meter = metrics.get_meter(__name__)
+# exporter to export metrics to OT Collector
+exporter = CollectorMetricsExporter(
+    service_name="basic-service", endpoint="localhost:55678"
+)
+# controller collects metrics created from meter and exports it via the
+# exporter every interval
+controller = PushController(meter, exporter, 5)
+
+counter = meter.create_metric(
+    "requests",
+    "number of requests",
+    "requests",
+    int,
+    Counter,
+    ("environment",),
+)
+
+# Labelsets are used to identify key-values that are associated with a specific
+# metric that you want to record. These are useful for pre-aggregation and can
+# be used to store custom dimensions pertaining to a metric
+label_set = meter.get_label_set({"environment": "staging"})
+
+counter.add(25, label_set)
+input("Press any key to exit...")
diff --git a/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py
new file mode 100644
--- /dev/null
+++ b/ext/opentelemetry-ext-otcollector/src/opentelemetry/ext/otcollector/metrics_exporter/__init__.py
@@ -0,0 +1,165 @@
+# Copyright 2020, OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""OpenTelemetry Collector Metrics Exporter."""
+
+import logging
+from typing import Sequence
+
+import grpc
+from opencensus.proto.agent.metrics.v1 import (
+    metrics_service_pb2,
+    metrics_service_pb2_grpc,
+)
+from opencensus.proto.metrics.v1 import metrics_pb2
+
+import opentelemetry.ext.otcollector.util as utils
+from opentelemetry.sdk.metrics import Counter, Metric
+from opentelemetry.sdk.metrics.export import (
+    MetricRecord,
+    MetricsExporter,
+    MetricsExportResult,
+    aggregate,
+)
+
+DEFAULT_ENDPOINT = "localhost:55678"
+
+logger = logging.getLogger(__name__)
+
+
+# pylint: disable=no-member
+class CollectorMetricsExporter(MetricsExporter):
+    """OpenTelemetry Collector metrics exporter.
+
+    Args:
+        endpoint: OpenTelemetry Collector OpenCensus receiver endpoint.
+        service_name: Name of Collector service.
+        host_name: Host name.
+        client: MetricsService client stub.
+    """
+
+    def __init__(
+        self,
+        endpoint: str = DEFAULT_ENDPOINT,
+        service_name: str = None,
+        host_name: str = None,
+        client: metrics_service_pb2_grpc.MetricsServiceStub = None,
+    ):
+        self.endpoint = endpoint
+        if client is None:
+            channel = grpc.insecure_channel(self.endpoint)
+            self.client = metrics_service_pb2_grpc.MetricsServiceStub(
+                channel=channel
+            )
+        else:
+            self.client = client
+
+        self.node = utils.get_node(service_name, host_name)
+
+    def export(
+        self, metric_records: Sequence[MetricRecord]
+    ) -> MetricsExportResult:
+        try:
+            responses = self.client.Export(
+                self.generate_metrics_requests(metric_records)
+            )
+
+            # Read response
+            for _ in responses:
+                pass
+
+        except grpc.RpcError:
+            return MetricsExportResult.FAILED_RETRYABLE
+
+        return MetricsExportResult.SUCCESS
+
+    def shutdown(self) -> None:
+        pass
+
+    def generate_metrics_requests(
+        self, metrics: Sequence[MetricRecord]
+    ) -> metrics_service_pb2.ExportMetricsServiceRequest:
+        collector_metrics = translate_to_collector(metrics)
+        service_request = metrics_service_pb2.ExportMetricsServiceRequest(
+            node=self.node, metrics=collector_metrics
+        )
+        yield service_request
+
+
+# pylint: disable=too-many-branches
+def translate_to_collector(
+    metric_records: Sequence[MetricRecord],
+) -> Sequence[metrics_pb2.Metric]:
+    collector_metrics = []
+    for metric_record in metric_records:
+
+        label_values = []
+        label_keys = []
+        for label_tuple in metric_record.label_set.labels:
+            label_keys.append(metrics_pb2.LabelKey(key=label_tuple[0]))
+            label_values.append(
+                metrics_pb2.LabelValue(
+                    has_value=label_tuple[1] is not None, value=label_tuple[1]
+                )
+            )
+
+        metric_descriptor = metrics_pb2.MetricDescriptor(
+            name=metric_record.metric.name,
+            description=metric_record.metric.description,
+            unit=metric_record.metric.unit,
+            type=get_collector_metric_type(metric_record.metric),
+            label_keys=label_keys,
+        )
+
+        timeseries = metrics_pb2.TimeSeries(
+            label_values=label_values,
+            points=[get_collector_point(metric_record)],
+        )
+        collector_metrics.append(
+            metrics_pb2.Metric(
+                metric_descriptor=metric_descriptor, timeseries=[timeseries]
+            )
+        )
+    return collector_metrics
+
+
+# pylint: disable=no-else-return
+def get_collector_metric_type(metric: Metric) -> metrics_pb2.MetricDescriptor:
+    if isinstance(metric, Counter):
+        if metric.value_type == int:
+            return metrics_pb2.MetricDescriptor.CUMULATIVE_INT64
+        elif metric.value_type == float:
+            return metrics_pb2.MetricDescriptor.CUMULATIVE_DOUBLE
+    return metrics_pb2.MetricDescriptor.UNSPECIFIED
+
+
+def get_collector_point(metric_record: MetricRecord) -> metrics_pb2.Point:
+    point = metrics_pb2.Point(
+        timestamp=utils.proto_timestamp_from_time_ns(
+            metric_record.metric.bind(
+                metric_record.label_set
+            ).last_update_timestamp
+        )
+    )
+    if metric_record.metric.value_type == int:
+        point.int64_value = metric_record.aggregator.checkpoint
+    elif metric_record.metric.value_type == float:
+        point.double_value = metric_record.aggregator.checkpoint
+    else:
+        raise TypeError(
+            "Unsupported metric type: {}".format(
+                metric_record.metric.value_type
+            )
+        )
+    return point


Test patch:
diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py
new file mode 100644
--- /dev/null
+++ b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_metrics_exporter.py
@@ -0,0 +1,202 @@
+# Copyright 2020, OpenTelemetry Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+from unittest import mock
+
+import grpc
+from google.protobuf.timestamp_pb2 import Timestamp
+from opencensus.proto.metrics.v1 import metrics_pb2
+
+from opentelemetry import metrics
+from opentelemetry.ext.otcollector import metrics_exporter
+from opentelemetry.sdk.metrics import Counter, Measure, MeterProvider
+from opentelemetry.sdk.metrics.export import (
+    MetricRecord,
+    MetricsExportResult,
+    aggregate,
+)
+
+
+# pylint: disable=no-member
+class TestCollectorMetricsExporter(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # pylint: disable=protected-access
+        cls._meter_defaults = (
+            metrics._METER_PROVIDER,
+            metrics._METER_PROVIDER_FACTORY,
+        )
+        metrics.set_preferred_meter_provider_implementation(
+            lambda _: MeterProvider()
+        )
+        cls._meter = metrics.get_meter(__name__)
+        kvp = {"environment": "staging"}
+        cls._test_label_set = cls._meter.get_label_set(kvp)
+
+    @classmethod
+    def tearDownClass(cls):
+        # pylint: disable=protected-access
+        (
+            metrics._METER_PROVIDER,
+            metrics._METER_PROVIDER_FACTORY,
+        ) = cls._meter_defaults
+
+    def test_constructor(self):
+        mock_get_node = mock.Mock()
+        patch = mock.patch(
+            "opentelemetry.ext.otcollector.util.get_node",
+            side_effect=mock_get_node,
+        )
+        service_name = "testServiceName"
+        host_name = "testHostName"
+        client = grpc.insecure_channel("")
+        endpoint = "testEndpoint"
+        with patch:
+            exporter = metrics_exporter.CollectorMetricsExporter(
+                service_name=service_name,
+                host_name=host_name,
+                endpoint=endpoint,
+                client=client,
+            )
+
+        self.assertIs(exporter.client, client)
+        self.assertEqual(exporter.endpoint, endpoint)
+        mock_get_node.assert_called_with(service_name, host_name)
+
+    def test_get_collector_metric_type(self):
+        result = metrics_exporter.get_collector_metric_type(
+            Counter("testName", "testDescription", "unit", int, None)
+        )
+        self.assertIs(result, metrics_pb2.MetricDescriptor.CUMULATIVE_INT64)
+        result = metrics_exporter.get_collector_metric_type(
+            Counter("testName", "testDescription", "unit", float, None)
+        )
+        self.assertIs(result, metrics_pb2.MetricDescriptor.CUMULATIVE_DOUBLE)
+        result = metrics_exporter.get_collector_metric_type(
+            Measure("testName", "testDescription", "unit", None, None)
+        )
+        self.assertIs(result, metrics_pb2.MetricDescriptor.UNSPECIFIED)
+
+    def test_get_collector_point(self):
+        aggregator = aggregate.CounterAggregator()
+        label_set = self._meter.get_label_set({"environment": "staging"})
+        int_counter = self._meter.create_metric(
+            "testName", "testDescription", "unit", int, Counter
+        )
+        float_counter = self._meter.create_metric(
+            "testName", "testDescription", "unit", float, Counter
+        )
+        measure = self._meter.create_metric(
+            "testName", "testDescription", "unit", float, Measure
+        )
+        result = metrics_exporter.get_collector_point(
+            MetricRecord(aggregator, label_set, int_counter)
+        )
+        self.assertIsInstance(result, metrics_pb2.Point)
+        self.assertIsInstance(result.timestamp, Timestamp)
+        self.assertEqual(result.int64_value, 0)
+        aggregator.update(123.5)
+        aggregator.take_checkpoint()
+        result = metrics_exporter.get_collector_point(
+            MetricRecord(aggregator, label_set, float_counter)
+        )
+        self.assertEqual(result.double_value, 123.5)
+        self.assertRaises(
+            TypeError,
+            metrics_exporter.get_collector_point(
+                MetricRecord(aggregator, label_set, measure)
+            ),
+        )
+
+    def test_export(self):
+        mock_client = mock.MagicMock()
+        mock_export = mock.MagicMock()
+        mock_client.Export = mock_export
+        host_name = "testHostName"
+        collector_exporter = metrics_exporter.CollectorMetricsExporter(
+            client=mock_client, host_name=host_name
+        )
+        test_metric = self._meter.create_metric(
+            "testname", "testdesc", "unit", int, Counter, ["environment"]
+        )
+        record = MetricRecord(
+            aggregate.CounterAggregator(), self._test_label_set, test_metric
+        )
+
+        result = collector_exporter.export([record])
+        self.assertIs(result, MetricsExportResult.SUCCESS)
+        # pylint: disable=unsubscriptable-object
+        export_arg = mock_export.call_args[0]
+        service_request = next(export_arg[0])
+        output_metrics = getattr(service_request, "metrics")
+        output_node = getattr(service_request, "node")
+        self.assertEqual(len(output_metrics), 1)
+        self.assertIsNotNone(getattr(output_node, "library_info"))
+        self.assertIsNotNone(getattr(output_node, "service_info"))
+        output_identifier = getattr(output_node, "identifier")
+        self.assertEqual(
+            getattr(output_identifier, "host_name"), "testHostName"
+        )
+
+    def test_translate_to_collector(self):
+
+        test_metric = self._meter.create_metric(
+            "testname", "testdesc", "unit", int, Counter, ["environment"]
+        )
+        aggregator = aggregate.CounterAggregator()
+        aggregator.update(123)
+        aggregator.take_checkpoint()
+        record = MetricRecord(aggregator, self._test_label_set, test_metric)
+        output_metrics = metrics_exporter.translate_to_collector([record])
+        self.assertEqual(len(output_metrics), 1)
+        self.assertIsInstance(output_metrics[0], metrics_pb2.Metric)
+        self.assertEqual(output_metrics[0].metric_descriptor.name, "testname")
+        self.assertEqual(
+            output_metrics[0].metric_descriptor.description, "testdesc"
+        )
+        self.assertEqual(output_metrics[0].metric_descriptor.unit, "unit")
+        self.assertEqual(
+            output_metrics[0].metric_descriptor.type,
+            metrics_pb2.MetricDescriptor.CUMULATIVE_INT64,
+        )
+        self.assertEqual(
+            len(output_metrics[0].metric_descriptor.label_keys), 1
+        )
+        self.assertEqual(
+            output_metrics[0].metric_descriptor.label_keys[0].key,
+            "environment",
+        )
+        self.assertEqual(len(output_metrics[0].timeseries), 1)
+        self.assertEqual(len(output_metrics[0].timeseries[0].label_values), 1)
+        self.assertEqual(
+            output_metrics[0].timeseries[0].label_values[0].has_value, True
+        )
+        self.assertEqual(
+            output_metrics[0].timeseries[0].label_values[0].value, "staging"
+        )
+        self.assertEqual(len(output_metrics[0].timeseries[0].points), 1)
+        self.assertEqual(
+            output_metrics[0].timeseries[0].points[0].timestamp.seconds,
+            record.metric.bind(record.label_set).last_update_timestamp
+            // 1000000000,
+        )
+        self.assertEqual(
+            output_metrics[0].timeseries[0].points[0].timestamp.nanos,
+            record.metric.bind(record.label_set).last_update_timestamp
+            % 1000000000,
+        )
+        self.assertEqual(
+            output_metrics[0].timeseries[0].points[0].int64_value, 123
+        )
diff --git a/ext/opentelemetry-ext-otcollector/tests/test_otcollector_exporter.py b/ext/opentelemetry-ext-otcollector/tests/test_otcollector_trace_exporter.py
similarity index 100%
rename from ext/opentelemetry-ext-otcollector/tests/test_otcollector_exporter.py
rename to ext/opentelemetry-ext-otcollector/tests/test_otcollector_trace_exporter.py