Repository: GoogleCloudPlatform/opentelemetry-operations-go
Pull request #418 (2022-06-09 17:04:28)

Record failed points in point_count metric
Fixes https://github.com/GoogleCloudPlatform/opentelemetry-operations-go/issues/410

damemi: Spent a lot of time trying to get this tested by passing a desired return code through `context.WithValue` but no luck. Ended up settling on this approach instead
damemi: /hold

Patch:
diff --git a/exporter/collector/logs.go b/exporter/collector/logs.go
index 8eb38f1..6b0e28a 100644
--- a/exporter/collector/logs.go
+++ b/exporter/collector/logs.go
@@ -21,6 +21,7 @@ import (
 	"math"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"cloud.google.com/go/logging"
@@ -310,15 +311,23 @@ func (l logMapper) logToSplitEntries(
 		entry.Timestamp = processTime
 	}
 
+	// build our own map off OTel attributes so we don't have to call .Get() for each special case
+	// (.Get() ranges over all attributes each time)
+	attrsMap := make(map[string]pcommon.Value)
+	log.Attributes().Range(func(k string, v pcommon.Value) bool {
+		attrsMap[k] = v
+		return true
+	})
+
 	// parse LogEntrySourceLocation struct from OTel attribute
-	sourceLocation, ok := log.Attributes().Get(SourceLocationAttributeKey)
-	if ok {
+	if sourceLocation, ok := attrsMap[SourceLocationAttributeKey]; ok {
 		var logEntrySourceLocation logpb.LogEntrySourceLocation
 		err := json.Unmarshal(sourceLocation.MBytesVal(), &logEntrySourceLocation)
 		if err != nil {
 			return []logging.Entry{entry}, err
 		}
 		entry.SourceLocation = &logEntrySourceLocation
+		delete(attrsMap, SourceLocationAttributeKey)
 	}
 
 	// parse TraceID and SpanID, if present
@@ -329,14 +338,14 @@ func (l logMapper) logToSplitEntries(
 		entry.SpanID = log.SpanID().HexString()
 	}
 
-	httpRequestAttr, ok := log.Attributes().Get(HTTPRequestAttributeKey)
-	if ok {
+	if httpRequestAttr, ok := attrsMap[HTTPRequestAttributeKey]; ok {
 		var httpRequest *logging.HTTPRequest
 		httpRequest, err := l.parseHTTPRequest(httpRequestAttr)
 		if err != nil {
 			l.obs.log.Debug("Unable to parse httpRequest", zap.Error(err))
 		}
 		entry.HTTPRequest = httpRequest
+		delete(attrsMap, HTTPRequestAttributeKey)
 	}
 
 	if log.SeverityNumber() < 0 || int(log.SeverityNumber()) > len(severityMapping)-1 {
@@ -344,6 +353,20 @@ func (l logMapper) logToSplitEntries(
 	}
 	entry.Severity = severityMapping[log.SeverityNumber()]
 
+	// parse remaining OTel attributes to GCP labels
+	for k, v := range attrsMap {
+		// skip "gcp.*" attributes since we process these to other fields
+		if strings.HasPrefix(k, "gcp.") {
+			continue
+		}
+		if entry.Labels == nil {
+			entry.Labels = make(map[string]string)
+		}
+		if _, ok := entry.Labels[k]; !ok {
+			entry.Labels[k] = v.AsString()
+		}
+	}
+
 	// Calculate the size of the internal log entry so this overhead can be accounted
 	// for when determining the need to split based on payload size
 	// TODO(damemi): Find an appropriate estimated buffer to account for the LogSplit struct as well
diff --git a/exporter/collector/metrics.go b/exporter/collector/metrics.go
index a8308f1..1cb00eb 100644
--- a/exporter/collector/metrics.go
+++ b/exporter/collector/metrics.go
@@ -245,7 +245,20 @@ func (me *MetricsExporter) PushMetrics(ctx context.Context, m pmetric.Metrics) e
 			s, _ := status.FromError(err)
 			st = statusCodeToString(s)
 
-			recordPointCountDataPoint(ctx, len(ts), st)
+			succeededPoints := len(ts)
+			failedPoints := 0
+			for _, detail := range s.Details() {
+				if summary, ok := detail.(*monitoringpb.CreateTimeSeriesSummary); ok {
+					failedPoints = int(summary.TotalPointCount - summary.SuccessPointCount)
+					succeededPoints = int(summary.SuccessPointCount)
+				}
+			}
+
+			// always record the number of successful points
+			recordPointCountDataPoint(ctx, succeededPoints, "OK")
+			if failedPoints > 0 {
+				recordPointCountDataPoint(ctx, failedPoints, st)
+			}
 			if err != nil {
 				errs = append(errs, fmt.Errorf("failed to export time series to GCM: %v", err))
 			}


Test patch:
diff --git a/exporter/collector/integrationtest/cmd/recordfixtures/main.go b/exporter/collector/integrationtest/cmd/recordfixtures/main.go
index babc056..cb1b280 100644
--- a/exporter/collector/integrationtest/cmd/recordfixtures/main.go
+++ b/exporter/collector/integrationtest/cmd/recordfixtures/main.go
@@ -102,7 +102,12 @@ func recordMetrics(ctx context.Context, t *FakeTesting, startTime, endTime time.
 			require.NoError(t, err)
 			defer inMemoryOCExporter.Shutdown(ctx)
 
-			require.NoError(t, testServerExporter.PushMetrics(ctx, metrics), "failed to export metrics to local test server")
+			err = testServerExporter.PushMetrics(ctx, metrics)
+			if !test.ExpectErr {
+				require.NoError(t, err, "failed to export metrics to local test server")
+			} else {
+				require.Error(t, err, "didn't record expected error")
+			}
 			require.NoError(t, testServerExporter.Shutdown(ctx))
 
 			selfObsMetrics, err := inMemoryOCExporter.Proto(ctx)
diff --git a/exporter/collector/integrationtest/metric_test_server.go b/exporter/collector/integrationtest/metric_test_server.go
index f7944dd..9823dcb 100644
--- a/exporter/collector/integrationtest/metric_test_server.go
+++ b/exporter/collector/integrationtest/metric_test_server.go
@@ -17,6 +17,7 @@ package integrationtest
 import (
 	"context"
 	"net"
+	"strings"
 	"sync"
 	"testing"
 
@@ -25,6 +26,8 @@ import (
 	"google.golang.org/genproto/googleapis/api/metric"
 	monitoringpb "google.golang.org/genproto/googleapis/monitoring/v3"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/emptypb"
 
 	"github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/collector"
@@ -102,7 +105,21 @@ func (f *fakeMetricServiceServer) CreateTimeSeries(
 	req *monitoringpb.CreateTimeSeriesRequest,
 ) (*emptypb.Empty, error) {
 	f.metricsTestServer.appendCreateTimeSeriesReq(req)
-	return &emptypb.Empty{}, nil
+
+	var code codes.Code
+	if strings.Contains(req.Name, "notfound") {
+		code = codes.NotFound
+	}
+
+	statusResp := status.New(code, "FORCED_TEST_ERROR")
+	if code == codes.NotFound {
+		statusResp, _ = statusResp.WithDetails(&monitoringpb.CreateTimeSeriesSummary{
+			TotalPointCount:   int32(len(req.TimeSeries)),
+			SuccessPointCount: 0,
+		})
+	}
+
+	return &emptypb.Empty{}, statusResp.Err()
 }
 
 func (f *fakeMetricServiceServer) CreateServiceTimeSeries(
@@ -150,7 +167,6 @@ func (m *MetricsTestServer) NewExporter(
 ) *collector.MetricsExporter {
 	cfg.MetricConfig.ClientConfig.Endpoint = m.Endpoint
 	cfg.MetricConfig.ClientConfig.UseInsecure = true
-	cfg.ProjectID = "fakeprojectid"
 
 	exporter, err := collector.NewGoogleCloudMetricsExporter(
 		ctx,
diff --git a/exporter/collector/integrationtest/metrics_test.go b/exporter/collector/integrationtest/metrics_test.go
index 86c4409..aaa2dc0 100644
--- a/exporter/collector/integrationtest/metrics_test.go
+++ b/exporter/collector/integrationtest/metrics_test.go
@@ -32,6 +32,7 @@ func TestMetrics(t *testing.T) {
 
 		t.Run(test.Name, func(t *testing.T) {
 			test.SkipIfNeeded(t)
+
 			metrics := test.LoadOTLPMetricsInput(t, startTime, endTime)
 
 			testServer, err := NewMetricTestServer()
@@ -44,11 +45,12 @@ func TestMetrics(t *testing.T) {
 			require.NoError(t, err)
 			defer inMemoryOCExporter.Shutdown(ctx)
 
-			require.NoError(
-				t,
-				testServerExporter.PushMetrics(ctx, metrics),
-				"Failed to export metrics to local test server",
-			)
+			err = testServerExporter.PushMetrics(ctx, metrics)
+			if !test.ExpectErr {
+				require.NoError(t, err, "Failed to export metrics to local test server")
+			} else {
+				require.Error(t, err, "Did not see expected error")
+			}
 			require.NoError(t, testServerExporter.Shutdown(ctx))
 
 			expectFixture := test.LoadMetricExpectFixture(
diff --git a/exporter/collector/integrationtest/testcase.go b/exporter/collector/integrationtest/testcase.go
index b143b53..5fe3acc 100644
--- a/exporter/collector/integrationtest/testcase.go
+++ b/exporter/collector/integrationtest/testcase.go
@@ -62,6 +62,8 @@ type TestCase struct {
 	ExpectFixturePath string
 	// Skip, if true, skips this test case
 	Skip bool
+	// ExpectErr sets whether the test is expected to fail
+	ExpectErr bool
 }
 
 func (tc *TestCase) LoadOTLPLogsInput(
@@ -357,7 +359,7 @@ func (tc *TestCase) SkipIfNeeded(t testing.TB) {
 
 func (tc *TestCase) CreateMetricConfig() collector.Config {
 	cfg := collector.DefaultConfig()
-	cfg.ProjectID = "fake-project"
+	cfg.ProjectID = "fakeprojectid"
 	// Set a big buffer to capture all CMD requests without dropping
 	cfg.MetricConfig.CreateMetricDescriptorBufferSize = 500
 	cfg.MetricConfig.InstrumentationLibraryLabels = false
diff --git a/exporter/collector/integrationtest/testcases.go b/exporter/collector/integrationtest/testcases.go
index daddaa1..643fce6 100644
--- a/exporter/collector/integrationtest/testcases.go
+++ b/exporter/collector/integrationtest/testcases.go
@@ -44,6 +44,15 @@ var (
 			OTLPInputFixturePath: "testdata/fixtures/basic_counter_metrics.json",
 			ExpectFixturePath:    "testdata/fixtures/basic_counter_metrics_expect.json",
 		},
+		{
+			Name:                 "Basic Counter with not found return code",
+			OTLPInputFixturePath: "testdata/fixtures/basic_counter_metrics.json",
+			ExpectFixturePath:    "testdata/fixtures/basic_counter_metrics_notfound_expect.json",
+			Configure: func(cfg *collector.Config) {
+				cfg.ProjectID = "notfoundproject"
+			},
+			ExpectErr: true,
+		},
 		{
 			Name:                 "Basic Prometheus metrics",
 			OTLPInputFixturePath: "testdata/fixtures/basic_prometheus_metrics.json",
diff --git a/exporter/collector/integrationtest/testdata/fixtures/basic_counter_metrics_notfound_expect.json b/exporter/collector/integrationtest/testdata/fixtures/basic_counter_metrics_notfound_expect.json
new file mode 100644
index 0000000..5f3dba6
--- /dev/null
+++ b/exporter/collector/integrationtest/testdata/fixtures/basic_counter_metrics_notfound_expect.json
@@ -0,0 +1,783 @@
+{
+  "createTimeSeriesRequests": [
+    {
+      "name": "projects/notfoundproject",
+      "timeSeries": [
+        {
+          "metric": {
+            "type": "workload.googleapis.com/testcounter",
+            "labels": {
+              "foo": "bar"
+            }
+          },
+          "resource": {
+            "type": "generic_node",
+            "labels": {
+              "location": "global",
+              "namespace": "",
+              "node_id": ""
+            }
+          },
+          "metricKind": "CUMULATIVE",
+          "valueType": "INT64",
+          "points": [
+            {
+              "interval": {
+                "endTime": "1970-01-01T00:00:00Z",
+                "startTime": "1970-01-01T00:00:00Z"
+              },
+              "value": {
+                "int64Value": "253"
+              }
+            }
+          ],
+          "unit": "1"
+        }
+      ]
+    }
+  ],
+  "createMetricDescriptorRequests": [
+    {
+      "name": "projects/notfoundproject",
+      "metricDescriptor": {
+        "name": "testcounter",
+        "type": "workload.googleapis.com/testcounter",
+        "labels": [
+          {
+            "key": "foo"
+          }
+        ],
+        "metricKind": "CUMULATIVE",
+        "valueType": "INT64",
+        "unit": "1",
+        "description": "This is a test counter",
+        "displayName": "testcounter"
+      }
+    }
+  ],
+  "selfObservabilityMetrics": {
+    "createTimeSeriesRequests": [
+      {
+        "name": "projects/myproject",
+        "timeSeries": [
+          {
+            "metric": {
+              "type": "custom.googleapis.com/opencensus/googlecloudmonitoring/point_count",
+              "labels": {
+                "status": "NOT_FOUND"
+              }
+            },
+            "resource": {
+              "type": "global"
+            },
+            "points": [
+              {
+                "interval": {
+                  "endTime": "1970-01-01T00:00:00Z",
+                  "startTime": "1970-01-01T00:00:00Z"
+                },
+                "value": {
+                  "int64Value": "1"
+                }
+              }
+            ]
+          },
+          {
+            "metric": {
+              "type": "custom.googleapis.com/opencensus/googlecloudmonitoring/point_count",
+              "labels": {
+                "status": "OK"
+              }
+            },
+            "resource": {
+              "type": "global"
+            },
+            "points": [
+              {
+                "interval": {
+                  "endTime": "1970-01-01T00:00:00Z",
+                  "startTime": "1970-01-01T00:00:00Z"
+                },
+                "value": {
+                  "int64Value": "0"
+                }
+              }
+            ]
+          },
+          {
+            "metric": {
+              "type": "custom.googleapis.com/opencensus/grpc.io/client/completed_rpcs",
+              "labels": {
+                "grpc_client_method": "google.monitoring.v3.MetricService/CreateMetricDescriptor",
+                "grpc_client_status": "OK"
+              }
+            },
+            "resource": {
+              "type": "global"
+            },
+            "points": [
+              {
+                "interval": {
+                  "endTime": "1970-01-01T00:00:00Z",
+                  "startTime": "1970-01-01T00:00:00Z"
+                },
+                "value": {
+                  "int64Value": "1"
+                }
+              }
+            ]
+          },
+          {
+            "metric": {
+              "type": "custom.googleapis.com/opencensus/grpc.io/client/completed_rpcs",
+              "labels": {
+                "grpc_client_method": "google.monitoring.v3.MetricService/CreateTimeSeries",
+                "grpc_client_status": "NOT_FOUND"
+              }
+            },
+            "resource": {
+              "type": "global"
+            },
+            "points": [
+              {
+                "interval": {
+                  "endTime": "1970-01-01T00:00:00Z",
+                  "startTime": "1970-01-01T00:00:00Z"
+                },
+                "value": {
+                  "int64Value": "1"
+                }
+              }
+            ]
+          },
+          {
+            "metric": {
+              "type": "custom.googleapis.com/opencensus/grpc.io/client/received_bytes_per_rpc",
+              "labels": {
+                "grpc_client_method": "google.monitoring.v3.MetricService/CreateMetricDescriptor"
+              }
+            },
+            "resource": {
+              "type": "global"
+            },
+            "points": [
+              {
+                "interval": {
+                  "endTime": "1970-01-01T00:00:00Z",
+                  "startTime": "1970-01-01T00:00:00Z"
+                },
+                "value": {
+                  "distributionValue": {
+                    "bucketOptions": {
+                      "explicitBuckets": {
+                        "bounds": [
+                          0,
+                          1024,
+                          2048,
+                          4096,
+                          16384,
+                          65536,
+                          262144,
+                          1048576,
+                          4194304,
+                          16777216,
+                          67108864,
+                          268435456,
+                          1073741824,
+                          4294967296
+                        ]
+                      }
+                    },
+                    "bucketCounts": [
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0"
+                    ]
+                  }
+                }
+              }
+            ]
+          },
+          {
+            "metric": {
+              "type": "custom.googleapis.com/opencensus/grpc.io/client/received_bytes_per_rpc",
+              "labels": {
+                "grpc_client_method": "google.monitoring.v3.MetricService/CreateTimeSeries"
+              }
+            },
+            "resource": {
+              "type": "global"
+            },
+            "points": [
+              {
+                "interval": {
+                  "endTime": "1970-01-01T00:00:00Z",
+                  "startTime": "1970-01-01T00:00:00Z"
+                },
+                "value": {
+                  "distributionValue": {
+                    "bucketOptions": {
+                      "explicitBuckets": {
+                        "bounds": [
+                          0,
+                          1024,
+                          2048,
+                          4096,
+                          16384,
+                          65536,
+                          262144,
+                          1048576,
+                          4194304,
+                          16777216,
+                          67108864,
+                          268435456,
+                          1073741824,
+                          4294967296
+                        ]
+                      }
+                    },
+                    "bucketCounts": [
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0"
+                    ]
+                  }
+                }
+              }
+            ]
+          },
+          {
+            "metric": {
+              "type": "custom.googleapis.com/opencensus/grpc.io/client/roundtrip_latency",
+              "labels": {
+                "grpc_client_method": "google.monitoring.v3.MetricService/CreateMetricDescriptor"
+              }
+            },
+            "resource": {
+              "type": "global"
+            },
+            "points": [
+              {
+                "interval": {
+                  "endTime": "1970-01-01T00:00:00Z",
+                  "startTime": "1970-01-01T00:00:00Z"
+                },
+                "value": {
+                  "distributionValue": {
+                    "bucketOptions": {
+                      "explicitBuckets": {
+                        "bounds": [
+                          0,
+                          0.01,
+                          0.05,
+                          0.1,
+                          0.3,
+                          0.6,
+                          0.8,
+                          1,
+                          2,
+                          3,
+                          4,
+                          5,
+                          6,
+                          8,
+                          10,
+                          13,
+                          16,
+                          20,
+                          25,
+                          30,
+                          40,
+                          50,
+                          65,
+                          80,
+                          100,
+                          130,
+                          160,
+                          200,
+                          250,
+                          300,
+                          400,
+                          500,
+                          650,
+                          800,
+                          1000,
+                          2000,
+                          5000,
+                          10000,
+                          20000,
+                          50000,
+                          100000
+                        ]
+                      }
+                    },
+                    "bucketCounts": [
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0"
+                    ]
+                  }
+                }
+              }
+            ]
+          },
+          {
+            "metric": {
+              "type": "custom.googleapis.com/opencensus/grpc.io/client/roundtrip_latency",
+              "labels": {
+                "grpc_client_method": "google.monitoring.v3.MetricService/CreateTimeSeries"
+              }
+            },
+            "resource": {
+              "type": "global"
+            },
+            "points": [
+              {
+                "interval": {
+                  "endTime": "1970-01-01T00:00:00Z",
+                  "startTime": "1970-01-01T00:00:00Z"
+                },
+                "value": {
+                  "distributionValue": {
+                    "bucketOptions": {
+                      "explicitBuckets": {
+                        "bounds": [
+                          0,
+                          0.01,
+                          0.05,
+                          0.1,
+                          0.3,
+                          0.6,
+                          0.8,
+                          1,
+                          2,
+                          3,
+                          4,
+                          5,
+                          6,
+                          8,
+                          10,
+                          13,
+                          16,
+                          20,
+                          25,
+                          30,
+                          40,
+                          50,
+                          65,
+                          80,
+                          100,
+                          130,
+                          160,
+                          200,
+                          250,
+                          300,
+                          400,
+                          500,
+                          650,
+                          800,
+                          1000,
+                          2000,
+                          5000,
+                          10000,
+                          20000,
+                          50000,
+                          100000
+                        ]
+                      }
+                    },
+                    "bucketCounts": [
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0"
+                    ]
+                  }
+                }
+              }
+            ]
+          },
+          {
+            "metric": {
+              "type": "custom.googleapis.com/opencensus/grpc.io/client/sent_bytes_per_rpc",
+              "labels": {
+                "grpc_client_method": "google.monitoring.v3.MetricService/CreateMetricDescriptor"
+              }
+            },
+            "resource": {
+              "type": "global"
+            },
+            "points": [
+              {
+                "interval": {
+                  "endTime": "1970-01-01T00:00:00Z",
+                  "startTime": "1970-01-01T00:00:00Z"
+                },
+                "value": {
+                  "distributionValue": {
+                    "bucketOptions": {
+                      "explicitBuckets": {
+                        "bounds": [
+                          0,
+                          1024,
+                          2048,
+                          4096,
+                          16384,
+                          65536,
+                          262144,
+                          1048576,
+                          4194304,
+                          16777216,
+                          67108864,
+                          268435456,
+                          1073741824,
+                          4294967296
+                        ]
+                      }
+                    },
+                    "bucketCounts": [
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0"
+                    ]
+                  }
+                }
+              }
+            ]
+          },
+          {
+            "metric": {
+              "type": "custom.googleapis.com/opencensus/grpc.io/client/sent_bytes_per_rpc",
+              "labels": {
+                "grpc_client_method": "google.monitoring.v3.MetricService/CreateTimeSeries"
+              }
+            },
+            "resource": {
+              "type": "global"
+            },
+            "points": [
+              {
+                "interval": {
+                  "endTime": "1970-01-01T00:00:00Z",
+                  "startTime": "1970-01-01T00:00:00Z"
+                },
+                "value": {
+                  "distributionValue": {
+                    "bucketOptions": {
+                      "explicitBuckets": {
+                        "bounds": [
+                          0,
+                          1024,
+                          2048,
+                          4096,
+                          16384,
+                          65536,
+                          262144,
+                          1048576,
+                          4194304,
+                          16777216,
+                          67108864,
+                          268435456,
+                          1073741824,
+                          4294967296
+                        ]
+                      }
+                    },
+                    "bucketCounts": [
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0",
+                      "0"
+                    ]
+                  }
+                }
+              }
+            ]
+          }
+        ]
+      }
+    ],
+    "createMetricDescriptorRequests": [
+      {
+        "name": "projects/myproject",
+        "metricDescriptor": {
+          "name": "projects/myproject/metricDescriptors/custom.googleapis.com/opencensus/exporter/enqueue_failed_log_records",
+          "type": "custom.googleapis.com/opencensus/exporter/enqueue_failed_log_records",
+          "labels": [
+            {
+              "key": "exporter"
+            }
+          ],
+          "metricKind": "CUMULATIVE",
+          "valueType": "INT64",
+          "unit": "1",
+          "description": "Number of log records failed to be added to the sending queue.",
+          "displayName": "OpenCensus/exporter/enqueue_failed_log_records"
+        }
+      },
+      {
+        "name": "projects/myproject",
+        "metricDescriptor": {
+          "name": "projects/myproject/metricDescriptors/custom.googleapis.com/opencensus/exporter/enqueue_failed_metric_points",
+          "type": "custom.googleapis.com/opencensus/exporter/enqueue_failed_metric_points",
+          "labels": [
+            {
+              "key": "exporter"
+            }
+          ],
+          "metricKind": "CUMULATIVE",
+          "valueType": "INT64",
+          "unit": "1",
+          "description": "Number of metric points failed to be added to the sending queue.",
+          "displayName": "OpenCensus/exporter/enqueue_failed_metric_points"
+        }
+      },
+      {
+        "name": "projects/myproject",
+        "metricDescriptor": {
+          "name": "projects/myproject/metricDescriptors/custom.googleapis.com/opencensus/exporter/enqueue_failed_spans",
+          "type": "custom.googleapis.com/opencensus/exporter/enqueue_failed_spans",
+          "labels": [
+            {
+              "key": "exporter"
+            }
+          ],
+          "metricKind": "CUMULATIVE",
+          "valueType": "INT64",
+          "unit": "1",
+          "description": "Number of spans failed to be added to the sending queue.",
+          "displayName": "OpenCensus/exporter/enqueue_failed_spans"
+        }
+      },
+      {
+        "name": "projects/myproject",
+        "metricDescriptor": {
+          "name": "projects/myproject/metricDescriptors/custom.googleapis.com/opencensus/exporter/queue_size",
+          "type": "custom.googleapis.com/opencensus/exporter/queue_size",
+          "labels": [
+            {
+              "key": "exporter"
+            }
+          ],
+          "metricKind": "GAUGE",
+          "valueType": "INT64",
+          "unit": "1",
+          "description": "Current size of the retry queue (in batches)",
+          "displayName": "OpenCensus/exporter/queue_size"
+        }
+      },
+      {
+        "name": "projects/myproject",
+        "metricDescriptor": {
+          "name": "projects/myproject/metricDescriptors/custom.googleapis.com/opencensus/googlecloudmonitoring/point_count",
+          "type": "custom.googleapis.com/opencensus/googlecloudmonitoring/point_count",
+          "labels": [
+            {
+              "key": "status"
+            }
+          ],
+          "metricKind": "CUMULATIVE",
+          "valueType": "INT64",
+          "unit": "1",
+          "description": "Count of metric points written to Cloud Monitoring.",
+          "displayName": "OpenCensus/googlecloudmonitoring/point_count"
+        }
+      },
+      {
+        "name": "projects/myproject",
+        "metricDescriptor": {
+          "name": "projects/myproject/metricDescriptors/custom.googleapis.com/opencensus/grpc.io/client/completed_rpcs",
+          "type": "custom.googleapis.com/opencensus/grpc.io/client/completed_rpcs",
+          "labels": [
+            {
+              "key": "grpc_client_method"
+            },
+            {
+              "key": "grpc_client_status"
+            }
+          ],
+          "metricKind": "CUMULATIVE",
+          "valueType": "INT64",
+          "unit": "1",
+          "description": "Count of RPCs by method and status.",
+          "displayName": "OpenCensus/grpc.io/client/completed_rpcs"
+        }
+      },
+      {
+        "name": "projects/myproject",
+        "metricDescriptor": {
+          "name": "projects/myproject/metricDescriptors/custom.googleapis.com/opencensus/grpc.io/client/received_bytes_per_rpc",
+          "type": "custom.googleapis.com/opencensus/grpc.io/client/received_bytes_per_rpc",
+          "labels": [
+            {
+              "key": "grpc_client_method"
+            }
+          ],
+          "metricKind": "CUMULATIVE",
+          "valueType": "DISTRIBUTION",
+          "unit": "By",
+          "description": "Distribution of bytes received per RPC, by method.",
+          "displayName": "OpenCensus/grpc.io/client/received_bytes_per_rpc"
+        }
+      },
+      {
+        "name": "projects/myproject",
+        "metricDescriptor": {
+          "name": "projects/myproject/metricDescriptors/custom.googleapis.com/opencensus/grpc.io/client/roundtrip_latency",
+          "type": "custom.googleapis.com/opencensus/grpc.io/client/roundtrip_latency",
+          "labels": [
+            {
+              "key": "grpc_client_method"
+            }
+          ],
+          "metricKind": "CUMULATIVE",
+          "valueType": "DISTRIBUTION",
+          "unit": "ms",
+          "description": "Distribution of round-trip latency, by method.",
+          "displayName": "OpenCensus/grpc.io/client/roundtrip_latency"
+        }
+      },
+      {
+        "name": "projects/myproject",
+        "metricDescriptor": {
+          "name": "projects/myproject/metricDescriptors/custom.googleapis.com/opencensus/grpc.io/client/sent_bytes_per_rpc",
+          "type": "custom.googleapis.com/opencensus/grpc.io/client/sent_bytes_per_rpc",
+          "labels": [
+            {
+              "key": "grpc_client_method"
+            }
+          ],
+          "metricKind": "CUMULATIVE",
+          "valueType": "DISTRIBUTION",
+          "unit": "By",
+          "description": "Distribution of bytes sent per RPC, by method.",
+          "displayName": "OpenCensus/grpc.io/client/sent_bytes_per_rpc"
+        }
+      }
+    ]
+  }
+}
diff --git a/exporter/collector/integrationtest/testdata/fixtures/logs_apache_access_expected.json b/exporter/collector/integrationtest/testdata/fixtures/logs_apache_access_expected.json
index 076ec2a..8cd93b9 100644
--- a/exporter/collector/integrationtest/testdata/fixtures/logs_apache_access_expected.json
+++ b/exporter/collector/integrationtest/testdata/fixtures/logs_apache_access_expected.json
@@ -20,6 +20,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -40,6 +43,9 @@
             "responseSize": "51164",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -60,6 +66,9 @@
             "responseSize": "3990",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -80,6 +89,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -100,6 +112,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -120,6 +135,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -140,6 +158,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -160,6 +181,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -180,6 +204,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -200,6 +227,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -220,6 +250,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -240,6 +273,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -260,6 +296,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -280,6 +319,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -300,6 +342,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -320,6 +365,9 @@
             "responseSize": "4429",
             "remoteIp": "127.0.0.2",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -332,7 +380,10 @@
             }
           },
           "textPayload": "127.0.0.1 - - [26/Apr/2022:22:54:43 +0800] \"-\" 408 -",
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         }
       ],
       "partialSuccess": true
diff --git a/exporter/collector/integrationtest/testdata/fixtures/logs_apache_error_expected.json b/exporter/collector/integrationtest/testdata/fixtures/logs_apache_error_expected.json
index 4440353..33fc5bb 100644
--- a/exporter/collector/integrationtest/testdata/fixtures/logs_apache_error_expected.json
+++ b/exporter/collector/integrationtest/testdata/fixtures/logs_apache_error_expected.json
@@ -16,7 +16,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 00:46:21.412645 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -33,7 +36,10 @@
             "time": "Tue Apr 26 00:46:21.457314 2022"
           },
           "timestamp": "1970-01-01T00:00:00Z",
-          "severity": "WARNING"
+          "severity": "WARNING",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -49,7 +55,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 00:46:21.502940 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -65,7 +74,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 00:46:21.503003 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -81,7 +93,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 00:46:38.988423 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -97,7 +112,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 22:48:34.466058 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -114,7 +132,10 @@
             "time": "Tue Apr 26 22:48:34.740290 2022"
           },
           "timestamp": "1970-01-01T00:00:00Z",
-          "severity": "WARNING"
+          "severity": "WARNING",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -130,7 +151,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 22:48:35.606223 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -146,7 +170,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 22:48:35.606298 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         }
       ],
       "partialSuccess": true
diff --git a/exporter/collector/integrationtest/testdata/fixtures/logs_multi_project_expected.json b/exporter/collector/integrationtest/testdata/fixtures/logs_multi_project_expected.json
index 4eedec4..8145d9b 100644
--- a/exporter/collector/integrationtest/testdata/fixtures/logs_multi_project_expected.json
+++ b/exporter/collector/integrationtest/testdata/fixtures/logs_multi_project_expected.json
@@ -20,6 +20,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -40,6 +43,9 @@
             "responseSize": "51164",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -60,6 +66,9 @@
             "responseSize": "3990",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -80,6 +89,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -100,6 +112,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -120,6 +135,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -140,6 +158,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -160,6 +181,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -180,6 +204,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -200,6 +227,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -220,6 +250,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -240,6 +273,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -260,6 +296,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -280,6 +319,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -300,6 +342,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -320,6 +365,9 @@
             "responseSize": "4429",
             "remoteIp": "127.0.0.2",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -332,7 +380,10 @@
             }
           },
           "textPayload": "127.0.0.1 - - [26/Apr/2022:22:54:43 +0800] \"-\" 408 -",
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fake-other-project/logs/multi-project",
@@ -352,6 +403,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -372,6 +426,9 @@
             "responseSize": "51164",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -392,6 +449,9 @@
             "responseSize": "3990",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -412,6 +472,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -432,6 +495,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -452,6 +518,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -472,6 +541,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -492,6 +564,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -512,6 +587,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -532,6 +610,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -552,6 +633,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -572,6 +656,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -592,6 +679,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -612,6 +702,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -632,6 +725,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -652,6 +748,9 @@
             "responseSize": "4429",
             "remoteIp": "127.0.0.2",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -664,7 +763,10 @@
             }
           },
           "textPayload": "127.0.0.1 - - [26/Apr/2022:22:54:43 +0800] \"-\" 408 -",
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         }
       ],
       "partialSuccess": true