Repository: GoogleCloudPlatform/opentelemetry-operations-go
Pull request #424 (2022-06-13 17:32:32)

Parse non-special log attributes to GCP labels
In the log exporter design, we intended to parse "additional attributes" (ie, non `gcp.*` special values) to their own Labels

Fixes https://github.com/GoogleCloudPlatform/opentelemetry-operations-go/issues/423


Patch:
diff --git a/exporter/collector/logs.go b/exporter/collector/logs.go
index 8eb38f1..6b0e28a 100644
--- a/exporter/collector/logs.go
+++ b/exporter/collector/logs.go
@@ -21,6 +21,7 @@ import (
 	"math"
 	"net/http"
 	"net/url"
+	"strings"
 	"time"
 
 	"cloud.google.com/go/logging"
@@ -310,15 +311,23 @@ func (l logMapper) logToSplitEntries(
 		entry.Timestamp = processTime
 	}
 
+	// build our own map off OTel attributes so we don't have to call .Get() for each special case
+	// (.Get() ranges over all attributes each time)
+	attrsMap := make(map[string]pcommon.Value)
+	log.Attributes().Range(func(k string, v pcommon.Value) bool {
+		attrsMap[k] = v
+		return true
+	})
+
 	// parse LogEntrySourceLocation struct from OTel attribute
-	sourceLocation, ok := log.Attributes().Get(SourceLocationAttributeKey)
-	if ok {
+	if sourceLocation, ok := attrsMap[SourceLocationAttributeKey]; ok {
 		var logEntrySourceLocation logpb.LogEntrySourceLocation
 		err := json.Unmarshal(sourceLocation.MBytesVal(), &logEntrySourceLocation)
 		if err != nil {
 			return []logging.Entry{entry}, err
 		}
 		entry.SourceLocation = &logEntrySourceLocation
+		delete(attrsMap, SourceLocationAttributeKey)
 	}
 
 	// parse TraceID and SpanID, if present
@@ -329,14 +338,14 @@ func (l logMapper) logToSplitEntries(
 		entry.SpanID = log.SpanID().HexString()
 	}
 
-	httpRequestAttr, ok := log.Attributes().Get(HTTPRequestAttributeKey)
-	if ok {
+	if httpRequestAttr, ok := attrsMap[HTTPRequestAttributeKey]; ok {
 		var httpRequest *logging.HTTPRequest
 		httpRequest, err := l.parseHTTPRequest(httpRequestAttr)
 		if err != nil {
 			l.obs.log.Debug("Unable to parse httpRequest", zap.Error(err))
 		}
 		entry.HTTPRequest = httpRequest
+		delete(attrsMap, HTTPRequestAttributeKey)
 	}
 
 	if log.SeverityNumber() < 0 || int(log.SeverityNumber()) > len(severityMapping)-1 {
@@ -344,6 +353,20 @@ func (l logMapper) logToSplitEntries(
 	}
 	entry.Severity = severityMapping[log.SeverityNumber()]
 
+	// parse remaining OTel attributes to GCP labels
+	for k, v := range attrsMap {
+		// skip "gcp.*" attributes since we process these to other fields
+		if strings.HasPrefix(k, "gcp.") {
+			continue
+		}
+		if entry.Labels == nil {
+			entry.Labels = make(map[string]string)
+		}
+		if _, ok := entry.Labels[k]; !ok {
+			entry.Labels[k] = v.AsString()
+		}
+	}
+
 	// Calculate the size of the internal log entry so this overhead can be accounted
 	// for when determining the need to split based on payload size
 	// TODO(damemi): Find an appropriate estimated buffer to account for the LogSplit struct as well


Test patch:
diff --git a/exporter/collector/integrationtest/testdata/fixtures/logs_apache_access_expected.json b/exporter/collector/integrationtest/testdata/fixtures/logs_apache_access_expected.json
index 076ec2a..8cd93b9 100644
--- a/exporter/collector/integrationtest/testdata/fixtures/logs_apache_access_expected.json
+++ b/exporter/collector/integrationtest/testdata/fixtures/logs_apache_access_expected.json
@@ -20,6 +20,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -40,6 +43,9 @@
             "responseSize": "51164",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -60,6 +66,9 @@
             "responseSize": "3990",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -80,6 +89,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -100,6 +112,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -120,6 +135,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -140,6 +158,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -160,6 +181,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -180,6 +204,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -200,6 +227,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -220,6 +250,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -240,6 +273,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -260,6 +296,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -280,6 +319,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -300,6 +342,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -320,6 +365,9 @@
             "responseSize": "4429",
             "remoteIp": "127.0.0.2",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -332,7 +380,10 @@
             }
           },
           "textPayload": "127.0.0.1 - - [26/Apr/2022:22:54:43 +0800] \"-\" 408 -",
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         }
       ],
       "partialSuccess": true
diff --git a/exporter/collector/integrationtest/testdata/fixtures/logs_apache_error_expected.json b/exporter/collector/integrationtest/testdata/fixtures/logs_apache_error_expected.json
index 4440353..33fc5bb 100644
--- a/exporter/collector/integrationtest/testdata/fixtures/logs_apache_error_expected.json
+++ b/exporter/collector/integrationtest/testdata/fixtures/logs_apache_error_expected.json
@@ -16,7 +16,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 00:46:21.412645 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -33,7 +36,10 @@
             "time": "Tue Apr 26 00:46:21.457314 2022"
           },
           "timestamp": "1970-01-01T00:00:00Z",
-          "severity": "WARNING"
+          "severity": "WARNING",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -49,7 +55,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 00:46:21.502940 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -65,7 +74,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 00:46:21.503003 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -81,7 +93,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 00:46:38.988423 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -97,7 +112,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 22:48:34.466058 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -114,7 +132,10 @@
             "time": "Tue Apr 26 22:48:34.740290 2022"
           },
           "timestamp": "1970-01-01T00:00:00Z",
-          "severity": "WARNING"
+          "severity": "WARNING",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -130,7 +151,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 22:48:35.606223 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fakeprojectid/logs/apache-error-fixture",
@@ -146,7 +170,10 @@
             "severity": "notice",
             "time": "Tue Apr 26 22:48:35.606298 2022"
           },
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         }
       ],
       "partialSuccess": true
diff --git a/exporter/collector/integrationtest/testdata/fixtures/logs_multi_project_expected.json b/exporter/collector/integrationtest/testdata/fixtures/logs_multi_project_expected.json
index 4eedec4..8145d9b 100644
--- a/exporter/collector/integrationtest/testdata/fixtures/logs_multi_project_expected.json
+++ b/exporter/collector/integrationtest/testdata/fixtures/logs_multi_project_expected.json
@@ -20,6 +20,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -40,6 +43,9 @@
             "responseSize": "51164",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -60,6 +66,9 @@
             "responseSize": "3990",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -80,6 +89,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -100,6 +112,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -120,6 +135,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -140,6 +158,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -160,6 +181,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -180,6 +204,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -200,6 +227,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -220,6 +250,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -240,6 +273,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -260,6 +296,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -280,6 +319,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -300,6 +342,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -320,6 +365,9 @@
             "responseSize": "4429",
             "remoteIp": "127.0.0.2",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -332,7 +380,10 @@
             }
           },
           "textPayload": "127.0.0.1 - - [26/Apr/2022:22:54:43 +0800] \"-\" 408 -",
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         },
         {
           "logName": "projects/fake-other-project/logs/multi-project",
@@ -352,6 +403,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -372,6 +426,9 @@
             "responseSize": "51164",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -392,6 +449,9 @@
             "responseSize": "3990",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -412,6 +472,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -432,6 +495,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -452,6 +518,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -472,6 +541,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -492,6 +564,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -512,6 +587,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -532,6 +610,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -552,6 +633,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -572,6 +656,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -592,6 +679,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -612,6 +702,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -632,6 +725,9 @@
             "responseSize": "1247",
             "remoteIp": "127.0.0.1",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -652,6 +748,9 @@
             "responseSize": "4429",
             "remoteIp": "127.0.0.2",
             "protocol": "HTTP/1.1"
+          },
+          "labels": {
+            "log.file.name": "test.log"
           }
         },
         {
@@ -664,7 +763,10 @@
             }
           },
           "textPayload": "127.0.0.1 - - [26/Apr/2022:22:54:43 +0800] \"-\" 408 -",
-          "timestamp": "1970-01-01T00:00:00Z"
+          "timestamp": "1970-01-01T00:00:00Z",
+          "labels": {
+            "log.file.name": "test.log"
+          }
         }
       ],
       "partialSuccess": true