Repository: fluxcd/notification-controller
Pull request #224 (2021-07-26 15:45:05)

providers/sentry: send traces
This is the last sentry PR, I promise 😄 

This PR adds support to send Sentry Traces instead of just errors. This means that normal info events are sent as traces and errors are sent as errors. 

This functionality can be disabled by setting `eventSeverity` in the related `Alert` object to `error`


Patch:
diff --git a/docs/spec/v1beta1/provider.md b/docs/spec/v1beta1/provider.md
index 62505c5..30b000d 100644
--- a/docs/spec/v1beta1/provider.md
+++ b/docs/spec/v1beta1/provider.md
@@ -241,6 +241,8 @@ spec:
   address: https://....@sentry.io/12341234
 ```
 
+The sentry provider also sends traces for events with the severity Info. This can be disabled by setting the `eventSeverity` field on the related `Alert` Rule to `error`.
+
 ### Azure Event Hub
 
 The Azure Event Hub supports two authentication methods, [JWT](https://docs.microsoft.com/en-us/azure/event-hubs/authenticate-application) and [SAS](https://docs.microsoft.com/en-us/azure/event-hubs/authorize-access-shared-access-signature) based.
diff --git a/internal/notifier/sentry.go b/internal/notifier/sentry.go
index b44dcb2..3b157bd 100644
--- a/internal/notifier/sentry.go
+++ b/internal/notifier/sentry.go
@@ -17,6 +17,7 @@ limitations under the License.
 package notifier
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
@@ -42,9 +43,10 @@ func NewSentry(certPool *x509.CertPool, dsn string, environment string) (*Sentry
 		}
 	}
 	client, err := sentry.NewClient(sentry.ClientOptions{
-		Dsn:           dsn,
-		Environment:   environment,
-		HTTPTransport: tr,
+		Dsn:              dsn,
+		Environment:      environment,
+		HTTPTransport:    tr,
+		TracesSampleRate: 1,
 	})
 	if err != nil {
 		return nil, err
@@ -57,16 +59,77 @@ func NewSentry(certPool *x509.CertPool, dsn string, environment string) (*Sentry
 
 // Post event to Sentry
 func (s *Sentry) Post(event events.Event) error {
-	// Skip any update events
-	if isCommitStatus(event.Metadata, "update") {
-		return nil
+	var sev *sentry.Event
+	// Send event to Sentry
+	switch event.Severity {
+	case events.EventSeverityInfo:
+		// Info is sent as a trace
+		sev = eventToSpan(event)
+		break
+	case events.EventSeverityError:
+		// Errors are sent as normal events
+		sev = toSentryEvent(event)
+		break
 	}
+	s.Client.CaptureEvent(sev, nil, nil)
 
-	// Send event to Sentry
-	s.Client.CaptureEvent(toSentryEvent(event), nil, nil)
 	return nil
 }
 
+// Convert a controller event to a Sentry trace
+// Sentry traces work slightly different compared to normal events, they don't cause
+// alerts by default and are saved differently.
+// They are shown in a dashobard with graphs, so they can be used to check if and how often
+// flux tasks are running
+func eventToSpan(event events.Event) *sentry.Event {
+	obj := event.InvolvedObject
+
+	// Sadly you can't create spans on specific clients, they are always auto-generated
+	// from the context, and the client saved within
+	span := sentry.StartSpan(context.Background(), "event")
+	// TODO: Maybe change the tag names?
+	span.SetTag("flux_involved_object_kind", obj.Kind)
+	span.SetTag("flux_involved_object_namespace", obj.Namespace)
+	span.SetTag("flux_involved_object_name", obj.Name)
+	span.SetTag("flux_reporting_controller", event.ReportingController)
+	span.SetTag("flux_reporting_instance", event.ReportingInstance)
+	span.SetTag("flux_reason", event.Reason)
+	span.StartTime = event.Timestamp.Time
+	span.EndTime = event.Timestamp.Time
+
+	for k, v := range event.Metadata {
+		span.SetTag(k, v)
+	}
+
+	// So because the sentry-go sdk has no way to send transactions
+	// with an explicit client, we have to do it ourselves
+	return &sentry.Event{
+		Type:        "transaction",
+		Transaction: eventSummary(event),
+		Message:     event.Message,
+		Contexts: map[string]interface{}{
+			"trace": &sentry.TraceContext{
+				TraceID:      span.TraceID,
+				SpanID:       span.SpanID,
+				ParentSpanID: span.ParentSpanID,
+				Op:           span.Op,
+				Description:  span.Description,
+				Status:       span.Status,
+			},
+		},
+		Tags:      span.Tags,
+		Extra:     span.Data,
+		Timestamp: span.EndTime,
+		StartTime: span.StartTime,
+		Spans:     []*sentry.Span{span},
+	}
+}
+
+func eventSummary(event events.Event) string {
+	obj := event.InvolvedObject
+	return fmt.Sprintf("%s: %s/%s", obj.Kind, obj.Namespace, obj.Name)
+}
+
 // Maps a controller-issued event to a Sentry event
 func toSentryEvent(event events.Event) *sentry.Event {
 	// Prepare Metadata
@@ -76,12 +139,11 @@ func toSentryEvent(event events.Event) *sentry.Event {
 	}
 
 	// Construct event
-	obj := event.InvolvedObject
 	return &sentry.Event{
 		Timestamp:   event.Timestamp.Time,
 		Level:       sentry.Level(event.Severity),
 		ServerName:  event.ReportingController,
-		Transaction: fmt.Sprintf("%s: %s/%s", obj.Kind, obj.Namespace, obj.Name),
+		Transaction: eventSummary(event),
 		Extra:       extra,
 		Message:     event.Message,
 	}


Test patch:
diff --git a/internal/notifier/sentry_test.go b/internal/notifier/sentry_test.go
index 8496c36..4e19551 100644
--- a/internal/notifier/sentry_test.go
+++ b/internal/notifier/sentry_test.go
@@ -68,3 +68,40 @@ func TestToSentryEvent(t *testing.T) {
 	}, s.Extra)
 	assert.Equal(t, "message", s.Message)
 }
+
+func TestToSentrySpan(t *testing.T) {
+	// Construct test event
+	e := events.Event{
+		InvolvedObject: corev1.ObjectReference{
+			Kind:      "GitRepository",
+			Namespace: "flux-system",
+			Name:      "test-app",
+		},
+		Severity:  "info",
+		Timestamp: metav1.Date(2020, 01, 01, 0, 0, 0, 0, time.UTC),
+		Message:   "message",
+		Metadata: map[string]string{
+			"key1": "val1",
+			"key2": "val2",
+		},
+		ReportingController: "source-controller",
+	}
+
+	// Map to Sentry event
+	s := eventToSpan(e)
+
+	// Assertions
+	assert.Equal(t, time.Date(2020, 01, 01, 0, 0, 0, 0, time.UTC), s.Timestamp)
+	assert.Equal(t, "transaction", s.Type)
+	assert.Equal(t, map[string]string{
+		"flux_involved_object_kind":      e.InvolvedObject.Kind,
+		"flux_involved_object_name":      e.InvolvedObject.Name,
+		"flux_involved_object_namespace": e.InvolvedObject.Namespace,
+		"flux_reason":                    e.Reason,
+		"flux_reporting_controller":      e.ReportingController,
+		"flux_reporting_instance":        e.ReportingInstance,
+		"key1":                           "val1",
+		"key2":                           "val2",
+	}, s.Tags)
+	assert.Equal(t, "message", s.Message)
+}