Repository: google/exposure-notifications-server
Pull request #1478 (2021-03-22 16:43:57)

Refactor cleanup jobs for forward-progress alerting
This does not actually change the alerts yet, just refactors the existing code to make it easier to export the new metrics and remove many unused metrics to simplify the code flow.

**Release Note**

<!-- Enter your extended release note in the below block. If the PR requires
additional action from users switching to the new release, include the string
"action required". If no release note is required, write "NONE". -->

```release-note
Refactor background cleanup jobs for forward-progress alerting. This removes unused metrics about the number of exposures/files deleted and simplifies the logic handling.
```


Patch:
diff --git a/docs/playbooks/alerts/ForwardProgressFailed.md b/docs/playbooks/alerts/ForwardProgressFailed.md
new file mode 100644
index 0000000..217a51c
--- /dev/null
+++ b/docs/playbooks/alerts/ForwardProgressFailed.md
@@ -0,0 +1,29 @@
+# ForwardProgressFailed
+
+This alert fires when background jobs have not made forward progress in an acceptable amount of time. The alert will include the name of the job that is failing to make forward progress. The jobs are invoked in the background.
+
+- `cleanup-export` - Purges old export files.
+
+- `cleanup-exposure` - Purges old exposures.
+
+Each job runs on a different interval. Check your Terraform configuration to see how frequently a specific job runs.
+
+## Triage Steps
+
+When one of the jobs does not return success within a configured interval, this alert will fire. For most cases, this means the job has already failed 2+ times.
+
+To begin triage, locate the logs for the corresponding service name using the Logs Explorer:
+
+```text
+resource.type="cloud_run_revision"
+resource.labels.service_name="<service>"
+```
+
+For example, if the failing service was `cleanup-export`:
+
+```text
+resource.type="cloud_run_revision"
+resource.labels.service_name="cleanup-export"
+```
+
+Check for errors in the logs.
diff --git a/internal/cleanup/cleanup.go b/internal/cleanup/cleanup.go
index 5528286..b855742 100644
--- a/internal/cleanup/cleanup.go
+++ b/internal/cleanup/cleanup.go
@@ -17,24 +17,24 @@ package cleanup
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"time"
 
 	"go.opencensus.io/stats"
-	"go.opencensus.io/trace"
 
 	"github.com/google/exposure-notifications-server/internal/export/database"
 	publishdb "github.com/google/exposure-notifications-server/internal/publish/database"
+	"github.com/hashicorp/go-multierror"
 
 	"github.com/google/exposure-notifications-server/internal/serverenv"
 	"github.com/google/exposure-notifications-server/internal/storage"
 	"github.com/google/exposure-notifications-server/pkg/logging"
 )
 
-const (
-	minTTL = 10 * 24 * time.Hour
-)
+const minTTL = 10 * 24 * time.Hour
 
 // NewExposureHandler creates a http.Handler for deleting exposure keys
 // from the database.
@@ -57,54 +57,52 @@ type exposureCleanupHandler struct {
 }
 
 func (h *exposureCleanupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	ctx, span := trace.StartSpan(r.Context(), "(*cleanup.exposureCleanupHandler).ServeHTTP")
-	defer span.End()
-
-	logger := logging.FromContext(ctx)
+	ctx := r.Context()
+	logger := logging.FromContext(ctx).Named("cleanup.exposure")
 
-	cutoff, err := cutoffDate(ctx, h.config.TTL, h.config.DebugOverrideCleanupMinDuration)
+	cutoff, err := cutoffDate(h.config.TTL, h.config.DebugOverrideCleanupMinDuration)
 	if err != nil {
-		message := fmt.Sprintf("error processing cutoff time: %v", err)
-		logger.Error(message)
-		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
-		stats.Record(ctx, mExposuresSetupFailed.M(1))
-		http.Error(w, "internal processing error", http.StatusInternalServerError)
+		logger.Errorw("failed to calculate cutoff date", "error", err)
+		respondWithError(w, http.StatusInternalServerError, err)
 		return
 	}
-	logger.Infof("Starting cleanup for records older than %v", cutoff.UTC())
-	stats.Record(ctx, mExposuresCleanupBefore.M(cutoff.Unix()))
 
-	// Set timeout
-	timeoutCtx, cancel := context.WithTimeout(ctx, h.config.Timeout)
-	defer cancel()
+	// Construct a multi-error. If one of the purges fails, we still want to
+	// attempt the other purges.
+	var merr *multierror.Error
 
-	count, err := h.database.DeleteExposuresBefore(timeoutCtx, cutoff)
-	if err != nil {
-		message := fmt.Sprintf("Failed deleting exposures: %v", err)
-		logger.Error(message)
-		stats.Record(ctx, mExposuresDeleteFailed.M(1))
-		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
-		http.Error(w, "internal processing error", http.StatusInternalServerError)
-		return
-	}
+	// Exposures
+	func() {
+		ctx, cancel := context.WithTimeout(ctx, h.config.Timeout)
+		defer cancel()
 
-	stats.Record(ctx, mExposuresDeleted.M(count))
-	logger.Infof("cleanup run complete, deleted %v records.", count)
+		if count, err := h.database.DeleteExposuresBefore(ctx, cutoff); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("failed to delete exposures: %w", err))
+		} else {
+			logger.Infow("purged exposures", "count", count)
+		}
+	}()
 
-	// Clean up associated stats.
-	statsCount, err := h.database.DeleteStatsBefore(timeoutCtx, cutoff)
-	if err != nil {
-		message := fmt.Sprintf("Failed deleting publish stats: %v", err)
-		logger.Error(message)
-		stats.Record(ctx, mExposuresStatsDeleteFailed.M(1))
-		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
-		http.Error(w, "internal processing error", http.StatusInternalServerError)
+	// Stats
+	func() {
+		ctx, cancel := context.WithTimeout(ctx, h.config.Timeout)
+		defer cancel()
+
+		if count, err := h.database.DeleteStatsBefore(ctx, cutoff); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("failed to delete stats: %w", err))
+		} else {
+			logger.Infow("purged statistics", "count", count)
+		}
+	}()
+
+	if merr != nil {
+		logger.Errorw("failed to cleanup exposures", "errors", merr.WrappedErrors())
+		respondWithError(w, http.StatusInternalServerError, merr)
 		return
 	}
-	stats.Record(ctx, mExposuresStatsDeleted.M(statsCount), mExposuresCleanupRun.M(1))
-	logger.Infof("stats cleanup run complete, deleted %v records.", statsCount)
 
-	w.WriteHeader(http.StatusOK)
+	stats.Record(ctx, mExposureSuccess.M(1))
+	respond(w, http.StatusOK)
 }
 
 // NewExportHandler creates a http.Handler that manages deletion of
@@ -133,49 +131,76 @@ type exportCleanupHandler struct {
 }
 
 func (h *exportCleanupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	ctx, span := trace.StartSpan(r.Context(), "(*cleanup.exportCleanupHandler).ServeHTTP")
-	defer span.End()
+	ctx := r.Context()
+	logger := logging.FromContext(ctx).Named("cleanup.export")
 
-	logger := logging.FromContext(ctx)
-
-	cutoff, err := cutoffDate(ctx, h.config.TTL, h.config.DebugOverrideCleanupMinDuration)
+	cutoff, err := cutoffDate(h.config.TTL, h.config.DebugOverrideCleanupMinDuration)
 	if err != nil {
-		message := fmt.Sprintf("error calculating cutoff time: %v", err)
-		stats.Record(ctx, mExportsSetupFailed.M(1))
-		logger.Error(message)
-		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
-		http.Error(w, "internal processing error", http.StatusInternalServerError)
+		logger.Errorw("failed to calculate cutoff date", "error", err)
+		respondWithError(w, http.StatusInternalServerError, err)
 		return
 	}
-	logger.Infof("Starting cleanup for export files older than %v", cutoff.UTC())
-	stats.Record(ctx, mExportsCleanupBefore.M(cutoff.Unix()))
 
-	// Set h.Timeout
-	timeoutCtx, cancel := context.WithTimeout(ctx, h.config.Timeout)
-	defer cancel()
+	// Construct a multi-error. If one of the purges fails, we still want to
+	// attempt the other purges.
+	var merr *multierror.Error
 
-	count, err := h.database.DeleteFilesBefore(timeoutCtx, cutoff, h.blobstore)
-	if err != nil {
-		message := fmt.Sprintf("Failed deleting export files: %v", err)
-		logger.Error(message)
-		stats.Record(ctx, mExposuresDeleteFailed.M(1))
-		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
-		http.Error(w, "internal processing error", http.StatusInternalServerError)
+	// Files
+	func() {
+		ctx, cancel := context.WithTimeout(ctx, h.config.Timeout)
+		defer cancel()
+
+		if count, err := h.database.DeleteFilesBefore(ctx, cutoff, h.blobstore); err != nil {
+			merr = multierror.Append(merr, fmt.Errorf("failed to delete files: %w", err))
+		} else {
+			logger.Infow("purged files", "count", count)
+		}
+	}()
+
+	if merr != nil {
+		logger.Errorw("failed to cleanup exports", "errors", merr.WrappedErrors())
+		respondWithError(w, http.StatusInternalServerError, merr)
 		return
 	}
 
-	stats.Record(ctx, mExportsDeleted.M(int64(count)), mExportsCleanupRun.M(1))
-	logger.Infof("cleanup run complete, deleted %v files.", count)
-	w.WriteHeader(http.StatusOK)
+	stats.Record(ctx, mExportSuccess.M(1))
+	respond(w, http.StatusOK)
+}
+
+func cutoffDate(d time.Duration, override bool) (time.Time, error) {
+	if d >= minTTL || override {
+		return time.Now().UTC().Add(-d), nil
+	}
+
+	return time.Time{}, fmt.Errorf("cleanup ttl %s is less than configured minimum ttl of %s", d, minTTL)
 }
 
-func cutoffDate(ctx context.Context, d time.Duration, overrideMinTTL bool) (time.Time, error) {
-	if d < minTTL {
-		if overrideMinTTL {
-			logging.FromContext(ctx).Warnf("Cleanup safety minimuim TTL is being overridden by the DEBUG_OVERRIDE_CLEANUP_MIN_DURATION=true environment variable")
+type result struct {
+	OK     bool     `json:"ok"`
+	Errors []string `json:"errors,omitempty"`
+}
+
+func respond(w http.ResponseWriter, code int) {
+	respondWithError(w, code, nil)
+}
+
+func respondWithError(w http.ResponseWriter, code int, err error) {
+	var r result
+	r.OK = err == nil
+
+	if err != nil {
+		var merr *multierror.Error
+		if errors.As(err, &merr) {
+			for _, err := range merr.WrappedErrors() {
+				if err != nil {
+					r.Errors = append(r.Errors, err.Error())
+				}
+			}
 		} else {
-			return time.Time{}, fmt.Errorf("cleanup ttl is less than configured minimum ttl")
+			r.Errors = append(r.Errors, err.Error())
 		}
 	}
-	return time.Now().Add(-d), nil
+
+	w.WriteHeader(code)
+	_ = json.NewEncoder(w).Encode(r)
 }
diff --git a/internal/cleanup/metrics.go b/internal/cleanup/metrics.go
index a0d0b79..17066c6 100644
--- a/internal/cleanup/metrics.go
+++ b/internal/cleanup/metrics.go
@@ -22,97 +22,26 @@ import (
 	"go.opencensus.io/stats/view"
 )
 
-var (
-	cleanupMetricsPrefix = metrics.MetricRoot + "cleanup"
-
-	mExposuresCleanupRun = stats.Int64(cleanupMetricsPrefix+"exposures_run",
-		"Number of times cleanup exposures has run", stats.UnitDimensionless)
-	mExportsCleanupRun = stats.Int64(cleanupMetricsPrefix+"exports_run",
-		"Number of times cleanup exports has run", stats.UnitDimensionless)
+const metricPrefix = metrics.MetricRoot + "cleanup"
 
-	mExposuresSetupFailed = stats.Int64(cleanupMetricsPrefix+"exposures_setup_failed",
-		"Instances of exposures setup failures", stats.UnitDimensionless)
-	mExposuresCleanupBefore = stats.Int64(cleanupMetricsPrefix+"exposures_cleanup_before",
-		"Exposures cleanup cutoff date", stats.UnitSeconds)
-	mExposuresDeleteFailed = stats.Int64(cleanupMetricsPrefix+"exposures_delete_failed",
-		"Instances of exposures delete failures", stats.UnitDimensionless)
-	mExposuresDeleted = stats.Int64(cleanupMetricsPrefix+"exposures_deleted",
-		"Exposures deletions", stats.UnitDimensionless)
-	mExposuresStatsDeleteFailed = stats.Int64(cleanupMetricsPrefix+"exposures_stats_delete_failed",
-		"Instances of exposures stats delete failures", stats.UnitDimensionless)
-	mExposuresStatsDeleted = stats.Int64(cleanupMetricsPrefix+"exposures_stats_deleted",
-		"Exposures stats deletions", stats.UnitDimensionless)
-	mExportsSetupFailed = stats.Int64(cleanupMetricsPrefix+"exports_setup_failed",
-		"Instances of export setup failures", stats.UnitDimensionless)
-	mExportsCleanupBefore = stats.Int64(cleanupMetricsPrefix+"exports_cleanup_before",
-		"Exports cleanup cutoff date", stats.UnitSeconds)
-	mExportsDeleteFailed = stats.Int64(cleanupMetricsPrefix+"exports_delete_failed",
-		"Instances of exports delete failures", stats.UnitDimensionless)
-	mExportsDeleted = stats.Int64(cleanupMetricsPrefix+"exports_deleted",
-		"Exports deletions", stats.UnitDimensionless)
+var (
+	mExportSuccess   = stats.Int64(metricPrefix+"/export_success", "successful execution", stats.UnitDimensionless)
+	mExposureSuccess = stats.Int64(metricPrefix+"/exposure_success", "successful execution", stats.UnitDimensionless)
 )
 
 func init() {
 	observability.CollectViews([]*view.View{
 		{
-			Name:        metrics.MetricRoot + "exposures_setup_failed_count",
-			Description: "Total count of exposures setup failures",
-			Measure:     mExposuresSetupFailed,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "exposure_cleanup_before_latest",
-			Description: "Last value of exposures cleanup cutoff date",
-			Measure:     mExposuresCleanupBefore,
-			Aggregation: view.LastValue(),
-		},
-		{
-			Name:        metrics.MetricRoot + "exposure_cleanup_delete_failed_count",
-			Description: "Total count of exposures delete failed",
-			Measure:     mExposuresDeleteFailed,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "exposures_deleted_count",
-			Description: "Total count of exposures deletions",
-			Measure:     mExposuresDeleted,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "exposure_stats_cleanup_delete_failed_count",
-			Description: "Total count of exposures stats delete failed",
-			Measure:     mExposuresStatsDeleteFailed,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "exposures_stats_deleted_count",
-			Description: "Total count of stats exposures deletions",
-			Measure:     mExposuresStatsDeleted,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "exports_setup_failed_count",
-			Description: "Total count of exports setup failures",
-			Measure:     mExportsSetupFailed,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "exports_cleanup_before_latest",
-			Description: "Last value of exports cleanup cutoff date",
-			Measure:     mExportsCleanupBefore,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "exports_delete_failed_count",
-			Description: "Total count of export deletion failures",
-			Measure:     mExportsDeleteFailed,
-			Aggregation: view.Sum(),
+			Name:        metricPrefix + "/export/success",
+			Description: "Number of export cleanup successes",
+			Measure:     mExportSuccess,
+			Aggregation: view.Count(),
 		},
 		{
-			Name:        metrics.MetricRoot + "exports_deleted_count",
-			Description: "Total count of exports deletions",
-			Measure:     mExportsDeleted,
-			Aggregation: view.Sum(),
+			Name:        metricPrefix + "/exposure/success",
+			Description: "Number of exposure cleanup successes",
+			Measure:     mExposureSuccess,
+			Aggregation: view.Count(),
 		},
 	}...)
 }
diff --git a/terraform/alerting/alerts.tf b/terraform/alerting/alerts.tf
index 0993188..03762ad 100644
--- a/terraform/alerting/alerts.tf
+++ b/terraform/alerting/alerts.tf
@@ -15,6 +15,7 @@
 
 locals {
   playbook_prefix = "https://github.com/google/exposure-notifications-server/blob/main/docs/playbooks/alerts"
+  custom_prefix   = "custom.googleapis.com/opencensus/en-server"
 }
 
 resource "google_monitoring_alert_policy" "CloudSchedulerJobFailed" {
@@ -63,6 +64,44 @@ resource "google_monitoring_alert_policy" "CloudSchedulerJobFailed" {
   ]
 }
 
+resource "google_monitoring_alert_policy" "ForwardProgressFailed" {
+  for_each = var.forward_progress_indicators
+
+  project      = var.project
+  display_name = "ForwardProgressFailed-${each.key}"
+  combiner     = "OR"
+
+  conditions {
+    display_name = each.key
+
+    condition_monitoring_query_language {
+      duration = "0s"
+      query    = <<-EOT
+      fetch generic_task
+      | metric '${local.custom_prefix}/${each.value.metric}'
+      | align delta_gauge(${each.value.window})
+      | group_by [], [val: aggregate(value.success)]
+      | absent_for ${each.value.window}
+      EOT
+
+      trigger {
+        count = 1
+      }
+    }
+  }
+
+  documentation {
+    content   = "${local.playbook_prefix}/ForwardProgressFailed.md"
+    mime_type = "text/markdown"
+  }
+
+  notification_channels = [for x in values(google_monitoring_notification_channel.paging) : x.id]
+
+  depends_on = [
+    null_resource.manual-step-to-enable-workspace,
+  ]
+}
+
 resource "google_monitoring_alert_policy" "probers" {
   project = var.project
 
diff --git a/terraform/alerting/variables.tf b/terraform/alerting/variables.tf
index 7007882..89e9fea 100644
--- a/terraform/alerting/variables.tf
+++ b/terraform/alerting/variables.tf
@@ -80,6 +80,21 @@ variable "capture_export_file_downloads" {
   description = "Capture metrics about mobile devices downloading export files. This can be used to create alerts when values drop below acceptable thresholds."
 }
 
+variable "forward_progress_indicators" {
+  type = map(object({
+    metric = string
+    window = string
+  }))
+
+  default = {
+    // cleanup-export runs every 4h, alert after 2 failures
+    "cleanup-export" = { metric = "cleanup/export/success", window = "485m" },
+
+    // cleanup-exposure runs every 4h, alert after 2 failures
+    "cleanup-exposure" = { metric = "cleanup/exposure/success", window = "485m" },
+  }
+}
+
 terraform {
   required_version = ">= 0.14.2"
 
diff --git a/terraform/vars.tf b/terraform/vars.tf
index 1b05c98..63b9525 100644
--- a/terraform/vars.tf
+++ b/terraform/vars.tf
@@ -195,7 +195,7 @@ variable "cleanup_exposure_worker_cron_schedule" {
 
 variable "cleanup_export_worker_cron_schedule" {
   type    = string
-  default = "0 */6 * * *"
+  default = "0 */4 * * *"
 
   description = "Schedule to execute the cleanup export worker service."
 }


Test patch:
diff --git a/docs/playbooks/playbook_test.go b/docs/playbooks/playbook_test.go
deleted file mode 100644
index d533bf7..0000000
--- a/docs/playbooks/playbook_test.go
+++ /dev/null
@@ -1,127 +0,0 @@
-// Copyright 2020 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"path/filepath"
-	"sort"
-	"testing"
-
-	"github.com/hashicorp/hcl/v2"
-	"github.com/hashicorp/hcl/v2/gohcl"
-	"github.com/hashicorp/hcl/v2/hclparse"
-)
-
-const (
-	alertTF = "../../terraform/alerting/alerts.tf"
-
-	// Ignore files not starting with an upper case letter.
-	playbooksGlob = "./alerts/[A-Z]*.md"
-)
-
-var alertPolicySchema = hcl.BodySchema{
-	Attributes: []hcl.AttributeSchema{
-		{Name: "display_name", Required: true},
-	},
-}
-
-type resource struct {
-	Type      string   `hcl:"type,label"`
-	Name      string   `hcl:"name,label"`
-	Remaining hcl.Body `hcl:",remain"`
-}
-
-type config struct {
-	Resources []*resource `hcl:"resource,block"`
-	Remaining hcl.Body    `hcl:",remain"`
-}
-
-func allPlaybookNames(t *testing.T) []string {
-	files, err := filepath.Glob(playbooksGlob)
-	if err != nil {
-		t.Fatalf("failed to find all playbooks: %s", err)
-	}
-	ret := make([]string, 0, len(files))
-	for _, x := range files {
-		basename := filepath.Base(x)
-		ret = append(ret, basename[:len(basename)-len(".md")])
-	}
-	return ret
-}
-
-func allAlertNames(t *testing.T) []string {
-	hcl, diag := hclparse.NewParser().ParseHCLFile(alertTF)
-	if diag.HasErrors() {
-		t.Fatalf("failed to parse alerts.tf: %v", diag)
-	}
-	var c config
-	diag = gohcl.DecodeBody(hcl.Body, nil, &c)
-	if diag.HasErrors() {
-		t.Fatalf("failed to decode body: %v", diag)
-	}
-	ret := make([]string, 0, len(c.Resources))
-	for _, res := range c.Resources {
-		if res.Type != "google_monitoring_alert_policy" {
-			continue
-		}
-		content, _, diag := res.Remaining.PartialContent(&alertPolicySchema)
-		if diag.HasErrors() {
-			t.Fatalf("failed to get attributes from block body: %v", diag)
-		}
-		v, diag := content.Attributes["display_name"].Expr.Value(nil)
-		if diag.HasErrors() {
-			t.Fatalf("failed to evaluate value of display_name in an alert policy block: %v", diag)
-		}
-		ret = append(ret, v.AsString())
-	}
-	return ret
-}
-
-func setDiff(xs, ys []string) []string {
-	s1 := make(map[string]bool)
-	s2 := make(map[string]bool)
-	for _, s := range xs {
-		s1[s] = true
-	}
-	for _, s := range ys {
-		s2[s] = true
-	}
-	for k := range s1 {
-		if s2[k] {
-			delete(s1, k)
-		}
-	}
-	ret := make([]string, 0, len(s1))
-	for k := range s1 {
-		ret = append(ret, k)
-	}
-	sort.Strings(ret)
-	return ret
-}
-
-// Test to ensure every playbook has a corresponding alert, and every alert has
-// a corresponding laybook.
-func TestPlaybooks(t *testing.T) {
-	t.Parallel()
-
-	playbooks := allPlaybookNames(t)
-	alerts := allAlertNames(t)
-	for _, x := range setDiff(playbooks, alerts) {
-		t.Errorf("Missing alert for playbook %q.", x)
-	}
-	for _, x := range setDiff(alerts, playbooks) {
-		t.Errorf("Missing playbook for alert %q.", x)
-	}
-}
diff --git a/internal/cleanup/cleanup_test.go b/internal/cleanup/cleanup_test.go
index abc4e34..d7123dd 100644
--- a/internal/cleanup/cleanup_test.go
+++ b/internal/cleanup/cleanup_test.go
@@ -16,6 +16,8 @@ package cleanup
 
 import (
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"testing"
 	"time"
 
@@ -81,6 +83,33 @@ func TestNewExposureHandler(t *testing.T) {
 	}
 }
 
+func TestExposureHandler_ServeHTTP(t *testing.T) {
+	t.Parallel()
+
+	ctx := project.TestContext(t)
+	testDB, _ := testDatabaseInstance.NewDatabase(t)
+	env := serverenv.New(ctx, serverenv.WithDatabase(testDB))
+
+	r, err := http.NewRequestWithContext(ctx, http.MethodGet, "/", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	w := httptest.NewRecorder()
+
+	handler, err := NewExposureHandler(&Config{
+		Timeout: 5 * time.Second,
+		TTL:     336 * time.Hour,
+	}, env)
+	if err != nil {
+		t.Fatal(err)
+	}
+	handler.ServeHTTP(w, r)
+
+	if got, want := w.Code, http.StatusOK; got != want {
+		t.Errorf("expected %d to be %d: %s", got, want, w.Body.String())
+	}
+}
+
 func TestNewExportHandler(t *testing.T) {
 	t.Parallel()
 
@@ -135,6 +164,43 @@ func TestNewExportHandler(t *testing.T) {
 	}
 }
 
+func TestExportHandler_ServeHTTP(t *testing.T) {
+	t.Parallel()
+
+	ctx := project.TestContext(t)
+	testDB, _ := testDatabaseInstance.NewDatabase(t)
+
+	bs, err := storage.NewMemory(ctx, &storage.Config{
+		Type: "MEMORY",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	env := serverenv.New(ctx,
+		serverenv.WithDatabase(testDB),
+		serverenv.WithBlobStorage(bs))
+
+	r, err := http.NewRequestWithContext(ctx, http.MethodGet, "/", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	w := httptest.NewRecorder()
+
+	handler, err := NewExportHandler(&Config{
+		Timeout: 5 * time.Second,
+		TTL:     336 * time.Hour,
+	}, env)
+	if err != nil {
+		t.Fatal(err)
+	}
+	handler.ServeHTTP(w, r)
+
+	if got, want := w.Code, http.StatusOK; got != want {
+		t.Errorf("expected %d to be %d: %s", got, want, w.Body.String())
+	}
+}
+
 func TestCutoffDate(t *testing.T) {
 	t.Parallel()
 
@@ -158,7 +224,7 @@ func TestCutoffDate(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 
-			got, err := cutoffDate(project.TestContext(t), tc.d, tc.override)
+			got, err := cutoffDate(tc.d, tc.override)
 			if tc.wantDur == 0 {
 				if err == nil {
 					t.Errorf("%q: got no error, wanted one", tc.d)