Repository: google/exposure-notifications-server
Pull request #1235 (2020-12-15 20:16:36)

Add a knob to enable Load Balancer logging.

google-oss-robot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/google/exposure-notifications-server/pull/1235#" title="Author self-approved">yegle</a>*
To complete the [pull request process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process), please assign **sethvargo** after the PR has been reviewed.
You can assign the PR to them by writing `/assign @sethvargo` in a comment when ready.

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=google%2Fexposure-notifications-server).

<details open>
Needs approval from an approver in each of these files:

- **[terraform/OWNERS](https://github.com/google/exposure-notifications-server/blob/main/terraform/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["sethvargo"]} -->

Patch:
diff --git a/internal/cleanup/cleanup.go b/internal/cleanup/cleanup.go
index ad60455..5bf76ee 100644
--- a/internal/cleanup/cleanup.go
+++ b/internal/cleanup/cleanup.go
@@ -21,12 +21,13 @@ import (
 	"net/http"
 	"time"
 
+	"go.opencensus.io/stats"
 	"go.opencensus.io/trace"
 
 	"github.com/google/exposure-notifications-server/internal/export/database"
 	publishdb "github.com/google/exposure-notifications-server/internal/publish/database"
 
-	"github.com/google/exposure-notifications-server/internal/metrics/metricsware"
+	"github.com/google/exposure-notifications-server/internal/metrics/cleanup"
 	"github.com/google/exposure-notifications-server/internal/serverenv"
 	"github.com/google/exposure-notifications-server/internal/storage"
 	"github.com/google/exposure-notifications-server/pkg/logging"
@@ -61,20 +62,18 @@ func (h *exposureCleanupHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 	defer span.End()
 
 	logger := logging.FromContext(ctx)
-	metrics := h.env.MetricsExporter(ctx)
-	metricsMiddleware := metricsware.NewMiddleWare(&metrics)
 
 	cutoff, err := cutoffDate(ctx, h.config.TTL, h.config.DebugOverrideCleanupMinDuration)
 	if err != nil {
 		message := fmt.Sprintf("error processing cutoff time: %v", err)
 		logger.Error(message)
 		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
-		metricsMiddleware.RecordExposuresSetupFailed(ctx)
+		stats.Record(ctx, cleanup.ExposuresSetupFailed.M(1))
 		http.Error(w, "internal processing error", http.StatusInternalServerError)
 		return
 	}
 	logger.Infof("Starting cleanup for records older than %v", cutoff.UTC())
-	metricsMiddleware.RecordExposuresCleanupCutoff(ctx, cutoff.Unix())
+	stats.Record(ctx, cleanup.ExposuresCleanupBefore.M(cutoff.Unix()))
 
 	// Set timeout
 	timeoutCtx, cancel := context.WithTimeout(ctx, h.config.Timeout)
@@ -84,13 +83,13 @@ func (h *exposureCleanupHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 	if err != nil {
 		message := fmt.Sprintf("Failed deleting exposures: %v", err)
 		logger.Error(message)
-		metricsMiddleware.RecordExposuresDeleteFailed(ctx)
+		stats.Record(ctx, cleanup.ExposuresDeleteFailed.M(1))
 		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
 		http.Error(w, "internal processing error", http.StatusInternalServerError)
 		return
 	}
 
-	metricsMiddleware.RecordExposuresDeleted(ctx, count)
+	stats.Record(ctx, cleanup.ExposuresDeleted.M(count))
 	logger.Infof("cleanup run complete, deleted %v records.", count)
 	w.WriteHeader(http.StatusOK)
 }
@@ -125,20 +124,18 @@ func (h *exportCleanupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	defer span.End()
 
 	logger := logging.FromContext(ctx)
-	metrics := h.env.MetricsExporter(ctx)
-	metricsMiddleware := metricsware.NewMiddleWare(&metrics)
 
 	cutoff, err := cutoffDate(ctx, h.config.TTL, h.config.DebugOverrideCleanupMinDuration)
 	if err != nil {
 		message := fmt.Sprintf("error calculating cutoff time: %v", err)
-		metricsMiddleware.RecordExportsSetupFailed(ctx)
+		stats.Record(ctx, cleanup.ExportsSetupFailed.M(1))
 		logger.Error(message)
 		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
 		http.Error(w, "internal processing error", http.StatusInternalServerError)
 		return
 	}
 	logger.Infof("Starting cleanup for export files older than %v", cutoff.UTC())
-	metricsMiddleware.RecordExportsCutoffDate(ctx, cutoff.Unix())
+	stats.Record(ctx, cleanup.ExportsCleanupBefore.M(cutoff.Unix()))
 
 	// Set h.Timeout
 	timeoutCtx, cancel := context.WithTimeout(ctx, h.config.Timeout)
@@ -148,13 +145,13 @@ func (h *exportCleanupHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	if err != nil {
 		message := fmt.Sprintf("Failed deleting export files: %v", err)
 		logger.Error(message)
-		metricsMiddleware.RecordExposuresDeleteFailed(ctx)
+		stats.Record(ctx, cleanup.ExposuresDeleteFailed.M(1))
 		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
 		http.Error(w, "internal processing error", http.StatusInternalServerError)
 		return
 	}
 
-	metricsMiddleware.RecordExportsDeleted(ctx, count)
+	stats.Record(ctx, cleanup.ExportsDeleted.M(int64(count)))
 	logger.Infof("cleanup run complete, deleted %v files.", count)
 	w.WriteHeader(http.StatusOK)
 }
diff --git a/internal/export/batcher.go b/internal/export/batcher.go
index 828e7d3..1c20c6a 100644
--- a/internal/export/batcher.go
+++ b/internal/export/batcher.go
@@ -25,8 +25,9 @@ import (
 	exportdatabase "github.com/google/exposure-notifications-server/internal/export/database"
 	"github.com/google/exposure-notifications-server/internal/export/model"
 	publishmodel "github.com/google/exposure-notifications-server/internal/publish/model"
+	"go.opencensus.io/stats"
 
-	"github.com/google/exposure-notifications-server/internal/metrics/metricsware"
+	"github.com/google/exposure-notifications-server/internal/metrics/export"
 	"github.com/google/exposure-notifications-server/pkg/logging"
 )
 
@@ -34,8 +35,6 @@ import (
 // create entries in ExportBatchJob as appropriate.
 func (s *Server) handleCreateBatches(ctx context.Context) http.HandlerFunc {
 	logger := logging.FromContext(ctx)
-	metrics := s.env.MetricsExporter(ctx)
-	metricsMiddleware := metricsware.NewMiddleWare(&metrics)
 	db := s.env.Database()
 
 	return func(w http.ResponseWriter, r *http.Request) {
@@ -47,7 +46,7 @@ func (s *Server) handleCreateBatches(ctx context.Context) http.HandlerFunc {
 		unlockFn, err := db.Lock(ctx, lock, s.config.CreateTimeout)
 		if err != nil {
 			if errors.Is(err, coredb.ErrAlreadyLocked) {
-				metricsMiddleware.RecordExportBatcherLockContention(ctx)
+				stats.Record(ctx, export.BatcherLockContention.M(1))
 				msg := fmt.Sprintf("Lock %s already in use, no work will be performed", lock)
 				logger.Infof(msg)
 				fmt.Fprint(w, msg) // We return status 200 here so that Cloud Scheduler does not retry.
@@ -85,7 +84,7 @@ func (s *Server) handleCreateBatches(ctx context.Context) http.HandlerFunc {
 		})
 		if err != nil {
 			// some specific error handling below, but just need one metric.
-			metricsMiddleware.RecordExportBatcherFailure(ctx)
+			stats.Record(ctx, export.BatcherFailure.M(1))
 		}
 		switch {
 		case err == nil:
@@ -103,8 +102,6 @@ func (s *Server) handleCreateBatches(ctx context.Context) http.HandlerFunc {
 
 func (s *Server) maybeCreateBatches(ctx context.Context, ec *model.ExportConfig, now time.Time) (int, error) {
 	logger := logging.FromContext(ctx)
-	metrics := s.env.MetricsExporter(ctx)
-	metricsMiddleware := metricsware.NewMiddleWare(&metrics)
 	exportDB := exportdatabase.New(s.env.Database())
 
 	latestEnd, err := exportDB.LatestExportBatchEnd(ctx, ec)
@@ -114,7 +111,7 @@ func (s *Server) maybeCreateBatches(ctx context.Context, ec *model.ExportConfig,
 
 	ranges := makeBatchRanges(ec.Period, latestEnd, now, s.config.TruncateWindow)
 	if len(ranges) == 0 {
-		metricsMiddleware.RecordExportBatcherNoWork(ctx)
+		stats.Record(ctx, export.BatcherNoWork.M(1))
 		logger.Debugf("Batch creation for config %d is not required, skipping", ec.ConfigID)
 		return 0, nil
 	}
@@ -144,7 +141,7 @@ func (s *Server) maybeCreateBatches(ctx context.Context, ec *model.ExportConfig,
 		return 0, fmt.Errorf("creating export batches for config %d: %w", ec.ConfigID, err)
 	}
 
-	metricsMiddleware.RecordExportBatcherCreation(ctx, len(batches))
+	stats.Record(ctx, export.BatcherCreated.M(int64(len(batches))))
 	logger.Infof("Created %d batch(es) for config %d", len(batches), ec.ConfigID)
 	return len(batches), nil
 }
diff --git a/internal/export/worker.go b/internal/export/worker.go
index 6b7c4c7..15c881e 100644
--- a/internal/export/worker.go
+++ b/internal/export/worker.go
@@ -27,13 +27,14 @@ import (
 
 	coredb "github.com/google/exposure-notifications-server/internal/database"
 	exportdatabase "github.com/google/exposure-notifications-server/internal/export/database"
+	"github.com/google/exposure-notifications-server/internal/metrics/export"
 	publishdatabase "github.com/google/exposure-notifications-server/internal/publish/database"
 	"github.com/google/exposure-notifications-server/internal/storage"
+	"go.opencensus.io/stats"
 
 	"github.com/google/exposure-notifications-server/internal/export/model"
 	publishmodel "github.com/google/exposure-notifications-server/internal/publish/model"
 
-	"github.com/google/exposure-notifications-server/internal/metrics/metricsware"
 	"github.com/google/exposure-notifications-server/pkg/logging"
 
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1alpha1"
@@ -189,9 +190,7 @@ func (s *Server) batchExposures(ctx context.Context, criteria publishdatabase.It
 
 	if droppedKeys > 0 {
 		logger.Errorf("Export found keys of invalid length, %v keys were dropped", droppedKeys)
-		metricsExporter := s.env.MetricsExporter(ctx)
-		metricsMiddleWare := metricsware.NewMiddleWare(&metricsExporter)
-		metricsMiddleWare.RecordExportWorkerBadKeyLength(ctx, droppedKeys)
+		stats.Record(ctx, export.WorkerBadKeyLength.M(int64(droppedKeys)))
 	}
 
 	// Sort all the keys that we got so that if the batch is re-run, the contents are stable IFF
diff --git a/internal/federationin/federationin.go b/internal/federationin/federationin.go
index 431376d..4b5a7ca 100644
--- a/internal/federationin/federationin.go
+++ b/internal/federationin/federationin.go
@@ -30,8 +30,7 @@ import (
 	coredb "github.com/google/exposure-notifications-server/internal/database"
 	"github.com/google/exposure-notifications-server/internal/federationin/database"
 	"github.com/google/exposure-notifications-server/internal/federationin/model"
-	"github.com/google/exposure-notifications-server/internal/metrics"
-	"github.com/google/exposure-notifications-server/internal/metrics/metricsware"
+	"github.com/google/exposure-notifications-server/internal/metrics/federationin"
 	"github.com/google/exposure-notifications-server/internal/pb/federation"
 	publishdb "github.com/google/exposure-notifications-server/internal/publish/database"
 	publishmodel "github.com/google/exposure-notifications-server/internal/publish/model"
@@ -45,6 +44,7 @@ import (
 	"google.golang.org/grpc/credentials/oauth"
 
 	"go.opencensus.io/plugin/ocgrpc"
+	"go.opencensus.io/stats"
 	"go.opencensus.io/trace"
 )
 
@@ -93,23 +93,21 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	defer span.End()
 
 	logger := logging.FromContext(ctx)
-	metrics := h.env.MetricsExporter(ctx)
-	metricsMiddleWare := metricsware.NewMiddleWare(&metrics)
 
 	queryIDs, ok := r.URL.Query()[queryParam]
 	if !ok {
-		metricsMiddleWare.RecordPullInvalidRequest(ctx)
+		stats.Record(ctx, federationin.PullInvalidRequest.M(1))
 		badRequestf(ctx, w, "%s is required", queryParam)
 		return
 	}
 	if len(queryIDs) > 1 {
-		metricsMiddleWare.RecordPullInvalidRequest(ctx)
+		stats.Record(ctx, federationin.PullInvalidRequest.M(1))
 		badRequestf(ctx, w, "only one %s allowed", queryParam)
 		return
 	}
 	queryID := queryIDs[0]
 	if queryID == "" {
-		metricsMiddleWare.RecordPullInvalidRequest(ctx)
+		stats.Record(ctx, federationin.PullInvalidRequest.M(1))
 		badRequestf(ctx, w, "%s is required", queryParam)
 		return
 	}
@@ -119,7 +117,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	unlockFn, err := h.db.Lock(ctx, lock, h.config.Timeout)
 	if err != nil {
 		if errors.Is(err, coredb.ErrAlreadyLocked) {
-			metricsMiddleWare.RecordPullLockContention(ctx)
+			stats.Record(ctx, federationin.PullLockContention.M(1))
 			msg := fmt.Sprintf("Lock %s already in use. No work will be performed.", lock)
 			logger.Infof(msg)
 			fmt.Fprint(w, msg) // We return status 200 here so that Cloud Scheduler does not retry.
@@ -206,7 +204,7 @@ func (h *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		maxMagnitudeSymptomOnsetDays: h.config.MaxMagnitudeSymptomOnsetDays,
 		debugReleaseSameDay:          h.config.ReleaseSameDayKeys,
 	}
-	if err := pull(timeoutContext, metrics, &opts); err != nil {
+	if err := pull(timeoutContext, &opts); err != nil {
 		internalErrorf(ctx, w, "Federation query %q failed: %v", queryID, err)
 		return
 	}
@@ -300,9 +298,7 @@ func buildExposure(e *federation.ExposureKey, config *Config) (*publishmodel.Exp
 	return &exposure, nil
 }
 
-func pull(ctx context.Context, metrics metrics.Exporter, opts *pullOptions) (err error) {
-	metricsMiddleWare := metricsware.NewMiddleWare(&metrics)
-
+func pull(ctx context.Context, opts *pullOptions) (err error) {
 	ctx, span := trace.StartSpan(ctx, "federationin.pull")
 	defer func() {
 		if err != nil {
@@ -393,8 +389,8 @@ func pull(ctx context.Context, metrics metrics.Exporter, opts *pullOptions) (err
 				return fmt.Errorf("inserting %d exposures: %w", len(newExposures), err)
 			}
 			// Success, update metrics
-			metricsMiddleWare.RecordPullInsertions(ctx, int(resp.Inserted))
-			metricsMiddleWare.RecordPullDropped(ctx, int(resp.Dropped))
+			stats.Record(ctx, federationin.PullInserts.M(int64(resp.Inserted)))
+			stats.Record(ctx, federationin.PullDropped.M(int64(resp.Dropped)))
 
 			total += int(resp.Inserted)
 		} else {
@@ -431,8 +427,8 @@ func pull(ctx context.Context, metrics metrics.Exporter, opts *pullOptions) (err
 				return fmt.Errorf("revising %d exposures: %w", len(revisedExposures), err)
 			}
 			// Success, update metrics
-			metricsMiddleWare.RecordPullRevisions(ctx, int(resp.Revised))
-			metricsMiddleWare.RecordPullDropped(ctx, int(resp.Dropped))
+			stats.Record(ctx, federationin.PullRevisions.M(int64(resp.Revised)))
+			stats.Record(ctx, federationin.PullDropped.M(int64(resp.Dropped)))
 
 			total += int(resp.Revised)
 		} else {
diff --git a/internal/federationout/federationout.go b/internal/federationout/federationout.go
index 5c53fb9..755bdee 100644
--- a/internal/federationout/federationout.go
+++ b/internal/federationout/federationout.go
@@ -27,12 +27,13 @@ import (
 	coredb "github.com/google/exposure-notifications-server/internal/database"
 	"github.com/google/exposure-notifications-server/internal/federationin/model"
 	"github.com/google/exposure-notifications-server/internal/federationout/database"
-	"github.com/google/exposure-notifications-server/internal/metrics/metricsware"
+	"github.com/google/exposure-notifications-server/internal/metrics/federationout"
 	"github.com/google/exposure-notifications-server/internal/pb/federation"
 	publishdb "github.com/google/exposure-notifications-server/internal/publish/database"
 	publishmodel "github.com/google/exposure-notifications-server/internal/publish/model"
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
 	"github.com/google/exposure-notifications-server/pkg/logging"
+	"go.opencensus.io/stats"
 
 	"github.com/google/exposure-notifications-server/internal/serverenv"
 	"google.golang.org/api/idtoken"
@@ -78,9 +79,7 @@ func (s Server) Fetch(ctx context.Context, req *federation.FederationFetchReques
 	logger := logging.FromContext(ctx)
 	response, err := s.fetch(ctx, req, s.publishdb.IterateExposures, publishmodel.TruncateWindow(time.Now(), s.config.TruncateWindow)) // Don't fetch the current window, which isn't complete yet. TODO(squee1945): should I double this for safety?
 	if err != nil {
-		metricsExporter := s.env.MetricsExporter(ctx)
-		metricsMiddleWare := metricsware.NewMiddleWare(&metricsExporter)
-		metricsMiddleWare.RecordFetchFailure(ctx)
+		stats.Record(ctx, federationout.FetchFailed.M(1))
 		logger.Errorf("Fetch error: %v", err)
 		return nil, errors.New("internal error")
 	}
@@ -89,8 +88,6 @@ func (s Server) Fetch(ctx context.Context, req *federation.FederationFetchReques
 
 func (s Server) fetch(ctx context.Context, req *federation.FederationFetchRequest, itFunc iterateExposuresFunc, fetchUntil time.Time) (*federation.FederationFetchResponse, error) {
 	logger := logging.FromContext(ctx)
-	metrics := s.env.MetricsExporter(ctx)
-	metricsMiddleWare := metricsware.NewMiddleWare(&metrics)
 
 	if in := intersect(req.IncludeRegions, req.ExcludeRegions); len(in) > 0 {
 		logger.Errorw("overlap in included and excluded regions", "intersection", in)
@@ -103,8 +100,8 @@ func (s Server) fetch(ctx context.Context, req *federation.FederationFetchReques
 	for i, exRegion := range req.ExcludeRegions {
 		req.ExcludeRegions[i] = strings.ToUpper(exRegion)
 	}
-	metricsMiddleWare.RecordFetchRegionsRequested(ctx, len(req.IncludeRegions))
-	metricsMiddleWare.RecordFetchRegionsExcluded(ctx, len(req.ExcludeRegions))
+	stats.Record(ctx, federationout.FetchRegionsRequested.M(int64(len(req.IncludeRegions))))
+	stats.Record(ctx, federationout.FetchRegionsExcluded.M(int64(len(req.ExcludeRegions))))
 
 	// Use configuration max or user provided max.
 	maxRecords := s.config.MaxRecords
@@ -174,7 +171,7 @@ func (s Server) fetch(ctx context.Context, req *federation.FederationFetchReques
 	}))
 	keepGoing := true
 	if err != nil {
-		metricsMiddleWare.RecordFetchError(ctx)
+		stats.Record(ctx, federationout.FetchError.M(1))
 		if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
 			logger.Infof("Fetch request reached time out, returning partial response.")
 			response.PartialResponse = true
@@ -211,7 +208,7 @@ func (s Server) fetch(ctx context.Context, req *federation.FederationFetchReques
 			excludeRegions: excludedRegions,
 		}))
 		if err != nil {
-			metricsMiddleWare.RecordFetchError(ctx)
+			stats.Record(ctx, federationout.FetchError.M(1))
 			if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
 				logger.Infof("Fetch request reached time out, returning partial response.")
 				response.PartialResponse = true
@@ -245,7 +242,7 @@ func (s Server) fetch(ctx context.Context, req *federation.FederationFetchReques
 		}
 	}
 
-	metricsMiddleWare.RecordFetchCount(ctx, count)
+	stats.Record(ctx, federationout.FetchCount.M(int64(count)))
 	logger.Infow("sent key", "keys", count)
 	return response, nil
 }
@@ -339,8 +336,6 @@ func buildIteratorFunction(request *BuildIteratorRequest) publishdb.IteratorFunc
 // AuthInterceptor validates incoming OIDC bearer token and adds corresponding FederationAuthorization record to the context.
 func (s Server) AuthInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
 	logger := logging.FromContext(ctx)
-	metrics := s.env.MetricsExporter(ctx)
-	metricsMiddleware := metricsware.NewMiddleWare(&metrics)
 
 	raw, err := rawToken(ctx)
 	if err != nil {
@@ -351,24 +346,24 @@ func (s Server) AuthInterceptor(ctx context.Context, req interface{}, info *grpc
 	token, err := idtoken.Validate(ctx, raw, "")
 	if err != nil {
 		logger.Infof("Invalid token: %v", err)
-		metricsMiddleware.RecordInvalidFetchAuthToken(ctx)
+		stats.Record(ctx, federationout.FetchInvalidAuthToken.M(1))
 		return nil, status.Errorf(codes.Unauthenticated, "Invalid token")
 	}
 
 	auth, err := s.db.GetFederationOutAuthorization(ctx, token.Issuer, token.Subject)
 	if err != nil {
 		if errors.Is(err, coredb.ErrNotFound) {
-			metricsMiddleware.RecordUnauthorizedFetchAttempt(ctx)
+			stats.Record(ctx, federationout.FetchUnauthorized.M(1))
 			logger.Infof("Authorization not found (issuer %q, subject %s)", token.Issuer, token.Subject)
 			return nil, status.Errorf(codes.Unauthenticated, "Invalid issuer/subject")
 		}
 		logger.Errorf("Failed to fetch authorization (issuer %q, subject %s): %v", token.Issuer, token.Subject, err)
-		metricsMiddleware.RecordInternalErrorDuringFetch(ctx)
+		stats.Record(ctx, federationout.FetchInternalError.M(1))
 		return nil, status.Errorf(codes.Internal, "Internal error")
 	}
 
 	if auth.Audience != "" && auth.Audience != token.Audience {
-		metricsMiddleware.RecordInvalidAudienceDuringFetch(ctx)
+		stats.Record(ctx, federationout.FetchInvalidAudience.M(1))
 		logger.Infof("Invalid audience, got %q, want %q", token.Audience, auth.Audience)
 		return nil, status.Errorf(codes.Unauthenticated, "Invalid audience")
 	}
diff --git a/internal/keyrotation/rotate.go b/internal/keyrotation/rotate.go
index 24db570..cd5c48b 100644
--- a/internal/keyrotation/rotate.go
+++ b/internal/keyrotation/rotate.go
@@ -21,10 +21,11 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/google/exposure-notifications-server/internal/metrics/metricsware"
+	"github.com/google/exposure-notifications-server/internal/metrics/rotate"
 	"github.com/google/exposure-notifications-server/internal/revision/database"
 	"github.com/google/exposure-notifications-server/pkg/logging"
 	"github.com/hashicorp/go-multierror"
+	"go.opencensus.io/stats"
 	"go.opencensus.io/trace"
 )
 
@@ -62,8 +63,6 @@ func (s *Server) handleRotateKeys(ctx context.Context) http.HandlerFunc {
 }
 
 func (s *Server) doRotate(ctx context.Context) error {
-	metrics := s.env.MetricsExporter(ctx)
-	metricsMiddleware := metricsware.NewMiddleWare(&metrics)
 	logger := logging.FromContext(ctx).Named("keyrotation.doRotate")
 
 	effectiveID, allowed, err := s.revisionDB.GetAllowedRevisionKeys(ctx)
@@ -81,7 +80,7 @@ func (s *Server) doRotate(ctx context.Context) error {
 		effectiveID = key.KeyID
 		previousCreated = key.CreatedAt
 		logger.Info("Created new revision key.")
-		metricsMiddleware.RecordRevisionKeyCreation(ctx)
+		stats.Record(ctx, rotate.RevisionKeysCreated.M(1))
 	} else {
 		previousCreated = allowed[0].CreatedAt
 	}
@@ -98,7 +97,7 @@ func (s *Server) doRotate(ctx context.Context) error {
 	}
 	if deleted > 0 {
 		logger.Infof("Deleted %d old revision keys.", deleted)
-		metricsMiddleware.RecordRevisionKeyDeletion(ctx, deleted)
+		stats.Record(ctx, rotate.RevisionKeysDeleted.M(int64(deleted)))
 	}
 	return result.ErrorOrNil()
 }
diff --git a/internal/publish/publish.go b/internal/publish/publish.go
index 1a0723b..7f25d0a 100644
--- a/internal/publish/publish.go
+++ b/internal/publish/publish.go
@@ -25,10 +25,11 @@ import (
 	"net/http"
 	"time"
 
+	"go.opencensus.io/stats"
 	"go.opencensus.io/trace"
 
 	"github.com/google/exposure-notifications-server/internal/authorizedapp"
-	"github.com/google/exposure-notifications-server/internal/metrics/metricsware"
+	"github.com/google/exposure-notifications-server/internal/metrics/publish"
 	"github.com/google/exposure-notifications-server/internal/pb"
 	"github.com/google/exposure-notifications-server/internal/publish/database"
 	"github.com/google/exposure-notifications-server/internal/publish/model"
@@ -189,8 +190,6 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 	defer span.End()
 
 	logger := logging.FromContext(ctx).With("health_authority_id", data.HealthAuthorityID)
-	metrics := h.serverenv.MetricsExporter(ctx)
-	metricsMiddleWare := metricsware.NewMiddleWare(&metrics)
 
 	logger.Info("publish API request")
 
@@ -209,7 +208,7 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 					Code:         verifyapi.ErrorUnknownHealthAuthorityID,
 				},
 				metrics: func() {
-					metricsMiddleWare.RecordHealthAuthorityNotAuthorized(ctx)
+					stats.Record(ctx, publish.HealthAuthorityNotAuthorized.M(1))
 				},
 			}
 		}
@@ -228,7 +227,7 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 				Code:         verifyapi.ErrorUnableToLoadHealthAuthority,
 			},
 			metrics: func() {
-				metricsMiddleWare.RecordErrorLoadingAuthorizedApp(ctx)
+				stats.Record(ctx, publish.ErrorLoadingAuthorizedApp.M(1))
 			},
 		}
 	}
@@ -250,7 +249,7 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 						ErrorMessage: message, // Error code omitted, since this isn't in the v1 path.
 					},
 					metrics: func() {
-						metricsMiddleWare.RecordRegionNotAuthorized(ctx)
+						stats.Record(ctx, publish.RegionNotAuthorized.M(1))
 					},
 				}
 			}
@@ -280,7 +279,7 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 				Code:         verifyapi.ErrorHealthAuthorityMissingRegionConfiguration,
 			},
 			metrics: func() {
-				metricsMiddleWare.RecordRegionNotSpecified(ctx)
+				stats.Record(ctx, publish.RegionNotSpecified.M(1))
 			},
 		}
 	}
@@ -290,7 +289,7 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 	if err != nil {
 		if appConfig.BypassHealthAuthorityVerification {
 			logger.Warnf("bypassing health authority certificate verification health authority: %v", appConfig.AppPackageName)
-			metricsMiddleWare.RecordVerificationBypassed(ctx)
+			stats.Record(ctx, publish.VerificationBypassed.M(1))
 		} else {
 			message := fmt.Sprintf("unable to validate diagnosis verification: %v", err)
 			if h.config.DebugLogBadCertificates {
@@ -306,7 +305,7 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 					Code:         verifyapi.ErrorVerificationCertificateInvalid,
 				},
 				metrics: func() {
-					metricsMiddleWare.RecordBadVerification(ctx)
+					stats.Record(ctx, publish.BadVerification.M(1))
 				},
 			}
 		}
@@ -346,7 +345,7 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 				Warnings:     transformWarnings,
 			},
 			metrics: func() {
-				metricsMiddleWare.RecordTransformFail(ctx)
+				stats.Record(ctx, publish.TransformFail.M(1))
 			},
 		}
 	}
@@ -396,7 +395,10 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 				Warnings:     transformWarnings,
 			},
 			metrics: func() {
-				metricsMiddleWare.RecordRevisionTokenIssue(ctx, metric)
+				// TODO(yegle): we want to use this string later when we
+				// refactor the metrics.
+				_ = metric
+				stats.Record(ctx, publish.RevisionTokenIssue.M(1))
 			},
 		}
 	}
@@ -443,9 +445,9 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 		status:      http.StatusOK,
 		pubResponse: &publishResponse,
 		metrics: func() {
-			metricsMiddleWare.RecordInsertions(ctx, int(resp.Inserted))
-			metricsMiddleWare.RecordRevisions(ctx, int(resp.Revised))
-			metricsMiddleWare.RecordDrops(ctx, int(resp.Dropped))
+			stats.Record(ctx, publish.ExposuresInserted.M(int64(resp.Inserted)))
+			stats.Record(ctx, publish.ExposuresInserted.M(int64(resp.Revised)))
+			stats.Record(ctx, publish.ExposuresInserted.M(int64(resp.Dropped)))
 		},
 	}
 }
diff --git a/internal/publish/publish_v1.go b/internal/publish/publish_v1.go
index 0a6fc4e..33f66af 100644
--- a/internal/publish/publish_v1.go
+++ b/internal/publish/publish_v1.go
@@ -20,11 +20,12 @@ import (
 	"fmt"
 	"net/http"
 
+	"go.opencensus.io/stats"
 	"go.opencensus.io/trace"
 
 	"github.com/google/exposure-notifications-server/internal/jsonutil"
 	"github.com/google/exposure-notifications-server/internal/maintenance"
-	"github.com/google/exposure-notifications-server/internal/metrics/metricsware"
+	"github.com/google/exposure-notifications-server/internal/metrics/publish"
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
 	"github.com/google/exposure-notifications-server/pkg/logging"
 	"github.com/mikehelmick/go-chaff"
@@ -34,9 +35,6 @@ func (h *PublishHandler) handleRequest(w http.ResponseWriter, r *http.Request) *
 	ctx, span := trace.StartSpan(r.Context(), "(*publish.PublishHandler).handleRequest")
 	defer span.End()
 
-	metrics := h.serverenv.MetricsExporter(ctx)
-	metricsMiddleware := metricsware.NewMiddleWare(&metrics)
-
 	w.Header().Set(HeaderAPIVersion, "v1")
 
 	var data verifyapi.Publish
@@ -55,7 +53,7 @@ func (h *PublishHandler) handleRequest(w http.ResponseWriter, r *http.Request) *
 				Code:         errorCode,
 			},
 			metrics: func() {
-				metricsMiddleware.RecordBadJSON(ctx)
+				stats.Record(ctx, publish.BadJSON.M(1))
 			},
 		}
 	}
@@ -72,11 +70,9 @@ func (h *PublishHandler) Handle() http.Handler {
 				response := h.handleRequest(w, r)
 
 				ctx := r.Context()
-				metrics := h.serverenv.MetricsExporter(ctx)
-				metricsMiddleware := metricsware.NewMiddleWare(&metrics)
 
 				if err := response.padResponse(h.config); err != nil {
-					metricsMiddleware.RecordPaddingFailure(ctx)
+					stats.Record(ctx, publish.PaddingFailed.M(1))
 					logging.FromContext(ctx).Errorw("failed to pad response", "error", err)
 				}
 
diff --git a/internal/publish/publish_v1alpha1.go b/internal/publish/publish_v1alpha1.go
index a461f8e..00e8691 100644
--- a/internal/publish/publish_v1alpha1.go
+++ b/internal/publish/publish_v1alpha1.go
@@ -20,11 +20,12 @@ import (
 	"fmt"
 	"net/http"
 
+	"go.opencensus.io/stats"
 	"go.opencensus.io/trace"
 
 	"github.com/google/exposure-notifications-server/internal/jsonutil"
 	"github.com/google/exposure-notifications-server/internal/maintenance"
-	"github.com/google/exposure-notifications-server/internal/metrics/metricsware"
+	"github.com/google/exposure-notifications-server/internal/metrics/publish"
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
 	"github.com/google/exposure-notifications-server/pkg/api/v1alpha1"
 	"github.com/google/exposure-notifications-server/pkg/logging"
@@ -35,9 +36,6 @@ func (h *PublishHandler) handleV1Apha1Request(w http.ResponseWriter, r *http.Req
 	ctx, span := trace.StartSpan(r.Context(), "(*publish.PublishHandler).handleRequest")
 	defer span.End()
 
-	metrics := h.serverenv.MetricsExporter(ctx)
-	metricsMiddleware := metricsware.NewMiddleWare(&metrics)
-
 	w.Header().Set(HeaderAPIVersion, "v1alpha")
 
 	var data v1alpha1.Publish
@@ -49,7 +47,7 @@ func (h *PublishHandler) handleV1Apha1Request(w http.ResponseWriter, r *http.Req
 			status:      code,
 			pubResponse: &verifyapi.PublishResponse{ErrorMessage: message}, // will be down-converted in ServeHTTP
 			metrics: func() {
-				metricsMiddleware.RecordBadJSON(ctx)
+				stats.Record(ctx, publish.BadJSON.M(1))
 			},
 		}
 	}
@@ -89,11 +87,9 @@ func (h *PublishHandler) HandleV1Alpha1() http.Handler {
 				response := h.handleV1Apha1Request(w, r)
 
 				ctx := r.Context()
-				metrics := h.serverenv.MetricsExporter(ctx)
-				metricsMiddleware := metricsware.NewMiddleWare(&metrics)
 
 				if err := response.padResponse(h.config); err != nil {
-					metricsMiddleware.RecordPaddingFailure(ctx)
+					stats.Record(ctx, publish.PaddingFailed.M(1))
 					logging.FromContext(ctx).Errorw("failed to padd response", "error", err)
 				}
 
diff --git a/terraform/service_debugger.tf b/terraform/service_debugger.tf
index dc34a17..7a94fd4 100644
--- a/terraform/service_debugger.tf
+++ b/terraform/service_debugger.tf
@@ -165,6 +165,9 @@ resource "google_compute_backend_service" "debugger" {
     group = google_compute_region_network_endpoint_group.debugger[0].id
   }
   security_policy = google_compute_security_policy.cloud-armor.name
+  log_config {
+    enable = var.enable_lb_logging
+  }
 }
 
 output "debugger_urls" {
diff --git a/terraform/service_exposure.tf b/terraform/service_exposure.tf
index 54ebf86..d1e21cc 100644
--- a/terraform/service_exposure.tf
+++ b/terraform/service_exposure.tf
@@ -171,6 +171,9 @@ resource "google_compute_backend_service" "exposure" {
     group = google_compute_region_network_endpoint_group.exposure[0].id
   }
   security_policy = google_compute_security_policy.cloud-armor.name
+  log_config {
+    enable = var.enable_lb_logging
+  }
 }
 
 output "exposure_urls" {
diff --git a/terraform/service_federationout.tf b/terraform/service_federationout.tf
index f5e8462..8d9a2cb 100644
--- a/terraform/service_federationout.tf
+++ b/terraform/service_federationout.tf
@@ -155,6 +155,9 @@ resource "google_compute_backend_service" "federationout" {
     group = google_compute_region_network_endpoint_group.federationout[0].id
   }
   security_policy = google_compute_security_policy.cloud-armor.name
+  log_config {
+    enable = var.enable_lb_logging
+  }
 }
 
 output "federationout_urls" {
diff --git a/terraform/vars.tf b/terraform/vars.tf
index 85b9e7d..8835334 100644
--- a/terraform/vars.tf
+++ b/terraform/vars.tf
@@ -249,6 +249,15 @@ variable "vpc_access_connector_max_throughput" {
   description = "Maximum provisioned traffic throughput in Mbps"
 }
 
+variable "enable_lb_logging" {
+  type        = bool
+  default     = false
+  description = <<-EOT
+  Whether to enable load balancer logging. This is useful for debugging Cloud
+  Armor issues.
+  EOT
+}
+
 terraform {
   required_version = ">= 0.13"
 


Test patch:
diff --git a/internal/federationin/federationin_test.go b/internal/federationin/federationin_test.go
index d88d354..1a60195 100644
--- a/internal/federationin/federationin_test.go
+++ b/internal/federationin/federationin_test.go
@@ -25,7 +25,6 @@ import (
 	publishmodel "github.com/google/exposure-notifications-server/internal/publish/model"
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
 
-	"github.com/google/exposure-notifications-server/internal/metrics"
 	"github.com/google/exposure-notifications-server/internal/pb/federation"
 
 	"github.com/google/go-cmp/cmp"
@@ -437,7 +436,7 @@ func TestFederationPull(t *testing.T) {
 				maxMagnitudeSymptomOnsetDays: 14,
 			}
 
-			err := pull(ctx, metrics.NewLogsBasedFromContext(ctx), &opts)
+			err := pull(ctx, &opts)
 			if err != nil {
 				t.Fatalf("pull returned err=%v, want err=nil", err)
 			}