Repository: google/exposure-notifications-server
Pull request #1388 (2021-02-17 00:22:08)

Clean up metrics in the Publish API
It should be safe to just remove the old metrics as we don't use them anywhere.

google-oss-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/google/exposure-notifications-server/pull/1388#" title="Author self-approved">yegle</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=google%2Fexposure-notifications-server).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/google/exposure-notifications-server/blob/main/OWNERS)~~ [yegle]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
google-oss-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/google/exposure-notifications-server/pull/1388#pullrequestreview-591762214" title="Approved">sethvargo</a>*, *<a href="https://github.com/google/exposure-notifications-server/pull/1388#" title="Author self-approved">yegle</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=google%2Fexposure-notifications-server).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/google/exposure-notifications-server/blob/main/OWNERS)~~ [sethvargo,yegle]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->
sethvargo: There are some unhandled errors

Patch:
diff --git a/builders/migrate.dockerfile b/builders/migrate.dockerfile
index 7d52ff5..4b53c98 100644
--- a/builders/migrate.dockerfile
+++ b/builders/migrate.dockerfile
@@ -15,7 +15,7 @@
 FROM alpine
 RUN apk add --no-cache bash
 
-ADD https://storage.googleapis.com/cloudsql-proxy/v1.19.1/cloud_sql_proxy.linux.amd64 /cloud-sql-proxy
+ADD https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 /cloud-sql-proxy
 COPY ./bin/migrate /migrate
 COPY ./builders/cloud-sql-exec /cloud-sql-exec
 COPY ./migrations /migrations
diff --git a/go.mod b/go.mod
index bdc53b9..85de17a 100644
--- a/go.mod
+++ b/go.mod
@@ -52,7 +52,7 @@ require (
 	github.com/lib/pq v1.9.0 // indirect
 	github.com/lstoll/awskms v0.0.0-20200603175638-a388516467f1
 	github.com/mattn/go-colorable v0.1.7 // indirect
-	github.com/mikehelmick/go-chaff v0.4.1
+	github.com/mikehelmick/go-chaff v0.5.0
 	github.com/mitchellh/mapstructure v1.4.1
 	github.com/opencontainers/runc v1.0.0-rc9 // indirect
 	github.com/ory/dockertest v3.3.5+incompatible
@@ -73,6 +73,7 @@ require (
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.16.0
 	golang.org/x/oauth2 v0.0.0-20210210192628-66670185b0cd
+	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
 	golang.org/x/tools v0.1.0
 	google.golang.org/api v0.39.0
@@ -80,5 +81,5 @@ require (
 	google.golang.org/grpc v1.35.0
 	google.golang.org/protobuf v1.25.0
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	honnef.co/go/tools v0.0.1-2020.1.4
+	honnef.co/go/tools v0.1.1
 )
diff --git a/go.sum b/go.sum
index edfde4e..7f6fe0a 100644
--- a/go.sum
+++ b/go.sum
@@ -606,8 +606,8 @@ github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/mikehelmick/go-chaff v0.4.1 h1:0UsypkU2XcsROrwvL+mN7mgRiiB6oEwt+iNVOHlXO0M=
-github.com/mikehelmick/go-chaff v0.4.1/go.mod h1:mFry3zNW17oxNGmZpQV3PEOmzTNyly3nLDYawCT/iCE=
+github.com/mikehelmick/go-chaff v0.5.0 h1:u8lrTCbUsyVBFRHPs8Nn3i0830XAOrbcA5dbQl8tk78=
+github.com/mikehelmick/go-chaff v0.5.0/go.mod h1:mFry3zNW17oxNGmZpQV3PEOmzTNyly3nLDYawCT/iCE=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -1337,6 +1337,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+honnef.co/go/tools v0.1.1 h1:EVDuO03OCZwpV2t/tLLxPmPiomagMoBOgfPt0FM+4IY=
+honnef.co/go/tools v0.1.1/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
 modernc.org/b v1.0.0/go.mod h1:uZWcZfRj1BpYzfN9JTerzlNUnnPsV9O2ZA8JsRcubNg=
 modernc.org/db v1.0.0/go.mod h1:kYD/cO29L/29RM0hXYl4i3+Q5VojL31kTUVpVJDw0s8=
 modernc.org/file v1.0.0/go.mod h1:uqEokAEn1u6e+J45e54dsEA/pw4o7zLrA2GwyntZzjw=
diff --git a/internal/jwks/config.go b/internal/jwks/config.go
index c789a2b..943bd3c 100644
--- a/internal/jwks/config.go
+++ b/internal/jwks/config.go
@@ -33,11 +33,17 @@ type Config struct {
 
 	Port string `env:"PORT, default=8080"`
 
-	// RequestTimeout is the per-request amount of time to wait for a response
-	// before timing out.
-	RequestTimeout time.Duration `env:"REQUEST_TIMEOUT, default=5s"`
+	// MaxRuntime is how long an individual handler should run.
+	MaxRuntime time.Duration `env:"MAX_RUNTIME, default=10m"`
+
+	// RequestTimeout is the client per-request timeout when accessing
+	// remote JWKS documents.
+	RequestTimeout time.Duration `env:"REQUEST_TIMEOUT, default=30s"`
 
 	KeyCleanupTTL time.Duration `env:"HEALTH_AUTHORITY_KEY_CLEANUP_TTL, default=720h"` // 30 days
+
+	// MaxWorkers is the number of parallel JWKS updates that can occur.
+	MaxWorkers uint `env:"MAX_WORKERS, default=5"`
 }
 
 func (c *Config) DatabaseConfig() *database.Config {
diff --git a/internal/jwks/handle_update_all.go b/internal/jwks/handle_update_all.go
index 51c09f9..b8153e4 100644
--- a/internal/jwks/handle_update_all.go
+++ b/internal/jwks/handle_update_all.go
@@ -15,6 +15,7 @@
 package jwks
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
@@ -29,7 +30,9 @@ const jwksLockID = "jwks-import"
 
 func (s *Server) handleUpdateAll() http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		ctx := r.Context()
+		ctx, cancel := context.WithTimeout(r.Context(), s.config.MaxRuntime)
+		defer cancel()
+
 		logger := logging.FromContext(ctx).Named("handleUpdateAll")
 
 		unlock, err := s.manager.db.Lock(ctx, jwksLockID, time.Minute)
diff --git a/internal/jwks/jwks.go b/internal/jwks/jwks.go
index 31a2981..6aa7725 100644
--- a/internal/jwks/jwks.go
+++ b/internal/jwks/jwks.go
@@ -23,6 +23,7 @@ import (
 	"encoding/pem"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"sort"
 	"strings"
@@ -33,9 +34,11 @@ import (
 	"github.com/google/exposure-notifications-server/internal/project"
 	hadb "github.com/google/exposure-notifications-server/internal/verification/database"
 	"github.com/google/exposure-notifications-server/internal/verification/model"
+	"github.com/google/exposure-notifications-server/pkg/cryptorand"
 	"github.com/google/exposure-notifications-server/pkg/logging"
 	"github.com/hashicorp/go-multierror"
 	"github.com/rakutentech/jwk-go/jwk"
+	"golang.org/x/sync/semaphore"
 )
 
 // Manager handles updating all HealthAuthorities if they've specified a JWKS
@@ -44,10 +47,11 @@ type Manager struct {
 	db         *database.DB
 	client     *http.Client
 	cleanupTTL time.Duration
+	maxWorkers uint
 }
 
 // NewManager creates a new Manager.
-func NewManager(db *database.DB, cleanupTTL, requestTimeout time.Duration) (*Manager, error) {
+func NewManager(db *database.DB, cleanupTTL, requestTimeout time.Duration, maxWorkers uint) (*Manager, error) {
 	if cleanupTTL < 0 {
 		cleanupTTL *= -1
 	}
@@ -60,6 +64,7 @@ func NewManager(db *database.DB, cleanupTTL, requestTimeout time.Duration) (*Man
 		db:         db,
 		client:     client,
 		cleanupTTL: cleanupTTL,
+		maxWorkers: maxWorkers,
 	}, nil
 }
 
@@ -73,7 +78,7 @@ func (mgr *Manager) getKeys(ctx context.Context, ha *model.HealthAuthority) ([]b
 		return nil, nil
 	}
 
-	req, err := http.NewRequestWithContext(ctx, "GET", jwksURI, nil)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, jwksURI, nil)
 	if err != nil {
 		return nil, fmt.Errorf("creating connection: %w", err)
 	}
@@ -171,8 +176,7 @@ func (mgr *Manager) updateHA(ctx context.Context, ha *model.HealthAuthority) err
 		With("health_authority_name", ha.Name).
 		With("health_authority_id", ha.ID)
 
-	if ha.JwksURI == nil || len(*ha.JwksURI) == 0 {
-		logger.Infow("skipping jwks, no URI specified")
+	if !ha.JWKSEnabled() {
 		return nil
 	}
 
@@ -248,18 +252,42 @@ func (mgr *Manager) UpdateAll(ctx context.Context) error {
 	defer logger.Debug("finished jwks update")
 
 	haDB := hadb.New(mgr.db)
-	healthAuthorities, err := haDB.ListAllHealthAuthoritiesWithoutKeys(ctx)
+	allHealthAuthorities, err := haDB.ListAllHealthAuthoritiesWithoutKeys(ctx)
 	if err != nil {
 		return fmt.Errorf("failed to query db: %w", err)
 	}
 
+	// Pre-filter out health authorities where discovery isn't enabled.
+	healthAuthorities := make([]*model.HealthAuthority, 0, len(allHealthAuthorities))
+	for _, ha := range allHealthAuthorities {
+		if ha.JWKSEnabled() {
+			healthAuthorities = append(healthAuthorities, ha)
+		}
+	}
+
+	//nolint:gosec
+	r := rand.New(cryptorand.NewSource())
+	r.Shuffle(len(healthAuthorities), func(i, j int) {
+		healthAuthorities[i], healthAuthorities[j] = healthAuthorities[j], healthAuthorities[i]
+	})
+
 	var merr *multierror.Error
 	var merrLock sync.Mutex
-
+	sem := semaphore.NewWeighted(int64(mgr.maxWorkers))
 	var wg sync.WaitGroup
 	for _, ha := range healthAuthorities {
+		// Rate limit the number of concurrent workers with a semaphore.
+		if err := sem.Acquire(ctx, 1); err != nil {
+			logger.Errorw("failed to acquire semaphore", "error", err)
+			merrLock.Lock()
+			merr = multierror.Append(merr, fmt.Errorf("failed to processes %v: %w", ha.Name, err))
+			merrLock.Unlock()
+			break
+		}
+
 		wg.Add(1)
 		go func(ha *model.HealthAuthority) {
+			defer sem.Release(1)
 			defer wg.Done()
 			err := mgr.updateHA(ctx, ha)
 			if err != nil {
diff --git a/internal/jwks/server.go b/internal/jwks/server.go
index ad660fc..d42044f 100644
--- a/internal/jwks/server.go
+++ b/internal/jwks/server.go
@@ -39,7 +39,7 @@ func NewServer(cfg *Config, env *serverenv.ServerEnv) (*Server, error) {
 		return nil, fmt.Errorf("missing database in server env")
 	}
 
-	manager, err := NewManager(env.Database(), cfg.KeyCleanupTTL, cfg.RequestTimeout)
+	manager, err := NewManager(env.Database(), cfg.KeyCleanupTTL, cfg.RequestTimeout, cfg.MaxWorkers)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create manager: %w", err)
 	}
diff --git a/internal/publish/metrics.go b/internal/publish/metrics.go
index 394b665..a6f7004 100644
--- a/internal/publish/metrics.go
+++ b/internal/publish/metrics.go
@@ -18,68 +18,72 @@ package publish
 import (
 	"github.com/google/exposure-notifications-server/internal/metrics"
 	"github.com/google/exposure-notifications-server/pkg/observability"
+	"go.opencensus.io/plugin/ochttp"
 	"go.opencensus.io/stats"
 	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
 )
 
 var (
 	publishMetricsPrefix = metrics.MetricRoot + "publish/"
 
-	mHealthAuthorityNotAuthorized = stats.Int64(publishMetricsPrefix+"pha_not_authorized",
-		"Public health authority authorized failures", stats.UnitDimensionless)
-	mErrorLoadingAuthorizedApp = stats.Int64(publishMetricsPrefix+"error_loading_app",
-		"Errors in loading authorized app", stats.UnitDimensionless)
-	mRegionNotAuthorized = stats.Int64(publishMetricsPrefix+"region_not_authorized",
-		"Instances of region not being authorized", stats.UnitDimensionless)
-	mRegionNotSpecified = stats.Int64(publishMetricsPrefix+"region_not_specified",
-		"Instances of region not being specified", stats.UnitDimensionless)
+	mLatencyMs = stats.Float64(publishMetricsPrefix+"requests", "publish requests", stats.UnitMilliseconds)
+
+	mExposuresCount = stats.Int64(publishMetricsPrefix+"exposures_count",
+		"exposure count", stats.UnitDimensionless)
+
 	mVerificationBypassed = stats.Int64(publishMetricsPrefix+"verification_bypassed",
 		"Instances of health authority verification being bypassed", stats.UnitDimensionless)
-	mBadVerification = stats.Int64(publishMetricsPrefix+"bad_verification",
-		"Instances of bad health authority verification", stats.UnitDimensionless)
-	mTransformFail = stats.Int64(publishMetricsPrefix+"transform_fail",
-		"Instances of failure fo transformation to exposure entities", stats.UnitDimensionless)
-	mRevisionTokenIssue = stats.Int64(publishMetricsPrefix+"revision_token_issue",
-		"Instances of issues in revision token processing", stats.UnitDimensionless)
-	mExposuresInserted = stats.Int64(publishMetricsPrefix+"exposures_insertions",
-		"Instances of exposure being inserted", stats.UnitDimensionless)
-	mExposuresRevised = stats.Int64(publishMetricsPrefix+"exposures_revisions",
-		"Instances of exposure being revised", stats.UnitDimensionless)
-	mExposuresDropped = stats.Int64(publishMetricsPrefix+"exposures_drops",
-		"Instances of exposure being dropped", stats.UnitDimensionless)
-	// v1 and v1alpha1
-	mBadJSON = stats.Int64(publishMetricsPrefix+"bad_json",
-		"Instances of bad JSON in incoming request", stats.UnitDimensionless)
 	// v1 and v1alpha1
 	mPaddingFailed = stats.Int64(publishMetricsPrefix+"padding_failed",
 		"Instances of response padding failures", stats.UnitDimensionless)
+
+	exposureTypeTag = tag.MustNewKey("type")
+
+	requestTagKeys = []tag.Key{
+		observability.BuildIDTagKey,
+		observability.BuildTagTagKey,
+		observability.BlameTagKey,
+		observability.ResultTagKey,
+	}
+	exposureTagKeys = []tag.Key{
+		observability.BuildIDTagKey,
+		observability.BuildTagTagKey,
+		exposureTypeTag,
+	}
+)
+
+func exposureType(s string) tag.Mutator {
+	return tag.Upsert(exposureTypeTag, s)
+}
+
+var (
+	exposuresInserted = exposureType("INSERTED")
+	exposuresRevised  = exposureType("REVISED")
+	exposuresDropped  = exposureType("DROPPED")
 )
 
 func init() {
 	observability.CollectViews([]*view.View{
 		{
-			Name:        metrics.MetricRoot + "pha_not_authorized_count",
-			Description: "Total count of authorization failures for the Public Health Authority",
-			Measure:     mHealthAuthorityNotAuthorized,
-			Aggregation: view.Sum(),
+			Name:        metrics.MetricRoot + "requests_count",
+			Description: "Total count of publish requests",
+			Measure:     mLatencyMs,
+			Aggregation: view.Count(),
+			TagKeys:     requestTagKeys,
 		},
 		{
-			Name:        metrics.MetricRoot + "error_loading_app_count",
-			Description: "Total count of errors in loading the authorized application",
-			Measure:     mErrorLoadingAuthorizedApp,
-			Aggregation: view.Sum(),
+			Name:        metrics.MetricRoot + "requests_latency",
+			Description: "Latency distribution of publish requests",
+			Measure:     mLatencyMs,
+			Aggregation: ochttp.DefaultLatencyDistribution,
+			TagKeys:     requestTagKeys,
 		},
 		{
-			Name:        metrics.MetricRoot + "region_not_authorized_count",
-			Description: "Total count of region not being authorized",
-			Measure:     mRegionNotAuthorized,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "region_not_specified_count",
-			Description: "Total count of region not being specified",
-			Measure:     mRegionNotSpecified,
-			Aggregation: view.Sum(),
+			Name:        metrics.MetricRoot + "exposures_count",
+			Description: "Total count of published exposures.",
+			Measure:     mExposuresCount,
+			TagKeys:     exposureTagKeys,
 		},
 		{
 			Name:        metrics.MetricRoot + "verification_bypassed_count",
@@ -87,49 +91,6 @@ func init() {
 			Measure:     mVerificationBypassed,
 			Aggregation: view.Sum(),
 		},
-		{
-			Name:        metrics.MetricRoot + "bad_verification_count",
-			Description: "Total count of bad health authority verification",
-			Measure:     mBadVerification,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "transform_fail_count",
-			Description: "Total count of transformations failures of exposure entities",
-			Measure:     mTransformFail,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "revision_token_issue_count",
-			Description: "Total count of revision token processing issues",
-			Measure:     mRevisionTokenIssue,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "exposures_insertions_count",
-			Description: "Total count of exposure insertions",
-			Measure:     mExposuresInserted,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "exposures_revisions_count",
-			Description: "Total count of exposure revisions",
-			Measure:     mExposuresRevised,
-			Aggregation: view.Sum(),
-		},
-		{
-			Name:        metrics.MetricRoot + "exposures_drops_count",
-			Description: "Total count of exposure drops",
-			Measure:     mExposuresDropped,
-			Aggregation: view.Sum(),
-		},
-		// v1 and v1alpha1
-		{
-			Name:        metrics.MetricRoot + "bad_json",
-			Description: "Total count of bad JSON in incoming requests",
-			Measure:     mBadJSON,
-			Aggregation: view.Sum(),
-		},
 		// v1 and v1alpha1
 		{
 			Name:        metrics.MetricRoot + "padding_failed",
diff --git a/internal/publish/publish.go b/internal/publish/publish.go
index fda9179..f23f250 100644
--- a/internal/publish/publish.go
+++ b/internal/publish/publish.go
@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"go.opencensus.io/stats"
+	"go.opencensus.io/tag"
 	"go.opencensus.io/trace"
 
 	"github.com/google/exposure-notifications-server/internal/authorizedapp"
@@ -41,6 +42,7 @@ import (
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
 	"github.com/google/exposure-notifications-server/pkg/base64util"
 	"github.com/google/exposure-notifications-server/pkg/logging"
+	obs "github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/server"
 	"github.com/gorilla/mux"
 	"github.com/mikehelmick/go-chaff"
@@ -162,9 +164,6 @@ func (s *Server) Routes(ctx context.Context) *mux.Router {
 type response struct {
 	status      int
 	pubResponse *verifyapi.PublishResponse
-
-	// metrics is a function to execute to publish metrics
-	metrics func()
 }
 
 func generatePadding(minPadding, paddingRange int64) (string, error) {
@@ -214,6 +213,10 @@ func (s *Server) process(ctx context.Context, data *verifyapi.Publish, platform
 	ctx, span := trace.StartSpan(ctx, "(*publish.PublishHandler).process")
 	defer span.End()
 
+	blame := obs.BlameNone
+	obsResult := obs.ResultOK
+	defer obs.RecordLatency(ctx, time.Now(), mLatencyMs, &blame, &obsResult)
+
 	logger := logging.FromContext(ctx).Named("process").
 		With("health_authority_id", data.HealthAuthorityID)
 
@@ -227,15 +230,14 @@ func (s *Server) process(ctx context.Context, data *verifyapi.Publish, platform
 		if errors.Is(err, authorizedapp.ErrAppNotFound) {
 			message := fmt.Sprintf("unauthorized health authority: %v", data.HealthAuthorityID)
 			span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
+			blame = obs.BlameClient
+			obsResult = obs.ResultError("ERROR_UNAUTHORIZED_HEALTH_AUTHORITY")
 			return &response{
 				status: http.StatusUnauthorized,
 				pubResponse: &verifyapi.PublishResponse{
 					ErrorMessage: message,
 					Code:         verifyapi.ErrorUnknownHealthAuthorityID,
 				},
-				metrics: func() {
-					stats.Record(ctx, mHealthAuthorityNotAuthorized.M(1))
-				},
 			}
 		}
 
@@ -246,15 +248,14 @@ func (s *Server) process(ctx context.Context, data *verifyapi.Publish, platform
 		// and no other data from the request.
 		message := fmt.Sprintf("error loading health authority config: %v", err)
 		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
+		blame = obs.BlameServer
+		obsResult = obs.ResultError("ERROR_LOADING_HEALTH_AUTHORITY")
 		return &response{
 			status: http.StatusNotFound,
 			pubResponse: &verifyapi.PublishResponse{
 				ErrorMessage: message,
 				Code:         verifyapi.ErrorUnableToLoadHealthAuthority,
 			},
-			metrics: func() {
-				stats.Record(ctx, mErrorLoadingAuthorizedApp.M(1))
-			},
 		}
 	}
 
@@ -269,14 +270,13 @@ func (s *Server) process(ctx context.Context, data *verifyapi.Publish, platform
 				err := fmt.Errorf("app %v tried to write to unauthorized region %v", appConfig.AppPackageName, r)
 				message := fmt.Sprintf("verifying allowed regions: %v", err)
 				span.SetStatus(trace.Status{Code: trace.StatusCodePermissionDenied, Message: message})
+				blame = obs.BlameClient
+				obsResult = obs.ResultError("ERROR_REGION_NOT_AUTHORIZED")
 				return &response{
 					status: http.StatusUnauthorized,
 					pubResponse: &verifyapi.PublishResponse{
 						ErrorMessage: message, // Error code omitted, since this isn't in the v1 path.
 					},
-					metrics: func() {
-						stats.Record(ctx, mRegionNotAuthorized.M(1))
-					},
 				}
 			}
 		}
@@ -298,15 +298,14 @@ func (s *Server) process(ctx context.Context, data *verifyapi.Publish, platform
 		logger.Errorf("No regions present in request or configured for HealthAuthorityID: %v", data.HealthAuthorityID)
 		message := fmt.Sprintf("unknown health authority regions for %v", data.HealthAuthorityID)
 		span.SetStatus(trace.Status{Code: trace.StatusCodePermissionDenied, Message: message})
+		blame = obs.BlameClient
+		obsResult = obs.ResultError("ERROR_REGION_NOT_SPECIFIED")
 		return &response{
 			status: http.StatusInternalServerError,
 			pubResponse: &verifyapi.PublishResponse{
 				ErrorMessage: message,
 				Code:         verifyapi.ErrorHealthAuthorityMissingRegionConfiguration,
 			},
-			metrics: func() {
-				stats.Record(ctx, mRegionNotSpecified.M(1))
-			},
 		}
 	}
 
@@ -324,15 +323,14 @@ func (s *Server) process(ctx context.Context, data *verifyapi.Publish, platform
 				logger.Errorw(message, "error", err)
 			}
 			span.SetStatus(trace.Status{Code: trace.StatusCodeInvalidArgument, Message: message})
+			blame = obs.BlameClient
+			obsResult = obs.ResultError("BAD_VERIFICATION")
 			return &response{
 				status: http.StatusUnauthorized,
 				pubResponse: &verifyapi.PublishResponse{
 					ErrorMessage: message,
 					Code:         verifyapi.ErrorVerificationCertificateInvalid,
 				},
-				metrics: func() {
-					stats.Record(ctx, mBadVerification.M(1))
-				},
 			}
 		}
 	}
@@ -367,6 +365,8 @@ func (s *Server) process(ctx context.Context, data *verifyapi.Publish, platform
 		message := fmt.Sprintf("unable to read request data: %v", transformError)
 		logger.Error(message)
 		span.SetStatus(trace.Status{Code: trace.StatusCodeInvalidArgument, Message: message})
+		blame = obs.BlameClient
+		obsResult = obs.ResultError("TRANSFORM_FAILED")
 		return &response{
 			status: http.StatusBadRequest,
 			pubResponse: &verifyapi.PublishResponse{
@@ -374,9 +374,6 @@ func (s *Server) process(ctx context.Context, data *verifyapi.Publish, platform
 				Code:         verifyapi.ErrorBadRequest,
 				Warnings:     transformWarnings,
 			},
-			metrics: func() {
-				stats.Record(ctx, mTransformFail.M(1))
-			},
 		}
 	}
 
@@ -397,30 +394,38 @@ func (s *Server) process(ctx context.Context, data *verifyapi.Publish, platform
 		status := http.StatusBadRequest
 		var logMessage, errorMessage, errorCode string
 		var errInvalidReportTypeTransition *model.ErrorKeyInvalidReportTypeTransition
-		metric := "publish-revision-token-issue"
 		switch {
 		case decryptFail || errors.Is(err, database.ErrExistingKeyNotInToken) || errors.Is(err, database.ErrRevisionTokenMetadataMismatch):
 			logMessage = fmt.Sprintf("revision token present, but invalid: %v", err)
 			errorMessage = "revision token is invalid"
 			errorCode = verifyapi.ErrorInvalidRevisionToken
+			blame = obs.BlameClient
+			obsResult = obs.ResultError("INVALID_REVISION_TOKEN")
 		case errors.Is(err, database.ErrNoRevisionToken):
 			logMessage = "no revision token"
 			errorMessage = "no revision token, but sent existing keys"
 			errorCode = verifyapi.ErrorMissingRevisionToken
+			blame = obs.BlameClient
+			obsResult = obs.ResultError("MISSING_REVISION_TOKEN")
 		case errors.Is(err, model.ErrorKeyAlreadyRevised):
 			logMessage = "key already revised"
 			errorMessage = "key was already revised"
 			errorCode = verifyapi.ErrorKeyAlreadyRevised
+			blame = obs.BlameClient
+			obsResult = obs.ResultError("KEY_ALREADY_REVISED")
 		case errors.As(err, &errInvalidReportTypeTransition):
 			logMessage = errInvalidReportTypeTransition.Error()
 			errorMessage = errInvalidReportTypeTransition.Error()
 			errorCode = verifyapi.ErrorInvalidReportTypeTransition
+			blame = obs.BlameClient
+			obsResult = obs.ResultError("INVALID_REPORT_TYPE_TRANSITION")
 		default:
 			logMessage = fmt.Sprintf("error writing exposure record: %v", err)
 			errorMessage = http.StatusText(http.StatusInternalServerError)
 			errorCode = verifyapi.ErrorInternalError
-			metric = "publish-db-write-error"
 			logger.Errorw("publish error", "error", logMessage)
+			blame = obs.BlameServer
+			obsResult = obs.ResultError("ERROR_DB_WRITE")
 		}
 		logger.Debugw("publish error", "error", logMessage)
 		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: logMessage})
@@ -431,12 +436,6 @@ func (s *Server) process(ctx context.Context, data *verifyapi.Publish, platform
 				Code:         errorCode,
 				Warnings:     transformWarnings,
 			},
-			metrics: func() {
-				// TODO(yegle): we want to use this string later when we
-				// refactor the metrics.
-				_ = metric
-				stats.Record(ctx, mRevisionTokenIssue.M(1))
-			},
 		}
 	}
 
@@ -478,14 +477,20 @@ func (s *Server) process(ctx context.Context, data *verifyapi.Publish, platform
 		publishResponse.ErrorMessage = transformError.Error()
 	}
 
+	exposureCounts := map[tag.Mutator]uint32{
+		exposuresInserted: resp.Inserted,
+		exposuresRevised:  resp.Revised,
+		exposuresDropped:  resp.Dropped,
+	}
+	for t, n := range exposureCounts {
+		if err := stats.RecordWithTags(ctx, []tag.Mutator{t}, mExposuresCount.M(int64(n))); err != nil {
+			logger.Errorw("failed to record stats", "error", err)
+		}
+	}
+
 	return &response{
 		status:      http.StatusOK,
 		pubResponse: &publishResponse,
-		metrics: func() {
-			stats.Record(ctx, mExposuresInserted.M(int64(resp.Inserted)))
-			stats.Record(ctx, mExposuresRevised.M(int64(resp.Revised)))
-			stats.Record(ctx, mExposuresDropped.M(int64(resp.Dropped)))
-		},
 	}
 }
 
diff --git a/internal/publish/publish_v1.go b/internal/publish/publish_v1.go
index 67c7e83..4bac729 100644
--- a/internal/publish/publish_v1.go
+++ b/internal/publish/publish_v1.go
@@ -18,6 +18,7 @@ package publish
 import (
 	"fmt"
 	"net/http"
+	"time"
 
 	"go.opencensus.io/stats"
 	"go.opencensus.io/trace"
@@ -25,6 +26,7 @@ import (
 	"github.com/google/exposure-notifications-server/internal/jsonutil"
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
 	"github.com/google/exposure-notifications-server/pkg/logging"
+	obs "github.com/google/exposure-notifications-server/pkg/observability"
 )
 
 func (s *Server) handleRequest(w http.ResponseWriter, r *http.Request) *response {
@@ -36,6 +38,9 @@ func (s *Server) handleRequest(w http.ResponseWriter, r *http.Request) *response
 	var data verifyapi.Publish
 	code, err := jsonutil.Unmarshal(w, r, &data)
 	if err != nil {
+		blame := obs.BlameClient
+		obsResult := obs.ResultError("BAD_JSON")
+		defer obs.RecordLatency(ctx, time.Now(), mLatencyMs, &blame, &obsResult)
 		message := fmt.Sprintf("error unmarshalling API call, code: %v: %v", code, err)
 		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
 		errorCode := verifyapi.ErrorBadRequest
@@ -48,9 +53,6 @@ func (s *Server) handleRequest(w http.ResponseWriter, r *http.Request) *response
 				ErrorMessage: message,
 				Code:         errorCode,
 			},
-			metrics: func() {
-				stats.Record(ctx, mBadJSON.M(1))
-			},
 		}
 	}
 
@@ -74,10 +76,6 @@ func (s *Server) handlePublishV1() http.Handler {
 			response.pubResponse.Padding = padding
 		}
 
-		if response.metrics != nil {
-			response.metrics()
-		}
-
 		jsonutil.MarshalResponse(w, response.status, response.pubResponse)
 	})
 }
diff --git a/internal/publish/publish_v1alpha1.go b/internal/publish/publish_v1alpha1.go
index e0bc9d9..6708494 100644
--- a/internal/publish/publish_v1alpha1.go
+++ b/internal/publish/publish_v1alpha1.go
@@ -18,6 +18,7 @@ package publish
 import (
 	"fmt"
 	"net/http"
+	"time"
 
 	"go.opencensus.io/stats"
 	"go.opencensus.io/trace"
@@ -26,6 +27,7 @@ import (
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
 	"github.com/google/exposure-notifications-server/pkg/api/v1alpha1"
 	"github.com/google/exposure-notifications-server/pkg/logging"
+	obs "github.com/google/exposure-notifications-server/pkg/observability"
 )
 
 func (s *Server) handleV1Apha1Request(w http.ResponseWriter, r *http.Request) *response {
@@ -39,12 +41,12 @@ func (s *Server) handleV1Apha1Request(w http.ResponseWriter, r *http.Request) *r
 	if err != nil {
 		message := fmt.Sprintf("error unmarshalling API call, code: %v: %v", code, err)
 		span.SetStatus(trace.Status{Code: trace.StatusCodeInternal, Message: message})
+		blame := obs.BlameClient
+		obsResult := obs.ResultError("BAD_JSON")
+		defer obs.RecordLatency(ctx, time.Now(), mLatencyMs, &blame, &obsResult)
 		return &response{
 			status:      code,
 			pubResponse: &verifyapi.PublishResponse{ErrorMessage: message}, // will be down-converted in ServeHTTP
-			metrics: func() {
-				stats.Record(ctx, mBadJSON.M(1))
-			},
 		}
 	}
 
@@ -100,10 +102,6 @@ func (s *Server) handlePublishV1Alpha1() http.Handler {
 			Warnings:          response.pubResponse.Warnings,
 		}
 
-		if response.metrics != nil {
-			response.metrics()
-		}
-
 		jsonutil.MarshalResponse(w, response.status, alpha1Response)
 	})
 }
diff --git a/internal/verification/model/health_authority.go b/internal/verification/model/health_authority.go
index 359f604..9fa2077 100644
--- a/internal/verification/model/health_authority.go
+++ b/internal/verification/model/health_authority.go
@@ -37,6 +37,11 @@ type HealthAuthority struct {
 	EnableStatsAPI bool
 }
 
+// JWKSEnabled returns true if JWKS discovery is enabled for this health authority.
+func (ha *HealthAuthority) JWKSEnabled() bool {
+	return !(ha.JwksURI == nil || len(*ha.JwksURI) == 0)
+}
+
 // SetJWKS sets the optional JwksURI property of the HealthAuthority.
 func (ha *HealthAuthority) SetJWKS(uri string) {
 	uri = project.TrimSpaceAndNonPrintable(uri)
diff --git a/pkg/errcmp/errcmp.go b/pkg/errcmp/errcmp.go
new file mode 100644
index 0000000..ca442bb
--- /dev/null
+++ b/pkg/errcmp/errcmp.go
@@ -0,0 +1,38 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package errcmp contins a convince helper for checking error conditions in tests
+package errcmp
+
+import (
+	"strings"
+	"testing"
+)
+
+func MustMatch(t testing.TB, err error, want string) {
+	t.Helper()
+
+	if err == nil {
+		if want != "" {
+			t.Fatalf("missing error, want: %q got: nil", want)
+		}
+		// want == "" is success
+	} else if err != nil {
+		if want == "" {
+			t.Fatalf("unexpected error: got: %v", err)
+		} else if !strings.Contains(err.Error(), want) {
+			t.Fatalf("wrong error; want: %q got: %v", want, err)
+		}
+	}
+}
diff --git a/terraform/service_jwks.tf b/terraform/service_jwks.tf
index 6327fa7..07eadc8 100644
--- a/terraform/service_jwks.tf
+++ b/terraform/service_jwks.tf
@@ -158,7 +158,7 @@ resource "google_cloud_run_service_iam_member" "jwks-invoker" {
 resource "google_cloud_scheduler_job" "jwks-worker" {
   name             = "jwks-worker"
   region           = var.cloudscheduler_location
-  schedule         = "*/5 * * * *"
+  schedule         = "*/2 * * * *"
   time_zone        = "America/Los_Angeles"
   attempt_deadline = "600s"
 


Test patch:
diff --git a/internal/authorizedapp/database/authorized_app_test.go b/internal/authorizedapp/database/authorized_app_test.go
index 2aa5eef..3492c01 100644
--- a/internal/authorizedapp/database/authorized_app_test.go
+++ b/internal/authorizedapp/database/authorized_app_test.go
@@ -16,11 +16,11 @@ package database
 
 import (
 	"crypto/ecdsa"
-	"strings"
 	"testing"
 
 	"github.com/google/exposure-notifications-server/internal/authorizedapp/model"
 	"github.com/google/exposure-notifications-server/internal/project"
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 )
@@ -34,12 +34,8 @@ func TestAuthorizedAppInsert_Errors(t *testing.T) {
 
 	source := &model.AuthorizedApp{}
 
-	validationError := "AuthorizedApp invalid: Health Authority ID cannot be empty"
-	if err := aadb.InsertAuthorizedApp(ctx, source); err == nil {
-		t.Fatal("expected validation error")
-	} else if !strings.Contains(err.Error(), validationError) {
-		t.Fatalf("wrong error, want: %q got: %q", validationError, err.Error())
-	}
+	err := aadb.InsertAuthorizedApp(ctx, source)
+	errcmp.MustMatch(t, err, "AuthorizedApp invalid: Health Authority ID cannot be empty")
 
 	source.AppPackageName = "foo"
 	source.AllowedRegions = map[string]struct{}{"US": {}}
@@ -103,12 +99,8 @@ func TestAuthorizedAppLifecycle(t *testing.T) {
 	}
 
 	// attempt to delete already deleted app.
-	expectedError := "no rows were deleted"
-	if err := aadb.DeleteAuthorizedApp(ctx, source.AppPackageName); err == nil {
-		t.Fatal("expected error deleting already deleted app")
-	} else if !strings.Contains(err.Error(), expectedError) {
-		t.Fatalf("wrong error, want: %q got: %q", expectedError, err.Error())
-	}
+	err = aadb.DeleteAuthorizedApp(ctx, source.AppPackageName)
+	errcmp.MustMatch(t, err, "no rows were deleted")
 }
 
 func TestUpdateAuthorizedApp_NoRows(t *testing.T) {
@@ -128,12 +120,8 @@ func TestUpdateAuthorizedApp_NoRows(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	expectedError := "no rows updated"
-	if err := aadb.UpdateAuthorizedApp(ctx, "wrongKey", source); err == nil {
-		t.Fatal("expected error, but didn't get one")
-	} else if !strings.Contains(err.Error(), expectedError) {
-		t.Fatalf("wrong error, want: %q got: %q", expectedError, err.Error())
-	}
+	err := aadb.UpdateAuthorizedApp(ctx, "wrongKey", source)
+	errcmp.MustMatch(t, err, "no rows updated")
 }
 
 func TestUpdateAuthorizedApp_Errors(t *testing.T) {
@@ -163,12 +151,8 @@ func TestUpdateAuthorizedApp_Errors(t *testing.T) {
 
 	appB.AppPackageName = appA.AppPackageName
 
-	expectedError := "updating authorizedapp: ERROR: duplicate key value violates unique constraint"
-	if err := aadb.UpdateAuthorizedApp(ctx, "appB", appB); err == nil {
-		t.Fatal("expected error, but didn't get one")
-	} else if !strings.Contains(err.Error(), expectedError) {
-		t.Fatalf("wrong error, want: %q got: %q", expectedError, err.Error())
-	}
+	err := aadb.UpdateAuthorizedApp(ctx, "appB", appB)
+	errcmp.MustMatch(t, err, "updating authorizedapp: ERROR: duplicate key value violates unique constraint")
 }
 
 func TestGetAuthorizedApp(t *testing.T) {
diff --git a/internal/exportimport/scheduler_test.go b/internal/exportimport/scheduler_test.go
index 5940f74..f76d88b 100644
--- a/internal/exportimport/scheduler_test.go
+++ b/internal/exportimport/scheduler_test.go
@@ -23,6 +23,7 @@ import (
 	exportimportdb "github.com/google/exposure-notifications-server/internal/exportimport/database"
 	"github.com/google/exposure-notifications-server/internal/exportimport/model"
 	"github.com/google/exposure-notifications-server/internal/project"
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 )
@@ -49,11 +50,8 @@ func TestSyncFileFromIndexErrorsInExportRoot(t *testing.T) {
 	// test data ensures that URL parsing strips extra slashes.
 	index := strings.Join([]string{"a.zip", "/b.zip", "//c.zip", ""}, "\n")
 
-	if _, _, err := syncFilesFromIndex(ctx, exportImportDB, config, index); err == nil {
-		t.Fatalf("expected error")
-	} else if !strings.Contains(err.Error(), "invalid URL escape") {
-		t.Fatalf("wrong error, wanted: invalid URL escape, got: %v", err)
-	}
+	_, _, err := syncFilesFromIndex(ctx, exportImportDB, config, index)
+	errcmp.MustMatch(t, err, "invalid URL escape")
 }
 
 func TestSyncFilenameShapes(t *testing.T) {
diff --git a/internal/jwks/jwks_test.go b/internal/jwks/jwks_test.go
index 0910040..9efa6c4 100644
--- a/internal/jwks/jwks_test.go
+++ b/internal/jwks/jwks_test.go
@@ -27,6 +27,7 @@ import (
 	"github.com/google/exposure-notifications-server/internal/project"
 	hadb "github.com/google/exposure-notifications-server/internal/verification/database"
 	"github.com/google/exposure-notifications-server/internal/verification/model"
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 )
 
 var testDatabaseInstance *database.TestInstance
@@ -142,7 +143,7 @@ func TestUpdateHA(t *testing.T) {
 			// Set up the tc.
 			ctx := project.TestContext(t)
 			testDB, _ := testDatabaseInstance.NewDatabase(t)
-			mgr, err := NewManager(testDB, time.Minute, 5*time.Second)
+			mgr, err := NewManager(testDB, time.Minute, 5*time.Second, 2)
 			if err != nil {
 				t.Fatalf("unexpected error: %v", err)
 			}
@@ -256,3 +257,64 @@ func TestStrip(t *testing.T) {
 		}
 	}
 }
+
+func TestUpdateAll(t *testing.T) {
+	t.Parallel()
+
+	docContents := encodeKeys(key1, key2)
+
+	cases := []struct {
+		name  string
+		delay time.Duration
+		err   string
+	}{
+		{
+			name:  "successful",
+			delay: time.Duration(0),
+			err:   "",
+		},
+		{
+			name:  "failure",
+			delay: 2 * time.Second,
+			err:   "failed to processes ha.1: reading connection",
+		},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			// Set up the tc.
+			ctx := project.TestContext(t)
+			testDB, _ := testDatabaseInstance.NewDatabase(t)
+			haDB := hadb.New(testDB)
+			mgr, err := NewManager(testDB, time.Minute, 1*time.Second, 2)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				time.Sleep(tc.delay)
+				fmt.Fprint(w, docContents)
+				w.Header().Set("Content-Type", "application/json")
+			}))
+			defer ts.Close()
+
+			// Add test health authorities.
+			healthAuthorities := make([]*model.HealthAuthority, 0, 5)
+			for i := 0; i < cap(healthAuthorities); i++ {
+				ha := &model.HealthAuthority{
+					Issuer:   fmt.Sprintf("iss.%d", i),
+					Audience: "aud",
+					Name:     fmt.Sprintf("ha.%d", i),
+					JwksURI:  &ts.URL,
+				}
+				if err := haDB.AddHealthAuthority(ctx, ha); err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			errcmp.MustMatch(t, mgr.UpdateAll(ctx), tc.err)
+		})
+	}
+}
diff --git a/internal/mirror/mirror_test.go b/internal/mirror/mirror_test.go
index 61364cf..c90d6bc 100644
--- a/internal/mirror/mirror_test.go
+++ b/internal/mirror/mirror_test.go
@@ -21,7 +21,6 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"sort"
-	"strings"
 	"testing"
 	"time"
 
@@ -31,6 +30,7 @@ import (
 	"github.com/google/exposure-notifications-server/internal/project"
 	"github.com/google/exposure-notifications-server/internal/serverenv"
 	"github.com/google/exposure-notifications-server/internal/storage"
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 	"github.com/google/go-cmp/cmp"
 	"github.com/gorilla/mux"
 	"github.com/sethvargo/go-envconfig"
@@ -315,12 +315,7 @@ func TestServer_DownloadIndex(t *testing.T) {
 		_, err = s.downloadIndex(ctx, &mirrormodel.Mirror{
 			IndexFile: ts.URL,
 		})
-		if err == nil {
-			t.Fatal("expected error")
-		}
-		if got, want := err.Error(), "failed to download"; !strings.Contains(got, want) {
-			t.Errorf("expected %q to contain %q", got, want)
-		}
+		errcmp.MustMatch(t, err, "failed to download")
 	})
 
 	// Client only reads up to configured bytes limit
@@ -343,9 +338,7 @@ func TestServer_DownloadIndex(t *testing.T) {
 		_, err = s.downloadIndex(ctx, &mirrormodel.Mirror{
 			IndexFile: ts.URL,
 		})
-		if got, want := err.Error(), "response exceeds 1 bytes"; !strings.Contains(got, want) {
-			t.Errorf("expected %q to contain %q", got, want)
-		}
+		errcmp.MustMatch(t, err, "response exceeds 1 bytes")
 	})
 
 	// Client handles empty index file
diff --git a/internal/mirror/server_test.go b/internal/mirror/server_test.go
index 082c1bc..18eb4a7 100644
--- a/internal/mirror/server_test.go
+++ b/internal/mirror/server_test.go
@@ -15,14 +15,13 @@
 package mirror
 
 import (
-	"fmt"
-	"strings"
 	"testing"
 
 	"github.com/google/exposure-notifications-server/internal/database"
 	"github.com/google/exposure-notifications-server/internal/project"
 	"github.com/google/exposure-notifications-server/internal/serverenv"
 	"github.com/google/exposure-notifications-server/internal/storage"
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 )
 
 func TestNewServer(t *testing.T) {
@@ -36,24 +35,24 @@ func TestNewServer(t *testing.T) {
 	testCases := []struct {
 		name string
 		env  *serverenv.ServerEnv
-		err  error
+		err  string
 	}{
 		{
 			name: "nil_database",
 			env:  serverenv.New(ctx),
-			err:  fmt.Errorf("missing database in server environment"),
+			err:  "missing database in server environment",
 		},
 		{
 			name: "nil_blobstore",
 			env:  serverenv.New(ctx, serverenv.WithDatabase(emptyDB)),
-			err:  fmt.Errorf("missing blobstore in server environment"),
+			err:  "missing blobstore in server environment",
 		},
 		{
 			name: "fully_specified",
 			env: serverenv.New(ctx,
 				serverenv.WithDatabase(emptyDB),
 				serverenv.WithBlobStorage(emptyBlobstore)),
-			err: nil,
+			err: "",
 		},
 	}
 
@@ -64,13 +63,8 @@ func TestNewServer(t *testing.T) {
 			t.Parallel()
 
 			got, err := NewServer(&Config{}, tc.env)
-			if tc.err != nil {
-				if got, want := err.Error(), tc.err.Error(); !strings.Contains(got, want) {
-					t.Fatalf("expected %q to be %q", got, want)
-				}
-			} else if err != nil {
-				t.Fatal(err)
-			} else {
+			errcmp.MustMatch(t, err, tc.err)
+			if err == nil {
 				if got, want := got.env, tc.env; got != want {
 					t.Fatalf("expected %#v to be %#v", got, want)
 				}
diff --git a/internal/publish/database/exposure_test.go b/internal/publish/database/exposure_test.go
index be6ffcf..ec5bd82 100644
--- a/internal/publish/database/exposure_test.go
+++ b/internal/publish/database/exposure_test.go
@@ -31,6 +31,7 @@ import (
 	"github.com/google/exposure-notifications-server/internal/project"
 	"github.com/google/exposure-notifications-server/internal/publish/model"
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1alpha1"
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 	pgx "github.com/jackc/pgx/v4"
 
 	"github.com/google/go-cmp/cmp"
@@ -328,11 +329,8 @@ func TestInsertAndReviseExposures_MissingRequest(t *testing.T) {
 	testDB, _ := testDatabaseInstance.NewDatabase(t)
 	pubDB := New(testDB)
 
-	if _, err := pubDB.InsertAndReviseExposures(ctx, nil); err == nil {
-		t.Fatal("missing error")
-	} else if !strings.Contains(err.Error(), "missing request") {
-		t.Fatalf("wrong error, want: %v got: %v", "missing request", err.Error())
-	}
+	_, err := pubDB.InsertAndReviseExposures(ctx, nil)
+	errcmp.MustMatch(t, err, "missing request")
 }
 
 func TestInsertAndReviseExposures_ConfigParadox(t *testing.T) {
@@ -347,11 +345,8 @@ func TestInsertAndReviseExposures_ConfigParadox(t *testing.T) {
 		OnlyRevisions: true,
 	}
 
-	if _, err := pubDB.InsertAndReviseExposures(ctx, req); err == nil {
-		t.Fatal("missing error")
-	} else if !strings.Contains(err.Error(), "configuration paradox") {
-		t.Fatalf("wrong error, want: %v got: %v", "configuration paradox", err.Error())
-	}
+	_, err := pubDB.InsertAndReviseExposures(ctx, req)
+	errcmp.MustMatch(t, err, "configuration paradox")
 }
 
 func TestReviseExposures(t *testing.T) {
diff --git a/internal/publish/model/exposure_model_test.go b/internal/publish/model/exposure_model_test.go
index 761af1f..3ae4042 100644
--- a/internal/publish/model/exposure_model_test.go
+++ b/internal/publish/model/exposure_model_test.go
@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"reflect"
 	"sort"
-	"strings"
 	"testing"
 	"time"
 
@@ -30,6 +29,7 @@ import (
 	"github.com/google/exposure-notifications-server/internal/verification"
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
 	"github.com/google/exposure-notifications-server/pkg/base64util"
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 	"github.com/google/exposure-notifications-server/pkg/timeutils"
 	"google.golang.org/protobuf/proto"
 
@@ -102,7 +102,6 @@ func TestIntervalNumber(t *testing.T) {
 func TestInvalidNew(t *testing.T) {
 	t.Parallel()
 
-	errMsg := "maxExposureKeys must be > 0"
 	cases := []struct {
 		maxKeys        uint
 		maxSameDayKeys uint
@@ -114,7 +113,7 @@ func TestInvalidNew(t *testing.T) {
 		{5, 0, "maxSameDayKeys must be >= 1, got"},
 	}
 
-	for i, c := range cases {
+	for _, c := range cases {
 		_, err := NewTransformer(&testConfig{
 			maxExposureKeys:     c.maxKeys,
 			maxSameDayKeys:      c.maxSameDayKeys,
@@ -122,11 +121,7 @@ func TestInvalidNew(t *testing.T) {
 			truncateWindow:      time.Hour,
 			maxSymptomOnsetDays: maxSymptomOnsetDays,
 		})
-		if err != nil && errMsg == "" {
-			t.Errorf("%v unexpected error: %v", i, err)
-		} else if err != nil && !strings.Contains(err.Error(), c.message) {
-			t.Errorf("%v error want '%v', got '%v'", i, c.message, err)
-		}
+		errcmp.MustMatch(t, err, c.message)
 	}
 }
 
@@ -157,10 +152,7 @@ func TestInvalidBase64(t *testing.T) {
 	batchTime := time.Date(2020, 3, 1, 10, 43, 1, 0, time.UTC)
 
 	_, err = transformer.TransformPublish(ctx, source, regions, nil, batchTime)
-	expErr := `key 0 cannot be imported: illegal base64 data at input byte 4`
-	if err == nil || !strings.Contains(err.Error(), expErr) {
-		t.Errorf("expected error '%v', got: %v", expErr, err)
-	}
+	errcmp.MustMatch(t, err, `key 0 cannot be imported: illegal base64 data at input byte 4`)
 }
 
 func TestDifferentEncodings(t *testing.T) {
@@ -363,15 +355,7 @@ func TestPublishValidation(t *testing.T) {
 			}
 
 			_, err = tf.TransformPublish(ctx, c.p, []string{}, nil, captureStartTime)
-			if err == nil {
-				if c.m != "" {
-					t.Errorf("want error '%v', got nil", c.m)
-				}
-			} else if !strings.Contains(err.Error(), c.m) {
-				t.Errorf("want error '%v', got '%v'", c.m, err)
-			} else if err != nil && c.m == "" {
-				t.Errorf("want error nil, got '%v'", err)
-			}
+			errcmp.MustMatch(t, err, c.m)
 		})
 	}
 }
@@ -1046,15 +1030,7 @@ func TestTransform(t *testing.T) {
 			t.Parallel()
 
 			result, err := transformer.TransformPublish(ctx, tc.Publish, tc.Regions, tc.Claims, batchTime)
-			if err != nil && tc.PartialError == "" {
-				t.Fatalf("TransformPublish returned unexpected error: %v", err)
-			} else if tc.PartialError != "" {
-				if err == nil {
-					t.Fatalf("TransformPublish didn't return expected error: %v", tc.PartialError)
-				} else if !strings.Contains(err.Error(), tc.PartialError) {
-					t.Fatalf("TransformPublish didn't return expected error: %q, got: %v", tc.PartialError, err)
-				}
-			}
+			errcmp.MustMatch(t, err, tc.PartialError)
 
 			if exp := tc.Warnings; len(exp) > 0 {
 				sort.Strings(exp)
@@ -1325,13 +1301,7 @@ func TestTransformOverlapping(t *testing.T) {
 				t.Fatalf("NewTransformer returned unexpected error: %v", err)
 			}
 			_, err = transformer.TransformPublish(ctx, &tc.source, tc.regions, nil, now)
-			if err != nil && tc.error == "" {
-				t.Fatalf("unexpected error, want: nil, got: %v", err)
-			} else if err != nil && !strings.Contains(err.Error(), tc.error) {
-				t.Fatalf("wrong error: want '%v', got: %v", tc.error, err.Error())
-			} else if err == nil && tc.error != "" {
-				t.Fatalf("missing error: want '%v', got: nil", tc.error)
-			}
+			errcmp.MustMatch(t, err, tc.error)
 		})
 	}
 }
@@ -1865,13 +1835,7 @@ func TestExposureReview(t *testing.T) {
 			if result != tc.needsRevision {
 				t.Errorf("revision decision mismatch: want: %v got: %v", tc.needsRevision, result)
 			}
-			if err != nil && tc.err == "" {
-				t.Fatalf("unexpected error: %v", err)
-			} else if err != nil && !strings.Contains(err.Error(), tc.err) {
-				t.Fatalf("wrong error: want '%v', got: '%v'", tc.err, err)
-			} else if err == nil && tc.err != "" {
-				t.Fatalf("expected error: want '%v', got: nil", tc.err)
-			}
+			errcmp.MustMatch(t, err, tc.err)
 			if tc.err != "" || !tc.needsRevision {
 				return
 			}
@@ -2140,15 +2104,7 @@ func TestExposureFromExportFile(t *testing.T) {
 			}
 
 			got, err := FromExportKey(tc.key, thisConfig)
-
-			if err != nil && tc.wantError == "" {
-				t.Fatalf("unexpected error: %v", err)
-			} else if err == nil && tc.wantError != "" {
-				t.Fatalf("missing expected error: %q", tc.wantError)
-			} else if err != nil && tc.wantError != "" && !strings.Contains(err.Error(), tc.wantError) {
-				t.Fatalf("wrong error, want: %q, got: %v", tc.wantError, err)
-			}
-
+			errcmp.MustMatch(t, err, tc.wantError)
 			if tc.wantError != "" {
 				return
 			}
diff --git a/internal/verification/authenticate_stats_test.go b/internal/verification/authenticate_stats_test.go
index 0b70db1..34d505b 100644
--- a/internal/verification/authenticate_stats_test.go
+++ b/internal/verification/authenticate_stats_test.go
@@ -21,13 +21,13 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
-	"strings"
 	"testing"
 	"time"
 
 	"github.com/google/exposure-notifications-server/internal/project"
 	"github.com/google/exposure-notifications-server/internal/verification/database"
 	"github.com/google/exposure-notifications-server/internal/verification/model"
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 
 	"github.com/dgrijalva/jwt-go"
 )
@@ -226,16 +226,7 @@ func TestAuthenticateStatsToken(t *testing.T) {
 			}
 
 			gotID, err := verifier.AuthenticateStatsToken(ctx, jwtString)
-
-			if err != nil {
-				if tc.Error == "" {
-					t.Fatalf("unexpected error: %v", err)
-				} else if !strings.Contains(err.Error(), tc.Error) {
-					t.Fatalf("wanted error '%v', got error '%v'", tc.Error, err.Error())
-				}
-			} else if tc.Error != "" {
-				t.Fatalf("wanted error '%v', but got nil", tc.Error)
-			}
+			errcmp.MustMatch(t, err, tc.Error)
 
 			if tc.Error == "" {
 				if gotID != healthAuthority.ID {
diff --git a/internal/verification/database/health_authority_db_test.go b/internal/verification/database/health_authority_db_test.go
index 1e5ef23..0f6a966 100644
--- a/internal/verification/database/health_authority_db_test.go
+++ b/internal/verification/database/health_authority_db_test.go
@@ -16,12 +16,12 @@ package database
 
 import (
 	"errors"
-	"strings"
 	"testing"
 	"time"
 
 	"github.com/google/exposure-notifications-server/internal/project"
 	"github.com/google/exposure-notifications-server/internal/verification/model"
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/google/go-cmp/cmp"
@@ -92,11 +92,8 @@ func TestAddHealthAuthorityErrors(t *testing.T) {
 			t.Parallel()
 
 			ctx := project.TestContext(t)
-			if err := haDB.AddHealthAuthority(ctx, tc.ha); err == nil {
-				t.Error("missing expected error")
-			} else if !strings.Contains(err.Error(), tc.want) {
-				t.Errorf("wrong error: want: %q got: %q", tc.want, err.Error())
-			}
+			err := haDB.AddHealthAuthority(ctx, tc.ha)
+			errcmp.MustMatch(t, err, tc.want)
 		})
 	}
 }
diff --git a/internal/verification/model/health_authority_test.go b/internal/verification/model/health_authority_test.go
index ba82a85..c5f35d8 100644
--- a/internal/verification/model/health_authority_test.go
+++ b/internal/verification/model/health_authority_test.go
@@ -15,10 +15,10 @@
 package model
 
 import (
-	"strings"
 	"testing"
 	"time"
 
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/protobuf/proto"
 )
@@ -196,13 +196,8 @@ NTG3+oqI0Q6a3kPOuAAAupr373j7O1YXrM2KAix966EPwTNlK7YCcJa0m6PKz9DT
 			}
 
 			k, err := hak.PublicKey()
-			if err != nil {
-				if tc.msg == "" {
-					t.Errorf("unexpected error: %v", err)
-				} else if !strings.Contains(err.Error(), tc.msg) {
-					t.Errorf("missing error text: want '%v', got '%v'", tc.msg, err.Error())
-				}
-			} else if k == nil {
+			errcmp.MustMatch(t, err, tc.msg)
+			if err == nil && k == nil {
 				t.Errorf("ECDSA public key is unexpectedly nil")
 			}
 		})
diff --git a/internal/verification/phaverify_test.go b/internal/verification/phaverify_test.go
index 548dbab..00e3f18 100644
--- a/internal/verification/phaverify_test.go
+++ b/internal/verification/phaverify_test.go
@@ -25,7 +25,6 @@ import (
 	"encoding/base64"
 	"encoding/pem"
 	"fmt"
-	"strings"
 	"testing"
 	"time"
 
@@ -39,6 +38,7 @@ import (
 
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
 	"github.com/google/exposure-notifications-server/pkg/api/v1alpha1"
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 	utils "github.com/google/exposure-notifications-server/pkg/verification"
 )
 
@@ -250,15 +250,7 @@ func TestVerifyCertificate(t *testing.T) {
 						t.Fatal(err)
 					}
 					verifiedClaims, err := verifier.VerifyDiagnosisCertificate(ctx, authApp, &publish)
-					if err != nil {
-						if tc.Error == "" {
-							t.Fatalf("unexpected error: %v", err)
-						} else if !strings.Contains(err.Error(), tc.Error) {
-							t.Fatalf("wanted error '%v', got error '%v'", tc.Error, err.Error())
-						}
-					} else if tc.Error != "" {
-						t.Fatalf("wanted error '%v', but got nil", tc.Error)
-					}
+					errcmp.MustMatch(t, err, tc.Error)
 
 					if tc.Error == "" {
 						if verifiedClaims == nil {
diff --git a/pkg/api/v1/veritifcation_types_test.go b/pkg/api/v1/veritifcation_types_test.go
index 70060fc..17ed51e 100644
--- a/pkg/api/v1/veritifcation_types_test.go
+++ b/pkg/api/v1/veritifcation_types_test.go
@@ -15,8 +15,9 @@
 package v1
 
 import (
-	"strings"
 	"testing"
+
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 )
 
 func TestValidateClaims(t *testing.T) {
@@ -25,11 +26,8 @@ func TestValidateClaims(t *testing.T) {
 	c := NewVerificationClaims()
 	c.ReportType = "bogus"
 
-	if err := c.CustomClaimsValid(); err == nil {
-		t.Fatal("expected an error, got nil")
-	} else if !strings.Contains(err.Error(), "bogus") {
-		t.Fatalf("wanted an error that contained bogus, got: %v", err)
-	}
+	err := c.CustomClaimsValid()
+	errcmp.MustMatch(t, err, "bogus")
 
 	for k := range ValidReportTypes {
 		c.ReportType = k
diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go
index f9f8344..cb8b643 100644
--- a/pkg/cache/cache_test.go
+++ b/pkg/cache/cache_test.go
@@ -17,10 +17,10 @@ package cache
 
 import (
 	"fmt"
-	"strings"
 	"testing"
 	"time"
 
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 	"github.com/google/go-cmp/cmp"
 )
 
@@ -137,11 +137,8 @@ func TestWriteThruError(t *testing.T) {
 func TestInvalidDuration(t *testing.T) {
 	t.Parallel()
 
-	if _, err := New(-1 * time.Second); err == nil {
-		t.Fatal("expected error, got nil")
-	} else if !strings.Contains(err.Error(), "duration cannot be negative") {
-		t.Fatalf("wrong error: want: `duration cannot be negative` got: %v", err.Error())
-	}
+	_, err := New(-1 * time.Second)
+	errcmp.MustMatch(t, err, "duration cannot be negative")
 }
 
 func TestConcurrentReaders(t *testing.T) {
diff --git a/pkg/keys/ecdsa_pem_test.go b/pkg/keys/ecdsa_pem_test.go
index 9b82ad3..59b683b 100644
--- a/pkg/keys/ecdsa_pem_test.go
+++ b/pkg/keys/ecdsa_pem_test.go
@@ -19,8 +19,9 @@ import (
 	"crypto/rsa"
 	"crypto/x509"
 	"encoding/pem"
-	"strings"
 	"testing"
+
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 )
 
 func TestParseECDSAPublicKey_DecodeError(t *testing.T) {
@@ -45,10 +46,5 @@ func TestParseECDSAPublicKey_WrongKeyType(t *testing.T) {
 	pemPublicKey := string(pemEncodedPub)
 
 	_, err = ParseECDSAPublicKey(pemPublicKey)
-	if err == nil {
-		t.Fatal("expected error, got nil")
-	}
-	if want := "x509.ParsePKIXPublicKey"; !strings.Contains(err.Error(), want) {
-		t.Fatalf("wrong error, want: %q got: %q", want, err.Error())
-	}
+	errcmp.MustMatch(t, err, "x509.ParsePKIXPublicKey")
 }
diff --git a/pkg/keys/filesystem_test.go b/pkg/keys/filesystem_test.go
index 458274b..e53ff74 100644
--- a/pkg/keys/filesystem_test.go
+++ b/pkg/keys/filesystem_test.go
@@ -23,10 +23,10 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 
 	"github.com/google/exposure-notifications-server/internal/project"
+	"github.com/google/exposure-notifications-server/pkg/errcmp"
 )
 
 func TestNewFilesystem(t *testing.T) {
@@ -177,15 +177,8 @@ func TestFilesystem_NewSigner(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			if _, err := fs.NewSigner(ctx, tc.keyID); err != nil {
-				if tc.err == "" {
-					t.Fatal(err)
-				}
-
-				if !strings.Contains(err.Error(), tc.err) {
-					t.Fatalf("expected %#v to contain %#v", err.Error(), tc.err)
-				}
-			}
+			_, err = fs.NewSigner(ctx, tc.keyID)
+			errcmp.MustMatch(t, err, tc.err)
 		})
 	}
 }
@@ -284,15 +277,7 @@ func TestFilesystem_EncryptDecrypt(t *testing.T) {
 			}
 
 			ciphertext, err := fs.Encrypt(ctx, tc.keyID, tc.plaintext, tc.aad)
-			if err != nil {
-				if tc.err == "" {
-					t.Fatal(err)
-				}
-
-				if !strings.Contains(err.Error(), tc.err) {
-					t.Fatalf("expected %#v to contain %#v", err.Error(), tc.err)
-				}
-			}
+			errcmp.MustMatch(t, err, tc.err)
 
 			if len(ciphertext) > 0 {
 				// Create another key version - this will ensure our ciphertext -> key
@@ -400,15 +385,7 @@ func TestFilesystem_SigningKeyVersions(t *testing.T) {
 			}
 
 			versions, err := fst.SigningKeyVersions(ctx, tc.keyID)
-			if err != nil {
-				if tc.err == "" {
-					t.Fatal(err)
-				}
-
-				if !strings.Contains(err.Error(), tc.err) {
-					t.Fatalf("expected %#v to contain %#v", err.Error(), tc.err)
-				}
-			}
+			errcmp.MustMatch(t, err, tc.err)
 
 			if got, want := len(versions), tc.exp; got != want {
 				t.Errorf("expected %d version to be %d", got, want)
diff --git a/pkg/keys/testing.go b/pkg/keys/testing.go
index 5f79e84..fd29640 100644
--- a/pkg/keys/testing.go
+++ b/pkg/keys/testing.go
@@ -16,6 +16,8 @@ package keys
 
 import (
 	"context"
+	"crypto/rand"
+	"encoding/hex"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -59,7 +61,7 @@ func TestEncryptionKey(tb testing.TB, kms KeyManager) string {
 	}
 
 	ctx := context.Background()
-	parent, err := typ.CreateEncryptionKey(ctx, "parent", "key")
+	parent, err := typ.CreateEncryptionKey(ctx, randomPrefix(tb, 8), "key")
 	if err != nil {
 		tb.Fatal(err)
 	}
@@ -82,7 +84,7 @@ func TestSigningKey(tb testing.TB, kms KeyManager) string {
 	}
 
 	ctx := context.Background()
-	parent, err := typ.CreateSigningKey(ctx, "parent", "key")
+	parent, err := typ.CreateSigningKey(ctx, randomPrefix(tb, 8), "key")
 	if err != nil {
 		tb.Fatal(err)
 	}
@@ -92,3 +94,17 @@ func TestSigningKey(tb testing.TB, kms KeyManager) string {
 
 	return parent
 }
+
+func randomPrefix(tb testing.TB, length int) string {
+	tb.Helper()
+
+	b := make([]byte, length)
+	n, err := rand.Read(b)
+	if err != nil {
+		tb.Fatalf("failed to generate random: %v", err)
+	}
+	if n < length {
+		tb.Fatalf("insufficient bytes read: %v, expected %v", n, length)
+	}
+	return hex.EncodeToString(b)[:length]
+}