Repository: google/exposure-notifications-server
Pull request #828 (2020-08-06 20:07:40)

Key Rotation E2E Test
Fixes https://github.com/google/exposure-notifications-server/issues/791

<!-- Please include the 'why' behind your changes if no issue exists -->
## Proposed Changes

* Start the key rotation endpoint in the testserver
    * Set it to rotate ~seconds
* Create an e2e test that asserts new keys created
* Change the error message if a revision token is passed, but doesn't decrypt

**Release Note**

<!-- Enter your extended release note in the below block. If the PR requires
additional action from users switching to the new release, include the string
"action required". If no release note is required, write "NONE". -->

```release-note
E2E test for key rotation
```

google-oss-robot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/google/exposure-notifications-server/pull/828#" title="Author self-approved">whaught</a>*
To complete the [pull request process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process), please assign **sethvargo**
You can assign the PR to them by writing `/assign @sethvargo` in a comment when ready.

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=google%2Fexposure-notifications-server).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/google/exposure-notifications-server/blob/main/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["sethvargo"]} -->
google-oss-robot: @whaught: The following test **failed**, say `/retest` to rerun all failed tests:

Test name | Commit | Details | Rerun command
--- | --- | --- | ---
pull-en-server-release-unit | 75e74198768d68c2bf3656324c12a9cf6d015af6 | [link](https://oss-prow.knative.dev/view/gs/oss-prow/pr-logs/pull/google_exposure-notifications-server/828/pull-en-server-release-unit/1291466168883744768) | `/test pull-en-server-release-unit`

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository. I understand the commands that are listed [here](https://go.k8s.io/bot-commands).
</details>
<!-- test report -->
google-oss-robot: @whaught: The following test **failed**, say `/retest` to rerun all failed tests:

Test name | Commit | Details | Rerun command
--- | --- | --- | ---
pull-en-server-release-unit | 6f55a6656885cf34a241776123ac4890b017ecd9 | [link](https://oss-prow.knative.dev/view/gs/oss-prow/pr-logs/pull/google_exposure-notifications-server/828/pull-en-server-release-unit/1291845166218350592) | `/test pull-en-server-release-unit`

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository. I understand the commands that are listed [here](https://go.k8s.io/bot-commands).
</details>
<!-- test report -->

Patch:
diff --git a/cmd/cleanup-export/main.go b/cmd/cleanup-export/main.go
index b8d1225..711cf62 100644
--- a/cmd/cleanup-export/main.go
+++ b/cmd/cleanup-export/main.go
@@ -22,8 +22,8 @@ import (
 
 	"github.com/google/exposure-notifications-server/internal/cleanup"
 	"github.com/google/exposure-notifications-server/internal/logging"
-	_ "github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/setup"
+	_ "github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/server"
 	"github.com/sethvargo/go-signalcontext"
 )
diff --git a/cmd/cleanup-exposure/main.go b/cmd/cleanup-exposure/main.go
index b44b256..0ef72f7 100644
--- a/cmd/cleanup-exposure/main.go
+++ b/cmd/cleanup-exposure/main.go
@@ -22,8 +22,8 @@ import (
 
 	"github.com/google/exposure-notifications-server/internal/cleanup"
 	"github.com/google/exposure-notifications-server/internal/logging"
-	_ "github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/setup"
+	_ "github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/server"
 	"github.com/sethvargo/go-signalcontext"
 )
diff --git a/cmd/export/main.go b/cmd/export/main.go
index db3d7db..5edbdae 100644
--- a/cmd/export/main.go
+++ b/cmd/export/main.go
@@ -21,8 +21,8 @@ import (
 
 	"github.com/google/exposure-notifications-server/internal/export"
 	"github.com/google/exposure-notifications-server/internal/logging"
-	_ "github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/setup"
+	_ "github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/server"
 	"github.com/sethvargo/go-signalcontext"
 )
diff --git a/cmd/exposure/main.go b/cmd/exposure/main.go
index eb53513..3d7e1fe 100644
--- a/cmd/exposure/main.go
+++ b/cmd/exposure/main.go
@@ -21,9 +21,9 @@ import (
 	"net/http"
 
 	"github.com/google/exposure-notifications-server/internal/logging"
-	_ "github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/publish"
 	"github.com/google/exposure-notifications-server/internal/setup"
+	_ "github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/server"
 	"github.com/sethvargo/go-signalcontext"
 )
diff --git a/cmd/federationin/main.go b/cmd/federationin/main.go
index c77e381..4c3edc9 100644
--- a/cmd/federationin/main.go
+++ b/cmd/federationin/main.go
@@ -23,8 +23,8 @@ import (
 
 	"github.com/google/exposure-notifications-server/internal/federationin"
 	"github.com/google/exposure-notifications-server/internal/logging"
-	_ "github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/setup"
+	_ "github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/server"
 	"github.com/sethvargo/go-signalcontext"
 )
diff --git a/cmd/federationout/main.go b/cmd/federationout/main.go
index 9815d06..f10bbc1 100644
--- a/cmd/federationout/main.go
+++ b/cmd/federationout/main.go
@@ -26,9 +26,9 @@ import (
 
 	"github.com/google/exposure-notifications-server/internal/federationout"
 	"github.com/google/exposure-notifications-server/internal/logging"
-	_ "github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/pb"
 	"github.com/google/exposure-notifications-server/internal/setup"
+	_ "github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/server"
 	"github.com/sethvargo/go-signalcontext"
 )
diff --git a/cmd/generate/main.go b/cmd/generate/main.go
index c5e42ad..309e09b 100644
--- a/cmd/generate/main.go
+++ b/cmd/generate/main.go
@@ -23,8 +23,8 @@ import (
 
 	"github.com/google/exposure-notifications-server/internal/generate"
 	"github.com/google/exposure-notifications-server/internal/logging"
-	_ "github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/setup"
+	_ "github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/server"
 	"github.com/sethvargo/go-signalcontext"
 )
diff --git a/cmd/key-rotation/main.go b/cmd/key-rotation/main.go
index 20058eb..a8fe294 100644
--- a/cmd/key-rotation/main.go
+++ b/cmd/key-rotation/main.go
@@ -21,8 +21,8 @@ import (
 
 	"github.com/google/exposure-notifications-server/internal/keyrotation"
 	"github.com/google/exposure-notifications-server/internal/logging"
-	_ "github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/setup"
+	_ "github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/server"
 	"github.com/sethvargo/go-signalcontext"
 )
diff --git a/docs/getting-started/deploying.md b/docs/getting-started/deploying.md
index b8cb000..e97c254 100644
--- a/docs/getting-started/deploying.md
+++ b/docs/getting-started/deploying.md
@@ -161,7 +161,7 @@ To enable distributed tracing, please ensure your environment has these variable
 Variable|Values|Comment
 ---|---|---
 OBSERVABILITY_EXPORTER|If unset, no exporting shall be done. Use any of "stackdriver", "prometheus", or "ocagent" otherwise|Note: when using "prometheus" a METRICS_PORT environment variable should also be set
-PROJECT_ID|The ProjectID of your associated Google Cloud Platform project on which this application shall be deployed|Required if you use "stackdrver"
+PROJECT_ID|The ProjectID of your associated Google Cloud Platform project on which this application shall be deployed|Required if you use "stackdriver"
 
 ## Running migrations
 
diff --git a/internal/cleanup/config.go b/internal/cleanup/config.go
index 99766bb..65879fd 100644
--- a/internal/cleanup/config.go
+++ b/internal/cleanup/config.go
@@ -18,9 +18,9 @@ import (
 	"time"
 
 	"github.com/google/exposure-notifications-server/internal/database"
-	"github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/setup"
 	"github.com/google/exposure-notifications-server/internal/storage"
+	"github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/secrets"
 )
 
diff --git a/internal/export/config.go b/internal/export/config.go
index aefffac..5b39669 100644
--- a/internal/export/config.go
+++ b/internal/export/config.go
@@ -19,10 +19,10 @@ import (
 	"time"
 
 	"github.com/google/exposure-notifications-server/internal/database"
-	"github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/setup"
 	"github.com/google/exposure-notifications-server/internal/storage"
 	"github.com/google/exposure-notifications-server/pkg/keys"
+	"github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/secrets"
 )
 
diff --git a/internal/federationin/config.go b/internal/federationin/config.go
index e7eb2e8..a853ffc 100644
--- a/internal/federationin/config.go
+++ b/internal/federationin/config.go
@@ -19,8 +19,8 @@ import (
 	"time"
 
 	"github.com/google/exposure-notifications-server/internal/database"
-	"github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/setup"
+	"github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/secrets"
 )
 
diff --git a/internal/federationout/config.go b/internal/federationout/config.go
index 8329d97..e60f2ac 100644
--- a/internal/federationout/config.go
+++ b/internal/federationout/config.go
@@ -18,8 +18,8 @@ import (
 	"time"
 
 	"github.com/google/exposure-notifications-server/internal/database"
-	"github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/setup"
+	"github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/secrets"
 )
 
diff --git a/internal/generate/config.go b/internal/generate/config.go
index 313b768..445195d 100644
--- a/internal/generate/config.go
+++ b/internal/generate/config.go
@@ -19,9 +19,9 @@ import (
 	"time"
 
 	"github.com/google/exposure-notifications-server/internal/database"
-	"github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/publish/model"
 	"github.com/google/exposure-notifications-server/internal/setup"
+	"github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/secrets"
 )
 
diff --git a/internal/integration/client.go b/internal/integration/client.go
index 2832d1c..6c46c46 100644
--- a/internal/integration/client.go
+++ b/internal/integration/client.go
@@ -94,6 +94,19 @@ func (c *Client) ExportBatches() error {
 	return nil
 }
 
+func (c *Client) RotateKeys() error {
+	resp, err := c.client.Get("/key-rotation/rotate-keys")
+	if err != nil {
+		return fmt.Errorf("failed to GET /key-rotation/rotate-keys: %w", err)
+	}
+
+	body, err := checkResp(resp)
+	if err != nil {
+		return fmt.Errorf("failed to GET /key-rotation/rotate-keys: %w: %s", err, body)
+	}
+	return nil
+}
+
 func (c *Client) StartExportWorkers() error {
 	resp, err := c.client.Get("/export/do-work")
 	if err != nil {
diff --git a/internal/integration/helpers.go b/internal/integration/helpers.go
index 298794a..3a4639d 100644
--- a/internal/integration/helpers.go
+++ b/internal/integration/helpers.go
@@ -30,6 +30,7 @@ import (
 	"github.com/google/exposure-notifications-server/internal/database"
 	"github.com/google/exposure-notifications-server/internal/export"
 	"github.com/google/exposure-notifications-server/internal/federationin"
+	"github.com/google/exposure-notifications-server/internal/keyrotation"
 	"github.com/google/exposure-notifications-server/internal/publish"
 	"github.com/google/exposure-notifications-server/internal/revision"
 	revdb "github.com/google/exposure-notifications-server/internal/revision/database"
@@ -52,13 +53,12 @@ import (
 var (
 	ExportDir    = "my-bucket"
 	FileNameRoot = "/"
-	jwtCfg       = testutil.JWTConfig{}
 )
 
 // NewTestServer sets up clients used for integration tests
-func NewTestServer(tb testing.TB, exportPeriod time.Duration) (*serverenv.ServerEnv, *Client, *database.DB) {
+func NewTestServer(tb testing.TB, exportPeriod time.Duration) (*serverenv.ServerEnv, *Client, *database.DB, testutil.JWTConfig) {
 	ctx := context.Background()
-	env, client := testServer(tb)
+	env, client, jwtCfg := testServer(tb)
 	db := env.Database()
 	enClient := &Client{client: client}
 
@@ -103,14 +103,17 @@ func NewTestServer(tb testing.TB, exportPeriod time.Duration) (*serverenv.Server
 		tb.Fatal(err)
 	}
 
-	return env, enClient, db
+	return env, enClient, db, jwtCfg
 }
 
 // testServer sets up mocked local servers for running tests
-func testServer(tb testing.TB) (*serverenv.ServerEnv, *http.Client) {
+func testServer(tb testing.TB) (*serverenv.ServerEnv, *http.Client, testutil.JWTConfig) {
 	tb.Helper()
 
-	ctx := context.Background()
+	var (
+		ctx    = context.Background()
+		jwtCfg = testutil.JWTConfig{}
+	)
 
 	aa, err := authorizedapp.NewMemoryProvider(ctx, nil)
 	if err != nil {
@@ -135,10 +138,10 @@ func testServer(tb testing.TB) (*serverenv.ServerEnv, *http.Client) {
 	if err != nil {
 		tb.Fatal(err)
 	}
-	if err := km.AddEncryptionKey("tokenkey"); err != nil {
+	if _, err := km.CreateEncryptionKey("tokenkey"); err != nil {
 		tb.Fatal(err)
 	}
-	if err := km.AddSigningKey("signingkey"); err != nil {
+	if _, err := km.CreateSigningKey("signingkey"); err != nil {
 		tb.Fatal(err)
 	}
 	// create an initial revision key.
@@ -248,13 +251,30 @@ func testServer(tb testing.TB) (*serverenv.ServerEnv, *http.Client) {
 	// Federation out
 	// TODO: this is a grpc listener and requires a lot of setup.
 
+	revConfig := revision.Config{
+		KeyID:     "tokenkey",
+		AAD:       []byte{1, 2, 3},
+		MinLength: 28,
+	}
+
+	// Key Rotation
+	keyRotationConfig := &keyrotation.Config{
+		RevisionToken: revConfig,
+
+		// Very accellerated schedule for testing.
+		NewKeyPeriod:       100 * time.Millisecond,
+		DeleteOldKeyPeriod: 100 * time.Millisecond,
+	}
+
+	rotationServer, err := keyrotation.NewServer(keyRotationConfig, env)
+	if err != nil {
+		tb.Fatal(err)
+	}
+	mux.Handle("/key-rotation/", http.StripPrefix("/key-rotation", rotationServer.Routes(ctx)))
+
 	// Publish
 	publishConfig := &publish.Config{
-		RevisionToken: revision.Config{
-			KeyID:     "tokenkey",
-			AAD:       []byte{1, 2, 3},
-			MinLength: 28,
-		},
+		RevisionToken:            revConfig,
 		MaxKeysOnPublish:         15,
 		MaxSameStartIntervalKeys: 2,
 		MaxIntervalAge:           360 * time.Hour,
@@ -288,7 +308,7 @@ func testServer(tb testing.TB) (*serverenv.ServerEnv, *http.Client) {
 	// Create a client
 	client := testClient(tb, srv)
 
-	return env, client
+	return env, client, jwtCfg
 }
 
 type prefixRoundTripper struct {
diff --git a/internal/keyrotation/config.go b/internal/keyrotation/config.go
index db39d01..c21dbd2 100644
--- a/internal/keyrotation/config.go
+++ b/internal/keyrotation/config.go
@@ -18,10 +18,10 @@ import (
 	"time"
 
 	"github.com/google/exposure-notifications-server/internal/database"
-	"github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/revision"
 	"github.com/google/exposure-notifications-server/internal/setup"
 	"github.com/google/exposure-notifications-server/pkg/keys"
+	"github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/secrets"
 )
 
diff --git a/internal/keyrotation/rotate.go b/internal/keyrotation/rotate.go
index e61ad5b..1f54a01 100644
--- a/internal/keyrotation/rotate.go
+++ b/internal/keyrotation/rotate.go
@@ -62,6 +62,7 @@ func (s *Server) handleRotateKeys(ctx context.Context) http.HandlerFunc {
 
 func (s *Server) doRotate(ctx context.Context) error {
 	metrics := s.env.MetricsExporter(ctx)
+	logger := logging.FromContext(ctx).Named("keyrotation.doRotate")
 
 	effectiveID, allowed, err := s.revisionDB.GetAllowedRevisionKeys(ctx)
 	if err != nil {
@@ -77,6 +78,7 @@ func (s *Server) doRotate(ctx context.Context) error {
 		}
 		effectiveID = key.KeyID
 		previousCreated = key.CreatedAt
+		logger.Info("Created new revision key.")
 		metrics.WriteInt("revision-keys-created", true, 1)
 	} else {
 		previousCreated = allowed[0].CreatedAt
@@ -92,8 +94,10 @@ func (s *Server) doRotate(ctx context.Context) error {
 		}
 		previousCreated = key.CreatedAt
 	}
-
-	metrics.WriteInt("revision-keys-deleted", true, deleted)
+	if deleted > 0 {
+		logger.Infof("Deleted %d old revision keys.", deleted)
+		metrics.WriteInt("revision-keys-deleted", true, deleted)
+	}
 	return result
 }
 
diff --git a/internal/observability/observability.go b/internal/observability/observability.go
deleted file mode 100644
index ae1d746..0000000
--- a/internal/observability/observability.go
+++ /dev/null
@@ -1,81 +0,0 @@
-// Copyright 2020 Google LLC
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// Package observability sets up and configures observability tools.
-package observability
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"contrib.go.opencensus.io/exporter/ocagent"
-	"contrib.go.opencensus.io/exporter/stackdriver"
-	"github.com/google/exposure-notifications-server/internal/logging"
-)
-
-// Exporter defines the minimum shared functionality for an observability exporter
-// used by this application.
-type Exporter interface {
-	InitExportOnce() error
-	Flush()
-}
-
-// NewFromEnv returns the observability exporter given the provided configuration, or an error
-// if it failed to be created.
-func NewFromEnv(ctx context.Context, config *Config) (Exporter, error) {
-	switch config.ExporterType {
-	default:
-		return &NoopExporter{}, nil
-
-	case ExporterNoop:
-		return &NoopExporter{}, nil
-
-	case ExporterStackdriver:
-		if config.StackdriverConfig.ProjectID == "" {
-			return nil, fmt.Errorf("configuration PROJECT_ID is required to use the Stackdriver observability exporter")
-		}
-		logger := logging.FromContext(ctx).Named("stackdriver")
-
-		monitoredResource := NewStackdriverMonitoredResoruce(&config.StackdriverConfig)
-
-		sde, err := stackdriver.NewExporter(stackdriver.Options{
-			ProjectID:         config.StackdriverConfig.ProjectID,
-			ReportingInterval: time.Minute, // stackdriver export interval minimum
-			MonitoredResource: monitoredResource,
-			OnError: func(err error) {
-				logger.Errorf("stackdriver export error: %v", err)
-			},
-		})
-		if err != nil {
-			return nil, fmt.Errorf("failed to create Stackdriver observability exporter: %v", err)
-		}
-		return &GenericExporter{sde, config.TraceProbabilitySampleRate, func() { sde.Flush() }}, nil
-
-	case ExporterOCAgent, ExporterPrometheus:
-		var opts []ocagent.ExporterOption
-		if config.OCAgentConfig.Insecure {
-			opts = append(opts, ocagent.WithInsecure())
-		}
-		if config.OCAgentConfig.Endpoint != "" {
-			opts = append(opts, ocagent.WithAddress(config.OCAgentConfig.Endpoint))
-		}
-
-		oce, err := ocagent.NewExporter(opts...)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create OpenCensus observability exporter: %v", err)
-		}
-		return &GenericExporter{oce, config.TraceProbabilitySampleRate, func() { oce.Flush() }}, nil
-	}
-}
diff --git a/internal/publish/config.go b/internal/publish/config.go
index a858cd5..40b0898 100644
--- a/internal/publish/config.go
+++ b/internal/publish/config.go
@@ -20,12 +20,12 @@ import (
 
 	"github.com/google/exposure-notifications-server/internal/authorizedapp"
 	"github.com/google/exposure-notifications-server/internal/database"
-	"github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/publish/model"
 	"github.com/google/exposure-notifications-server/internal/revision"
 	"github.com/google/exposure-notifications-server/internal/setup"
 	"github.com/google/exposure-notifications-server/internal/verification"
 	"github.com/google/exposure-notifications-server/pkg/keys"
+	"github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/secrets"
 )
 
diff --git a/internal/publish/model/exposure_model.go b/internal/publish/model/exposure_model.go
index d8a325f..db34e5e 100644
--- a/internal/publish/model/exposure_model.go
+++ b/internal/publish/model/exposure_model.go
@@ -53,7 +53,7 @@ type Exposure struct {
 	LocalProvenance  bool
 	FederationSyncID int64
 
-	// These fileds are nullable to maintain backwards compatibility with
+	// These fields are nullable to maintain backwards compatibility with
 	// older versions that predate their existence.
 	HealthAuthorityID     *int64
 	ReportType            string
diff --git a/internal/publish/publish.go b/internal/publish/publish.go
index 94fbb8c..7dec68c 100644
--- a/internal/publish/publish.go
+++ b/internal/publish/publish.go
@@ -249,6 +249,7 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 
 	// Examine the revision token. It is expected that it is missing in most cases.
 	var token *pb.RevisionTokenData
+	decryptFail := false
 	if len(data.RevisionToken) != 0 {
 		encryptedToken, err := base64util.DecodeString(data.RevisionToken)
 		if err != nil {
@@ -256,8 +257,9 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 		} else {
 			token, err = h.tokenManager.UnmarshalRevisionToken(ctx, encryptedToken, h.tokenAAD)
 			if err != nil {
-				logger.Errorf("unable to unmarshall / descrypt revision token: %v", err)
+				logger.Errorf("unable to unmarshall / decrypt revision token: %v", err)
 				token = nil // just in case.
+				decryptFail = true
 			}
 		}
 	}
@@ -283,14 +285,14 @@ func (h *PublishHandler) process(ctx context.Context, data *verifyapi.Publish, b
 		var logMessage, errorMessage, errorCode string
 		metric := "publish-revision-token-issue"
 		switch {
+		case decryptFail || errors.Is(err, database.ErrExistingKeyNotInToken) || errors.Is(err, database.ErrRevisionTokenMetadataMismatch):
+			logMessage = fmt.Sprintf("revision token present, but invalid: %v", err)
+			errorMessage = "revision token is invalid"
+			errorCode = verifyapi.ErrorInvalidRevisionToken
 		case errors.Is(err, database.ErrNoRevisionToken):
 			logMessage = "no revision token"
 			errorMessage = "no revision token, but sent existing keys"
 			errorCode = verifyapi.ErrorMissingRevisionToken
-		case errors.Is(err, database.ErrExistingKeyNotInToken) || errors.Is(err, database.ErrRevisionTokenMetadataMismatch):
-			logMessage = fmt.Sprintf("revision token present, but invalid: %v", err)
-			errorMessage = "revision token is invalid"
-			errorCode = verifyapi.ErrorInvalidRevisionToken
 		default:
 			logMessage = fmt.Sprintf("error writing exposure record: %v", err)
 			errorMessage = http.StatusText(http.StatusInternalServerError)
diff --git a/internal/serverenv/env.go b/internal/serverenv/env.go
index 36cabbe..caf86f8 100644
--- a/internal/serverenv/env.go
+++ b/internal/serverenv/env.go
@@ -23,9 +23,9 @@ import (
 	"github.com/google/exposure-notifications-server/internal/authorizedapp"
 	"github.com/google/exposure-notifications-server/internal/database"
 	"github.com/google/exposure-notifications-server/internal/metrics"
-	"github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/storage"
 	"github.com/google/exposure-notifications-server/pkg/keys"
+	"github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/secrets"
 )
 
@@ -178,7 +178,9 @@ func (s *ServerEnv) Close(ctx context.Context) error {
 	}
 
 	if s.observabilityExporter != nil {
-		s.observabilityExporter.Flush()
+		if err := s.observabilityExporter.Close(); err != nil {
+			return nil
+		}
 	}
 
 	return nil
diff --git a/internal/setup/setup.go b/internal/setup/setup.go
index 81f9ab4..28574f8 100644
--- a/internal/setup/setup.go
+++ b/internal/setup/setup.go
@@ -23,10 +23,10 @@ import (
 	"github.com/google/exposure-notifications-server/internal/database"
 	"github.com/google/exposure-notifications-server/internal/logging"
 	"github.com/google/exposure-notifications-server/internal/metrics"
-	"github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/serverenv"
 	"github.com/google/exposure-notifications-server/internal/storage"
 	"github.com/google/exposure-notifications-server/pkg/keys"
+	"github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/secrets"
 	"github.com/sethvargo/go-envconfig"
 )
@@ -170,18 +170,21 @@ func SetupWith(ctx context.Context, config interface{}, l envconfig.Lookuper) (*
 	// Configure and initialize the observability exporter.
 	if provider, ok := config.(ObservabilityExporterConfigProvider); ok {
 		logger.Info("configuring observability exporter")
+
 		oeConfig := provider.ObservabilityExporterConfig()
 		oe, err := observability.NewFromEnv(ctx, oeConfig)
 		if err != nil {
-			return nil, fmt.Errorf("unable to create ObservabilityExporter provider: %v", err)
+			return nil, fmt.Errorf("unable to create observability provider: %v", err)
 		}
-		if err := oe.InitExportOnce(); err != nil {
-			return nil, fmt.Errorf("error initializing observability exporter: %v", err)
+		if err := oe.StartExporter(); err != nil {
+			return nil, fmt.Errorf("failed to start observability: %w", err)
 		}
+		exporter := serverenv.WithObservabilityExporter(oe)
 
-		serverEnvOpts = append(serverEnvOpts, serverenv.WithObservabilityExporter(oe))
+		// Update serverEnv setup.
+		serverEnvOpts = append(serverEnvOpts, exporter)
 
-		logger.Infow("observability exporter", "config", oeConfig)
+		logger.Infow("observability", "config", oeConfig)
 	}
 
 	// Configure blob storage.
diff --git a/pkg/api/v1/exposure_types.go b/pkg/api/v1/exposure_types.go
index 4650ee4..766263c 100644
--- a/pkg/api/v1/exposure_types.go
+++ b/pkg/api/v1/exposure_types.go
@@ -62,7 +62,7 @@ const (
 //  - Android: The App Package AppPackageName
 //  - iOS: The BundleID
 // verificationPayload: The Verification Certificate from a verification server.
-// hmacKey: the device generated secret that is used to recalcualte the HMAC value
+// hmacKey: the device generated secret that is used to recalculate the HMAC value
 //  that is present in the verification payload.
 //
 // symptomOnsetInterval: An interval number that aligns with the symptom onset date.
@@ -77,8 +77,8 @@ const (
 //  keys of a "traveler" who has left the home region represented by this server
 //  (or by the home health authority in case of a multi-tenant installation).
 //
-// revisionToken: An opaque string that must be passed in-tact from on additional
-//   publish requests from the same device, there the same TEKs may be published
+// revisionToken: An opaque string that must be passed intact on additional
+//   publish requests from the same device, where the same TEKs may be published
 //   again.
 //
 // Padding: random base64 encoded data to obscure the request size. The server will
diff --git a/pkg/api/v1alpha1/exposure_types.go b/pkg/api/v1alpha1/exposure_types.go
index 8b492f7..b797c50 100644
--- a/pkg/api/v1alpha1/exposure_types.go
+++ b/pkg/api/v1alpha1/exposure_types.go
@@ -45,7 +45,7 @@ const (
 //  - Android: The App Package AppPackageName
 //  - iOS: The BundleID
 // VerifcationPayload: The Verification Certificate from a verification server.
-// HMACKey: the device generated secret that is used to recalcualte the HMAC value
+// HMACKey: the device generated secret that is used to recalculate the HMAC value
 //  that is present in the verification payload.
 //
 // SymptomOnsetInterval: An interval number that aligns with the symptom onset date.
@@ -56,8 +56,8 @@ const (
 //  - Does not have to be within range of any of the provided keys (i.e. future
 //    key uploads)
 //
-// revisionToken: An opaque string that must be passed in-tact from on additional
-//   publish requests from the same device, there the same TEKs may be published
+// revisionToken: An opaque string that must be passed intact on additional
+//   publish requests from the same device, where the same TEKs may be published
 //   again.
 //
 // Padding: random base64 encoded data to obscure the request size.
diff --git a/pkg/keys/in_memory.go b/pkg/keys/in_memory.go
index 47498ea..5b43ae3 100644
--- a/pkg/keys/in_memory.go
+++ b/pkg/keys/in_memory.go
@@ -27,6 +27,12 @@ import (
 	"sync"
 )
 
+var _ KeyManager = (*InMemory)(nil)
+var _ SigningKeyCreator = (*InMemory)(nil)
+var _ SigningKeyAdder = (*InMemory)(nil)
+var _ EncryptionKeyCreator = (*InMemory)(nil)
+var _ EncryptionKeyAdder = (*InMemory)(nil)
+
 // InMemory is useful for testing. Do NOT use in a running system as all
 // keys are only kept in memory and will be lost across server reboots.
 type InMemory struct {
@@ -43,42 +49,68 @@ func NewInMemory(ctx context.Context) (*InMemory, error) {
 	}, nil
 }
 
-// AddSigningKey generates a new ECDSA P256 Signing Key identified by
+// CreateSigningKey generates a new ECDSA P256 Signing Key identified by
 // the provided keyID
-func (k *InMemory) AddSigningKey(keyID string) error {
+func (k *InMemory) CreateSigningKey(keyID string) (*ecdsa.PrivateKey, error) {
 	k.mu.Lock()
 	defer k.mu.Unlock()
 
 	if _, ok := k.signingKeys[keyID]; ok {
-		return fmt.Errorf("key already exists: %v", keyID)
+		return nil, fmt.Errorf("key already exists: %v", keyID)
 	}
 
 	pk, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 	if err != nil {
-		return fmt.Errorf("unable to generate private key: %w", err)
+		return nil, fmt.Errorf("unable to generate private key: %w", err)
+	}
+
+	k.signingKeys[keyID] = pk
+	return pk, nil
+}
+
+// AddSigningKey adds a new ECDSA P256 Signing Key identified by the provided
+// keyID.
+func (k *InMemory) AddSigningKey(keyID string, pk *ecdsa.PrivateKey) error {
+	k.mu.Lock()
+	defer k.mu.Unlock()
+
+	if _, ok := k.signingKeys[keyID]; ok {
+		return fmt.Errorf("key already exists: %v", keyID)
 	}
 
 	k.signingKeys[keyID] = pk
 	return nil
 }
 
-// AddEncryptionKey generates a new encryption key identified by
-// the provided keyID.
-func (k *InMemory) AddEncryptionKey(keyID string) error {
+// CreateEncryptionKey generates and stores new encryption key identified by the
+// provided keyID.
+func (k *InMemory) CreateEncryptionKey(keyID string) ([]byte, error) {
 	k.mu.Lock()
 	defer k.mu.Unlock()
 
 	if _, ok := k.cryptoKeys[keyID]; ok {
-		return fmt.Errorf("key already exists: %v", keyID)
+		return nil, fmt.Errorf("key already exists: %v", keyID)
 	}
 
 	key := make([]byte, 32)
 	if _, err := io.ReadFull(rand.Reader, key); err != nil {
-		return fmt.Errorf("failed to read random bytes: %w", err)
+		return nil, fmt.Errorf("failed to read random bytes: %w", err)
 	}
 
 	k.cryptoKeys[keyID] = key
+	return key, nil
+}
+
+// AddEncryptionKey stores the key on the system.
+func (k *InMemory) AddEncryptionKey(keyID string, key []byte) error {
+	k.mu.Lock()
+	defer k.mu.Unlock()
 
+	if _, ok := k.cryptoKeys[keyID]; ok {
+		return fmt.Errorf("key already exists: %v", keyID)
+	}
+
+	k.cryptoKeys[keyID] = key
 	return nil
 }
 
diff --git a/pkg/keys/keys.go b/pkg/keys/keys.go
index 1ae1e93..d8bdf15 100644
--- a/pkg/keys/keys.go
+++ b/pkg/keys/keys.go
@@ -23,6 +23,7 @@ package keys
 import (
 	"context"
 	"crypto"
+	"crypto/ecdsa"
 	"fmt"
 )
 
@@ -48,6 +49,26 @@ type KeyManager interface {
 	Decrypt(ctx context.Context, keyID string, ciphertext []byte, aad []byte) ([]byte, error)
 }
 
+// EncryptionKeyCreator supports creating encryption keys.
+type EncryptionKeyCreator interface {
+	CreateEncryptionKey(string) ([]byte, error)
+}
+
+// EncryptionKeyAdder supports creating encryption keys.
+type EncryptionKeyAdder interface {
+	AddEncryptionKey(string, []byte) error
+}
+
+// SigningKeyCreator supports creating signing keys.
+type SigningKeyCreator interface {
+	CreateSigningKey(string) (*ecdsa.PrivateKey, error)
+}
+
+// SigningKeyAdder supports creating signing keys.
+type SigningKeyAdder interface {
+	AddSigningKey(string, *ecdsa.PrivateKey) error
+}
+
 // KeyManagerFor returns the appropriate key manager for the given type.
 func KeyManagerFor(ctx context.Context, typ KeyManagerType) (KeyManager, error) {
 	switch typ {
diff --git a/internal/observability/config.go b/pkg/observability/config.go
similarity index 76%
rename from internal/observability/config.go
rename to pkg/observability/config.go
index 316d477..cd39a3e 100644
--- a/internal/observability/config.go
+++ b/pkg/observability/config.go
@@ -29,30 +29,31 @@ const (
 type Config struct {
 	ExporterType ExporterType `env:"OBSERVABILITY_EXPORTER, default=STACKDRIVER"`
 
-	OpenCensusConfig
-	StackdriverConfig
-	OCAgentConfig
+	OpenCensus  *OpenCensusConfig
+	Stackdriver *StackdriverConfig
 }
 
 // OpenCensusConfig holds the configuration options for the open census exporter
 type OpenCensusConfig struct {
-	TraceProbabilitySampleRate float64 `env:"TRACE_PROBABILITY, default=0.40"`
+	SampleRate float64 `env:"TRACE_PROBABILITY, default=0.40"`
+
+	Insecure bool   `env:"OCAGENT_INSECURE"`
+	Endpoint string `env:"OCAGENT_TRACE_EXPORTER_ENDPOINT"`
 }
 
 // StackdriverConfig holds the configuration options for the stackdriver exporter
 type StackdriverConfig struct {
-	ProjectID string `env:"PROJECT_ID, default=$GOOGLE_CLOUD_PROJECT"`
-	// Knative+Cloud Run container contract env vars
-	// https://cloud.google.com/run/docs/reference/container-contract#env-vars
-	// If present, can be used to configured the Stackdriver MonitoredResource correctly.
-	Service  string `env:"K_SERVICE"`
-	Revision string `env:"K_REVISION"`
+	SampleRate float64 `env:"TRACE_PROBABILITY, default=0.40"`
 
-	Namespace string `env:"K_CONFIGURATION, default=ens"`
-}
+	ProjectID string `env:"PROJECT_ID, default=$GOOGLE_CLOUD_PROJECT"`
 
-// OCAgentConfig holds the configuration options for the default opencensus exporter
-type OCAgentConfig struct {
-	Insecure bool   `env:"OCAGENT_INSECURE, default=true"`
-	Endpoint string `env:"OCAGENT_TRACE_EXPORTER_ENDPOINT"`
+	// Knative+Cloud Run container contract envvars:
+	//
+	// https://cloud.google.com/run/docs/reference/container-contract#env-vars
+	//
+	// If present, can be used to configured the Stackdriver MonitoredResource
+	// correctly.
+	Service   string `env:"K_SERVICE"`
+	Revision  string `env:"K_REVISION"`
+	Namespace string `env:"K_CONFIGURATION, default=en"`
 }
diff --git a/internal/observability/noop.go b/pkg/observability/noop.go
similarity index 69%
rename from internal/observability/noop.go
rename to pkg/observability/noop.go
index c66633d..93a4e63 100644
--- a/internal/observability/noop.go
+++ b/pkg/observability/noop.go
@@ -15,14 +15,22 @@
 // Package observability sets up and configures observability tools.
 package observability
 
+import "context"
+
 // Compile-time check to verify implements interface.
-var _ Exporter = (*NoopExporter)(nil)
+var _ Exporter = (*noopExporter)(nil)
+
+// noopExporter is an observability exporter that does nothing.
+type noopExporter struct{}
 
-// NoopExporter is an observability exporter that does nothing.
-type NoopExporter struct{}
+func NewNoop(_ context.Context) (Exporter, error) {
+	return &noopExporter{}, nil
+}
 
-func (g *NoopExporter) InitExportOnce() error {
+func (g *noopExporter) StartExporter() error {
 	return nil
 }
 
-func (g *NoopExporter) Flush() {}
+func (g *noopExporter) Close() error {
+	return nil
+}
diff --git a/internal/observability/generic.go b/pkg/observability/observability.go
similarity index 52%
rename from internal/observability/generic.go
rename to pkg/observability/observability.go
index 95102bd..425d907 100644
--- a/internal/observability/generic.go
+++ b/pkg/observability/observability.go
@@ -16,66 +16,50 @@
 package observability
 
 import (
+	"context"
 	"fmt"
-	"sync"
+	"io"
 
 	"go.opencensus.io/plugin/ocgrpc"
 	"go.opencensus.io/plugin/ochttp"
 	"go.opencensus.io/stats/view"
-	"go.opencensus.io/trace"
 )
 
-// Compile-time check to verify implements interface.
-var _ Exporter = (*GenericExporter)(nil)
-
-var initExporterOnce sync.Once
-
-type FlushFn func()
-
-type traceAndViewExporter interface {
-	trace.Exporter
-	view.Exporter
-}
-
-// GenericExporter is a standard implementation of an exporter that wraps the opencensus interfaces
-// with custom configuration
-type GenericExporter struct {
-	exporter   traceAndViewExporter
-	sampleRate float64
-	flushFn    FlushFn
-}
-
-func (g *GenericExporter) InitExportOnce() error {
-	var err error
-	initExporterOnce.Do(func() {
-		err = g.initExporter()
-	})
-	return err
-}
-
-func (g *GenericExporter) Flush() {
-	g.flushFn()
+// Exporter defines the minimum shared functionality for an observability exporter
+// used by this application.
+type Exporter interface {
+	io.Closer
+	StartExporter() error
 }
 
-func (g *GenericExporter) initExporter() error {
-	if g.exporter == nil {
-		return nil
+// NewFromEnv returns the observability exporter given the provided configuration, or an error
+// if it failed to be created.
+func NewFromEnv(ctx context.Context, config *Config) (Exporter, error) {
+	switch config.ExporterType {
+	case ExporterNoop:
+		return NewNoop(ctx)
+	case ExporterStackdriver:
+		return NewStackdriver(ctx, config.Stackdriver)
+	case ExporterOCAgent, ExporterPrometheus:
+		return NewOpenCensus(ctx, config.OpenCensus)
+	default:
+		return nil, fmt.Errorf("unknown observability exporter type %v", config.ExporterType)
 	}
-	trace.ApplyConfig(trace.Config{
-		DefaultSampler: trace.ProbabilitySampler(g.sampleRate),
-	})
-	trace.RegisterExporter(g.exporter)
-	view.RegisterExporter(g.exporter)
+}
 
+// registerViews registers the necessary tracing views.
+func registerViews() error {
 	// Record the various HTTP view to collect metrics.
 	httpViews := append(ochttp.DefaultServerViews, ochttp.DefaultClientViews...)
 	if err := view.Register(httpViews...); err != nil {
-		return fmt.Errorf("failed to register http views for observability exporter: %v", err)
+		return fmt.Errorf("failed to register http views: %w", err)
 	}
+
 	// Register the various gRPC views to collect metrics.
 	gRPCViews := append(ocgrpc.DefaultServerViews, ocgrpc.DefaultClientViews...)
 	if err := view.Register(gRPCViews...); err != nil {
-		return fmt.Errorf("failed to register grpc views for observability exporter: %v", err)
+		return fmt.Errorf("failed to register grpc views: %w", err)
 	}
+
 	return nil
 }
diff --git a/pkg/observability/opencensus.go b/pkg/observability/opencensus.go
new file mode 100644
index 0000000..24ce19c
--- /dev/null
+++ b/pkg/observability/opencensus.go
@@ -0,0 +1,76 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package observability
+
+import (
+	"context"
+	"fmt"
+
+	"contrib.go.opencensus.io/exporter/ocagent"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/trace"
+)
+
+var _ Exporter = (*opencensusExporter)(nil)
+
+type opencensusExporter struct {
+	exporter *ocagent.Exporter
+	config   *OpenCensusConfig
+}
+
+// NewOpenCensus creates a new metrics and trace exporter for OpenCensus.
+func NewOpenCensus(ctx context.Context, config *OpenCensusConfig) (Exporter, error) {
+	var opts []ocagent.ExporterOption
+	if config.Insecure {
+		opts = append(opts, ocagent.WithInsecure())
+	}
+	if config.Endpoint != "" {
+		opts = append(opts, ocagent.WithAddress(config.Endpoint))
+	}
+
+	oc, err := ocagent.NewExporter(opts...)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create opencensus exporter: %w", err)
+	}
+	return &opencensusExporter{oc, config}, nil
+}
+
+// StartExporter starts the exporter.
+func (e *opencensusExporter) StartExporter() error {
+	trace.ApplyConfig(trace.Config{
+		DefaultSampler: trace.ProbabilitySampler(e.config.SampleRate),
+	})
+	trace.RegisterExporter(e.exporter)
+	view.RegisterExporter(e.exporter)
+
+	if err := registerViews(); err != nil {
+		return fmt.Errorf("failed to start opencensus exporter: %w", err)
+	}
+
+	return nil
+}
+
+// Close halts the exporter.
+func (e *opencensusExporter) Close() error {
+	if err := e.exporter.Stop(); err != nil {
+		return fmt.Errorf("failed to stop exporter: %w", err)
+	}
+
+	// Unregister the exporter
+	trace.UnregisterExporter(e.exporter)
+	view.UnregisterExporter(e.exporter)
+
+	return nil
+}
diff --git a/internal/observability/resource.go b/pkg/observability/resource.go
similarity index 90%
rename from internal/observability/resource.go
rename to pkg/observability/resource.go
index 46e297a..c195c78 100644
--- a/internal/observability/resource.go
+++ b/pkg/observability/resource.go
@@ -22,9 +22,9 @@ import (
 	"github.com/google/uuid"
 )
 
-var _ monitoredresource.Interface = (*StackdriverMonitoredResoruce)(nil)
+var _ monitoredresource.Interface = (*stackdriverMonitoredResource)(nil)
 
-type StackdriverMonitoredResoruce struct {
+type stackdriverMonitoredResource struct {
 	resource string
 	labels   map[string]string
 }
@@ -65,12 +65,12 @@ func NewStackdriverMonitoredResoruce(c *StackdriverConfig) monitoredresource.Int
 	}
 	labels["namespace"] = c.Namespace
 
-	return &StackdriverMonitoredResoruce{
+	return &stackdriverMonitoredResource{
 		resource: resource,
 		labels:   labels,
 	}
 }
 
-func (s *StackdriverMonitoredResoruce) MonitoredResource() (string, map[string]string) {
+func (s *stackdriverMonitoredResource) MonitoredResource() (string, map[string]string) {
 	return s.resource, s.labels
 }
diff --git a/pkg/observability/stackdriver.go b/pkg/observability/stackdriver.go
new file mode 100644
index 0000000..fa9e066
--- /dev/null
+++ b/pkg/observability/stackdriver.go
@@ -0,0 +1,90 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package observability sets up and configures observability tools.
+package observability
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"contrib.go.opencensus.io/exporter/stackdriver"
+	"github.com/google/exposure-notifications-server/internal/logging"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/trace"
+)
+
+var _ Exporter = (*stackdriverExporter)(nil)
+
+type stackdriverExporter struct {
+	exporter *stackdriver.Exporter
+	config   *StackdriverConfig
+}
+
+// NewStackdriver creates a new metrics and trace exporter for Stackdriver.
+func NewStackdriver(ctx context.Context, config *StackdriverConfig) (Exporter, error) {
+	logger := logging.FromContext(ctx).Named("stackdriver")
+
+	projectID := config.ProjectID
+	if projectID == "" {
+		return nil, fmt.Errorf("missing PROJECT_ID in Stackdriver exporter")
+	}
+
+	monitoredResource := NewStackdriverMonitoredResoruce(config)
+
+	exporter, err := stackdriver.NewExporter(stackdriver.Options{
+		ProjectID:         projectID,
+		ReportingInterval: time.Minute, // stackdriver export interval minimum
+		MonitoredResource: monitoredResource,
+		OnError: func(err error) {
+			logger.Errorw("failed to export metric", "error", err)
+		},
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to create Stackdriver exporter: %w", err)
+	}
+	return &stackdriverExporter{exporter, config}, nil
+}
+
+// StartExporter starts the exporter.
+func (e *stackdriverExporter) StartExporter() error {
+	if err := e.exporter.StartMetricsExporter(); err != nil {
+		return fmt.Errorf("failed to start stackdriver exporter: %w", err)
+	}
+
+	trace.ApplyConfig(trace.Config{
+		DefaultSampler: trace.ProbabilitySampler(e.config.SampleRate),
+	})
+	trace.RegisterExporter(e.exporter)
+	view.RegisterExporter(e.exporter)
+
+	return nil
+}
+
+// Close halts the exporter.
+func (e *stackdriverExporter) Close() error {
+	// Flush any existing metrics
+	e.exporter.Flush()
+
+	// Stop the exporter
+	e.exporter.StopMetricsExporter()
+
+	// Unregister the exporter
+	trace.UnregisterExporter(e.exporter)
+	view.UnregisterExporter(e.exporter)
+
+	return nil
+}


Test patch:
diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go
index 88faa93..59848f8 100644
--- a/internal/integration/integration_test.go
+++ b/internal/integration/integration_test.go
@@ -46,7 +46,7 @@ func TestIntegration(t *testing.T) {
 	t.Parallel()
 
 	ctx := context.Background()
-	env, client, db := NewTestServer(t, 2*time.Second)
+	env, client, db, jwtCfg := NewTestServer(t, 2*time.Second)
 
 	// Set query criteria (used throughout)
 	criteria := publishdb.IterateExposuresCriteria{
@@ -64,9 +64,14 @@ func TestIntegration(t *testing.T) {
 	verification, salt := testutil.IssueJWT(t, jwtCfg)
 	payload.VerificationPayload = verification
 	payload.HMACKey = salt
+	var revisionToken string
 	if resp, err := client.PublishKeys(payload); err != nil {
 		t.Fatal(err)
 	} else {
+		if resp.RevisionToken == "" {
+			t.Fatal("empty revision token")
+		}
+		revisionToken = resp.RevisionToken
 		t.Logf("response: %+v", resp)
 	}
 
@@ -208,6 +213,26 @@ func TestIntegration(t *testing.T) {
 		}
 	}
 
+	// Rotate Keys. Should Genereate a new key.
+	time.Sleep(200 * time.Millisecond) // Ensure DeleteOldKeyPeriod is elapsed
+	if err := client.RotateKeys(); err != nil {
+		t.Fatalf("Error rotating keys: %v", err)
+	}
+
+	// Rotate Keys. Should Delete the original key.
+	time.Sleep(200 * time.Millisecond) // Ensure DeleteOldKeyPeriod is elapsed
+	if err := client.RotateKeys(); err != nil {
+		t.Fatalf("Error rotating keys: %v", err)
+	}
+
+	// Re-publish with the original token. This key is now not-allowed.
+	payload.RevisionToken = revisionToken
+	if _, err := client.PublishKeys(payload); err == nil {
+		t.Fatal(err)
+	} else if !strings.Contains(err.Error(), verifyapi.ErrorInvalidRevisionToken) {
+		t.Fatal(err)
+	}
+
 	// Publish some new keys so we can generate a new batch
 	keys = util.GenerateExposureKeys(3, -1, false)
 	payload.Keys = keys
diff --git a/internal/keyrotation/rotate_test.go b/internal/keyrotation/rotate_test.go
index f1587cc..41b4147 100644
--- a/internal/keyrotation/rotate_test.go
+++ b/internal/keyrotation/rotate_test.go
@@ -37,7 +37,9 @@ func TestRotateKeys(t *testing.T) {
 
 	kms, _ := keys.NewInMemory(context.Background())
 	keyID := "testKeyID"
-	kms.AddEncryptionKey(keyID)
+	if _, err := kms.CreateEncryptionKey(keyID); err != nil {
+		t.Fatal(err)
+	}
 	config := &Config{
 		RevisionToken:      revision.Config{KeyID: keyID},
 		DeleteOldKeyPeriod: 14 * 24 * time.Hour, // two weeks
diff --git a/internal/keyrotation/server_test.go b/internal/keyrotation/server_test.go
index 7511d18..3a36d5f 100644
--- a/internal/keyrotation/server_test.go
+++ b/internal/keyrotation/server_test.go
@@ -62,7 +62,9 @@ func TestNewRotationHandler(t *testing.T) {
 			t.Parallel()
 
 			keyID := "test" + t.Name()
-			kms.AddEncryptionKey(keyID)
+			if _, err := kms.CreateEncryptionKey(keyID); err != nil {
+				t.Fatal(err)
+			}
 			config := &Config{
 				RevisionToken: revision.Config{KeyID: keyID},
 			}
diff --git a/internal/performance/export_test.go b/internal/performance/export_test.go
index c8878b5..edd87f5 100644
--- a/internal/performance/export_test.go
+++ b/internal/performance/export_test.go
@@ -30,6 +30,7 @@ import (
 	publishdb "github.com/google/exposure-notifications-server/internal/publish/database"
 	publishmodel "github.com/google/exposure-notifications-server/internal/publish/model"
 	"github.com/google/exposure-notifications-server/internal/storage"
+	testutil "github.com/google/exposure-notifications-server/internal/utils"
 	verifyapi "github.com/google/exposure-notifications-server/pkg/api/v1"
 	"github.com/google/exposure-notifications-server/pkg/base64util"
 	"github.com/google/exposure-notifications-server/pkg/util"
@@ -60,14 +61,16 @@ func TestExport(t *testing.T) {
 	makoQuickstore, cancel := setup(t)
 	defer cancel(context.Background())
 
-	env, client, db := integration.NewTestServer(t, exportPeriod)
+	env, client, db, jwtCfg := integration.NewTestServer(t, exportPeriod)
+	keys := util.GenerateExposureKeys(keysPerPublish, -1, false)
 	payload := &verifyapi.Publish{
-		Keys:              util.GenerateExposureKeys(keysPerPublish, -1, false),
+		Keys:              keys,
 		HealthAuthorityID: "com.example.app",
-
-		// TODO: hook up verification
-		VerificationPayload: "TODO: ",
 	}
+	jwtCfg.ExposureKeys = keys
+	verification, salt := testutil.IssueJWT(t, jwtCfg)
+	payload.VerificationPayload = verification
+	payload.HMACKey = salt
 	if _, err := client.PublishKeys(payload); err != nil {
 		t.Fatal(err)
 	}
diff --git a/internal/publish/publish_test.go b/internal/publish/publish_test.go
index 6445efa..644b764 100644
--- a/internal/publish/publish_test.go
+++ b/internal/publish/publish_test.go
@@ -371,7 +371,9 @@ func TestPublishWithBypass(t *testing.T) {
 			t.Fatalf("can't make kms: %v", err)
 		}
 		keyID := "rev"
-		kms.AddEncryptionKey(keyID)
+		if _, err := kms.CreateEncryptionKey(keyID); err != nil {
+			t.Fatal(err)
+		}
 		tokenAAD := make([]byte, 16)
 		if _, err := rand.Read(tokenAAD); err != nil {
 			t.Fatalf("not enough entropy: %v", err)
@@ -682,7 +684,9 @@ func TestKeyRevision(t *testing.T) {
 		t.Fatalf("can't make kms: %v", err)
 	}
 	keyID := "rev"
-	kms.AddEncryptionKey(keyID)
+	if _, err := kms.CreateEncryptionKey(keyID); err != nil {
+		t.Fatal(err)
+	}
 	tokenAAD := make([]byte, 16)
 	if _, err := rand.Read(tokenAAD); err != nil {
 		t.Fatalf("not enough entropy: %v", err)
diff --git a/internal/revision/database/revision_test.go b/internal/revision/database/revision_test.go
index a020e94..2e1d8a2 100644
--- a/internal/revision/database/revision_test.go
+++ b/internal/revision/database/revision_test.go
@@ -40,7 +40,7 @@ func TestRevisionKey(t *testing.T) {
 		t.Fatalf("unable to cerate in memory KMS")
 	}
 	keyID := "funkey"
-	if err := kms.AddEncryptionKey(keyID); err != nil {
+	if _, err := kms.CreateEncryptionKey(keyID); err != nil {
 		t.Fatalf("unable to generate key: %v", err)
 	}
 
@@ -76,7 +76,7 @@ func TestMultipleRevisionKeys(t *testing.T) {
 		t.Fatalf("unable to cerate in memory KMS")
 	}
 	keyID := "funkey"
-	if err := kms.AddEncryptionKey(keyID); err != nil {
+	if _, err := kms.CreateEncryptionKey(keyID); err != nil {
 		t.Fatalf("unable to generate key: %v", err)
 	}
 
diff --git a/internal/revision/revision_test.go b/internal/revision/revision_test.go
index 379dc66..91817eb 100644
--- a/internal/revision/revision_test.go
+++ b/internal/revision/revision_test.go
@@ -128,7 +128,7 @@ func TestEncryptDecrypt(t *testing.T) {
 		t.Fatalf("unable to cerate in memory KMS")
 	}
 	keyID := "skeleton"
-	if err := kms.AddEncryptionKey(keyID); err != nil {
+	if _, err := kms.CreateEncryptionKey(keyID); err != nil {
 		t.Fatalf("unable to generate key: %v", err)
 	}
 	key := make([]byte, 32)
diff --git a/internal/setup/setup_test.go b/internal/setup/setup_test.go
index f20afa5..809bfcf 100644
--- a/internal/setup/setup_test.go
+++ b/internal/setup/setup_test.go
@@ -23,10 +23,10 @@ import (
 
 	"github.com/google/exposure-notifications-server/internal/authorizedapp"
 	"github.com/google/exposure-notifications-server/internal/database"
-	"github.com/google/exposure-notifications-server/internal/observability"
 	"github.com/google/exposure-notifications-server/internal/setup"
 	"github.com/google/exposure-notifications-server/internal/storage"
 	"github.com/google/exposure-notifications-server/pkg/keys"
+	"github.com/google/exposure-notifications-server/pkg/observability"
 	"github.com/google/exposure-notifications-server/pkg/secrets"
 	"github.com/sethvargo/go-envconfig"
 )
@@ -231,9 +231,10 @@ func TestSetupWith(t *testing.T) {
 		if oe == nil {
 			t.Errorf("expected observability exporter to exist")
 		}
-
-		if _, ok := oe.(*observability.NoopExporter); !ok {
-			t.Errorf("expected %T to be GenericExporter", oe)
-		}
+		defer func() {
+			if err := oe.Close(); err != nil {
+				t.Fatal(err)
+			}
+		}()
 	})
 }